Pgstac cql2text (#346)

* add support for cql2-text with pgstac

* update changelog

* peg pygeofilter tag, remove comments in docker-compose

Co-authored-by: Rob Emanuele <[email protected]>

Author: bitner

CHANGES.md

@@ -11,12 +11,15 @@
 ### Changed
 
 * update FastAPI requirement to allow version >=0.73 ([#337](https://github.com/stac-utils/stac-fastapi/pull/337))
+* Bump version of PGStac to 0.4.5  ([#346](https://github.com/stac-utils/stac-fastapi/pull/346))
+* Add support for PGStac Backend to use PyGeofilter to convert Get Request with cql2-text into cql2-json to send to PGStac backend ([#346](https://github.com/stac-utils/stac-fastapi/pull/346))
 
 ### Removed
 
 ### Fixed
 * Bumped uvicorn version to 0.17 (from >=0.12, <=0.14) to resolve security vulnerability related to websockets dependency version ([#343](https://github.com/stac-utils/stac-fastapi/pull/343))
 * `AttributeError` and/or missing properties when requesting the complete `properties`-field in searches. Added test. ([#339](https://github.com/stac-utils/stac-fastapi/pull/339))
+* Fixes issues (and adds tests) for issues caused by regression in pgstac ([#345](https://github.com/stac-utils/stac-fastapi/issues/345)
 
 
 ## [2.3.0]

Makefile

@@ -40,7 +40,7 @@ test-sqlalchemy: run-joplin-sqlalchemy
 
 .PHONY: test-pgstac
 test-pgstac:
-	$(run_pgstac) /bin/bash -c 'export && ./scripts/wait-for-it.sh database:5432 && cd /app/stac_fastapi/pgstac/tests/ && pytest'
+	$(run_pgstac) /bin/bash -c 'export && ./scripts/wait-for-it.sh database:5432 && cd /app/stac_fastapi/pgstac/tests/ && pytest -vvv --asyncio-mode=auto'
 
 .PHONY: run-database
 run-database:

docker-compose.yml

@@ -32,9 +32,6 @@ services:
   app-pgstac:
     container_name: stac-fastapi-pgstac
     image: stac-utils/stac-fastapi
-    build:
-      context: .
-      dockerfile: Dockerfile
     platform: linux/amd64
     environment:
       - APP_HOST=0.0.0.0
@@ -65,7 +62,7 @@ services:
 
   database:
     container_name: stac-db
-    image: ghcr.io/stac-utils/pgstac:v0.4.3
+    image: ghcr.io/stac-utils/pgstac:v0.4.5
     environment:
       - POSTGRES_USER=username
       - POSTGRES_PASSWORD=password

stac_fastapi/extensions/stac_fastapi/extensions/core/filter/request.py

@@ -20,19 +20,21 @@ class FilterLang(str, Enum):
 
     cql_json = "cql-json"
     cql2_json = "cql2-json"
-    cql_text = "cql-text"
+    cql2_text = "cql2-text"
 
 
 @attr.s
 class FilterExtensionGetRequest(APIRequest):
     """Filter extension GET request model."""
 
     filter: Optional[str] = attr.ib(default=None)
+    filter_crs: Optional[str] = Field(alias="filter-crs", default=None)
+    filter_lang: Optional[FilterLang] = Field(alias="filter-lang", default="cql2-text")
 
 
 class FilterExtensionPostRequest(BaseModel):
     """Filter extension POST request model."""
 
     filter: Optional[Dict[str, Any]] = None
     filter_crs: Optional[str] = Field(alias="filter-crs", default=None)
-    filter_lang: Optional[FilterLang] = Field(alias="filter-lang", default=None)
+    filter_lang: Optional[FilterLang] = Field(alias="filter-lang", default="cql-json")

stac_fastapi/pgstac/setup.py

@@ -16,6 +16,7 @@
     "asyncpg",
     "buildpg",
     "brotli_asgi",
+    "pygeofilter @ git+https://github.com/geopython/[email protected]#egg=pygeofilter",
 ]
 
 extra_reqs = {
@@ -25,9 +26,8 @@
         "pytest-asyncio",
         "pre-commit",
         "requests",
-        "pypgstac==0.4.3",
+        "pypgstac==0.4.5",
         "httpx",
-        "shapely",
     ],
     "docs": ["mkdocs", "mkdocs-material", "pdocs"],
     "server": ["uvicorn[standard]==0.17.0"],
@@ -49,7 +49,7 @@
         "License :: OSI Approved :: MIT License",
     ],
     keywords="STAC FastAPI COG",
-    author=u"David Bitner",
+    author="David Bitner",
     author_email="[email protected]",
     url="https://github.com/stac-utils/stac-fastapi",
     license="MIT",

stac_fastapi/pgstac/stac_fastapi/pgstac/core.py

@@ -10,6 +10,8 @@
 from buildpg import render
 from fastapi import HTTPException
 from pydantic import ValidationError
+from pygeofilter.backends.cql2_json import to_cql2
+from pygeofilter.parsers.cql2_text import parse as parse_cql2_text
 from stac_pydantic.links import Relations
 from stac_pydantic.shared import MimeTypes
 from starlette.requests import Request
@@ -256,6 +258,8 @@ async def get_search(
         token: Optional[str] = None,
         fields: Optional[List[str]] = None,
         sortby: Optional[str] = None,
+        filter: Optional[str] = None,
+        filter_lang: Optional[str] = None,
         **kwargs,
     ) -> ItemCollection:
         """Cross catalog search (GET).
@@ -265,6 +269,15 @@ async def get_search(
         Returns:
             ItemCollection containing items which match the search criteria.
         """
+        request = kwargs["request"]
+        query_params = str(request.query_params)
+
+        # Kludgy fix because using factory does not allow alias for filter-lang
+        if filter_lang is None:
+            match = re.search(r"filter-lang=([a-z0-9-]+)", query_params, re.IGNORECASE)
+            if match:
+                filter_lang = match.group(1)
+
         # Parse request parameters
         base_args = {
             "collections": collections,
@@ -275,6 +288,12 @@ async def get_search(
             "query": orjson.loads(query) if query else query,
         }
 
+        if filter:
+            if filter_lang == "cql2-text":
+                ast = parse_cql2_text(filter)
+                base_args["filter"] = orjson.loads(to_cql2(ast))
+                base_args["filter-lang"] = "cql2-json"
+
         if datetime:
             base_args["datetime"] = datetime
 
@@ -304,9 +323,17 @@ async def get_search(
                     includes.add(field)
             base_args["fields"] = {"include": includes, "exclude": excludes}
 
+        # Remove None values from dict
+        clean = {}
+        for k, v in base_args.items():
+            if v is not None and v != []:
+                clean[k] = v
+
         # Do the request
         try:
-            search_request = self.post_request_model(**base_args)
-        except ValidationError:
-            raise HTTPException(status_code=400, detail="Invalid parameters provided")
+            search_request = self.post_request_model(**clean)
+        except ValidationError as e:
+            raise HTTPException(
+                status_code=400, detail=f"Invalid parameters provided {e}"
+            )
         return await self.post_search(search_request, request=kwargs["request"])

stac_fastapi/pgstac/stac_fastapi/pgstac/types/search.py

@@ -17,21 +17,10 @@ class PgstacSearch(BaseSearchPostRequest):
 
     @validator("filter_lang", pre=False, check_fields=False, always=True)
     def validate_query_uses_cql(cls, v, values):
-        """If using query syntax, forces cql-json."""
-        retval = v
-        if values.get("query", None) is not None:
-            retval = "cql-json"
-        if values.get("collections", None) is not None:
-            retval = "cql-json"
-        if values.get("ids", None) is not None:
-            retval = "cql-json"
-        if values.get("datetime", None) is not None:
-            retval = "cql-json"
-        if values.get("bbox", None) is not None:
-            retval = "cql-json"
-        if v == "cql2-json" and retval == "cql-json":
+        """Use of Query Extension is not allowed with cql2."""
+        if values.get("query", None) is not None and v != "cql-json":
             raise ValueError(
-                "query, collections, ids, datetime, and bbox"
-                "parameters are not available in cql2-json"
+                "Query extension is not available when using pgstac with cql2"
             )
-        return retval
+
+        return v

stac_fastapi/pgstac/tests/clients/test_postgres.py

@@ -97,7 +97,6 @@ async def test_delete_item(app_client, load_test_collection, load_test_item):
     item = load_test_item
 
     resp = await app_client.delete(f"/collections/{coll.id}/items/{item.id}")
-    print(resp.content)
     assert resp.status_code == 200
 
     resp = await app_client.get(f"/collections/{coll.id}/items/{item.id}")

stac_fastapi/pgstac/tests/resources/test_item.py

@@ -810,6 +810,98 @@ async def test_item_search_get_filter_extension_cql(
     )
 
 
+@pytest.mark.asyncio
+async def test_item_search_get_filter_extension_cql2(
+    app_client, load_test_data, load_test_collection
+):
+    """Test GET search with JSONB query (cql2 json filter extension)"""
+    test_item = load_test_data("test_item.json")
+    resp = await app_client.post(
+        f"/collections/{test_item['collection']}/items", json=test_item
+    )
+    assert resp.status_code == 200
+
+    second_test_item = load_test_data("test_item2.json")
+    resp = await app_client.post(
+        f"/collections/{test_item['collection']}/items", json=second_test_item
+    )
+    assert resp.status_code == 200
+
+    # EPSG is a JSONB key
+    params = {
+        "collections": [test_item["collection"]],
+        "filter-lang": "cql2-json",
+        "filter": {
+            "op": "gt",
+            "args": [
+                {"property": "proj:epsg"},
+                test_item["properties"]["proj:epsg"] + 1,
+            ],
+        },
+    }
+    print(params)
+    resp = await app_client.post("/search", json=params)
+    resp_json = resp.json()
+
+    assert resp.status_code == 200
+    assert len(resp_json.get("features")) == 0
+
+    params = {
+        "collections": [test_item["collection"]],
+        "filter-lang": "cql2-json",
+        "filter": {
+            "op": "eq",
+            "args": [
+                {"property": "proj:epsg"},
+                test_item["properties"]["proj:epsg"],
+            ],
+        },
+    }
+    resp = await app_client.post("/search", json=params)
+    resp_json = resp.json()
+    assert len(resp.json()["features"]) == 1
+    assert (
+        resp_json["features"][0]["properties"]["proj:epsg"]
+        == test_item["properties"]["proj:epsg"]
+    )
+
+
+@pytest.mark.asyncio
+async def test_item_search_get_filter_extension_cql2_with_query_fails(
+    app_client, load_test_data, load_test_collection
+):
+    """Test GET search with JSONB query (cql2 json filter extension)"""
+    test_item = load_test_data("test_item.json")
+    resp = await app_client.post(
+        f"/collections/{test_item['collection']}/items", json=test_item
+    )
+    assert resp.status_code == 200
+
+    second_test_item = load_test_data("test_item2.json")
+    resp = await app_client.post(
+        f"/collections/{test_item['collection']}/items", json=second_test_item
+    )
+    assert resp.status_code == 200
+
+    # EPSG is a JSONB key
+    params = {
+        "collections": [test_item["collection"]],
+        "filter-lang": "cql2-json",
+        "filter": {
+            "op": "gt",
+            "args": [
+                {"property": "proj:epsg"},
+                test_item["properties"]["proj:epsg"] + 1,
+            ],
+        },
+        "query": {"eo:cloud_cover": {"eq": 0}},
+    }
+    print(params)
+    resp = await app_client.post("/search", json=params)
+    print(resp.content)
+    assert resp.status_code == 400
+
+
 @pytest.mark.asyncio
 async def test_get_missing_item_collection(app_client):
     """Test reading a collection which does not exist"""
@@ -888,14 +980,20 @@ async def test_pagination_post(app_client, load_test_data, load_test_collection)
         ids.append(uid)
 
     # Paginate through all 5 items with a limit of 1 (expecting 5 requests)
-    request_body = {"ids": ids, "limit": 1}
+    request_body = {
+        "filter-lang": "cql2-json",
+        "filter": {"op": "in", "args": [{"property": "id"}, ids]},
+        "limit": 1,
+    }
+    print(f"REQUEST BODY: {request_body}")
     page = await app_client.post("/search", json=request_body)
     idx = 0
     item_ids = []
     while True:
         idx += 1
         page_data = page.json()
         item_ids.append(page_data["features"][0]["id"])
+        print(f"PAGING: {page_data['links']}")
         next_link = list(filter(lambda l: l["rel"] == "next", page_data["links"]))
         if not next_link:
             break
@@ -907,6 +1005,7 @@ async def test_pagination_post(app_client, load_test_data, load_test_collection)
             assert False
 
     # Our limit is 1 so we expect len(ids) number of requests before we run out of pages
+    print(idx, ids)
     assert idx == len(ids)
 
     # Confirm we have paginated through all items
@@ -931,8 +1030,16 @@ async def test_pagination_token_idempotent(
         assert resp.status_code == 200
         ids.append(uid)
 
-    page = await app_client.get("/search", params={"ids": ",".join(ids), "limit": 3})
+    page = await app_client.post(
+        "/search",
+        json={
+            "filter-lang": "cql2-json",
+            "filter": {"op": "in", "args": [{"property": "id"}, ids]},
+            "limit": 3,
+        },
+    )
     page_data = page.json()
+    print(f"LINKS: {page_data['links']}")
     next_link = list(filter(lambda l: l["rel"] == "next", page_data["links"]))
 
     # Confirm token is idempotent
@@ -1161,7 +1268,7 @@ async def test_item_search_get_filter_extension_cql_explicitlang(
 
 
 @pytest.mark.asyncio
-async def test_item_search_get_filter_extension_cql2(
+async def test_item_search_get_filter_extension_cql2_2(
     app_client, load_test_data, load_test_collection
 ):
     """Test GET search with JSONB query (cql json filter extension)"""
@@ -1253,3 +1360,34 @@ async def test_search_datetime_validation_errors(app_client):
 
         resp = await app_client.get("/search?datetime={}".format(dt))
         assert resp.status_code == 400
+
+
+@pytest.mark.asyncio
+async def test_filter_cql2text(app_client, load_test_data, load_test_collection):
+    """Test GET search with cql2-text"""
+    test_item = load_test_data("test_item.json")
+    resp = await app_client.post(
+        f"/collections/{test_item['collection']}/items", json=test_item
+    )
+    assert resp.status_code == 200
+
+    epsg = test_item["properties"]["proj:epsg"]
+    collection = test_item["collection"]
+
+    filter = f"proj:epsg={epsg} AND collection = '{collection}'"
+    params = {"filter": filter, "filter-lang": "cql2-text"}
+    resp = await app_client.get("/search", params=params)
+    resp_json = resp.json()
+    print(resp_json)
+    assert len(resp.json()["features"]) == 1
+    assert (
+        resp_json["features"][0]["properties"]["proj:epsg"]
+        == test_item["properties"]["proj:epsg"]
+    )
+
+    filter = f"proj:epsg={epsg + 1} AND collection = '{collection}'"
+    params = {"filter": filter, "filter-lang": "cql2-text"}
+    resp = await app_client.get("/search", params=params)
+    resp_json = resp.json()
+    print(resp_json)
+    assert len(resp.json()["features"]) == 0