Use stac-api-validator in ci (#508)

* feat: add validation via stac-api-validator

Includes a local validation script and a CI job.

* fix: get_item response is geojson

* fix, pgstac: self link, item collection endpoint

* fix: don't add context extension to landing page

* deps: update uvicorn to non-yanked version

* fix: exclude optional collection values

If we're not using response models, we can't automagically exclude none values.

* fix: add required links to item collection

sqlalchemy

Author: gadomski

.github/workflows/cicd.yaml

@@ -121,6 +121,66 @@ jobs:
           POSTGRES_HOST_READER: localhost
           POSTGRES_HOST_WRITER: localhost
           POSTGRES_PORT: 5432
+
+  validate:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        backend: ["sqlalchemy", "pgstac"]
+    services:
+      pgstac:
+        image: ghcr.io/stac-utils/pgstac:v0.6.11
+        env:
+          POSTGRES_USER: username
+          POSTGRES_PASSWORD: password
+          POSTGRES_DB: postgis
+          PGUSER: username
+          PGPASSWORD: password
+          PGDATABASE: postgis
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          - 5432:5432
+    steps:
+      - name: Check out repository code
+        uses: actions/checkout@v3
+      - name: Setup Python
+        uses: actions/setup-python@v3
+        with:
+          python-version: "3.10"
+          cache: pip
+          cache-dependency-path: stac_fastapi/pgstac/setup.cfg
+      - name: Install stac-fastapi and stac-api-validator
+        run: pip install ./stac_fastapi/api ./stac_fastapi/types ./stac_fastapi/${{ matrix.backend }}[server] stac-api-validator==0.4.1
+      - name: Run migration
+        if: ${{ matrix.backend == 'sqlalchemy' }}
+        run: cd stac_fastapi/sqlalchemy && alembic upgrade head
+        env:
+          POSTGRES_USER: username
+          POSTGRES_PASS: password
+          POSTGRES_DBNAME: postgis
+          POSTGRES_HOST: localhost
+          POSTGRES_PORT: 5432
+      - name: Load data and validate
+        run: python -m stac_fastapi.${{ matrix.backend }}.app & ./scripts/wait-for-it.sh localhost:8080 && python ./scripts/ingest_joplin.py http://localhost:8080 && ./scripts/validate http://localhost:8080
+        env:
+          POSTGRES_USER: username
+          POSTGRES_PASS: password
+          POSTGRES_DBNAME: postgis
+          POSTGRES_HOST_READER: localhost
+          POSTGRES_HOST_WRITER: localhost
+          POSTGRES_PORT: 5432
+          PGUSER: username
+          PGPASSWORD: password
+          PGHOST: localhost
+          PGDATABASE: postgis
+          APP_HOST: 0.0.0.0
+          APP_PORT: 8080
+
   test-docs:
     runs-on: ubuntu-latest
     steps:

scripts/validate

@@ -0,0 +1,28 @@
+#!/usr/bin/env sh
+#
+# Validate a STAC server using [stac-api-validator](https://github.com/stac-utils/stac-api-validator).
+#
+# Assumptions:
+#
+#   - You have stac-api-validator installed, e.g. via `pip install stac-api-validator`
+#   - You've loaded the joplin data, probably using `python ./scripts/ingest_joplin.py http://localhost:8080``
+#
+# Currently, item-search is not checked, because it crashes stac-api-validator (probably a problem on our side).
+
+set -e
+
+if [ $# -eq 0 ]; then
+    root_url=http://localhost:8080
+else
+    root_url="$1"
+fi
+geometry='{"type":"Polygon","coordinates":[[[-94.6884155,37.0595608],[-94.6884155,37.0332547],[-94.6554565,37.0332547],[-94.6554565,37.0595608],[-94.6884155,37.0595608]]]}'
+
+stac-api-validator --root-url "$root_url" \
+    --conformance core \
+    --conformance collections \
+    --conformance features \
+    --conformance filter \
+    --collection joplin \
+    --geometry "$geometry"
+    # --conformance item-search # currently breaks stac-api-validator

stac_fastapi/api/stac_fastapi/api/app.py

@@ -158,12 +158,12 @@ def register_get_item(self):
             name="Get Item",
             path="/collections/{collection_id}/items/{item_id}",
             response_model=Item if self.settings.enable_response_models else None,
-            response_class=self.response_class,
+            response_class=GeoJSONResponse,
             response_model_exclude_unset=True,
             response_model_exclude_none=True,
             methods=["GET"],
             endpoint=create_async_endpoint(
-                self.client.get_item, ItemUri, self.response_class
+                self.client.get_item, ItemUri, GeoJSONResponse
             ),
         )
 

stac_fastapi/pgstac/setup.py

@@ -31,7 +31,7 @@
         "httpx",
     ],
     "docs": ["mkdocs", "mkdocs-material", "pdocs"],
-    "server": ["uvicorn[standard]==0.17.0"],
+    "server": ["uvicorn[standard]==0.17.0.post1"],
     "awslambda": ["mangum"],
 }
 

stac_fastapi/pgstac/stac_fastapi/pgstac/core.py

@@ -18,7 +18,12 @@
 from starlette.requests import Request
 
 from stac_fastapi.pgstac.config import Settings
-from stac_fastapi.pgstac.models.links import CollectionLinks, ItemLinks, PagingLinks
+from stac_fastapi.pgstac.models.links import (
+    CollectionLinks,
+    ItemCollectionLinks,
+    ItemLinks,
+    PagingLinks,
+)
 from stac_fastapi.pgstac.types.search import PgstacSearch
 from stac_fastapi.pgstac.utils import filter_fields
 from stac_fastapi.types.core import AsyncBaseCoreClient
@@ -286,7 +291,7 @@ async def item_collection(
             **clean,
         )
         item_collection = await self._search_base(req, **kwargs)
-        links = await CollectionLinks(
+        links = await ItemCollectionLinks(
             collection_id=collection_id, request=kwargs["request"]
         ).get_links(extra_links=item_collection["links"])
         item_collection["links"] = links

stac_fastapi/pgstac/stac_fastapi/pgstac/models/links.py

@@ -206,6 +206,27 @@ def link_items(self) -> Dict:
         )
 
 
+@attr.s
+class ItemCollectionLinks(CollectionLinksBase):
+    """Create inferred links specific to collections."""
+
+    def link_self(self) -> Dict:
+        """Return the self link."""
+        return dict(
+            rel=Relations.self.value,
+            type=MimeTypes.geojson.value,
+            href=self.resolve(f"collections/{self.collection_id}/items"),
+        )
+
+    def link_parent(self) -> Dict:
+        """Create the `parent` link."""
+        return self.collection_link(rel=Relations.parent.value)
+
+    def link_collection(self) -> Dict:
+        """Create the `collection` link."""
+        return self.collection_link()
+
+
 @attr.s
 class ItemLinks(CollectionLinksBase):
     """Create inferred links specific to items."""

stac_fastapi/pgstac/tests/api/test_api.py

@@ -51,6 +51,27 @@ async def test_get_features_content_type(app_client, load_test_collection):
     assert resp.headers["content-type"] == "application/geo+json"
 
 
+async def test_get_features_self_link(app_client, load_test_collection):
+    # https://github.com/stac-utils/stac-fastapi/issues/483
+    resp = await app_client.get(f"collections/{load_test_collection.id}/items")
+    assert resp.status_code == 200
+    resp_json = resp.json()
+    self_link = next(
+        (link for link in resp_json["links"] if link["rel"] == "self"), None
+    )
+    assert self_link is not None
+    assert self_link["href"].endswith("/items")
+
+
+async def test_get_feature_content_type(
+    app_client, load_test_collection, load_test_item
+):
+    resp = await app_client.get(
+        f"collections/{load_test_collection.id}/items/{load_test_item.id}"
+    )
+    assert resp.headers["content-type"] == "application/geo+json"
+
+
 async def test_api_headers(app_client):
     resp = await app_client.get("/api")
     assert (
@@ -71,6 +92,13 @@ async def test_core_router(api_client, app):
     assert not core_routes - api_routes
 
 
+async def test_landing_page_stac_extensions(app_client):
+    resp = await app_client.get("/")
+    assert resp.status_code == 200
+    resp_json = resp.json()
+    assert not resp_json["stac_extensions"]
+
+
 async def test_transactions_router(api_client, app):
     transaction_routes = set()
     for transaction_route in STAC_TRANSACTION_ROUTES:

stac_fastapi/sqlalchemy/setup.py

@@ -29,7 +29,7 @@
         "requests",
     ],
     "docs": ["mkdocs", "mkdocs-material", "pdocs"],
-    "server": ["uvicorn[standard]==0.17.0"],
+    "server": ["uvicorn[standard]==0.17.0.post1"],
 }
 
 

stac_fastapi/sqlalchemy/stac_fastapi/sqlalchemy/core.py

@@ -169,7 +169,23 @@ def item_collection(
                 else None
             )
 
-            links = []
+            links = [
+                {
+                    "rel": Relations.self.value,
+                    "type": "application/geo+json",
+                    "href": str(kwargs["request"].url),
+                },
+                {
+                    "rel": Relations.root.value,
+                    "type": "application/json",
+                    "href": str(kwargs["request"].base_url),
+                },
+                {
+                    "rel": Relations.parent.value,
+                    "type": "application/json",
+                    "href": str(kwargs["request"].base_url),
+                },
+            ]
             if page.next:
                 links.append(
                     {

stac_fastapi/sqlalchemy/stac_fastapi/sqlalchemy/serializers.py

@@ -146,22 +146,28 @@ def db_to_stac(cls, db_model: database.Collection, base_url: str) -> TypedDict:
         if db_links:
             collection_links += resolve_links(db_links, base_url)
 
-        stac_extensions = db_model.stac_extensions or []
-
-        return stac_types.Collection(
+        collection = stac_types.Collection(
             type="Collection",
             id=db_model.id,
-            stac_extensions=stac_extensions,
             stac_version=db_model.stac_version,
-            title=db_model.title,
             description=db_model.description,
-            keywords=db_model.keywords,
             license=db_model.license,
-            providers=db_model.providers,
-            summaries=db_model.summaries,
             extent=db_model.extent,
             links=collection_links,
         )
+        # We need to manually include optional values to ensure they are
+        # excluded if we're not using response models.
+        if db_model.stac_extensions:
+            collection["stac_extensions"] = db_model.stac_extensions
+        if db_model.title:
+            collection["title"] = db_model.title
+        if db_model.keywords:
+            collection["keywords"] = db_model.keywords
+        if db_model.providers:
+            collection["providers"] = db_model.providers
+        if db_model.summaries:
+            collection["summaries"] = db_model.summaries
+        return collection
 
     @classmethod
     def stac_to_db(

stac_fastapi/sqlalchemy/tests/api/test_api.py

@@ -53,6 +53,13 @@ def test_core_router(api_client):
     assert not core_routes - api_routes
 
 
+def test_landing_page_stac_extensions(app_client):
+    resp = app_client.get("/")
+    assert resp.status_code == 200
+    resp_json = resp.json()
+    assert not resp_json["stac_extensions"]
+
+
 def test_transactions_router(api_client):
     transaction_routes = set(STAC_TRANSACTION_ROUTES)
     api_routes = set(
@@ -445,9 +452,18 @@ def test_app_search_response_duplicate_forwarded_headers(
             assert link["href"].startswith("https://testserver:1234/")
 
 
-async def test_get_features_content_type(app_client, load_test_data):
+def test_get_features_content_type(app_client, load_test_data):
     item = load_test_data("test_item.json")
-    resp = await app_client.get(f"collections/{item['collection']}/items")
+    resp = app_client.get(f"collections/{item['collection']}/items")
+    assert resp.headers["content-type"] == "application/geo+json"
+
+
+def test_get_feature_content_type(app_client, load_test_data, postgres_transactions):
+    item = load_test_data("test_item.json")
+    postgres_transactions.create_item(
+        item["collection"], item, request=MockStarletteRequest
+    )
+    resp = app_client.get(f"collections/{item['collection']}/items/{item['id']}")
     assert resp.headers["content-type"] == "application/geo+json"
 
 

stac_fastapi/sqlalchemy/tests/conftest.py

@@ -70,6 +70,7 @@ def load_file(filename: str) -> Dict:
 
 class MockStarletteRequest:
     base_url = "http://test-server"
+    url = "http://test-server/some/endpoint"
 
 
 @pytest.fixture

stac_fastapi/types/stac_fastapi/types/core.py

@@ -351,13 +351,10 @@ def landing_page(self, **kwargs) -> stac_types.LandingPage:
         """
         request: Request = kwargs["request"]
         base_url = get_base_url(request)
-        extension_schemas = [
-            schema.schema_href for schema in self.extensions if schema.schema_href
-        ]
         landing_page = self._landing_page(
             base_url=base_url,
             conformance_classes=self.conformance_classes(),
-            extension_schemas=extension_schemas,
+            extension_schemas=[],
         )
 
         # Add Collections links
@@ -550,13 +547,10 @@ async def landing_page(self, **kwargs) -> stac_types.LandingPage:
         """
         request: Request = kwargs["request"]
         base_url = get_base_url(request)
-        extension_schemas = [
-            schema.schema_href for schema in self.extensions if schema.schema_href
-        ]
         landing_page = self._landing_page(
             base_url=base_url,
             conformance_classes=self.conformance_classes(),
-            extension_schemas=extension_schemas,
+            extension_schemas=[],
         )
         collections = await self.all_collections(request=kwargs["request"])
         for collection in collections["collections"]: