Pydantic 2 and STAC API 1.0 (#127)

* Migrate to Pydantic v2

* PEP 621 - use pyproject.toml

* Fix pydantic 2 issues, single out item collection into its own module

* fix cli module

* move api specific link logic into API subpackage

* move catalog, collection, and item specific API logic into API subpackage

* single out API tests, add more tests

* fix context and mark as deprecated

* add more text data

* lint

* add model serializer to Item to address developmentseed/geojson-pydantic#147

* update CI and readme

* Fix type hints for py3.8

* use literals for Catalog type and Link Relations and Methods

* add inline comment

* untangle item_collection and api.item_collectoin tests

---------

Co-authored-by: David Huard <[email protected]>

Author: thomas-maschler

.github/workflows/cicd.yml

@@ -14,20 +14,21 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ["3.7", "3.8", "3.9", "3.10"]
+        python-version: ["3.8", "3.9", "3.10", "3.11"]
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
 
       - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python-version }}
 
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          python -m pip install tox codecov pre-commit
+          python -m pip install tox pre-commit
+          pre-commit install
 
       # Run tox using the version of Python in `PATH`
       - name: Run Tox

.github/workflows/release.yml

@@ -11,22 +11,22 @@ jobs:
     runs-on: ubuntu-latest
     if: ${{ github.repository }} == 'stac-utils/stac-pydantic'
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
 
       - name: Set up Python 3.x
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v4
         with:
           python-version: "3.x"
 
       - name: Install release dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install setuptools wheel twine
+          pip install build
 
       - name: Build and publish package
         env:
           TWINE_USERNAME: ${{ secrets.PYPI_STACUTILS_USERNAME }}
           TWINE_PASSWORD: ${{ secrets.PYPI_STACUTILS_PASSWORD }}
         run: |
-          python setup.py sdist bdist_wheel
+          python -m build
           twine upload dist/*

.gitignore

@@ -106,6 +106,9 @@ celerybeat-schedule
 # dotenv
 .env
 
+# venv
+.venv
+
 # Spyder project settings
 .spyderproject
 .spyproject
@@ -117,4 +120,4 @@ celerybeat-schedule
 /site
 
 # skaffold temporary build/deploy files
-build.out
+build.out

.pre-commit-config.yaml

@@ -1,27 +1,37 @@
+# See https://pre-commit.com for more information
+# See https://pre-commit.com/hooks.html for more hooks
 repos:
-  - repo: https://github.com/psf/black
-    rev: 22.3.0
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.0.1
     hooks:
-      - id: black
-        language_version: python3
-        args: ["--safe"]
+      - id: end-of-file-fixer
+      - id: trailing-whitespace
+      - id: check-yaml
+      - id: check-added-large-files
+        args: ["--maxkb=40960"]
 
-  - repo: https://github.com/PyCQA/isort
-    rev: 5.10.1
+  - repo: local
     hooks:
+      - id: black
+        name: Black Formatting
+        language: system
+        types: [python]
+        entry: black
+
       - id: isort
-        language_version: python3
+        name: iSort Import Sorting
+        language: system
+        types: [python]
+        entry: isort
 
-  - repo: https://github.com/PyCQA/flake8
-    rev: 4.0.1
-    hooks:
       - id: flake8
-        language_version: python3
+        name: Flake8 Formatting
+        language: system
+        types: [python]
+        entry: flake8 --config=./pyproject.toml
 
-  - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v0.931
-    hooks:
       - id: mypy
-        exclude: ^tests/
-        args: [--strict]
-        additional_dependencies: ["pydantic", "types-jsonschema", "types-setuptools", "types-requests"]
+        name: MyPy Typecheck
+        language: system
+        types: [python]
+        entry: mypy --config-file=./pyproject.toml

CHANGELOG.txt

@@ -1,5 +1,6 @@
 Unreleased
 ------------------
+- Support pydantic>2.0 (@huard)
 
 2.0.3 (2022-5-3)
 ------------------
@@ -105,4 +106,4 @@ Unreleased
 - Allow extra asset-level fields (#1)
 - Fix population by field name model config, allowing model creation without extension namespaces (#2)
 - Add enum of commonly used asset media types (#3)
-- Move geojson models to `geojson-pydantic` library (#4)
+- Move geojson models to `geojson-pydantic` library (#4)

README.md

@@ -8,7 +8,7 @@ pip install stac-pydantic
 
 For local development:
 ```
-pip install -e .["dev"]
+pip install -e ".[all]"
 ```
 
 | stac-pydantic | STAC Version |
@@ -31,11 +31,11 @@ pre-commit install
 Ensure you have all Python versions installed that the tests will be run against. If using pyenv, run:
 
 ```shell
-pyenv install 3.7.12
-pyenv install 3.8.12 
-pyenv install 3.9.10
-pyenv install 3.10.2
-pyenv local 3.7.12 3.8.12 3.9.10 3.10.2
+pyenv install 3.8.18
+pyenv install 3.9.18
+pyenv install 3.10.13
+pyenv install 3.11.5
+pyenv local 3.8.18 3.9.18 3.10.13 3.11.5
 ```
 
 Run the entire test suite:
@@ -78,7 +78,7 @@ assert catalog.links[0].href == "item.json"
 ```
 
 ### Extensions
-STAC defines many extensions which let the user customize the data in their catalog. `stac-pydantic.extensions.validate_extensions` will validate a `dict`, `Item`, `Collection` or `Catalog` against the schema urls provided in the `stac_extensions` property: 
+STAC defines many extensions which let the user customize the data in their catalog. `stac-pydantic.extensions.validate_extensions` will validate a `dict`, `Item`, `Collection` or `Catalog` against the schema urls provided in the `stac_extensions` property:
 
 ```python
 from stac_pydantic import Item
@@ -88,36 +88,87 @@ stac_item = {
     "id": "12345",
     "type": "Feature",
     "stac_extensions": [
-        "https://stac-extensions.github.io/eo/v1.0.0/schema.json" 
+        "https://stac-extensions.github.io/eo/v1.0.0/schema.json"
     ],
     "geometry": { "type": "Point", "coordinates": [0, 0] },
+    "bbox": [0.0, 0.0, 0.0, 0.0],
     "properties": {
         "datetime": "2020-03-09T14:53:23.262208+00:00",
         "eo:cloud_cover": 25,
     },
     "links": [],
-    "assets": [],
+    "assets": {},
 }
 
-model = Item(**stac_item) 
+model = Item(**stac_item)
 validate_extensions(model, reraise_exception=True)
-assert getattr(model.properties, "eo:cloud_cover") == 25 
+assert getattr(model.properties, "eo:cloud_cover") == 25
 ```
 
 The complete list of current STAC Extensions can be found [here](https://stac-extensions.github.io/).
 
 #### Vendor Extensions
 The same procedure described above works for any STAC Extension schema as long as it can be loaded from a public url.
 
+### STAC API
+The [STAC API Specs](https://github.com/radiantearth/stac-api-spec) extent the core STAC specification for implementing dynamic catalogs. STAC Objects used in an API context should always import models from the `api` subpackage. This package extends
+Catalog, Collection, and Item models with additional fields and validation rules and introduces Collections and ItemCollections models and Pagination/ Search Links.
+It also implements models for defining ItemSeach queries.
+
+```python
+from stac_pydantic.api import Item, ItemCollection
+
+stac_item = Item(**{
+    "id": "12345",
+    "type": "Feature",
+    "stac_extensions": [],
+    "geometry": { "type": "Point", "coordinates": [0, 0] },
+    "bbox": [0.0, 0.0, 0.0, 0.0],
+    "properties": {
+        "datetime": "2020-03-09T14:53:23.262208+00:00",
+    },
+    "collection": "CS3",
+    "links": [
+          {
+            "rel": "self",
+            "href": "http://stac.example.com/catalog/collections/CS3-20160503_132130_04/items/CS3-20160503_132130_04.json"
+          },
+          {
+            "rel": "collection",
+            "href": "http://stac.example.com/catalog/CS3-20160503_132130_04/catalog.json"
+          },
+          {
+            "rel": "root",
+            "href": "http://stac.example.com/catalog"
+          }],
+    "assets": {},
+    })
+
+stac_item_collection = ItemCollection(**{
+    "type": "FeatureCollection",
+    "features": [stac_item],
+    "links": [
+          {
+            "rel": "self",
+            "href": "http://stac.example.com/catalog/search?collection=CS3",
+            "type": "application/geo+json"
+          },
+          {
+            "rel": "root",
+            "href": "http://stac.example.com/catalog",
+            "type": "application/json"
+          }],
+    })
+```
+
 ### Exporting Models
 Most STAC extensions are namespaced with a colon (ex `eo:gsd`) to keep them distinct from other extensions.  Because
 Python doesn't support the use of colons in variable names, we use [Pydantic aliasing](https://pydantic-docs.helpmanual.io/usage/model_config/#alias-generator)
 to add the namespace upon model export.  This requires [exporting](https://pydantic-docs.helpmanual.io/usage/exporting_models/)
-the model with the `by_alias = True` parameter.  A convenience method (``to_dict()``) is provided to export models with
-extension namespaces:
+the model with the `by_alias = True` parameter. Export methods (`model_dump()` and `model_dump_json()`) for models in this library have `by_alias` and `exclude_unset` st to `True` by default:
 
 ```python
-item_dict = item.to_dict()
+item_dict = item.model_dump()
 assert item_dict['properties']['landsat:row'] == item.properties.row == 250
 ```
 

pyproject.toml

@@ -0,0 +1,86 @@
+[build-system]
+requires = ["setuptools>=61.0"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name="stac-pydantic"
+description="Pydantic data models for the STAC spec"
+classifiers=[
+        "Intended Audience :: Developers",
+        "Intended Audience :: Information Technology",
+        "Intended Audience :: Science/Research",
+        "Programming Language :: Python :: 3.8",
+        "Programming Language :: Python :: 3.9",
+        "Programming Language :: Python :: 3.10",
+        "Programming Language :: Python :: 3.11",
+        "License :: OSI Approved :: MIT License",
+    ]
+keywords=["stac", "pydantic", "validation"]
+authors=[{ name = "Arturo Engineering", email = "[email protected]"}]
+license= { text = "MIT" }
+requires-python=">=3.8"
+dependencies = ["click>=8.1.7", "pydantic>=2.4.1", "geojson-pydantic>=1.0.0"]
+dynamic = ["version", "readme"]
+
+[project.scripts]
+stac-pydantic = "stac_pydantic.scripts.cli:app"
+
+[project.urls]
+homepage = "https://github.com/stac-utils/stac-pydantic"
+repository ="https://github.com/stac-utils/stac-pydantic.git"
+
+[project.optional-dependencies]
+dev = ["arrow>=1.2.3",
+        "pytest>=7.4.2",
+        "pytest-cov>=4.1.0",
+        "pytest-icdiff>=0.8",
+        "requests>=2.31.0",
+        "shapely>=2.0.1",
+        "dictdiffer>=0.9.0",
+        "jsonschema>=4.19.1",
+        "pyyaml>=6.0.1"]
+lint = ["types-requests>=2.31.0.5",
+        "types-jsonschema>=4.19.0.3",
+        "types-PyYAML>=6.0.12.12",
+        "black>=23.9.1",
+        "isort>=5.12.0",
+        "flake8>=6.1.0",
+        "Flake8-pyproject>=1.2.3",
+        "mypy>=1.5.1",
+        "pre-commit>=3.4.0",
+        "tox>=4.11.3"]
+
+[tool.setuptools.dynamic]
+version = { attr = "stac_pydantic.version.__version__" }
+readme = {file = ["README.md"], content-type = "text/markdown"}
+
+[tool.setuptools.package-data]
+stac_pydantic= ["*.typed"]
+
+[tool.setuptools.packages.find]
+include = ["stac_pydantic*"]
+exclude = ["tests*"]
+
+[tool.pytest.ini_options]
+addopts = "-sv --cov stac_pydantic --cov-report xml --cov-report term-missing  --cov-fail-under 95"
+
+[tool.black]
+line-length = 88
+target-version = ["py311"]
+
+[tool.isort]
+profile = "black"
+known_first_party = "stac_pydantic"
+known_third_party = ["pydantic", "geojson-pydantic", "click"]
+default_section = "THIRDPARTY"
+
+[tool.mypy]
+plugins = "pydantic.mypy"
+ignore_missing_imports = true
+exclude = ["tests", ".venv"]
+
+[tool.flake8]
+ignore = ["E203", "E501"]
+select = ["C","E","F","W","B","B950"]
+exclude = ["tests", ".venv"]
+max-line-length = 88