enable hot reloading (#392)

* * refactor makefile targets to be more clear and work better for dev
* rework multi-stage Dockerfile build to run faster and remove unused
  directives
* update README with better instructions for running

* readd pgstac asyncio config

* enable hot-reloading for apps

* update changelog

Author: philvarner

CHANGES.md

@@ -15,6 +15,7 @@
 * Add support for PGStac Backend to use PyGeofilter to convert Get Request with cql2-text into cql2-json to send to PGStac backend ([#346](https://github.com/stac-utils/stac-fastapi/pull/346))
 * Bulk Transactions object Items iterator now returns the Item objects rather than the string IDs of the Item objects
   ([#355](https://github.com/stac-utils/stac-fastapi/issues/355))
+* docker-compose now runs uvicorn with hot-reloading enabled
 
 ### Removed
 

Dockerfile

@@ -1,23 +1,22 @@
 FROM python:3.8-slim as base
 
-FROM base as builder
 # Any python libraries that require system libraries to be installed will likely
 # need the following packages in order to build
-RUN apt-get update && apt-get install -y build-essential git
+RUN apt-get update && \
+    apt-get -y upgrade && \
+    apt-get install -y build-essential git && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
 
 ENV CURL_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt
 
-ARG install_dev_dependencies=true
+FROM base as builder
 
 WORKDIR /app
 
-# Install stac_fastapi.types
 COPY . /app
 
-ENV PATH=$PATH:/install/bin
-
-RUN mkdir -p /install && \
-    pip install -e ./stac_fastapi/types[dev] && \
+RUN pip install -e ./stac_fastapi/types[dev] && \
     pip install -e ./stac_fastapi/api[dev] && \
     pip install -e ./stac_fastapi/extensions[dev] && \
     pip install -e ./stac_fastapi/sqlalchemy[dev,server] && \

Makefile

@@ -2,7 +2,7 @@
 APP_HOST ?= 0.0.0.0
 APP_PORT ?= 8080
 EXTERNAL_APP_PORT ?= ${APP_PORT}
-run_docker = docker-compose run --rm \
+run_sqlalchemy = docker-compose run --rm \
 				-p ${EXTERNAL_APP_PORT}:${APP_PORT} \
 				-e APP_HOST=${APP_HOST} \
 				-e APP_PORT=${APP_PORT} \
@@ -18,25 +18,29 @@ run_pgstac = docker-compose run --rm \
 image:
 	docker-compose build
 
-.PHONY: docker-run
-docker-run: image
-	$(run_docker)
+.PHONY: docker-run-all
+docker-run-all:
+	docker-compose up
+
+.PHONY: docker-run-sqlalchemy
+docker-run-sqlalchemy: image
+	$(run_sqlalchemy)
 
 .PHONY: docker-run-pgstac
 docker-run-pgstac: image
 	$(run_pgstac)
 
-.PHONY: docker-shell
-docker-shell:
-	$(run_docker) /bin/bash
+.PHONY: docker-shell-sqlalchemy
+docker-shell-sqlalchemy:
+	$(run_sqlalchemy) /bin/bash
 
 .PHONY: docker-shell-pgstac
 docker-shell-pgstac:
 	$(run_pgstac) /bin/bash
 
 .PHONY: test-sqlalchemy
 test-sqlalchemy: run-joplin-sqlalchemy
-	$(run_docker) /bin/bash -c 'export && ./scripts/wait-for-it.sh database:5432 && cd /app/stac_fastapi/sqlalchemy/tests/ && pytest'
+	$(run_sqlalchemy) /bin/bash -c 'export && ./scripts/wait-for-it.sh database:5432 && cd /app/stac_fastapi/sqlalchemy/tests/ && pytest -vvv'
 
 .PHONY: test-pgstac
 test-pgstac:

README.md

@@ -63,46 +63,70 @@ pip install -e stac_fastapi/pgstac
 
 ## Local Development
 
-Use docker-compose to deploy the application, migrate the database, and ingest some example data:
+Use docker-compose via make to start the application, migrate the database, and ingest some example data:
 ```bash
-docker-compose build
-docker-compose up
+make image
+make docker-run-all
+```
 
-# You can also launch application with specific backend (PGSTac or sqlalchemy)
-docker-compose up app-sqlalchemy
-# or
-docker-compose up app-pgstac
+- The SQLAlchemy backend app will be available on <http://localhost:8081>.
+- The PGStac backend app will be available on <http://localhost:8082>.
+
+You can also launch only one of the applications with either of these commands:
+
+```shell
+make docker-run-pgstac
+make docker-run-sqlalchemy
 ```
 
-For local development it is often more convenient to run the application outside docker-compose:
-```bash
-make docker-run
+The application will be started on <http://localhost:8080>.
+
+By default, the apps are run with uvicorn hot-reloading enabled. This can be turned off by changing the value
+of the `RELOAD` env var in docker-compose.yml to `false`.
+
+#### Note to Docker for Windows users
+
+You'll need to enable experimental features on Docker for Windows in order to run the docker-compose,
+due to the "--platform" flag that is required to allow the project to run on some Apple architectures.
+To do this, open Docker Desktop, go to settings, select "Docker Engine", and modify the configuration
+JSON to have `"experimental": true`.
+
+### Testing
+
+Before running the tests, ensure the database and apps run with docker-compose are down:
+
+```shell
+docker-compose down
 ```
 
-Before commit, install the [pre-commit](https://pre-commit.com) hooks with:
+The database container provided by the docker-compose stack must be running. This can be started with:
 
 ```shell
-pre-commit install
+make run-database
 ```
 
-The pre-commit hooks can be run manually with:
+To run tests for both the pgstac and sqlalchemy backends, execute:
 
 ```shell
-pre-commit run --all-files
+make test
 ```
 
-#### Note to Docker for Windows users
+To only run pgstac backend tests:
 
-You'll need to enable experimental features on Docker for Windows in order to run the docker-compose, due to the "--platform" flag that is required to allow the project to run on some Apple architectures. To do this, open Docker Desktop, go to settings, select "Docker Engine", and modify the configuration JSON to have `"experimental": true`.
+```shell
+make test-pgstac
+```
 
-### Testing
-The database container provided by the docker-compose stack must be running.  Run all tests:
-```bash
-make test
+To only run sqlalchemy backend tests:
+
+```shell
+make test-sqlalchemy
 ```
 
-Run individual tests by running pytest within the docker container:
-```bash
-make docker-shell
-$ pytest -v
+Run individual tests by running pytest within a docker container:
+
+```shell
+make docker-shell-pgstac # or docker-shell-sqlalchemy
+$ pip install -e stac_fastapi/pgstac[dev]
+$ pytest -v stac_fastapi/pgstac/tests/api/test_api.py 
 ```

docker-compose.yml

@@ -10,7 +10,7 @@ services:
     environment:
       - APP_HOST=0.0.0.0
       - APP_PORT=8081
-      - RELOAD=false
+      - RELOAD=true
       - ENVIRONMENT=local
       - POSTGRES_USER=username
       - POSTGRES_PASS=password
@@ -36,7 +36,7 @@ services:
     environment:
       - APP_HOST=0.0.0.0
       - APP_PORT=8082
-      - RELOAD=false
+      - RELOAD=true
       - ENVIRONMENT=local
       - POSTGRES_USER=username
       - POSTGRES_PASS=password

stac_fastapi/pgstac/README.md

@@ -32,12 +32,11 @@ PGStac stores all collection and item records as jsonb fields exactly as they co
 
 While the Stac Sort Extension is fully supported, [PGStac](https://github.com/stac-utils/pgstac) is particularly enhanced to be able to sort by datetime (either ascending or descending). Sorting by anything other than datetime (the default if no sort is specified) on very large Stac repositories without very specific query limits (ie selecting a single day date range) will not have the same performance. For more than millions of records it is recommended to either set a low connection timeout on PostgreSQL or to disable use of the Sort Extension.
 
-
 `stac-fastapi pgstac` was initially added to `stac-fastapi` by [developmentseed](https://github.com/developmentseed).
 
 ## Installation
 
-```
+```shell
 git clone https://github.com/stac-utils/stac-fastapi.git
 cd stac-fastapi
 pip install -e \
@@ -47,35 +46,16 @@ pip install -e \
     stac_fastapi/pgstac[dev,server]
 ```
 
-## Local Development
-Use docker-compose to deploy the application with a PGStac database. The docker-compose environment will run stac-fastapi/sqlalchemy on port 8081 and stac-fastapi/pgstac on port 8082.
-```bash
-docker-compose build
-docker-compose up
-```
-
-
-### Testing
-The test suite will create a new database on the currently set up database that will have all data truncated between tests. The database must be running with docker-compose or you must have your environment pointing to a running database with PGStac installed to run the tests. All tests are run in the database named "pgstactestdb".
-
-Run all tests:
-```bash
-make test-pgstac
-```
-
-Run individual tests by running pytest within the docker container:
-```bash
-make docker-shell
-$ pytest -v
-```
-
 ### Migrations
 PGStac is an external project and the may be used by multiple front ends.
-For Stac FastAPI development, a docker image (which is pulled as part of the docker-compose) is available at bitner/pgstac:[version] that has the full database already set up for PGStac.
+For Stac FastAPI development, a docker image (which is pulled as part of the docker-compose) is available at
+bitner/pgstac:[version] that has the full database already set up for PGStac.
 
-There is also a python utility as part of PGStac (pypgstac) that includes a migration utility. The pgstac version required by stac-fastapi/pgstac is pinned by using the pinned version of pypgstac in the [setup](setup.py) file.
+There is also a python utility as part of PGStac (pypgstac) that includes a migration utility. The pgstac
+version required by stac-fastapi/pgstac is pinned by using the pinned version of pypgstac in the [setup](setup.py) file.
 
 In order to migrate database versions you can use the migration utility:
-```bash
+
+```shell
 pypgstac migrate
 ```