diff --git a/.github/workflows/cicd.yaml b/.github/workflows/cicd.yaml index 96ab9b73..0c20216f 100644 --- a/.github/workflows/cicd.yaml +++ b/.github/workflows/cicd.yaml @@ -6,18 +6,12 @@ on: branches: [main] workflow_dispatch: - jobs: test: runs-on: ubuntu-latest strategy: matrix: - include: - - {python: '3.12', pypgstac: '0.9.*'} - - {python: '3.12', pypgstac: '0.8.*'} - - {python: '3.11', pypgstac: '0.8.*'} - - {python: '3.10', pypgstac: '0.8.*'} - - {python: '3.9', pypgstac: '0.8.*'} + python-version: ['3.9', '3.10', '3.11', '3.12', '3.13', '3.14'] timeout-minutes: 20 @@ -46,7 +40,6 @@ jobs: run: | python -m pip install --upgrade pip python -m pip install .[dev,server,validation] - python -m pip install "pypgstac==${{ matrix.pypgstac }}" - name: Run test suite run: python -m pytest --cov stac_fastapi.pgstac --cov-report xml --cov-report term-missing diff --git a/README.md b/README.md index f7414cac..91f158bf 100644 --- a/README.md +++ b/README.md @@ -43,7 +43,12 @@ For more than millions of records it is recommended to either set a low connecti ### Hydration -To configure **stac-fastapi-pgstac** to [hydrate search result items in the API](https://stac-utils.github.io/pgstac/pgstac/#runtime-configurations), set the `USE_API_HYDRATE` environment variable to `true` or explicitly set the option in the PGStac Settings object. +To configure **stac-fastapi-pgstac** to [hydrate search result items at the API level](https://stac-utils.github.io/pgstac/pgstac/#runtime-configurations), set the `USE_API_HYDRATE` environment variable to `true`. If `false` (default) the hydration will be done in the database. + +| use_api_hydrate (API) | nohydrate (PgSTAC) | Hydration | +| --- | --- | --- | +| False | False | PgSTAC | +| True | True | API | ### Migrations diff --git a/setup.py b/setup.py index ab3cbf86..994cbf62 100644 --- a/setup.py +++ b/setup.py @@ -16,7 +16,8 @@ "buildpg", "brotli_asgi", "cql2>=0.3.6", - "pypgstac>=0.8,<0.10", + "pypgstac>=0.9,<0.10", + "hydraters>=0.1.3", "typing_extensions>=4.9.0", "jsonpatch>=1.33.0", "json-merge-patch>=0.3.0", @@ -25,7 +26,6 @@ extra_reqs = { "dev": [ "pystac[validation]", - "pypgstac[psycopg]==0.9.*", "pytest-postgresql", "pytest", "pytest-cov", @@ -36,6 +36,8 @@ "httpx", "twine", "wheel", + "psycopg[binary]==3.1.*", + "psycopg-pool==3.1.*", ], "docs": [ "black>=23.10.1", @@ -67,6 +69,8 @@ "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", "License :: OSI Approved :: MIT License", ], keywords="STAC FastAPI COG", diff --git a/stac_fastapi/pgstac/config.py b/stac_fastapi/pgstac/config.py index 74fa2717..9af6513c 100644 --- a/stac_fastapi/pgstac/config.py +++ b/stac_fastapi/pgstac/config.py @@ -174,6 +174,23 @@ class Settings(ApiSettings): prefix_path: str = "" use_api_hydrate: bool = False + """ + When USE_API_HYDRATE=TRUE, PgSTAC database will receive `NO_HYDRATE=TRUE` + + | use_api_hydrate | nohydrate | Hydration | + | --- | --- | --- | + | False | False | PgSTAC | + | True | True | API | + + ref: https://stac-utils.github.io/pgstac/pgstac/#runtime-configurations + """ + exclude_hydrate_markers: bool = True + """ + In some case, PgSTAC can return `DO_NOT_MERGE_MARKER` markers (`𒍟※`). + If `EXCLUDE_HYDRATE_MARKERS=TRUE` and `USE_API_HYDRATE=TRUE`, stac-fastapi-pgstac + will exclude those values from the responses. + """ + invalid_id_chars: List[str] = DEFAULT_INVALID_ID_CHARS base_item_cache: Type[BaseItemCache] = DefaultBaseItemCache diff --git a/stac_fastapi/pgstac/core.py b/stac_fastapi/pgstac/core.py index d159ba67..ad5864d7 100644 --- a/stac_fastapi/pgstac/core.py +++ b/stac_fastapi/pgstac/core.py @@ -331,7 +331,11 @@ async def _get_base_item(collection_id: str) -> Dict[str, Any]: # Exclude None values base_item = {k: v for k, v in base_item.items() if v is not None} - feature = hydrate(base_item, feature) + feature = hydrate( + base_item, + feature, + strip_unmatched_markers=settings.exclude_hydrate_markers, + ) # Grab ids needed for links that may be removed by the fields extension. collection_id = feature.get("collection") diff --git a/tests/conftest.py b/tests/conftest.py index dbbb5979..4a71d05a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -9,7 +9,6 @@ import pytest from fastapi import APIRouter from httpx import ASGITransport, AsyncClient -from pypgstac import __version__ as pgstac_version from pypgstac.db import PgstacDB from pypgstac.migrate import Migrate from pytest_postgresql.janitor import DatabaseJanitor @@ -54,12 +53,6 @@ logger = logging.getLogger(__name__) -requires_pgstac_0_9_2 = pytest.mark.skipif( - tuple(map(int, pgstac_version.split("."))) < (0, 9, 2), - reason="PgSTAC>=0.9.2 required", -) - - @pytest.fixture(scope="session") def database(postgresql_proc): with DatabaseJanitor( @@ -79,7 +72,13 @@ def database(postgresql_proc): yield jan -@pytest.fixture(autouse=True) +@pytest.fixture( + params=[ + "0.8.6", + "0.9.8", + ], + autouse=True, +) async def pgstac(database): connection = f"postgresql://{database.user}:{quote(database.password)}@{database.host}:{database.port}/{database.dbname}" yield @@ -100,7 +99,7 @@ async def pgstac(database): # Run all the tests that use the api_client in both db hydrate and api hydrate mode @pytest.fixture( params=[ - # hydratation, prefix, model_validation + # API hydratation, prefix, model_validation (False, "", False), (False, "/router_prefix", False), (True, "", False), diff --git a/tests/data/test2_item.json b/tests/data/test2_item.json index 62fa2521..9bd7a1ed 100644 --- a/tests/data/test2_item.json +++ b/tests/data/test2_item.json @@ -64,7 +64,7 @@ "nodata": 0, "offset": 2.03976, "data_type": "uint8", - "spatial_resolution": 60 + "spatial_resolution": 80 } ] }, @@ -172,7 +172,6 @@ "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["cloud"], "title": "Pixel Quality Assessment Band (QA_PIXEL)", - "description": "Collection 2 Level-1 Pixel Quality Assessment Band", "raster:bands": [ { "unit": "bit index", diff --git a/tests/resources/test_collection.py b/tests/resources/test_collection.py index 013f9baa..049ff823 100644 --- a/tests/resources/test_collection.py +++ b/tests/resources/test_collection.py @@ -4,8 +4,6 @@ import pytest from stac_pydantic import Collection -from ..conftest import requires_pgstac_0_9_2 - async def test_create_collection(app_client, load_test_data: Callable): in_json = load_test_data("test_collection.json") @@ -349,11 +347,15 @@ async def test_get_collections_search( assert len(resp.json()["collections"]) == 2 -@requires_pgstac_0_9_2 @pytest.mark.asyncio async def test_collection_search_freetext( app_client, load_test_collection, load_test2_collection ): + res = await app_client.get("/_mgmt/health") + pgstac_version = res.json()["pgstac"]["pgstac_version"] + if tuple(map(int, pgstac_version.split("."))) < (0, 9, 2): + pass + # free-text resp = await app_client.get( "/collections", @@ -388,11 +390,15 @@ async def test_collection_search_freetext( assert len(resp.json()["collections"]) == 0 -@requires_pgstac_0_9_2 @pytest.mark.asyncio async def test_collection_search_freetext_advanced( app_client_advanced_freetext, load_test_collection, load_test2_collection ): + res = await app_client_advanced_freetext.get("/_mgmt/health") + pgstac_version = res.json()["pgstac"]["pgstac_version"] + if tuple(map(int, pgstac_version.split("."))) < (0, 9, 2): + pass + # free-text resp = await app_client_advanced_freetext.get( "/collections", @@ -436,9 +442,13 @@ async def test_collection_search_freetext_advanced( assert len(resp.json()["collections"]) == 0 -@requires_pgstac_0_9_2 @pytest.mark.asyncio async def test_all_collections_with_pagination(app_client, load_test_data): + res = await app_client.get("/_mgmt/health") + pgstac_version = res.json()["pgstac"]["pgstac_version"] + if tuple(map(int, pgstac_version.split("."))) < (0, 9, 2): + pass + data = load_test_data("test_collection.json") collection_id = data["id"] for ii in range(0, 12): @@ -468,9 +478,13 @@ async def test_all_collections_with_pagination(app_client, load_test_data): assert {"root", "self"} == {link["rel"] for link in links} -@requires_pgstac_0_9_2 @pytest.mark.asyncio async def test_all_collections_without_pagination(app_client_no_ext, load_test_data): + res = await app_client_no_ext.get("/_mgmt/health") + pgstac_version = res.json()["pgstac"]["pgstac_version"] + if tuple(map(int, pgstac_version.split("."))) < (0, 9, 2): + pass + data = load_test_data("test_collection.json") collection_id = data["id"] for ii in range(0, 12): @@ -491,11 +505,15 @@ async def test_all_collections_without_pagination(app_client_no_ext, load_test_d assert {"root", "self"} == {link["rel"] for link in links} -@requires_pgstac_0_9_2 @pytest.mark.asyncio async def test_get_collections_search_pagination( app_client, load_test_collection, load_test2_collection ): + res = await app_client.get("/_mgmt/health") + pgstac_version = res.json()["pgstac"]["pgstac_version"] + if tuple(map(int, pgstac_version.split("."))) < (0, 9, 2): + pass + resp = await app_client.get("/collections") assert resp.json()["numberReturned"] == 2 assert resp.json()["numberMatched"] == 2 @@ -621,12 +639,16 @@ async def test_get_collections_search_pagination( assert {"root", "self"} == {link["rel"] for link in links} -@requires_pgstac_0_9_2 @pytest.mark.xfail(strict=False) @pytest.mark.asyncio async def test_get_collections_search_offset_1( app_client, load_test_collection, load_test2_collection ): + res = await app_client.get("/_mgmt/health") + pgstac_version = res.json()["pgstac"]["pgstac_version"] + if tuple(map(int, pgstac_version.split("."))) < (0, 9, 2): + pass + # BUG: pgstac doesn't return a `prev` link when limit is not set # offset=1, should have a `previous` link resp = await app_client.get( diff --git a/tests/resources/test_item.py b/tests/resources/test_item.py index 490d652a..b7795fba 100644 --- a/tests/resources/test_item.py +++ b/tests/resources/test_item.py @@ -18,8 +18,6 @@ from stac_fastapi.pgstac.models.links import CollectionLinks -from ..conftest import requires_pgstac_0_9_2 - async def test_create_collection(app_client, load_test_data: Callable): in_json = load_test_data("test_collection.json") @@ -1693,9 +1691,13 @@ async def test_get_search_link_media(app_client): assert get_self_link["type"] == "application/geo+json" -@requires_pgstac_0_9_2 @pytest.mark.asyncio async def test_item_search_freetext(app_client, load_test_data, load_test_collection): + res = await app_client.get("/_mgmt/health") + pgstac_version = res.json()["pgstac"]["pgstac_version"] + if tuple(map(int, pgstac_version.split("."))) < (0, 9, 2): + pass + test_item = load_test_data("test_item.json") resp = await app_client.post( f"/collections/{test_item['collection']}/items", json=test_item @@ -1722,3 +1724,73 @@ async def test_item_search_freetext(app_client, load_test_data, load_test_collec params={"q": "yo"}, ) assert resp.json()["numberReturned"] == 0 + + +@pytest.mark.asyncio +async def test_item_asset_change(app_client, load_test_data): + """Check that changing item_assets in collection does + not affect existing items if hydration should not occur. + + """ + # load collection + data = load_test_data("test2_collection.json") + collection_id = data["id"] + + resp = await app_client.post("/collections", json=data) + assert "item_assets" in data + assert resp.status_code == 201 + assert "item_assets" in resp.json() + + # load items + test_item = load_test_data("test2_item.json") + resp = await app_client.post(f"/collections/{collection_id}/items", json=test_item) + assert resp.status_code == 201 + + # check list of items + resp = await app_client.get( + f"/collections/{collection_id}/items", params={"limit": 1} + ) + assert len(resp.json()["features"]) == 1 + assert resp.status_code == 200 + + # NOTE: API or PgSTAC Hydration we should get the same values as original Item + assert ( + test_item["assets"]["red"]["raster:bands"] + == resp.json()["features"][0]["assets"]["red"]["raster:bands"] + ) + + # NOTE: `description` is not in the item body but in the collection's item-assets + # because it's not in the original item it won't be hydrated + assert not resp.json()["features"][0]["assets"]["qa_pixel"].get("description") + + ########################################################################### + # Remove item_assets in collection + operations = [{"op": "remove", "path": "/item_assets"}] + resp = await app_client.patch(f"/collections/{collection_id}", json=operations) + assert resp.status_code == 200 + + # Make sure item_assets is not in collection response + resp = await app_client.get(f"/collections/{collection_id}") + assert resp.status_code == 200 + assert "item_assets" not in resp.json() + ########################################################################### + + resp = await app_client.get( + f"/collections/{collection_id}/items", params={"limit": 1} + ) + assert len(resp.json()["features"]) == 1 + assert resp.status_code == 200 + + # NOTE: here we should only get `scale`, `offset` and `spatial_resolution` + # because the other values were stripped on ingestion (dehydration is a default in PgSTAC) + # scale and offset are no in item-asset and spatial_resolution is different, so the value in the item body is kept + assert ["scale", "offset", "spatial_resolution"] == list( + resp.json()["features"][0]["assets"]["red"]["raster:bands"][0] + ) + + # Only run this test for PgSTAC hydratation because `exclude_hydrate_markers=True` by default + if not app_client._transport.app.state.settings.use_api_hydrate: + # NOTE: `description` is not in the original item but in the collection's item-assets + # We get "𒍟※" because PgSTAC set it when ingesting (`description`is item-assets) + # because we removed item-assets, pgstac cannot hydrate this field, and thus return "𒍟※" + assert resp.json()["features"][0]["assets"]["qa_pixel"]["description"] == "𒍟※"