From bbb0ea6674e4beb016653a5344cfe514a12457ce Mon Sep 17 00:00:00 2001 From: Kenzie Reading Date: Sun, 30 Aug 2026 18:24:19 +0100 Subject: [PATCH] feat: add created_at feed filters --- api/src/feeds/impl/feeds_api_impl.py | 41 +++++- api/src/shared/common/db_utils.py | 12 ++ api/src/shared/feed_filters/feed_filter.py | 3 + .../shared/feed_filters/gbfs_feed_filter.py | 3 + .../shared/feed_filters/gtfs_feed_filter.py | 3 + .../feed_filters/gtfs_rt_feed_filter.py | 3 + api/tests/integration/test_feeds_api.py | 135 +++++++++++++++++- docs/DatabaseCatalogAPI.yaml | 28 ++++ 8 files changed, 226 insertions(+), 2 deletions(-) diff --git a/api/src/feeds/impl/feeds_api_impl.py b/api/src/feeds/impl/feeds_api_impl.py index 65341158e..60f748db4 100644 --- a/api/src/feeds/impl/feeds_api_impl.py +++ b/api/src/feeds/impl/feeds_api_impl.py @@ -106,14 +106,26 @@ def get_feeds( status: str, provider: str, producer_url: str, + created_after: str, + created_before: str, is_official: bool, db_session: Session, ) -> List[Feed]: """Get some (or all) feeds from the Mobility Database.""" is_email_restricted = is_user_email_restricted() self.logger.debug(f"User email is restricted: {is_email_restricted}") + if created_after and not valid_iso_date(created_after): + raise_http_validation_error(invalid_date_message.format("created_after")) + if created_before and not valid_iso_date(created_before): + raise_http_validation_error(invalid_date_message.format("created_before")) + feed_filter = FeedFilter( - status=status, provider__ilike=provider, producer_url__ilike=producer_url, stable_id=None + status=status, + provider__ilike=provider, + producer_url__ilike=producer_url, + stable_id=None, + created_at__gte=parse_iso_datetime(created_after), + created_at__lte=parse_iso_datetime(created_before), ) feed_query = feed_filter.filter(Database().get_query_model(db_session, FeedOrm)) feed_query = add_official_filter(feed_query, is_official) @@ -199,6 +211,8 @@ def get_gtfs_feeds( offset: int, provider: str, producer_url: str, + created_after: str, + created_before: str, country_code: str, subdivision_name: str, municipality: str, @@ -208,6 +222,11 @@ def get_gtfs_feeds( is_official: bool, db_session: Session, ) -> List[GtfsFeed]: + if created_after and not valid_iso_date(created_after): + raise_http_validation_error(invalid_date_message.format("created_after")) + if created_before and not valid_iso_date(created_before): + raise_http_validation_error(invalid_date_message.format("created_before")) + try: published_only = is_user_email_restricted() feed_query = get_gtfs_feeds_query( @@ -215,6 +234,8 @@ def get_gtfs_feeds( offset=offset, provider=provider, producer_url=producer_url, + created_after=parse_iso_datetime(created_after), + created_before=parse_iso_datetime(created_before), country_code=country_code, subdivision_name=subdivision_name, municipality=municipality, @@ -270,6 +291,8 @@ def get_gtfs_rt_feeds( offset: int, provider: str, producer_url: str, + created_after: str, + created_before: str, entity_types: str, country_code: str, subdivision_name: str, @@ -278,6 +301,11 @@ def get_gtfs_rt_feeds( db_session: Session, ) -> List[GtfsRTFeed]: """Get some (or all) GTFS Realtime feeds from the Mobility Database.""" + if created_after and not valid_iso_date(created_after): + raise_http_validation_error(invalid_date_message.format("created_after")) + if created_before and not valid_iso_date(created_before): + raise_http_validation_error(invalid_date_message.format("created_before")) + try: published_only = is_user_email_restricted() feed_query = get_gtfs_rt_feeds_query( @@ -285,6 +313,8 @@ def get_gtfs_rt_feeds( offset=offset, provider=provider, producer_url=producer_url, + created_after=parse_iso_datetime(created_after), + created_before=parse_iso_datetime(created_before), entity_types=entity_types, country_code=country_code, subdivision_name=subdivision_name, @@ -545,6 +575,8 @@ def get_gbfs_feeds( offset: int, provider: str, producer_url: str, + created_after: str, + created_before: str, country_code: str, subdivision_name: str, municipality: str, @@ -552,10 +584,17 @@ def get_gbfs_feeds( version: str, db_session: Session, ) -> List[GbfsFeed]: + if created_after and not valid_iso_date(created_after): + raise_http_validation_error(invalid_date_message.format("created_after")) + if created_before and not valid_iso_date(created_before): + raise_http_validation_error(invalid_date_message.format("created_before")) + query = get_gbfs_feeds_query( db_session=db_session, provider=provider, producer_url=producer_url, + created_after=parse_iso_datetime(created_after), + created_before=parse_iso_datetime(created_before), country_code=country_code, subdivision_name=subdivision_name, municipality=municipality, diff --git a/api/src/shared/common/db_utils.py b/api/src/shared/common/db_utils.py index 761d25216..992668875 100644 --- a/api/src/shared/common/db_utils.py +++ b/api/src/shared/common/db_utils.py @@ -41,6 +41,8 @@ def get_gtfs_feeds_query( offset: int | None = None, provider: str | None = None, producer_url: str | None = None, + created_after=None, + created_before=None, country_code: str | None = None, subdivision_name: str | None = None, municipality: str | None = None, @@ -56,6 +58,8 @@ def get_gtfs_feeds_query( stable_id=stable_id, provider__ilike=provider, producer_url__ilike=producer_url, + created_at__gte=created_after, + created_at__lte=created_before, location=None, ) @@ -235,6 +239,8 @@ def get_gtfs_rt_feeds_query( offset: int | None, provider: str | None, producer_url: str | None, + created_after, + created_before, entity_types: str | None, country_code: str | None, subdivision_name: str | None, @@ -260,6 +266,8 @@ def get_gtfs_rt_feeds_query( stable_id=None, provider__ilike=provider, producer_url__ilike=producer_url, + created_at__gte=created_after, + created_at__lte=created_before, entity_types=EntityTypeFilter(name__in=entity_types_list), location=LocationFilter( country_code=country_code, @@ -462,6 +470,8 @@ def get_gbfs_feeds_query( stable_id: Optional[str] = None, provider: Optional[str] = None, producer_url: Optional[str] = None, + created_after=None, + created_before=None, country_code: Optional[str] = None, subdivision_name: Optional[str] = None, municipality: Optional[str] = None, @@ -472,6 +482,8 @@ def get_gbfs_feeds_query( stable_id=stable_id, provider__ilike=provider, producer_url__ilike=producer_url, + created_at__gte=created_after, + created_at__lte=created_before, system_id=system_id, location=( LocationFilter( diff --git a/api/src/shared/feed_filters/feed_filter.py b/api/src/shared/feed_filters/feed_filter.py index 16058efda..465e7d1bc 100644 --- a/api/src/shared/feed_filters/feed_filter.py +++ b/api/src/shared/feed_filters/feed_filter.py @@ -1,4 +1,5 @@ from typing import Optional +from datetime import datetime from fastapi_filter.contrib.sqlalchemy import Filter @@ -11,6 +12,8 @@ class FeedFilter(Filter): stable_id: Optional[str] provider__ilike: Optional[str] # case insensitive producer_url__ilike: Optional[str] # case insensitive + created_at__gte: Optional[datetime] = None + created_at__lte: Optional[datetime] = None def __init__(self, *args, **kwargs): kwargs_normalized = normalize_str_parameter("status", **kwargs) diff --git a/api/src/shared/feed_filters/gbfs_feed_filter.py b/api/src/shared/feed_filters/gbfs_feed_filter.py index 5ee1a73e5..4ec8e573f 100644 --- a/api/src/shared/feed_filters/gbfs_feed_filter.py +++ b/api/src/shared/feed_filters/gbfs_feed_filter.py @@ -1,4 +1,5 @@ from typing import Optional +from datetime import datetime from fastapi_filter.contrib.sqlalchemy import Filter @@ -17,6 +18,8 @@ class GbfsFeedFilter(Filter): stable_id: Optional[str] = None provider__ilike: Optional[str] = None # case-insensitive producer_url__ilike: Optional[str] = None # case-insensitive + created_at__gte: Optional[datetime] = None + created_at__lte: Optional[datetime] = None location: Optional[LocationFilter] = None system_id: Optional[str] = None version: Optional[GbfsVersionFilter] = None diff --git a/api/src/shared/feed_filters/gtfs_feed_filter.py b/api/src/shared/feed_filters/gtfs_feed_filter.py index b4e3e6ae9..a205b53e1 100644 --- a/api/src/shared/feed_filters/gtfs_feed_filter.py +++ b/api/src/shared/feed_filters/gtfs_feed_filter.py @@ -1,4 +1,5 @@ from typing import Optional +from datetime import datetime from fastapi_filter.contrib.sqlalchemy import Filter @@ -25,6 +26,8 @@ class GtfsFeedFilter(Filter): stable_id: Optional[str] provider__ilike: Optional[str] # case insensitive producer_url__ilike: Optional[str] # case insensitive + created_at__gte: Optional[datetime] = None + created_at__lte: Optional[datetime] = None location: Optional[LocationFilter] class Constants(Filter.Constants): diff --git a/api/src/shared/feed_filters/gtfs_rt_feed_filter.py b/api/src/shared/feed_filters/gtfs_rt_feed_filter.py index ae7cabfa4..96b5349f2 100644 --- a/api/src/shared/feed_filters/gtfs_rt_feed_filter.py +++ b/api/src/shared/feed_filters/gtfs_rt_feed_filter.py @@ -1,4 +1,5 @@ from typing import Optional, List +from datetime import datetime from fastapi_filter.contrib.sqlalchemy import Filter @@ -22,6 +23,8 @@ class GtfsRtFeedFilter(Filter): stable_id: Optional[str] provider__ilike: Optional[str] # case insensitive producer_url__ilike: Optional[str] # case insensitive + created_at__gte: Optional[datetime] = None + created_at__lte: Optional[datetime] = None entity_types: Optional[EntityTypeFilter] location: Optional[LocationFilter] diff --git a/api/tests/integration/test_feeds_api.py b/api/tests/integration/test_feeds_api.py index fe336b18c..d919d9f98 100644 --- a/api/tests/integration/test_feeds_api.py +++ b/api/tests/integration/test_feeds_api.py @@ -1,7 +1,7 @@ # coding: utf-8 +from datetime import datetime, timedelta import pytest from fastapi.testclient import TestClient -from datetime import timedelta from tests.test_utils.database import TEST_GTFS_FEED_STABLE_IDS, TEST_GTFS_RT_FEED_STABLE_ID, TEST_DATASET_STABLE_IDS from tests.test_utils.token import authHeaders @@ -1172,3 +1172,136 @@ def test_gtfs_feed_availability_invalid_date_returns_422(client: TestClient): params={"from": "not-a-date"}, ) assert response.status_code == 422 + + +@pytest.mark.parametrize( + "endpoint", + [ + "/v1/feeds", + "/v1/gtfs_feeds", + "/v1/gtfs_rt_feeds", + "/v1/gbfs_feeds", + ], + ids=["feeds", "gtfs_feeds", "gtfs_rt_feeds", "gbfs_feeds"], +) +def test_feed_list_created_at_filters_are_inclusive(client: TestClient, endpoint): + """Feed-list created_at bounds include feeds exactly on either boundary.""" + response = client.request( + "GET", + endpoint, + headers=authHeaders, + ) + assert response.status_code == 200 + + feeds = response.json() + assert feeds, f"Expected test data for {endpoint}" + assert all(feed["created_at"] is not None for feed in feeds) + + created_at_by_id = {feed["id"]: datetime.fromisoformat(feed["created_at"].replace("Z", "+00:00")) for feed in feeds} + + boundary = max(created_at_by_id.values()) + boundary_iso = boundary.isoformat().replace("+00:00", "Z") + + expected_ids = {feed_id for feed_id, created_at in created_at_by_id.items() if created_at == boundary} + + after_response = client.request( + "GET", + endpoint, + headers=authHeaders, + params={"created_after": boundary_iso}, + ) + assert after_response.status_code == 200 + + after_ids = {feed["id"] for feed in after_response.json()} + assert after_ids == expected_ids + + before_response = client.request( + "GET", + endpoint, + headers=authHeaders, + params={"created_before": boundary_iso}, + ) + assert before_response.status_code == 200 + + before_ids = {feed["id"] for feed in before_response.json()} + assert expected_ids.issubset(before_ids) + + +@pytest.mark.parametrize( + "endpoint", + [ + "/v1/feeds", + "/v1/gtfs_feeds", + "/v1/gtfs_rt_feeds", + "/v1/gbfs_feeds", + ], + ids=["feeds", "gtfs_feeds", "gtfs_rt_feeds", "gbfs_feeds"], +) +def test_feed_list_created_at_range(client: TestClient, endpoint): + """Combined created_at bounds return exactly the inclusive temporal range.""" + response = client.request( + "GET", + endpoint, + headers=authHeaders, + ) + assert response.status_code == 200 + + feeds = response.json() + assert feeds, f"Expected test data for {endpoint}" + assert all(feed["created_at"] is not None for feed in feeds) + + created_at_by_id = {feed["id"]: datetime.fromisoformat(feed["created_at"].replace("Z", "+00:00")) for feed in feeds} + + ordered_dates = sorted(set(created_at_by_id.values())) + + lower = ordered_dates[0] + upper = ordered_dates[-1] + + lower_iso = lower.isoformat().replace("+00:00", "Z") + upper_iso = upper.isoformat().replace("+00:00", "Z") + + expected_ids = {feed_id for feed_id, created_at in created_at_by_id.items() if lower <= created_at <= upper} + + filtered_response = client.request( + "GET", + endpoint, + headers=authHeaders, + params={ + "created_after": lower_iso, + "created_before": upper_iso, + }, + ) + assert filtered_response.status_code == 200 + + filtered_ids = {feed["id"] for feed in filtered_response.json()} + assert filtered_ids == expected_ids + + +@pytest.mark.parametrize( + "endpoint", + [ + "/v1/feeds", + "/v1/gtfs_feeds", + "/v1/gtfs_rt_feeds", + "/v1/gbfs_feeds", + ], + ids=["feeds", "gtfs_feeds", "gtfs_rt_feeds", "gbfs_feeds"], +) +@pytest.mark.parametrize( + "parameter", + ["created_after", "created_before"], +) +def test_feed_list_created_at_invalid_date_returns_422( + client: TestClient, + endpoint, + parameter, +): + """Invalid created_at bounds are rejected consistently by feed-list endpoints.""" + response = client.request( + "GET", + endpoint, + headers=authHeaders, + params={parameter: "invalid_date"}, + ) + + assert response.status_code == 422 diff --git a/docs/DatabaseCatalogAPI.yaml b/docs/DatabaseCatalogAPI.yaml index f47442dff..a377e8633 100644 --- a/docs/DatabaseCatalogAPI.yaml +++ b/docs/DatabaseCatalogAPI.yaml @@ -53,6 +53,8 @@ paths: - $ref: "#/components/parameters/status" - $ref: "#/components/parameters/provider" - $ref: "#/components/parameters/producer_url" + - $ref: "#/components/parameters/created_after" + - $ref: "#/components/parameters/created_before" - $ref: "#/components/parameters/is_official_query_param" security: @@ -98,6 +100,8 @@ paths: - $ref: "#/components/parameters/offset" - $ref: "#/components/parameters/provider" - $ref: "#/components/parameters/producer_url" + - $ref: "#/components/parameters/created_after" + - $ref: "#/components/parameters/created_before" - $ref: "#/components/parameters/country_code" - $ref: "#/components/parameters/subdivision_name" - $ref: "#/components/parameters/municipality" @@ -127,6 +131,8 @@ paths: - $ref: "#/components/parameters/offset" - $ref: "#/components/parameters/provider" - $ref: "#/components/parameters/producer_url" + - $ref: "#/components/parameters/created_after" + - $ref: "#/components/parameters/created_before" - $ref: "#/components/parameters/entity_types" - $ref: "#/components/parameters/country_code" - $ref: "#/components/parameters/subdivision_name" @@ -153,6 +159,8 @@ paths: - $ref: "#/components/parameters/offset" - $ref: "#/components/parameters/provider" - $ref: "#/components/parameters/producer_url" + - $ref: "#/components/parameters/created_after" + - $ref: "#/components/parameters/created_before" - $ref: "#/components/parameters/country_code" - $ref: "#/components/parameters/subdivision_name" - $ref: "#/components/parameters/municipality" @@ -2375,6 +2383,26 @@ components: schema: type: string example: Los Angeles + created_after: + name: created_after + in: query + description: Filter feeds added to the database at or after this timestamp. Date should be in ISO 8601 date-time format. + required: False + schema: + type: string + format: date-time + example: "2026-01-01T00:00:00Z" + + created_before: + name: created_before + in: query + description: Filter feeds added to the database at or before this timestamp. Date should be in ISO 8601 date-time format. + required: False + schema: + type: string + format: date-time + example: "2026-08-01T00:00:00Z" + downloaded_after: name: downloaded_after in: query