From dd5dec9e22dd21a2d5ab9600f63116d9f4dfbeb0 Mon Sep 17 00:00:00 2001 From: thodson-usgs Date: Mon, 3 Aug 2026 11:58:24 -0500 Subject: [PATCH 1/4] refactor(waterdata): split collection-family adapters --- .github/workflows/python-package.yml | 8 + CONTRIBUTING.md | 3 + NEWS.md | 2 + dataretrieval/ngwmn.py | 15 +- dataretrieval/nldi.py | 9 + dataretrieval/ogc/context.py | 15 + dataretrieval/ogc/requests.py | 67 +- dataretrieval/ogc/schema.py | 28 + dataretrieval/ogc/shaping.py | 2 +- dataretrieval/streamstats.py | 2 + dataretrieval/transport/__init__.py | 2 + dataretrieval/waterdata/api.py | 3492 +---------------- dataretrieval/waterdata/cql.py | 166 + dataretrieval/waterdata/measurements.py | 576 +++ dataretrieval/waterdata/metadata.py | 994 +++++ dataretrieval/waterdata/nearest.py | 5 +- dataretrieval/waterdata/ratings.py | 3 + dataretrieval/waterdata/reference.py | 174 + dataretrieval/waterdata/samples.py | 432 ++ dataretrieval/waterdata/stats.py | 3 + dataretrieval/waterdata/time_series.py | 1197 ++++++ dataretrieval/waterdata/types.py | 10 + dataretrieval/wateruse.py | 9 + dataretrieval/wqp.py | 16 + .../decisions/0007-adapter-facades.rst | 64 + docs/source/architecture/decisions/index.rst | 1 + docs/source/architecture/index.rst | 64 +- tests/architecture_test.py | 201 + tests/contracts/README.md | 19 + tests/contracts/public_api_test.py | 309 ++ tests/waterdata_test.py | 4 +- tests/waterdata_utils_test.py | 9 +- 32 files changed, 4389 insertions(+), 3512 deletions(-) create mode 100644 dataretrieval/ogc/context.py create mode 100644 dataretrieval/ogc/schema.py create mode 100644 dataretrieval/waterdata/cql.py create mode 100644 dataretrieval/waterdata/measurements.py create mode 100644 dataretrieval/waterdata/metadata.py create mode 100644 dataretrieval/waterdata/reference.py create mode 100644 dataretrieval/waterdata/samples.py create mode 100644 dataretrieval/waterdata/time_series.py create mode 100644 docs/source/architecture/decisions/0007-adapter-facades.rst create mode 100644 tests/contracts/README.md create mode 100644 tests/contracts/public_api_test.py diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index c56bdf39..6537f9c3 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -61,7 +61,15 @@ jobs: installed = Path(dataretrieval.__file__).resolve() assert not installed.is_relative_to(checkout), (installed, checkout) assert importlib.util.find_spec("dataretrieval.waterdata.api") is not None + assert importlib.util.find_spec("dataretrieval.waterdata.time_series") is not None + assert importlib.util.find_spec("dataretrieval.waterdata.metadata") is not None + assert importlib.util.find_spec("dataretrieval.waterdata.measurements") is not None + assert importlib.util.find_spec("dataretrieval.waterdata.reference") is not None + assert importlib.util.find_spec("dataretrieval.waterdata.samples") is not None + assert importlib.util.find_spec("dataretrieval.waterdata.cql") is not None assert importlib.util.find_spec("dataretrieval.ogc.engine") is not None + assert importlib.util.find_spec("dataretrieval.ogc.context") is not None + assert importlib.util.find_spec("dataretrieval.ogc.schema") is not None assert files("dataretrieval").joinpath("py.typed").is_file() assert waterdata.get_daily assert ngwmn.get_sites diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 024d6192..f8667780 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -117,6 +117,9 @@ link checking. * Group public download functions by data portal. For example, modern Water Data functions belong in `dataretrieval.waterdata`; legacy NWIS functions remain quarantined in `dataretrieval.nwis` during deprecation. +* Treat a change to a service's documented return shape or metadata type as a + public compatibility change; update contract tests and architecture + documentation and follow the deprecation process where required. * Preserve the dependency direction documented in [`docs/source/architecture`](docs/source/architecture/index.rst): public facades depend on service/protocol adapters, which depend on stable shared diff --git a/NEWS.md b/NEWS.md index a9a63718..7c926476 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,3 +1,5 @@ +**08/03/2026:** Split the Water Data implementation into focused time-series, metadata, measurements, reference, samples, and CQL collection-family modules behind the unchanged `waterdata.api` facade. Active service modules now declare explicit exports; public Water Data imports, signatures, function identities, deprecations, and return contracts are protected by executable contract snapshots. OGC ambient context and schema/queryables execution are separated from request construction, adapter-to-adapter reach-through is prohibited by architecture tests, and service-specific output shapes are documented rather than forced into one model. + **08/02/2026:** Added an internal API-neutral transport layer for guarded HTTP clients, host-scoped authentication, cursor pagination, bounded retry, response aggregation, progress, and sync-over-async dispatch. Water Use and the non-OGC Statistics API now consume transport directly instead of private OGC execution helpers; WQP, NLDI, and StreamStats opt into bounded transient retry while deprecated NWIS behavior remains unchanged. OGC retains CQL2, request construction, feature shaping, chunk planning, resumable calls, and interruption types, with compatibility imports at previous private paths. Failed pagination and fan-out still raise rather than returning partial data, and no public signatures or return shapes changed. **08/02/2026:** Phase 1 OGC boundary stabilization: the `dataretrieval.ogc` package now exposes a deliberate, small facade (`OgcDialect`, `prepare_request_args`, `get_ogc_data`, `fetch_ogc_request`) that service adapters (NGWMN, Water Data's generic wrapper) import. Internal request-construction helpers moved to a new `ogc.requests` module; the dialect type and endpoint constants live in the leaf `ogc.policy` module. `ogc.shaping` no longer depends on `ogc.engine` at all, and the complete runtime OGC import graph is now acyclic. `_default_headers` now accepts a target URL and adds `X-Api-Key` only for `api.waterdata.usgs.gov`; shared sync and async clients also strip the key before following any cross-host redirect, including external rating-asset downloads. `waterdata.utils` no longer bulk-re-exports private OGC symbols, and consumers import implementation helpers from their canonical modules. No public API, return-value, or deprecation changes. diff --git a/dataretrieval/ngwmn.py b/dataretrieval/ngwmn.py index c4522e08..c2b437b9 100644 --- a/dataretrieval/ngwmn.py +++ b/dataretrieval/ngwmn.py @@ -28,8 +28,21 @@ from dataretrieval.ogc import OgcDialect, get_ogc_data, prepare_request_args from dataretrieval.utils import BaseMetadata +__all__ = [ + "get_sites", + "get_water_level", + "get_lithology", + "get_well_construction", + "get_providers", +] + + # The Water Data API base URL, from the credentials leaf rather than OGC policy -# internals: it names the same authority the API key is scoped to. +# internals: it names the same authority the API key is scoped to. Spelling the +# host out here instead would put a second copy of it in the package, which +# ``tests/architecture_test.py::test_credential_policy_has_one_definition`` +# rejects: the code that attaches the API key and the code that strips it at +# redirect time must not be able to disagree about which host is authorized. BASE_URL = WATERDATA_BASE_URL # The National Ground-Water Monitoring Network exposes its own OGC API at a diff --git a/dataretrieval/nldi.py b/dataretrieval/nldi.py index 57d6048d..04a0b477 100644 --- a/dataretrieval/nldi.py +++ b/dataretrieval/nldi.py @@ -5,6 +5,15 @@ from dataretrieval.utils import _query_with_retry +__all__ = [ + "get_flowlines", + "get_basin", + "get_features", + "get_features_by_data_source", + "search", +] + + try: import geopandas as gpd except ImportError as err: diff --git a/dataretrieval/ogc/context.py b/dataretrieval/ogc/context.py new file mode 100644 index 00000000..8b607509 --- /dev/null +++ b/dataretrieval/ogc/context.py @@ -0,0 +1,15 @@ +"""Ambient per-call OGC request context.""" + +from dataretrieval.ogc.policy import DEFAULT_DIALECT, OGC_API_URL, OgcDialect +from dataretrieval.utils import Ambient + +# Optional cap on rows accumulated by one paginated request. +_row_cap: Ambient[int | None] = Ambient("ogc_row_cap", None) + +# OGC base URL targeted by request construction and schema lookup. +_ogc_base_url: Ambient[str] = Ambient("ogc_base_url", OGC_API_URL) + +# Per-call request and response dialect. +_dialect: Ambient[OgcDialect] = Ambient("ogc_dialect", DEFAULT_DIALECT) + +__all__: list[str] = [] diff --git a/dataretrieval/ogc/requests.py b/dataretrieval/ogc/requests.py index 462cfe4b..2efc7a02 100644 --- a/dataretrieval/ogc/requests.py +++ b/dataretrieval/ogc/requests.py @@ -1,58 +1,29 @@ -"""OGC request preparation, construction, and schema/queryables lookup. +"""OGC argument normalization and HTTP request construction. -This module owns the machinery for building OGC API requests (both GET and -POST/CQL2 paths), the ambient base-URL and dialect state that request builders -read, and the queryables/schema request helper used by empty-result shaping. - -It depends on :mod:`~dataretrieval.ogc.policy` (the dialect type and endpoint -constants), :mod:`~dataretrieval.ogc.dates`, :mod:`~dataretrieval.ogc.errors`, -and :mod:`~dataretrieval.utils` (shared HTTP primitives). It must NOT import -engine or shaping. +Ambient request state lives in :mod:`dataretrieval.ogc.context`; queryables and +schema execution live in :mod:`dataretrieval.ogc.schema`. The schema helper is +imported here only to preserve its previous private path. """ from __future__ import annotations import json -import logging import re from collections.abc import Iterable, Mapping -from typing import Any, cast +from typing import Any import httpx +from dataretrieval.ogc.context import _dialect as _dialect +from dataretrieval.ogc.context import _ogc_base_url as _ogc_base_url +from dataretrieval.ogc.context import _row_cap as _context_row_cap from dataretrieval.ogc.dates import _DATE_RANGE_PARAMS, _format_api_dates -from dataretrieval.ogc.errors import _raise_for_non_200 -from dataretrieval.ogc.policy import DEFAULT_DIALECT, OGC_API_URL, OgcDialect -from dataretrieval.transport.http import ( - HTTPX_DEFAULTS, -) -from dataretrieval.transport.http import ( - default_headers as _default_headers, -) -from dataretrieval.transport.http import ( - get as _get, -) -from dataretrieval.utils import Ambient - -logger = logging.getLogger(__name__) - -# --------------------------------------------------------------------------- -# Ambient per-call state -# --------------------------------------------------------------------------- - -# Optional cap on the rows one paginated call accumulates before it stops -# following ``next`` links (``None`` = uncapped). Set by :func:`get_reference_table` -# to preview large tables without downloading every page. -_row_cap: Ambient[int | None] = Ambient("ogc_row_cap", None) - -# OGC base URL the shared request builder (:func:`_construct_api_requests`) -# targets — the main Water Data API or, for NGWMN collections, their own base. -_ogc_base_url: Ambient[str] = Ambient("ogc_base_url", OGC_API_URL) - -# Per-call OGC dialect the request builder reads for CQL2-vs-GET routing and -# date-only formatting (default: a plain OGC API). -_dialect: Ambient[OgcDialect] = Ambient("ogc_dialect", DEFAULT_DIALECT) +from dataretrieval.ogc.schema import _check_ogc_requests as _schema_check_ogc_requests +from dataretrieval.transport.http import default_headers as _default_headers +# Previous private paths remain available while ownership lives in context/schema. +_row_cap = _context_row_cap +_check_ogc_requests = _schema_check_ogc_requests # --------------------------------------------------------------------------- # Monitoring location ID validation @@ -221,18 +192,6 @@ def _construct_cql_request( ) -def _check_ogc_requests( - endpoint: str, req_type: str = "queryables" -) -> tuple[dict[str, Any], httpx.Response]: - """Send an HTTP GET request to the OGC endpoint for queryables/schema.""" - if req_type not in ("queryables", "schema"): - raise ValueError(f"req_type must be 'queryables' or 'schema', got {req_type!r}") - url = f"{_ogc_base_url.get()}/collections/{endpoint}/{req_type}" - resp = _get(url, headers=_default_headers(url), **HTTPX_DEFAULTS) - _raise_for_non_200(resp) - return cast("dict[str, Any]", resp.json()), resp - - # --------------------------------------------------------------------------- # Argument normalization helpers # --------------------------------------------------------------------------- diff --git a/dataretrieval/ogc/schema.py b/dataretrieval/ogc/schema.py new file mode 100644 index 00000000..1b92ae64 --- /dev/null +++ b/dataretrieval/ogc/schema.py @@ -0,0 +1,28 @@ +"""OGC queryables and schema retrieval.""" + +from __future__ import annotations + +from typing import Any, cast + +import httpx + +from dataretrieval.ogc.context import _ogc_base_url +from dataretrieval.ogc.errors import _raise_for_non_200 +from dataretrieval.transport.http import HTTPX_DEFAULTS +from dataretrieval.transport.http import default_headers as _default_headers +from dataretrieval.transport.http import get as _get + + +def _check_ogc_requests( + endpoint: str, req_type: str = "queryables" +) -> tuple[dict[str, Any], httpx.Response]: + """Retrieve one collection's queryables or response schema.""" + if req_type not in ("queryables", "schema"): + raise ValueError(f"req_type must be 'queryables' or 'schema', got {req_type!r}") + url = f"{_ogc_base_url.get()}/collections/{endpoint}/{req_type}" + response = _get(url, headers=_default_headers(url), **HTTPX_DEFAULTS) + _raise_for_non_200(response) + return cast("dict[str, Any]", response.json()), response + + +__all__: list[str] = [] diff --git a/dataretrieval/ogc/shaping.py b/dataretrieval/ogc/shaping.py index 383e54f1..00cd3470 100644 --- a/dataretrieval/ogc/shaping.py +++ b/dataretrieval/ogc/shaping.py @@ -184,7 +184,7 @@ def _deal_with_empty( if return_list.empty: if not properties or all(pd.isna(properties)): # Import from requests module (no engine dependency). - from dataretrieval.ogc.requests import _check_ogc_requests + from dataretrieval.ogc.schema import _check_ogc_requests schema, _ = _check_ogc_requests(endpoint=service, req_type="schema") properties = list(schema.get("properties", {}).keys()) diff --git a/dataretrieval/streamstats.py b/dataretrieval/streamstats.py index 6727c2bd..6681191f 100644 --- a/dataretrieval/streamstats.py +++ b/dataretrieval/streamstats.py @@ -15,6 +15,8 @@ from dataretrieval.transport.http import HTTPX_DEFAULTS from dataretrieval.utils import _get_with_retry +__all__ = ["download_workspace", "get_sample_watershed", "get_watershed", "Watershed"] + def download_workspace(workspaceID: str, format: str = "") -> httpx.Response: """Function to download a StreamStats workspace. diff --git a/dataretrieval/transport/__init__.py b/dataretrieval/transport/__init__.py index e437aa13..6f8e1aa7 100644 --- a/dataretrieval/transport/__init__.py +++ b/dataretrieval/transport/__init__.py @@ -5,3 +5,5 @@ Service and protocol adapters consume these components; this package is not a public framework API. """ + +__all__: list[str] = [] diff --git a/dataretrieval/waterdata/api.py b/dataretrieval/waterdata/api.py index 3f2e8dcb..71fe61af 100644 --- a/dataretrieval/waterdata/api.py +++ b/dataretrieval/waterdata/api.py @@ -1,3441 +1,65 @@ -"""Functions for downloading data from the Water Data APIs, including the USGS -Aquarius Samples database. - -See https://api.waterdata.usgs.gov/ for API reference. -""" +"""Backward-compatible facade for Water Data collection-family adapters.""" from __future__ import annotations -import json -import logging -from collections.abc import Iterable -from io import StringIO -from typing import Any, get_args -from urllib.parse import quote - -import httpx -import pandas as pd - -from dataretrieval.ogc import fetch_ogc_request -from dataretrieval.ogc.errors import _raise_for_non_200 -from dataretrieval.ogc.filters import FILTER_LANG -from dataretrieval.ogc.requests import ( - _as_str_list, - _check_ogc_requests, - _construct_cql_request, - _switch_properties_id, -) -from dataretrieval.transport.http import ( - HTTPX_DEFAULTS, -) -from dataretrieval.transport.http import ( - default_headers as _default_headers, +from dataretrieval.waterdata import samples as _samples +from dataretrieval.waterdata.cql import get_cql +from dataretrieval.waterdata.measurements import ( + get_channel, + get_field_measurements, + get_peaks, ) -from dataretrieval.transport.http import ( - get as _get, +from dataretrieval.waterdata.metadata import ( + get_combined_metadata, + get_field_measurements_metadata, + get_monitoring_locations, + get_time_series_metadata, ) -from dataretrieval.utils import BaseMetadata, _attach_datetime_columns, to_str -from dataretrieval.waterdata import stats -from dataretrieval.waterdata.types import ( - CODE_SERVICES, - METADATA_COLLECTIONS, - PROFILES, - SERVICES, - WATERDATA_SERVICES, +from dataretrieval.waterdata.reference import get_queryables, get_reference_table +from dataretrieval.waterdata.samples import ( + get_codes, + get_samples, + get_samples_summary, ) -from dataretrieval.waterdata.utils import ( - _OUTPUT_ID_BY_SERVICE, - SAMPLES_URL, - _accept_legacy_kwargs, - _check_profiles, - _finalize_ogc, - _get_args, - _with_state, - get_ogc_data, +from dataretrieval.waterdata.time_series import ( + get_continuous, + get_daily, + get_latest_continuous, + get_latest_daily, + get_stats_date_range, + get_stats_por, ) - -# Set up logger for this module -logger = logging.getLogger(__name__) - - -def get_daily( - monitoring_location_id: str | Iterable[str] | None = None, - parameter_code: str | Iterable[str] | None = None, - statistic_id: str | Iterable[str] | None = None, - properties: str | Iterable[str] | None = None, - time_series_id: str | Iterable[str] | None = None, - daily_id: str | Iterable[str] | None = None, - approval_status: str | Iterable[str] | None = None, - unit_of_measure: str | Iterable[str] | None = None, - qualifier: str | Iterable[str] | None = None, - value: str | Iterable[str] | None = None, - last_modified: str | Iterable[str] | None = None, - skip_geometry: bool | None = None, - time: str | Iterable[str] | None = None, - bbox: list[float] | None = None, - limit: int | None = None, - filter: str | None = None, - filter_lang: FILTER_LANG | None = None, - convert_type: bool = True, - max_rows: int | None = None, - **queryables: Any, -) -> tuple[pd.DataFrame, BaseMetadata]: - """Daily data provide one data value to represent water conditions for the - day. - - Throughout much of the history of the USGS, the primary water data available - was daily data collected manually at the monitoring location once each day. - With improved availability of computer storage and automated transmission of - data, the daily data published today are generally a statistical summary or - metric of the continuous data collected each day, such as the daily mean, - minimum, or maximum value. Daily data are automatically calculated from the - continuous data of the same parameter code and are described by parameter - code and a statistic code. These data have also been referred to as “daily - values” or “DV”. - - Parameters - ---------- - monitoring_location_id : string or iterable of strings, optional - A unique identifier representing a single monitoring location. This - corresponds to the id field in the monitoring-locations endpoint. - Monitoring location IDs are created by combining the agency code of - the agency responsible for the monitoring location (e.g. USGS) with - the ID number of the monitoring location (e.g. 02238500), separated - by a hyphen (e.g. USGS-02238500). - parameter_code : string or iterable of strings, optional - Parameter codes are 5-digit codes used to identify the constituent - measured and the units of measure. A complete list of parameter - codes and associated groupings can be found at - https://help.waterdata.usgs.gov/codes-and-parameters/parameters. - statistic_id : string or iterable of strings, optional - A code corresponding to the statistic an observation represents. - Example codes include 00001 (max), 00002 (min), and 00003 (mean). - A complete list of codes and their descriptions can be found at - https://help.waterdata.usgs.gov/code/stat_cd_nm_query?stat_nm_cd=%25&fmt=html. - properties : string or iterable of strings, optional - A list of requested columns to be returned from the query. - Available options are: geometry, id, time_series_id, - monitoring_location_id, parameter_code, statistic_id, time, value, - unit_of_measure, approval_status, qualifier, last_modified - time_series_id : string or iterable of strings, optional - A unique identifier representing a single time series. This - corresponds to the id field in the time-series-metadata endpoint. - daily_id : string or iterable of strings, optional - A universally unique identifier (UUID) representing a single version of - a record. It is not stable over time. Every time the record is refreshed - in our database (which may happen as part of normal operations and does - not imply any change to the data itself) a new ID will be generated. To - uniquely identify a single observation over time, compare the time and - time_series_id fields; each time series will only have a single - observation at a given time. - approval_status : string or iterable of strings, optional - Some of the data that you have obtained from this U.S. Geological Survey - database may not have received Director's approval. Any such data values - are qualified as provisional and are subject to revision. Provisional - data are released on the condition that neither the USGS nor the United - States Government may be held liable for any damages resulting from its - use. This field reflects the approval status of each record, and is either - "Approved", meaning processing review has been completed and the data is - approved for publication, or "Provisional" and subject to revision. For - more information about provisional data, go to: - https://waterdata.usgs.gov/provisional-data-statement/. - unit_of_measure : string or iterable of strings, optional - A human-readable description of the units of measurement associated - with an observation. - qualifier : string or iterable of strings, optional - This field indicates any qualifiers associated with an observation, for - instance if a sensor may have been impacted by ice or if values were - estimated. - value : string or iterable of strings, optional - The value of the observation. Values are transmitted as strings in - the JSON response format in order to preserve precision. - last_modified : string, optional - The last time a record was refreshed in our database. This may happen - due to regular operational processes and does not necessarily indicate - that anything about the measurement has changed. You can query this field - using date-times or intervals, adhering to RFC 3339, or using ISO 8601 - duration objects. Intervals may be bounded or half-bounded (double-dots - at start or end). - Examples: - - * A date-time: "2018-02-12T23:20:50Z" - * A bounded interval: "2018-02-12T00:00:00Z/2018-03-18T12:31:12Z" - * Half-bounded intervals: "2018-02-12T00:00:00Z/.." or - "../2018-03-18T12:31:12Z" - * Duration objects: "P1M" for data from the past month or - "PT36H" for the last 36 hours - - Only features that have a last_modified that intersects the value of - datetime are selected. - skip_geometry : boolean, optional - This option can be used to skip response geometries for each feature. - The returning object will be a data frame with no spatial information. - Note that the USGS Water Data APIs use camelCase "skipGeometry" in - CQL2 queries. - time : string, optional - The date an observation represents. You can query this field using - date-times or intervals, adhering to RFC 3339, or using ISO 8601 - duration objects. Intervals may be bounded or half-bounded (double-dots - at start or end). Only features that have a time that intersects the - value of datetime are selected. If a feature has multiple temporal - properties, it is the decision of the server whether only a single - temporal property is used to determine the extent or all relevant - temporal properties. - Examples: - - * A date-time: "2018-02-12T23:20:50Z" - * A bounded interval: "2018-02-12T00:00:00Z/2018-03-18T12:31:12Z" - * Half-bounded intervals: "2018-02-12T00:00:00Z/.." or - "../2018-03-18T12:31:12Z" - * Duration objects: "P1M" for data from the past month or - "PT36H" for the last 36 hours - - bbox : list of numbers, optional - Only features that have a geometry that intersects the bounding box are - selected. The bounding box is provided as four or six numbers, - depending on whether the coordinate reference system includes a vertical - axis (height or depth). Coordinates are assumed to be in crs 4326. The - expected format is ``[xmin, ymin, xmax, ymax]``, i.e. - ``[Western-most longitude, Southern-most latitude, Eastern-most - longitude, Northern-most latitude]``. - limit : int, optional - The optional limit parameter is used to control the subset of the - selected features that should be returned in each page. The maximum - allowable limit is 50000. It may be beneficial to set this number lower - if your internet connection is spotty. The default (None) will set the - limit to the maximum allowable limit for the service. - This is a per-page size, not a cap on the total result: a query - matching more rows than ``limit`` still returns every matching row - across multiple pages. Use ``max_rows`` to cap the total instead. - filter, filter_lang : optional - Server-side CQL filter passed through as the OGC ``filter`` / - ``filter-lang`` query parameters. See - :mod:`dataretrieval.ogc.filters` for syntax, auto-chunking, - and the lexicographic-comparison pitfall. - convert_type : boolean, optional - If True, converts columns to appropriate types. - max_rows : int, optional - Cap the total number of rows returned, stopping pagination early - instead of downloading the whole result. Unlike ``limit`` (the - per-page size), this bounds the total result across every page. - The default (None) follows pagination to completion. - **queryables : string or iterable of strings, optional - Any other queryable property of this collection, passed through as a - server-side filter. Call :func:`get_queryables` to see the queryables a - collection supports. - - Returns - ------- - df : ``pandas.DataFrame`` or ``geopandas.GeoDataFrame`` - Formatted data returned from the API query. - md: :obj:`dataretrieval.utils.BaseMetadata` - A custom metadata object - - Raises - ------ - ChunkInterrupted - A transient failure (429 / 5xx / timeout) interrupted the request - after the built-in retries. Completed work is preserved; resume - with ``exc.call.resume()`` (see :doc:`/userguide/errors`). - - Examples - -------- - .. code:: - - >>> # Get daily flow data from a single site - >>> # over a yearlong period - >>> df, md = dataretrieval.waterdata.get_daily( - ... monitoring_location_id="USGS-02238500", - ... parameter_code="00060", - ... time="2021-01-01T00:00:00Z/2022-01-01T00:00:00Z", - ... ) - - >>> # Quick "show me the last week" idiom (ISO 8601 duration) - >>> df, md = dataretrieval.waterdata.get_daily( - ... monitoring_location_id="USGS-02238500", - ... parameter_code="00060", - ... time="P7D", - ... ) - - >>> # Get approved daily flow data from multiple sites - >>> df, md = dataretrieval.waterdata.get_daily( - ... monitoring_location_id=["USGS-05114000", "USGS-09423350"], - ... approval_status="Approved", - ... time="2024-01-01/..", - ... ) - - >>> # Pull only rows whose underlying record was refreshed in the - >>> # last 7 days — handy for incremental ETL polling - >>> df, md = dataretrieval.waterdata.get_daily( - ... monitoring_location_id="USGS-02238500", - ... parameter_code="00060", - ... last_modified="P7D", - ... ) - - >>> # Chain queries: pull all stream sites in a state, then their - >>> # daily discharge for the last week. The site list can be hundreds - >>> # of values long — the request is transparently chunked across - >>> # multiple sub-requests so the URL stays under the server's byte - >>> # limit. Combined output looks like a single query. - >>> sites_df, _ = dataretrieval.waterdata.get_monitoring_locations( - ... state="Ohio", - ... site_type="Stream", - ... ) - >>> df, md = dataretrieval.waterdata.get_daily( - ... monitoring_location_id=sites_df["monitoring_location_id"].tolist(), - ... parameter_code="00060", - ... time="P7D", - ... ) - """ - service = "daily" - - # Build argument dictionary, omitting None values - args = _get_args(locals(), exclude={"max_rows"}) - - return get_ogc_data(args, service, max_rows=max_rows) - - -def get_continuous( - monitoring_location_id: str | Iterable[str] | None = None, - parameter_code: str | Iterable[str] | None = None, - statistic_id: str | Iterable[str] | None = None, - properties: str | Iterable[str] | None = None, - time_series_id: str | Iterable[str] | None = None, - continuous_id: str | Iterable[str] | None = None, - approval_status: str | Iterable[str] | None = None, - unit_of_measure: str | Iterable[str] | None = None, - qualifier: str | Iterable[str] | None = None, - value: str | Iterable[str] | None = None, - last_modified: str | Iterable[str] | None = None, - time: str | Iterable[str] | None = None, - limit: int | None = None, - filter: str | None = None, - filter_lang: FILTER_LANG | None = None, - convert_type: bool = True, - max_rows: int | None = None, - **queryables: Any, -) -> tuple[pd.DataFrame, BaseMetadata]: - """ - Continuous data provide instantaneous water conditions. - - This is an early version of the continuous endpoint that is feature-complete - and is being made available for limited use. Geometries are not included - with the continuous endpoint. If the "time" input is left blank, the service - will return the most recent year of measurements. Users may request no more - than three years of data with each function call. - - Continuous data are collected at a high frequency, typically 15-minute - intervals. Depending on the specific monitoring location, the data may be - transmitted automatically via telemetry and be available on WDFN within - minutes of collection, while other times the delivery of data may be delayed - if the monitoring location does not have the capacity to automatically - transmit data. Continuous data are described by parameter name and - parameter code (pcode). These data might also be referred to as - "instantaneous values" or "IV". - - Parameters - ---------- - monitoring_location_id : string or iterable of strings, optional - A unique identifier representing a single monitoring location. This - corresponds to the id field in the monitoring-locations endpoint. - Monitoring location IDs are created by combining the agency code of - the agency responsible for the monitoring location (e.g. USGS) with - the ID number of the monitoring location (e.g. 02238500), separated - by a hyphen (e.g. USGS-02238500). - parameter_code : string or iterable of strings, optional - Parameter codes are 5-digit codes used to identify the constituent - measured and the units of measure. A complete list of parameter - codes and associated groupings can be found at - https://help.waterdata.usgs.gov/codes-and-parameters/parameters. - statistic_id : string or iterable of strings, optional - A code corresponding to the statistic an observation represents. - Continuous data are nearly always associated with statistic id - 00011. Using a different code (such as 00003 for mean) will - typically return no results. A complete list of codes and their - descriptions can be found at - https://help.waterdata.usgs.gov/code/stat_cd_nm_query?stat_nm_cd=%25&fmt=html. - properties : string or iterable of strings, optional - A list of requested columns to be returned from the query. - Available options are: geometry, id, time_series_id, - monitoring_location_id, parameter_code, statistic_id, time, value, - unit_of_measure, approval_status, qualifier, last_modified - time_series_id : string or iterable of strings, optional - A unique identifier representing a single time series. This - corresponds to the id field in the time-series-metadata endpoint. - continuous_id : string or iterable of strings, optional - A universally unique identifier (UUID) representing a single version of - a record. It is not stable over time. Every time the record is refreshed - in our database (which may happen as part of normal operations and does - not imply any change to the data itself) a new ID will be generated. To - uniquely identify a single observation over time, compare the time and - time_series_id fields; each time series will only have a single - observation at a given time. - approval_status : string or iterable of strings, optional - Some of the data that you have obtained from this U.S. Geological Survey - database may not have received Director's approval. Any such data values - are qualified as provisional and are subject to revision. Provisional - data are released on the condition that neither the USGS nor the United - States Government may be held liable for any damages resulting from its - use. This field reflects the approval status of each record, and is either - "Approved", meaning processing review has been completed and the data is - approved for publication, or "Provisional" and subject to revision. For - more information about provisional data, go to: - https://waterdata.usgs.gov/provisional-data-statement/. - unit_of_measure : string or iterable of strings, optional - A human-readable description of the units of measurement associated - with an observation. - qualifier : string or iterable of strings, optional - This field indicates any qualifiers associated with an observation, for - instance if a sensor may have been impacted by ice or if values were - estimated. - value : string or iterable of strings, optional - The value of the observation. Values are transmitted as strings in - the JSON response format in order to preserve precision. - last_modified : string, optional - The last time a record was refreshed in our database. This may happen - due to regular operational processes and does not necessarily indicate - that anything about the measurement has changed. You can query this field - using date-times or intervals, adhering to RFC 3339, or using ISO 8601 - duration objects. Intervals may be bounded or half-bounded (double-dots - at start or end). - Examples: - - * A date-time: "2018-02-12T23:20:50Z" - * A bounded interval: "2018-02-12T00:00:00Z/2018-03-18T12:31:12Z" - * Half-bounded intervals: "2018-02-12T00:00:00Z/.." or - "../2018-03-18T12:31:12Z" - * Duration objects: "P1M" for data from the past month or - "PT36H" for the last 36 hours - - Only features that have a last_modified that intersects the value of - datetime are selected. - time : string, optional - The date an observation represents. You can query this field using - date-times or intervals, adhering to RFC 3339, or using ISO 8601 - duration objects. Intervals may be bounded or half-bounded (double-dots - at start or end). Only features that have a time that intersects the - value of datetime are selected. If a feature has multiple temporal - properties, it is the decision of the server whether only a single - temporal property is used to determine the extent or all relevant - temporal properties. - Examples: - - * A date-time: "2018-02-12T23:20:50Z" - * A bounded interval: "2018-02-12T00:00:00Z/2018-03-18T12:31:12Z" - * Half-bounded intervals: "2018-02-12T00:00:00Z/.." or - "../2018-03-18T12:31:12Z" - * Duration objects: "P1M" for data from the past month or - "PT36H" for the last 36 hours - - limit : int, optional - The optional limit parameter is used to control the subset of the - selected features that should be returned in each page. The maximum - allowable limit is 10000. It may be beneficial to set this number lower - if your internet connection is spotty. The default (None) will set the - limit to the maximum allowable limit for the service. - This is a per-page size, not a cap on the total result: a query - matching more rows than ``limit`` still returns every matching row - across multiple pages. Use ``max_rows`` to cap the total instead. - filter, filter_lang : optional - Server-side CQL filter passed through as the OGC ``filter`` / - ``filter-lang`` query parameters. See - :mod:`dataretrieval.ogc.filters` for syntax, auto-chunking, - and the lexicographic-comparison pitfall. - convert_type : boolean, optional - If True, converts columns to appropriate types. - max_rows : int, optional - Cap the total number of rows returned, stopping pagination early - instead of downloading the whole result. Unlike ``limit`` (the - per-page size), this bounds the total result across every page. - The default (None) follows pagination to completion. - **queryables : string or iterable of strings, optional - Any other queryable property of this collection, passed through as a - server-side filter. Call :func:`get_queryables` to see the queryables a - collection supports. - - Returns - ------- - df : ``pandas.DataFrame`` or ``geopandas.GeoDataFrame`` - Formatted data returned from the API query. - md: :obj:`dataretrieval.utils.BaseMetadata` - A custom metadata object - - Raises - ------ - ChunkInterrupted - A transient failure (429 / 5xx / timeout) interrupted the request - after the built-in retries. Completed work is preserved; resume - with ``exc.call.resume()`` (see :doc:`/userguide/errors`). - - Examples - -------- - .. code:: - - >>> # Get instantaneous gage height data from a - >>> # single site from a single year - >>> df, md = dataretrieval.waterdata.get_continuous( - ... monitoring_location_id="USGS-02238500", - ... parameter_code="00065", - ... time="2021-01-01T00:00:00Z/2022-01-01T00:00:00Z", - ... ) - - >>> # Pull several disjoint time windows in one call via a CQL - >>> # ``filter``. See ``dataretrieval.ogc.filters`` for the - >>> # full grammar, auto-chunking, and pitfalls. - >>> df, md = dataretrieval.waterdata.get_continuous( - ... monitoring_location_id="USGS-02238500", - ... parameter_code="00060", - ... filter=( - ... "(time >= '2023-06-01T12:00:00Z' " - ... "AND time <= '2023-06-01T13:00:00Z') " - ... "OR (time >= '2023-06-15T12:00:00Z' " - ... "AND time <= '2023-06-15T13:00:00Z')" - ... ), - ... filter_lang="cql-text", - ... ) - """ - service = "continuous" - - # Build argument dictionary, omitting None values - args = _get_args(locals(), exclude={"max_rows"}) - - return get_ogc_data(args, service, max_rows=max_rows) - - -def get_monitoring_locations( - monitoring_location_id: str | Iterable[str] | None = None, - agency_code: str | Iterable[str] | None = None, - agency_name: str | Iterable[str] | None = None, - monitoring_location_number: str | Iterable[str] | None = None, - monitoring_location_name: str | Iterable[str] | None = None, - district_code: str | Iterable[str] | None = None, - country_code: str | Iterable[str] | None = None, - country_name: str | Iterable[str] | None = None, - state: str | Iterable[str] | None = None, - state_code: str | Iterable[str] | None = None, - state_name: str | Iterable[str] | None = None, - county_code: str | Iterable[str] | None = None, - county_name: str | Iterable[str] | None = None, - minor_civil_division_code: str | Iterable[str] | None = None, - site_type_code: str | Iterable[str] | None = None, - site_type: str | Iterable[str] | None = None, - hydrologic_unit_code: str | Iterable[str] | None = None, - basin_code: str | Iterable[str] | None = None, - altitude: str | Iterable[str] | None = None, - altitude_accuracy: str | Iterable[str] | None = None, - altitude_method_code: str | Iterable[str] | None = None, - altitude_method_name: str | Iterable[str] | None = None, - vertical_datum: str | Iterable[str] | None = None, - vertical_datum_name: str | Iterable[str] | None = None, - horizontal_positional_accuracy_code: str | Iterable[str] | None = None, - horizontal_positional_accuracy: str | Iterable[str] | None = None, - horizontal_position_method_code: str | Iterable[str] | None = None, - horizontal_position_method_name: str | Iterable[str] | None = None, - original_horizontal_datum: str | Iterable[str] | None = None, - original_horizontal_datum_name: str | Iterable[str] | None = None, - drainage_area: str | Iterable[str] | None = None, - contributing_drainage_area: str | Iterable[str] | None = None, - time_zone_abbreviation: str | Iterable[str] | None = None, - uses_daylight_savings: str | Iterable[str] | None = None, - construction_date: str | Iterable[str] | None = None, - aquifer_code: str | Iterable[str] | None = None, - national_aquifer_code: str | Iterable[str] | None = None, - aquifer_type_code: str | Iterable[str] | None = None, - well_constructed_depth: str | Iterable[str] | None = None, - hole_constructed_depth: str | Iterable[str] | None = None, - depth_source_code: str | Iterable[str] | None = None, - properties: str | Iterable[str] | None = None, - skip_geometry: bool | None = None, - bbox: list[float] | None = None, - limit: int | None = None, - filter: str | None = None, - filter_lang: FILTER_LANG | None = None, - convert_type: bool = True, - max_rows: int | None = None, - **queryables: Any, -) -> tuple[pd.DataFrame, BaseMetadata]: - """Location information is basic information about the monitoring location - including the name, identifier, agency responsible for data collection, and - the date the location was established. It also includes information about - the type of location, such as stream, lake, or groundwater, and geographic - information about the location, such as state, county, latitude and - longitude, and hydrologic unit code (HUC). - - Parameters - ---------- - monitoring_location_id : string or iterable of strings, optional - A unique identifier representing a single monitoring location. This - corresponds to the id field in the monitoring-locations endpoint. - Monitoring location IDs are created by combining the agency code of - the agency responsible for the monitoring location (e.g. USGS) with - the ID number of the monitoring location (e.g. 02238500), separated - by a hyphen (e.g. USGS-02238500). - agency_code : string or iterable of strings, optional - The agency that is reporting the data. Agency codes are fixed values - assigned by the National Water Information System (NWIS). - agency_name : string or iterable of strings, optional - The name of the agency that is reporting the data. - monitoring_location_number : string or iterable of strings, optional - Each monitoring location in the USGS data base has a unique 8- to - 15-digit identification number. Monitoring location numbers are - assigned based on this logic: - https://help.waterdata.usgs.gov/faq/sites/do-station-numbers-have-any-particular-meaning. - monitoring_location_name : string or iterable of strings, optional - This is the official name of the monitoring location in the database. - For well information this can be a district-assigned local number. - district_code : string or iterable of strings, optional - The Water Science Centers (WSCs) across the United States use the FIPS - state code as the district code. In some cases, monitoring locations and - samples may be managed by a water science center that is adjacent to the - state in which the monitoring location actually resides. For example a - monitoring location may have a district code of 30 which translates to - Montana, but the state code could be 56 for Wyoming because that is where - the monitoring location actually is located. - country_code : string or iterable of strings, optional - The code for the country in which the monitoring location is located. - country_name : string or iterable of strings, optional - The name of the country in which the monitoring location is located. - state : string or iterable of strings, optional - State/territory filter (the recommended parameter). Accepts a full name - (``"Wisconsin"``), a two-letter postal code (``"WI"``), or a two-digit - ANSI/FIPS code (``"55"``). - state_code : string or iterable of strings, optional - State code. A two-digit ANSI code (formerly FIPS code) as defined by - the American National Standards Institute, to define States and - equivalents. A three-digit ANSI code is used to define counties and - county equivalents. A `lookup table - `_ - is available. The only countries with - political subdivisions other than the US are Mexico and Canada. The Mexican - states have US state codes ranging from 81-86 and Canadian provinces have - state codes ranging from 90-98. - state_name : string or iterable of strings, optional - The name of the state or state equivalent in which the monitoring location - is located. - county_code : string or iterable of strings, optional - The code for the county or county equivalent (parish, borough, etc.) in which - the monitoring location is located. A `list of codes - `__ is available. - county_name : string or iterable of strings, optional - The name of the county or county equivalent (parish, borough, etc.) in which - the monitoring location is located. A `list of codes - `__ is available. - minor_civil_division_code : string or iterable of strings, optional - Codes for primary governmental or administrative divisions of the county or - county equivalent in which the monitoring location is located. - site_type_code : string or iterable of strings, optional - A code describing the hydrologic setting of the monitoring location. - site_type : string or iterable of strings, optional - A description of the hydrologic setting of the monitoring location. - hydrologic_unit_code : string or iterable of strings, optional - The United States is divided and sub-divided into successively smaller - hydrologic units which are classified into four levels: regions, - sub-regions, accounting units, and cataloging units. The hydrologic - units are arranged within each other, from the smallest (cataloging - units) to the largest (regions). Each hydrologic unit is identified by a - unique hydrologic unit code (HUC) consisting of two to eight digits - based on the four levels of classification in the hydrologic unit - system. - basin_code : string or iterable of strings, optional - The Basin Code or "drainage basin code" is a two-digit code that further - subdivides the 8-digit hydrologic-unit code. The drainage basin code is - defined by the USGS State Office where the monitoring location is - located. - altitude : string or iterable of strings, optional - Altitude of the monitoring location referenced to the specified Vertical - Datum. - altitude_accuracy : string or iterable of strings, optional - Accuracy of the altitude, in feet. An accuracy of +/- 0.1 foot would be - entered as “.1”. Many altitudes are interpolated from the contours on - topographic maps; accuracies determined in this way are generally - entered as one-half of the contour interval. - altitude_method_code : string or iterable of strings, optional - Codes representing the method used to measure altitude. - altitude_method_name : string or iterable of strings, optional - The name of the method used to measure altitude. - vertical_datum : string or iterable of strings, optional - The datum used to determine altitude and vertical position at the - monitoring location. - vertical_datum_name : string or iterable of strings, optional - The datum used to determine altitude and vertical position at the - monitoring location. - horizontal_positional_accuracy_code : string or iterable of strings, optional - Indicates the accuracy of the latitude longitude values. - horizontal_positional_accuracy : string or iterable of strings, optional - Indicates the accuracy of the latitude longitude values. - horizontal_position_method_code : string or iterable of strings, optional - Indicates the method used to determine latitude longitude values. - horizontal_position_method_name : string or iterable of strings, optional - Indicates the method used to determine latitude longitude values. - original_horizontal_datum : string or iterable of strings, optional - Coordinates are published in EPSG:4326 / WGS84 / World Geodetic System - 1984. This field indicates the original datum used to determine - coordinates before they were converted. - original_horizontal_datum_name : string or iterable of strings, optional - Coordinates are published in EPSG:4326 / WGS84 / World Geodetic System - 1984. This field indicates the original datum used to determine coordinates - before they were converted. - drainage_area : string or iterable of strings, optional - The area enclosed by a topographic divide from which direct surface runoff - from precipitation normally drains by gravity into the stream above that - point. - contributing_drainage_area : string or iterable of strings, optional - The contributing drainage area of a lake, stream, wetland, or estuary - monitoring location, in square miles. This item should be present only - if the contributing area is different from the total drainage area. This - situation can occur when part of the drainage area consists of very - porous soil or depressions that either allow all runoff to enter the - groundwater or trap the water in ponds so that rainfall does not - contribute to runoff. A transbasin diversion can also affect the total - drainage area. - time_zone_abbreviation : string or iterable of strings, optional - A short code describing the time zone used by a monitoring location. - uses_daylight_savings : string or iterable of strings, optional - A flag indicating whether or not a monitoring location uses daylight savings. - construction_date : string or iterable of strings, optional - Date the well was completed. - aquifer_code : string or iterable of strings, optional - Local aquifers in the USGS water resources data base are identified by a - geohydrologic unit code (a three-digit number related to the age of the - formation, followed by a 4 or 5 character abbreviation for the geologic - unit or aquifer name). Additional information is available - `at this link `_. - national_aquifer_code : string or iterable of strings, optional - National aquifers are the principal aquifers or aquifer systems in the United - States, defined as regionally extensive aquifers or aquifer systems that have - the potential to be used as a source of potable water. Not all groundwater - monitoring locations can be associated with a National Aquifer. Such - monitoring locations will not be retrieved using this search criteria. A `list - of National aquifer codes and names `_ - is available. - aquifer_type_code : string or iterable of strings, optional - Groundwater occurs in aquifers under two different conditions. Where water - only partly fills an aquifer, the upper surface is free to rise and decline. - These aquifers are referred to as unconfined (or water-table) aquifers. Where - water completely fills an aquifer that is overlain by a confining bed, the - aquifer is referred to as a confined (or artesian) aquifer. When a confined - aquifer is penetrated by a well, the water level in the well will rise above - the top of the aquifer (but not necessarily above land surface). Additional - information is available `at this link `_. - well_constructed_depth : string or iterable of strings, optional - The depth of the finished well, in feet below land surface datum. Note: Not - all groundwater monitoring locations have information on Well Depth. Such - monitoring locations will not be retrieved using this search criteria. - hole_constructed_depth : string or iterable of strings, optional - The total depth to which the hole is drilled, in feet below land surface datum. - Note: Not all groundwater monitoring locations have information on Hole Depth. - Such monitoring locations will not be retrieved using this search criteria. - depth_source_code : string or iterable of strings, optional - A code indicating the source of water-level data. A `list of - codes `_ - is available. - properties : string or iterable of strings, optional - A list of requested columns to be returned from the query. Available - options are: geometry, id, agency_code, agency_name, - monitoring_location_number, monitoring_location_name, district_code, - country_code, country_name, state_code, state_name, county_code, - county_name, minor_civil_division_code, site_type_code, site_type, - hydrologic_unit_code, basin_code, altitude, altitude_accuracy, - altitude_method_code, altitude_method_name, vertical_datum, - vertical_datum_name, horizontal_positional_accuracy_code, - horizontal_positional_accuracy, horizontal_position_method_code, - horizontal_position_method_name, original_horizontal_datum, - original_horizontal_datum_name, drainage_area, - contributing_drainage_area, time_zone_abbreviation, - uses_daylight_savings, construction_date, aquifer_code, - national_aquifer_code, aquifer_type_code, well_constructed_depth, - hole_constructed_depth, depth_source_code. - bbox : list of numbers, optional - Only features that have a geometry that intersects the bounding box are - selected. The bounding box is provided as four or six numbers, - depending on whether the coordinate reference system includes a vertical - axis (height or depth). Coordinates are assumed to be in crs 4326. The - expected format is ``[xmin, ymin, xmax, ymax]``, i.e. - ``[Western-most longitude, Southern-most latitude, Eastern-most - longitude, Northern-most latitude]``. - limit : int, optional - The optional limit parameter is used to control the subset of the - selected features that should be returned in each page. The maximum - allowable limit is 50000. It may be beneficial to set this number lower - if your internet connection is spotty. The default (None) will set the - limit to the maximum allowable limit for the service. - This is a per-page size, not a cap on the total result: a query - matching more rows than ``limit`` still returns every matching row - across multiple pages. Use ``max_rows`` to cap the total instead. - skip_geometry : boolean, optional - This option can be used to skip response geometries for each feature. - The returning object will be a data frame with no spatial information. - Note that the USGS Water Data APIs use camelCase "skipGeometry" in - CQL2 queries. - filter, filter_lang : optional - Server-side CQL filter passed through as the OGC ``filter`` / - ``filter-lang`` query parameters. See - :mod:`dataretrieval.ogc.filters` for syntax, auto-chunking, - and the lexicographic-comparison pitfall. - convert_type : boolean, optional - If True, converts columns to appropriate types. - max_rows : int, optional - Cap the total number of rows returned, stopping pagination early - instead of downloading the whole result. Unlike ``limit`` (the - per-page size), this bounds the total result across every page. - The default (None) follows pagination to completion. - **queryables : string or iterable of strings, optional - Any other queryable property of this collection, passed through as a - server-side filter. Call :func:`get_queryables` to see the queryables a - collection supports. - - Returns - ------- - df : ``pandas.DataFrame`` or ``geopandas.GeoDataFrame`` - Formatted data returned from the API query. - md: :obj:`dataretrieval.utils.BaseMetadata` - A custom metadata object - - Raises - ------ - ChunkInterrupted - A transient failure (429 / 5xx / timeout) interrupted the request - after the built-in retries. Completed work is preserved; resume - with ``exc.call.resume()`` (see :doc:`/userguide/errors`). - - Examples - -------- - .. code:: - - >>> # Get monitoring locations within a bounding box - >>> # and leave out geometry - >>> df, md = dataretrieval.waterdata.get_monitoring_locations( - ... bbox=[-90.2, 42.6, -88.7, 43.2], skip_geometry=True - ... ) - - >>> # Get monitoring location info for specific sites - >>> # and only specific properties - >>> df, md = dataretrieval.waterdata.get_monitoring_locations( - ... monitoring_location_id=["USGS-05114000", "USGS-09423350"], - ... properties=["monitoring_location_id", "state_name", "country_name"], - ... ) - """ - service = "monitoring-locations" - - # Build argument dictionary, omitting None values (resolving the unified - # `state` argument into the OGC `state_name` queryable). - args = _get_args( - _with_state(locals(), to="name", into="state_name"), exclude={"max_rows"} - ) - - return get_ogc_data(args, service, max_rows=max_rows) - - -def get_time_series_metadata( - monitoring_location_id: str | Iterable[str] | None = None, - parameter_code: str | Iterable[str] | None = None, - parameter_name: str | Iterable[str] | None = None, - properties: str | Iterable[str] | None = None, - statistic_id: str | Iterable[str] | None = None, - hydrologic_unit_code: str | Iterable[str] | None = None, - state: str | Iterable[str] | None = None, - state_name: str | Iterable[str] | None = None, - last_modified: str | Iterable[str] | None = None, - begin: str | Iterable[str] | None = None, - end: str | Iterable[str] | None = None, - begin_utc: str | Iterable[str] | None = None, - end_utc: str | Iterable[str] | None = None, - unit_of_measure: str | Iterable[str] | None = None, - computation_period_identifier: str | Iterable[str] | None = None, - computation_identifier: str | Iterable[str] | None = None, - thresholds: float | list[float] | None = None, - sublocation_identifier: str | Iterable[str] | None = None, - primary: str | Iterable[str] | None = None, - parent_time_series_id: str | Iterable[str] | None = None, - time_series_id: str | Iterable[str] | None = None, - web_description: str | Iterable[str] | None = None, - skip_geometry: bool | None = None, - bbox: list[float] | None = None, - limit: int | None = None, - filter: str | None = None, - filter_lang: FILTER_LANG | None = None, - convert_type: bool = True, - max_rows: int | None = None, - **queryables: Any, -) -> tuple[pd.DataFrame, BaseMetadata]: - """Daily data and continuous measurements are grouped into time series, - which represent a collection of observations of a single parameter, - potentially aggregated using a standard statistic, at a single monitoring - location. This endpoint provides metadata about those time series, - including their operational thresholds, units of measurement, and when - the earliest and most recent observations in a time series occurred. - - Parameters - ---------- - monitoring_location_id : string or iterable of strings, optional - A unique identifier representing a single monitoring location. This - corresponds to the id field in the monitoring-locations endpoint. - Monitoring location IDs are created by combining the agency code of - the agency responsible for the monitoring location (e.g. USGS) with - the ID number of the monitoring location (e.g. 02238500), separated - by a hyphen (e.g. USGS-02238500). - parameter_code : string or iterable of strings, optional - Parameter codes are 5-digit codes used to identify the constituent - measured and the units of measure. A complete list of parameter - codes and associated groupings can be found at - https://help.waterdata.usgs.gov/codes-and-parameters/parameters. - parameter_name : string or iterable of strings, optional - A human-understandable name corresponding to parameter_code. - properties : string or iterable of strings, optional - A list of requested columns to be returned from the query. - Available options are: begin, begin_utc, computation_identifier, - computation_period_identifier, end, end_utc, geometry, - hydrologic_unit_code, id, last_modified, monitoring_location_id, - parameter_code, parameter_description, parameter_name, - parent_time_series_id, primary, state_name, statistic_id, - sublocation_identifier, thresholds, unit_of_measure, web_description - statistic_id : string or iterable of strings, optional - A code corresponding to the statistic an observation represents. - Example codes include 00001 (max), 00002 (min), and 00003 (mean). - A complete list of codes and their descriptions can be found at - https://help.waterdata.usgs.gov/code/stat_cd_nm_query?stat_nm_cd=%25&fmt=html. - hydrologic_unit_code : string or iterable of strings, optional - The United States is divided and sub-divided into successively smaller - hydrologic units which are classified into four levels: regions, - sub-regions, accounting units, and cataloging units. The hydrologic - units are arranged within each other, from the smallest (cataloging units) - to the largest (regions). Each hydrologic unit is identified by a unique - hydrologic unit code (HUC) consisting of two to eight digits based on the - four levels of classification in the hydrologic unit system. - state : string or iterable of strings, optional - State/territory filter (the recommended parameter). Accepts a full name - (``"Wisconsin"``), a two-letter postal code (``"WI"``), or a two-digit - ANSI/FIPS code (``"55"``). - state_name : string or iterable of strings, optional - The name of the state or state equivalent in which the monitoring location - is located. - last_modified : string, optional - The last time a record was refreshed in our database. This may happen - due to regular operational processes and does not necessarily indicate - that anything about the measurement has changed. You can query this field - using date-times or intervals, adhering to RFC 3339, or using ISO 8601 - duration objects. Intervals may be bounded or half-bounded (double-dots - at start or end). Only features that have a last_modified that - intersects the value of datetime are selected. - Examples: - - * A date-time: "2018-02-12T23:20:50Z" - * A bounded interval: "2018-02-12T00:00:00Z/2018-03-18T12:31:12Z" - * Half-bounded intervals: "2018-02-12T00:00:00Z/.." or - "../2018-03-18T12:31:12Z" - * Duration objects: "P1M" for data from the past month or "PT36H" - for the last 36 hours - - begin : string or iterable of strings, optional - This field contains the same information as "begin_utc", but in the - local time of the monitoring location. It is retained for backwards - compatibility, but will be removed in V1 of these APIs. - end : string or iterable of strings, optional - This field contains the same information as "end_utc", but in the - local time of the monitoring location. It is retained for backwards - compatibility, but will be removed in V1 of these APIs. - begin_utc : string or iterable of strings, optional - The datetime of the earliest observation in the time series. Together - with end, this field represents the period of record of a time series. - Note that some time series may have large gaps in their collection - record. This field is currently in the local time of the monitoring - location. We intend to update this in version v0 to use UTC with a time - zone. You can query this field using date-times or intervals, adhering - to RFC 3339, or using ISO 8601 duration objects. Intervals may be - bounded or half-bounded (double-dots at start or end). Only features - that have a begin that intersects the value of datetime are selected. - Examples: - - * A date-time: "2018-02-12T23:20:50Z" - * A bounded interval: "2018-02-12T00:00:00Z/2018-03-18T12:31:12Z" - * Half-bounded intervals: "2018-02-12T00:00:00Z/.." or - "../2018-03-18T12:31:12Z" - * Duration objects: "P1M" for data from the past month or - "PT36H" for the last 36 hours - - end_utc : string or iterable of strings, optional - The datetime of the most recent observation in the time series. Data returned by - this endpoint updates at most once per day, and potentially less frequently than - that, and as such there may be more recent observations within a time series - than the time series end value reflects. Together with begin, this field - represents the period of record of a time series. It is additionally used to - determine whether a time series is "active". We intend to update this in - version v0 to use UTC with a time zone. - You can query this field using date-times or intervals, - adhering to RFC 3339, or using ISO 8601 duration objects. Intervals - may be bounded or half-bounded (double-dots at start or end). Only - features that have an end that intersects the value of datetime are - selected. - Examples: - - * A date-time: "2018-02-12T23:20:50Z" - * A bounded interval: "2018-02-12T00:00:00Z/2018-03-18T12:31:12Z" - * Half-bounded intervals: "2018-02-12T00:00:00Z/.." or - "../2018-03-18T12:31:12Z" - * Duration objects: "P1M" for data from the past month or - "PT36H" for the last 36 hours - - unit_of_measure : string or iterable of strings, optional - A human-readable description of the units of measurement associated - with an observation. - computation_period_identifier : string or iterable of strings, optional - Indicates the period of data used for any statistical computations. - computation_identifier : string or iterable of strings, optional - Indicates whether the data from this time series represent a specific - statistical computation. - thresholds : number or list of numbers, optional - Thresholds represent known numeric limits for a time series, for example - the historic maximum value for a parameter or a level below which a - sensor is non-operative. These thresholds are sometimes used to - automatically determine if an observation is erroneous due to sensor - error, and therefore shouldn't be included in the time series. - sublocation_identifier : string or iterable of strings, optional - primary : string or iterable of strings, optional - parent_time_series_id : string or iterable of strings, optional - time_series_id : string or iterable of strings, optional - A unique identifier representing a single time series. This - corresponds to the id field in the time-series-metadata endpoint. - web_description : string or iterable of strings, optional - A description of what this time series represents, as used by WDFN and - other USGS data dissemination products. - skip_geometry : boolean, optional - This option can be used to skip response geometries for each feature. - The returning object will be a data frame with no spatial information. - Note that the USGS Water Data APIs use camelCase "skipGeometry" in - CQL2 queries. - bbox : list of numbers, optional - Only features that have a geometry that intersects the bounding box are - selected. The bounding box is provided as four or six numbers, - depending on whether the coordinate reference system includes a vertical - axis (height or depth). Coordinates are assumed to be in crs 4326. The - expected format is ``[xmin, ymin, xmax, ymax]``, i.e. - ``[Western-most longitude, Southern-most latitude, Eastern-most - longitude, Northern-most latitude]``. - limit : int, optional - The optional limit parameter is used to control the subset of the - selected features that should be returned in each page. The maximum - allowable limit is 50000. It may be beneficial to set this number lower - if your internet connection is spotty. The default (None) will set the - limit to the maximum allowable limit for the service. - This is a per-page size, not a cap on the total result: a query - matching more rows than ``limit`` still returns every matching row - across multiple pages. Use ``max_rows`` to cap the total instead. - filter, filter_lang : optional - Server-side CQL filter passed through as the OGC ``filter`` / - ``filter-lang`` query parameters. See - :mod:`dataretrieval.ogc.filters` for syntax, auto-chunking, - and the lexicographic-comparison pitfall. - convert_type : boolean, optional - If True, converts columns to appropriate types. - max_rows : int, optional - Cap the total number of rows returned, stopping pagination early - instead of downloading the whole result. Unlike ``limit`` (the - per-page size), this bounds the total result across every page. - The default (None) follows pagination to completion. - **queryables : string or iterable of strings, optional - Any other queryable property of this collection, passed through as a - server-side filter. Call :func:`get_queryables` to see the queryables a - collection supports. - - Returns - ------- - df : ``pandas.DataFrame`` or ``geopandas.GeoDataFrame`` - Formatted data returned from the API query. - md: :obj:`dataretrieval.utils.BaseMetadata` - A custom metadata object - - Raises - ------ - ChunkInterrupted - A transient failure (429 / 5xx / timeout) interrupted the request - after the built-in retries. Completed work is preserved; resume - with ``exc.call.resume()`` (see :doc:`/userguide/errors`). - - Examples - -------- - .. code:: - - >>> # Get timeseries metadata information from a single site - >>> # over a yearlong period - >>> df, md = dataretrieval.waterdata.get_time_series_metadata( - ... monitoring_location_id="USGS-02238500" - ... ) - - >>> # Get timeseries metadata information from multiple sites - >>> # that begin after January 1, 1990. - >>> df, md = dataretrieval.waterdata.get_time_series_metadata( - ... monitoring_location_id=["USGS-05114000", "USGS-09423350"], - ... begin="1990-01-01/..", - ... ) - """ - service = "time-series-metadata" - - # Build argument dictionary, omitting None values (resolving the unified - # `state` argument into the OGC `state_name` queryable). - args = _get_args( - _with_state(locals(), to="name", into="state_name"), exclude={"max_rows"} - ) - - return get_ogc_data(args, service, max_rows=max_rows) - - -def get_combined_metadata( - monitoring_location_id: str | Iterable[str] | None = None, - parameter_code: str | Iterable[str] | None = None, - parameter_name: str | Iterable[str] | None = None, - parameter_description: str | Iterable[str] | None = None, - unit_of_measure: str | Iterable[str] | None = None, - statistic_id: str | Iterable[str] | None = None, - data_type: str | Iterable[str] | None = None, - computation_identifier: str | Iterable[str] | None = None, - thresholds: float | list[float] | None = None, - sublocation_identifier: str | Iterable[str] | None = None, - primary: str | Iterable[str] | None = None, - parent_time_series_id: str | Iterable[str] | None = None, - web_description: str | Iterable[str] | None = None, - last_modified: str | Iterable[str] | None = None, - begin: str | Iterable[str] | None = None, - end: str | Iterable[str] | None = None, - agency_code: str | Iterable[str] | None = None, - agency_name: str | Iterable[str] | None = None, - monitoring_location_number: str | Iterable[str] | None = None, - monitoring_location_name: str | Iterable[str] | None = None, - district_code: str | Iterable[str] | None = None, - country_code: str | Iterable[str] | None = None, - country_name: str | Iterable[str] | None = None, - state: str | Iterable[str] | None = None, - state_code: str | Iterable[str] | None = None, - state_name: str | Iterable[str] | None = None, - county_code: str | Iterable[str] | None = None, - county_name: str | Iterable[str] | None = None, - minor_civil_division_code: str | Iterable[str] | None = None, - site_type_code: str | Iterable[str] | None = None, - site_type: str | Iterable[str] | None = None, - hydrologic_unit_code: str | Iterable[str] | None = None, - basin_code: str | Iterable[str] | None = None, - altitude: str | Iterable[str] | None = None, - altitude_accuracy: str | Iterable[str] | None = None, - altitude_method_code: str | Iterable[str] | None = None, - altitude_method_name: str | Iterable[str] | None = None, - vertical_datum: str | Iterable[str] | None = None, - vertical_datum_name: str | Iterable[str] | None = None, - horizontal_positional_accuracy_code: str | Iterable[str] | None = None, - horizontal_positional_accuracy: str | Iterable[str] | None = None, - horizontal_position_method_code: str | Iterable[str] | None = None, - horizontal_position_method_name: str | Iterable[str] | None = None, - original_horizontal_datum: str | Iterable[str] | None = None, - original_horizontal_datum_name: str | Iterable[str] | None = None, - drainage_area: str | Iterable[str] | None = None, - contributing_drainage_area: str | Iterable[str] | None = None, - time_zone_abbreviation: str | Iterable[str] | None = None, - uses_daylight_savings: str | Iterable[str] | None = None, - construction_date: str | Iterable[str] | None = None, - aquifer_code: str | Iterable[str] | None = None, - national_aquifer_code: str | Iterable[str] | None = None, - aquifer_type_code: str | Iterable[str] | None = None, - well_constructed_depth: str | Iterable[str] | None = None, - hole_constructed_depth: str | Iterable[str] | None = None, - depth_source_code: str | Iterable[str] | None = None, - properties: str | Iterable[str] | None = None, - skip_geometry: bool | None = None, - bbox: list[float] | None = None, - limit: int | None = None, - filter: str | None = None, - filter_lang: FILTER_LANG | None = None, - convert_type: bool = True, - max_rows: int | None = None, - **queryables: Any, -) -> tuple[pd.DataFrame, BaseMetadata]: - """Get combined monitoring-location and time-series metadata. - - The ``combined-metadata`` collection joins the monitoring-locations - catalog with the time-series-metadata catalog so that one row is - returned per (location, parameter, statistic) inventory entry, - carrying every column from both source endpoints. This makes it the - most flexible "what data is available" endpoint in the Water Data - API: any monitoring-location attribute (state, HUC, site type, - drainage area, well-construction depth, …) can be combined with any - time-series attribute (parameter code, statistic, data type, period - of record, …) in a single query. - - See the OpenAPI reference for the full list of supported fields: - https://api.waterdata.usgs.gov/ogcapi/v0/openapi?f=html#/combined-metadata - - All ~35 location-catalog kwargs are accepted (``agency_code``, - ``state_name``, ``drainage_area``, ``aquifer_code``, …) but only - the most-used ones are documented below; see - :func:`get_monitoring_locations` for per-field descriptions. - - Parameters - ---------- - monitoring_location_id : string or iterable of strings, optional - A unique identifier representing a single monitoring location. - Created by combining the agency code (e.g. ``USGS``) with the ID - number (e.g. ``02238500``), separated by a hyphen - (e.g. ``"USGS-02238500"``). - parameter_code : string or iterable of strings, optional - 5-digit codes used to identify the constituent measured and the - units of measure. See - https://help.waterdata.usgs.gov/codes-and-parameters/parameters. - parameter_name : string or iterable of strings, optional - A human-understandable name corresponding to ``parameter_code``. - parameter_description : string or iterable of strings, optional - A human-readable description of what is being measured. - unit_of_measure : string or iterable of strings, optional - A human-readable description of the units of measurement - associated with an observation. - statistic_id : string or iterable of strings, optional - A code corresponding to the statistic an observation represents - (e.g. ``00001`` max, ``00002`` min, ``00003`` mean). Full list at - https://help.waterdata.usgs.gov/code/stat_cd_nm_query?stat_nm_cd=%25&fmt=html. - data_type : string or iterable of strings, optional - The type of data the time series represents, e.g. - ``"Continuous values"``, ``"Daily values"``, - ``"Field measurements"``. - computation_identifier : string or iterable of strings, optional - Indicates whether the data from this time series represent a - specific statistical computation. - thresholds : number or list of numbers, optional - Numeric limits known for a time series (e.g. historic maximum, - below-which-the-sensor-is-non-operative). - sublocation_identifier : string or iterable of strings, optional - primary : string or iterable of strings, optional - A flag identifying whether the time series is "primary". Primary - time series are standard observations that have undergone Bureau - review and approval. Non-primary (provisional) time series have a - missing ``primary`` value, are produced for timely best-science - use, and are retained by this system for only 120 days. - parent_time_series_id : string or iterable of strings, optional - web_description : string or iterable of strings, optional - A description of what this time series represents, as used by - WDFN and other USGS data dissemination products. - last_modified, begin, end : string, optional - Datetime fields that accept either an RFC 3339 datetime, an - interval (``"start/end"``, optionally half-bounded with ``..``), - or an ISO 8601 duration (e.g. ``"P1M"``, ``"PT36H"``). See - :func:`get_time_series_metadata` for the full grammar. - state : string or iterable of strings, optional - State/territory filter (the recommended parameter). Accepts a full - name (``"Wisconsin"``), a two-letter postal code (``"WI"``), or a - two-digit ANSI/FIPS code (``"55"``). - state_name, county_name, hydrologic_unit_code, site_type, \ -site_type_code : string or iterable of strings, optional - Common location-catalog filters carried over from the - ``monitoring-locations`` collection. The function also accepts - the full list of location-catalog kwargs (agency, district, - altitude, vertical/horizontal datum, drainage area, aquifer, - well construction, …); see :func:`get_monitoring_locations` for - descriptions of each. - properties : string or iterable of strings, optional - Subset of columns to return. Defaults to every available - property. - skip_geometry : boolean, optional - Skip per-feature geometries; the returned object will be a plain - ``DataFrame`` with no spatial information. The Water Data APIs - use camelCase ``skipGeometry`` in CQL2 queries. - bbox : list of numbers, optional - Only features whose geometry intersects the bounding box are - selected. Format: ``[xmin, ymin, xmax, ymax]`` in CRS 4326 - (longitude/latitude, west-south-east-north). - limit : int, optional - Page size; the maximum allowable value is 50000. Default - (``None``) requests the maximum allowable limit. This is a - per-page size, not a cap on the total result: a query matching more - rows than ``limit`` still returns every matching row across - multiple pages. Use ``max_rows`` to cap the total instead. - filter, filter_lang : optional - Server-side CQL filter passed through as the OGC ``filter`` / - ``filter-lang`` query parameters. See - :mod:`dataretrieval.ogc.filters` for syntax, auto-chunking, - and the lexicographic-comparison pitfall. - convert_type : boolean, optional - If True, converts columns to appropriate types. - max_rows : int, optional - Cap the total number of rows returned, stopping pagination early - instead of downloading the whole result. Unlike ``limit`` (the - per-page size), this bounds the total result across every page. - The default (None) follows pagination to completion. - **queryables : string or iterable of strings, optional - Any other queryable property of this collection, passed through as a - server-side filter. Call :func:`get_queryables` to see the queryables a - collection supports. - - Returns - ------- - df : ``pandas.DataFrame`` or ``geopandas.GeoDataFrame`` - Formatted data returned from the API query. - md : :obj:`dataretrieval.utils.BaseMetadata` - A custom metadata object pertaining to the query. - - Raises - ------ - ChunkInterrupted - A transient failure (429 / 5xx / timeout) interrupted the request - after the built-in retries. Completed work is preserved; resume - with ``exc.call.resume()`` (see :doc:`/userguide/errors`). - - Examples - -------- - .. code:: - - >>> # All time series and field measurements at a single surface-water site - >>> df, md = dataretrieval.waterdata.get_combined_metadata( - ... monitoring_location_id="USGS-05407000" - ... ) - - >>> # Same, for a groundwater well — water-level and aquifer columns - >>> # are populated where the surface-water example has nulls - >>> df, md = dataretrieval.waterdata.get_combined_metadata( - ... monitoring_location_id="USGS-375907091432201" - ... ) - - >>> # Every series in a single county, useful for area-of-interest workflows - >>> df, md = dataretrieval.waterdata.get_combined_metadata( - ... state="Wisconsin", county_name="Dane County" - ... ) - - >>> # Inventory across multiple HUCs, restricted to streams and springs - >>> df, md = dataretrieval.waterdata.get_combined_metadata( - ... hydrologic_unit_code=["11010008", "11010009"], - ... site_type=["Stream", "Spring"], - ... ) - - >>> # Discharge time series at three sites with at least one - >>> # observation in the past month - >>> df, md = dataretrieval.waterdata.get_combined_metadata( - ... monitoring_location_id=[ - ... "USGS-07069000", - ... "USGS-07064000", - ... "USGS-07068000", - ... ], - ... end="P1M", - ... parameter_code="00060", - ... ) - - >>> # Two-step "what's available?" → "fetch it" workflow: - >>> # 1. inventory the sites in two HUCs - >>> hucs, _ = dataretrieval.waterdata.get_combined_metadata( - ... hydrologic_unit_code=["11010008", "11010009"], - ... site_type="Stream", - ... ) - >>> # 2. pull continuous discharge at every distinct site found - >>> sites = hucs["monitoring_location_id"].unique().tolist() - >>> df, md = dataretrieval.waterdata.get_continuous( - ... monitoring_location_id=sites, - ... parameter_code="00060", - ... time="P1D", - ... ) - - """ - service = "combined-metadata" - - # Resolve the unified `state` argument into the OGC `state_name` queryable. - args = _get_args( - _with_state(locals(), to="name", into="state_name"), exclude={"max_rows"} - ) - - return get_ogc_data(args, service, max_rows=max_rows) - - -def get_latest_continuous( - monitoring_location_id: str | Iterable[str] | None = None, - parameter_code: str | Iterable[str] | None = None, - statistic_id: str | Iterable[str] | None = None, - properties: str | Iterable[str] | None = None, - time_series_id: str | Iterable[str] | None = None, - latest_continuous_id: str | Iterable[str] | None = None, - approval_status: str | Iterable[str] | None = None, - unit_of_measure: str | Iterable[str] | None = None, - qualifier: str | Iterable[str] | None = None, - value: str | Iterable[str] | None = None, - last_modified: str | Iterable[str] | None = None, - skip_geometry: bool | None = None, - time: str | Iterable[str] | None = None, - bbox: list[float] | None = None, - limit: int | None = None, - filter: str | None = None, - filter_lang: FILTER_LANG | None = None, - convert_type: bool = True, - max_rows: int | None = None, - **queryables: Any, -) -> tuple[pd.DataFrame, BaseMetadata]: - """This endpoint provides the most recent observation for each time series - of continuous data. Continuous data are collected via automated sensors - installed at a monitoring location. They are collected at a high frequency - and often at a fixed 15-minute interval. Depending on the specific monitoring - location, the data may be transmitted automatically via telemetry and be - available on WDFN within minutes of collection, while other times the delivery - of data may be delayed if the monitoring location does not have the capacity to - automatically transmit data. Continuous data are described by parameter name - and parameter code. These data might also be referred to as "instantaneous - values" or "IV". - - Parameters - ---------- - monitoring_location_id : string or iterable of strings, optional - A unique identifier representing a single monitoring location. This - corresponds to the id field in the monitoring-locations endpoint. - Monitoring location IDs are created by combining the agency code of the - agency responsible for the monitoring location (e.g. USGS) with the ID - number of the monitoring location (e.g. 02238500), separated by a hyphen - (e.g. USGS-02238500). - parameter_code : string or iterable of strings, optional - Parameter codes are 5-digit codes used to identify the constituent - measured and the units of measure. A complete list of parameter codes - and associated groupings can be found at - https://help.waterdata.usgs.gov/codes-and-parameters/parameters. - statistic_id : string or iterable of strings, optional - A code corresponding to the statistic an observation represents. - Example codes include 00001 (max), 00002 (min), and 00003 (mean). - A complete list of codes and their descriptions can be found at - https://help.waterdata.usgs.gov/code/stat_cd_nm_query?stat_nm_cd=%25&fmt=html. - properties : string or iterable of strings, optional - A list of requested columns to be returned from the query. Available - options are: geometry, id, time_series_id, monitoring_location_id, - parameter_code, statistic_id, time, value, unit_of_measure, - approval_status, qualifier, last_modified - time_series_id : string or iterable of strings, optional - A unique identifier representing a single time series. This - corresponds to the id field in the time-series-metadata endpoint. - latest_continuous_id : string or iterable of strings, optional - A universally unique identifier (UUID) representing a single version of - a record. It is not stable over time. Every time the record is refreshed - in our database (which may happen as part of normal operations and does - not imply any change to the data itself) a new ID will be generated. To - uniquely identify a single observation over time, compare the time and - time_series_id fields; each time series will only have a single - observation at a given time. - approval_status : string or iterable of strings, optional - Some of the data that you have obtained from this U.S. Geological Survey - database may not have received Director's approval. Any such data values - are qualified as provisional and are subject to revision. Provisional - data are released on the condition that neither the USGS nor the United - States Government may be held liable for any damages resulting from its - use. This field reflects the approval status of each record, and is either - "Approved", meaning processing review has been completed and the data is - approved for publication, or "Provisional" and subject to revision. For - more information about provisional data, go to: - https://waterdata.usgs.gov/provisional-data-statement/. - unit_of_measure : string or iterable of strings, optional - A human-readable description of the units of measurement associated - with an observation. - qualifier : string or iterable of strings, optional - This field indicates any qualifiers associated with an observation, for - instance if a sensor may have been impacted by ice or if values were - estimated. - value : string or iterable of strings, optional - The value of the observation. Values are transmitted as strings in - the JSON response format in order to preserve precision. - last_modified : string, optional - The last time a record was refreshed in our database. This may happen - due to regular operational processes and does not necessarily indicate - that anything about the measurement has changed. You can query this field - using date-times or intervals, adhering to RFC 3339, or using ISO 8601 - duration objects. Intervals may be bounded or half-bounded (double-dots - at start or end). Only features that have a last_modified that - intersects the value of datetime are selected. - Examples: - - * A date-time: "2018-02-12T23:20:50Z" - * A bounded interval: "2018-02-12T00:00:00Z/2018-03-18T12:31:12Z" - * Half-bounded intervals: "2018-02-12T00:00:00Z/.." or - "../2018-03-18T12:31:12Z" - * Duration objects: "P1M" for data from the past month or - "PT36H" for the last 36 hours - - skip_geometry : boolean, optional - This option can be used to skip response geometries for each feature. - The returning object will be a data frame with no spatial information. - Note that the USGS Water Data APIs use camelCase "skipGeometry" in - CQL2 queries. - time : string, optional - The date an observation represents. You can query this field using - date-times or intervals, adhering to RFC 3339, or using ISO 8601 - duration objects. Intervals may be bounded or half-bounded (double-dots - at start or end). Only features that have a time that intersects the - value of datetime are selected. If a feature has multiple temporal - properties, it is the decision of the server whether only a single - temporal property is used to determine the extent or all relevant - temporal properties. - Examples: - - * A date-time: "2018-02-12T23:20:50Z" - * A bounded interval: "2018-02-12T00:00:00Z/2018-03-18T12:31:12Z" - * Half-bounded intervals: "2018-02-12T00:00:00Z/.." or - "../2018-03-18T12:31:12Z" - * Duration objects: "P1M" for data from the past month or - "PT36H" for the last 36 hours - - bbox : list of numbers, optional - Only features that have a geometry that intersects the bounding box are - selected. The bounding box is provided as four or six numbers, - depending on whether the coordinate reference system includes a vertical - axis (height or depth). Coordinates are assumed to be in crs 4326. The - expected format is ``[xmin, ymin, xmax, ymax]``, i.e. - ``[Western-most longitude, Southern-most latitude, Eastern-most - longitude, Northern-most latitude]``. - limit : int, optional - The optional limit parameter is used to control the subset of the - selected features that should be returned in each page. The maximum - allowable limit is 50000. It may be beneficial to set this number lower - if your internet connection is spotty. The default (None) will set the - limit to the maximum allowable limit for the service. - This is a per-page size, not a cap on the total result: a query - matching more rows than ``limit`` still returns every matching row - across multiple pages. Use ``max_rows`` to cap the total instead. - filter, filter_lang : optional - Server-side CQL filter passed through as the OGC ``filter`` / - ``filter-lang`` query parameters. See - :mod:`dataretrieval.ogc.filters` for syntax, auto-chunking, - and the lexicographic-comparison pitfall. - convert_type : boolean, optional - If True, converts columns to appropriate types. - max_rows : int, optional - Cap the total number of rows returned, stopping pagination early - instead of downloading the whole result. Unlike ``limit`` (the - per-page size), this bounds the total result across every page. - The default (None) follows pagination to completion. - **queryables : string or iterable of strings, optional - Any other queryable property of this collection, passed through as a - server-side filter. Call :func:`get_queryables` to see the queryables a - collection supports. - - Returns - ------- - df : ``pandas.DataFrame`` or ``geopandas.GeoDataFrame`` - Formatted data returned from the API query. - md: :obj:`dataretrieval.utils.BaseMetadata` - A custom metadata object - - Raises - ------ - ChunkInterrupted - A transient failure (429 / 5xx / timeout) interrupted the request - after the built-in retries. Completed work is preserved; resume - with ``exc.call.resume()`` (see :doc:`/userguide/errors`). - - Examples - -------- - .. code:: - - >>> # Get latest flow data from a single site - >>> df, md = dataretrieval.waterdata.get_latest_continuous( - ... monitoring_location_id="USGS-02238500", parameter_code="00060" - ... ) - - >>> # Restrict to the last 7 days; sites with no observation in that - >>> # window are dropped instead of returned with stale values - >>> df, md = dataretrieval.waterdata.get_latest_continuous( - ... monitoring_location_id="USGS-02238500", - ... parameter_code="00060", - ... time="P7D", - ... ) - - >>> # Pull only rows whose underlying record was refreshed in the - >>> # last 7 days, across multiple sites and parameters - >>> df, md = dataretrieval.waterdata.get_latest_continuous( - ... monitoring_location_id=["USGS-451605097071701", "USGS-14181500"], - ... parameter_code=["00060", "72019"], - ... last_modified="P7D", - ... ) - - >>> # Get latest continuous measurements for multiple sites - >>> df, md = dataretrieval.waterdata.get_latest_continuous( - ... monitoring_location_id=["USGS-05114000", "USGS-09423350"] - ... ) - """ - service = "latest-continuous" - - # Build argument dictionary, omitting None values - args = _get_args(locals(), exclude={"max_rows"}) - - return get_ogc_data(args, service, max_rows=max_rows) - - -def get_latest_daily( - monitoring_location_id: str | Iterable[str] | None = None, - parameter_code: str | Iterable[str] | None = None, - statistic_id: str | Iterable[str] | None = None, - properties: str | Iterable[str] | None = None, - time_series_id: str | Iterable[str] | None = None, - latest_daily_id: str | Iterable[str] | None = None, - approval_status: str | Iterable[str] | None = None, - unit_of_measure: str | Iterable[str] | None = None, - qualifier: str | Iterable[str] | None = None, - value: str | Iterable[str] | None = None, - last_modified: str | Iterable[str] | None = None, - skip_geometry: bool | None = None, - time: str | Iterable[str] | None = None, - bbox: list[float] | None = None, - limit: int | None = None, - filter: str | None = None, - filter_lang: FILTER_LANG | None = None, - convert_type: bool = True, - max_rows: int | None = None, - **queryables: Any, -) -> tuple[pd.DataFrame, BaseMetadata]: - """Daily data provide one data value to represent water conditions for the - day. - - Throughout much of the history of the USGS, the primary water data available - was daily data collected manually at the monitoring location once each day. - With improved availability of computer storage and automated transmission of - data, the daily data published today are generally a statistical summary or - metric of the continuous data collected each day, such as the daily mean, - minimum, or maximum value. Daily data are automatically calculated from the - continuous data of the same parameter code and are described by parameter - code and a statistic code. These data have also been referred to as “daily - values” or “DV”. - - Parameters - ---------- - monitoring_location_id : string or iterable of strings, optional - A unique identifier representing a single monitoring location. This - corresponds to the id field in the monitoring-locations endpoint. - Monitoring location IDs are created by combining the agency code of the - agency responsible for the monitoring location (e.g. USGS) with the ID - number of the monitoring location (e.g. 02238500), separated by a hyphen - (e.g. USGS-02238500). - parameter_code : string or iterable of strings, optional - Parameter codes are 5-digit codes used to identify the constituent - measured and the units of measure. A complete list of parameter codes - and associated groupings can be found at - https://help.waterdata.usgs.gov/codes-and-parameters/parameters. - statistic_id : string or iterable of strings, optional - A code corresponding to the statistic an observation represents. - Example codes include 00001 (max), 00002 (min), and 00003 (mean). - A complete list of codes and their descriptions can be found at - https://help.waterdata.usgs.gov/code/stat_cd_nm_query?stat_nm_cd=%25&fmt=html. - properties : string or iterable of strings, optional - A list of requested columns to be returned from the query. Available - options are: geometry, id, time_series_id, monitoring_location_id, - parameter_code, statistic_id, time, value, unit_of_measure, - approval_status, qualifier, last_modified - time_series_id : string or iterable of strings, optional - A unique identifier representing a single time series. This - corresponds to the id field in the time-series-metadata endpoint. - latest_daily_id : string or iterable of strings, optional - A universally unique identifier (UUID) representing a single version of - a record. It is not stable over time. Every time the record is refreshed - in our database (which may happen as part of normal operations and does - not imply any change to the data itself) a new ID will be generated. To - uniquely identify a single observation over time, compare the time and - time_series_id fields; each time series will only have a single - observation at a given time. - approval_status : string or iterable of strings, optional - Some of the data that you have obtained from this U.S. Geological Survey - database may not have received Director's approval. Any such data values - are qualified as provisional and are subject to revision. Provisional - data are released on the condition that neither the USGS nor the United - States Government may be held liable for any damages resulting from its - use. This field reflects the approval status of each record, and is either - "Approved", meaning processing review has been completed and the data is - approved for publication, or "Provisional" and subject to revision. For - more information about provisional data, go to: - https://waterdata.usgs.gov/provisional-data-statement/. - unit_of_measure : string or iterable of strings, optional - A human-readable description of the units of measurement associated - with an observation. - qualifier : string or iterable of strings, optional - This field indicates any qualifiers associated with an observation, for - instance if a sensor may have been impacted by ice or if values were - estimated. - value : string or iterable of strings, optional - The value of the observation. Values are transmitted as strings in - the JSON response format in order to preserve precision. - last_modified : string, optional - The last time a record was refreshed in our database. This may happen - due to regular operational processes and does not necessarily indicate - that anything about the measurement has changed. You can query this field - using date-times or intervals, adhering to RFC 3339, or using ISO 8601 - duration objects. Intervals may be bounded or half-bounded (double-dots - at start or end). Only features that have a last_modified that - intersects the value of datetime are selected. - Examples: - - * A date-time: "2018-02-12T23:20:50Z" - * A bounded interval: "2018-02-12T00:00:00Z/2018-03-18T12:31:12Z" - * Half-bounded intervals: "2018-02-12T00:00:00Z/.." or - "../2018-03-18T12:31:12Z" - * Duration objects: "P1M" for data from the past month or - "PT36H" for the last 36 hours - - skip_geometry : boolean, optional - This option can be used to skip response geometries for each feature. - The returning object will be a data frame with no spatial information. - Note that the USGS Water Data APIs use camelCase "skipGeometry" in - CQL2 queries. - time : string, optional - The date an observation represents. You can query this field using - date-times or intervals, adhering to RFC 3339, or using ISO 8601 - duration objects. Intervals may be bounded or half-bounded (double-dots - at start or end). Only features that have a time that intersects the - value of datetime are selected. If a feature has multiple temporal - properties, it is the decision of the server whether only a single - temporal property is used to determine the extent or all relevant - temporal properties. - Examples: - - * A date-time: "2018-02-12T23:20:50Z" - * A bounded interval: "2018-02-12T00:00:00Z/2018-03-18T12:31:12Z" - * Half-bounded intervals: "2018-02-12T00:00:00Z/.." or - "../2018-03-18T12:31:12Z" - * Duration objects: "P1M" for data from the past month or - "PT36H" for the last 36 hours - - bbox : list of numbers, optional - Only features that have a geometry that intersects the bounding box are - selected. The bounding box is provided as four or six numbers, - depending on whether the coordinate reference system includes a vertical - axis (height or depth). Coordinates are assumed to be in crs 4326. The - expected format is ``[xmin, ymin, xmax, ymax]``, i.e. - ``[Western-most longitude, Southern-most latitude, Eastern-most - longitude, Northern-most latitude]``. - limit : int, optional - The optional limit parameter is used to control the subset of the - selected features that should be returned in each page. The maximum - allowable limit is 50000. It may be beneficial to set this number lower - if your internet connection is spotty. The default (None) will set the - limit to the maximum allowable limit for the service. - This is a per-page size, not a cap on the total result: a query - matching more rows than ``limit`` still returns every matching row - across multiple pages. Use ``max_rows`` to cap the total instead. - filter, filter_lang : optional - Server-side CQL filter passed through as the OGC ``filter`` / - ``filter-lang`` query parameters. See - :mod:`dataretrieval.ogc.filters` for syntax, auto-chunking, - and the lexicographic-comparison pitfall. - convert_type : boolean, optional - If True, converts columns to appropriate types. - max_rows : int, optional - Cap the total number of rows returned, stopping pagination early - instead of downloading the whole result. Unlike ``limit`` (the - per-page size), this bounds the total result across every page. - The default (None) follows pagination to completion. - **queryables : string or iterable of strings, optional - Any other queryable property of this collection, passed through as a - server-side filter. Call :func:`get_queryables` to see the queryables a - collection supports. - - Returns - ------- - df : ``pandas.DataFrame`` or ``geopandas.GeoDataFrame`` - Formatted data returned from the API query. - md: :obj:`dataretrieval.utils.BaseMetadata` - A custom metadata object - - Raises - ------ - ChunkInterrupted - A transient failure (429 / 5xx / timeout) interrupted the request - after the built-in retries. Completed work is preserved; resume - with ``exc.call.resume()`` (see :doc:`/userguide/errors`). - - Examples - -------- - .. code:: - - >>> # Get most recent daily flow data from a single site - >>> df, md = dataretrieval.waterdata.get_latest_daily( - ... monitoring_location_id="USGS-02238500", parameter_code="00060" - ... ) - - >>> # Restrict to rows whose underlying record was refreshed in the - >>> # last 7 days - >>> df, md = dataretrieval.waterdata.get_latest_daily( - ... monitoring_location_id="USGS-02238500", - ... parameter_code="00060", - ... last_modified="P7D", - ... ) - - >>> # Multi-site, multi-parameter — discharge and water temperature - >>> # at two sites in a single round-trip - >>> df, md = dataretrieval.waterdata.get_latest_daily( - ... monitoring_location_id=["USGS-01491000", "USGS-01645000"], - ... parameter_code=["00060", "00010"], - ... ) - - >>> # Get most recent daily measurements for two sites - >>> df, md = dataretrieval.waterdata.get_latest_daily( - ... monitoring_location_id=["USGS-05114000", "USGS-09423350"] - ... ) - """ - service = "latest-daily" - - # Build argument dictionary, omitting None values - args = _get_args(locals(), exclude={"max_rows"}) - - return get_ogc_data(args, service, max_rows=max_rows) - - -def get_field_measurements( - monitoring_location_id: str | Iterable[str] | None = None, - parameter_code: str | Iterable[str] | None = None, - observing_procedure_code: str | Iterable[str] | None = None, - properties: str | Iterable[str] | None = None, - field_visit_id: str | Iterable[str] | None = None, - approval_status: str | Iterable[str] | None = None, - unit_of_measure: str | Iterable[str] | None = None, - qualifier: str | Iterable[str] | None = None, - value: str | Iterable[str] | None = None, - last_modified: str | Iterable[str] | None = None, - observing_procedure: str | Iterable[str] | None = None, - vertical_datum: str | Iterable[str] | None = None, - measuring_agency: str | Iterable[str] | None = None, - skip_geometry: bool | None = None, - time: str | Iterable[str] | None = None, - bbox: list[float] | None = None, - limit: int | None = None, - filter: str | None = None, - filter_lang: FILTER_LANG | None = None, - convert_type: bool = True, - max_rows: int | None = None, - **queryables: Any, -) -> tuple[pd.DataFrame, BaseMetadata]: - """Field measurements are physically measured values collected during a - visit to the monitoring location. Field measurements consist of measurements - of gage height and discharge, and readings of groundwater levels, and are - primarily used as calibration readings for the automated sensors collecting - continuous data. They are collected at a low frequency, and delivery of the - data in WDFN may be delayed due to data processing time. - - Parameters - ---------- - monitoring_location_id : string or iterable of strings, optional - A unique identifier representing a single monitoring location. This - corresponds to the id field in the monitoring-locations endpoint. - Monitoring location IDs are created by combining the agency code of the - agency responsible for the monitoring location (e.g. USGS) with the ID - number of the monitoring location (e.g. 02238500), separated by a hyphen - (e.g. USGS-02238500). - parameter_code : string or iterable of strings, optional - Parameter codes are 5-digit codes used to identify the constituent - measured and the units of measure. A complete list of parameter codes - and associated groupings can be found at - https://help.waterdata.usgs.gov/codes-and-parameters/parameters. - observing_procedure_code : string or iterable of strings, optional - A short code corresponding to the observing procedure for the field - measurement. - properties : string or iterable of strings, optional - A list of requested columns to be returned from the query. See the - field-measurements schema in the OpenAPI reference for the available - columns (e.g. geometry, id, monitoring_location_id, parameter_code, - value, unit_of_measure, approval_status, qualifier, last_modified): - https://api.waterdata.usgs.gov/ogcapi/v0/openapi?f=html#/field-measurements - field_visit_id : string or iterable of strings, optional - A universally unique identifier (UUID) for the field visit. - Multiple measurements may be made during a single field visit. - approval_status : string or iterable of strings, optional - Some of the data that you have obtained from this U.S. Geological Survey - database may not have received Director's approval. Any such data values - are qualified as provisional and are subject to revision. Provisional - data are released on the condition that neither the USGS nor the United - States Government may be held liable for any damages resulting from its - use. This field reflects the approval status of each record, and is either - "Approved", meaning processing review has been completed and the data is - approved for publication, or "Provisional" and subject to revision. For - more information about provisional data, go to: - https://waterdata.usgs.gov/provisional-data-statement/. - unit_of_measure : string or iterable of strings, optional - A human-readable description of the units of measurement associated - with an observation. - qualifier : string or iterable of strings, optional - This field indicates any qualifiers associated with an observation, for - instance if a sensor may have been impacted by ice or if values were - estimated. - value : string or iterable of strings, optional - The value of the observation. Values are transmitted as strings in - the JSON response format in order to preserve precision. - last_modified : string, optional - The last time a record was refreshed in our database. This may happen - due to regular operational processes and does not necessarily indicate - that anything about the measurement has changed. You can query this field - using date-times or intervals, adhering to RFC 3339, or using ISO 8601 - duration objects. Intervals may be bounded or half-bounded (double-dots - at start or end). Only features that have a last_modified that - intersects the value of datetime are selected. - Examples: - - * A date-time: "2018-02-12T23:20:50Z" - * A bounded interval: "2018-02-12T00:00:00Z/2018-03-18T12:31:12Z" - * Half-bounded intervals: "2018-02-12T00:00:00Z/.." or - "../2018-03-18T12:31:12Z" - * Duration objects: "P1M" for data from the past month or - "PT36H" for the last 36 hours - - observing_procedure : string or iterable of strings, optional - Water measurement or water-quality observing procedure descriptions. - vertical_datum : string or iterable of strings, optional - The datum used to determine altitude and vertical position at the - monitoring location. - measuring_agency : string or iterable of strings, optional - The agency performing the measurement. - skip_geometry : boolean, optional - This option can be used to skip response geometries for each feature. - The returning object will be a data frame with no spatial information. - Note that the USGS Water Data APIs use camelCase "skipGeometry" in - CQL2 queries. - time : string, optional - The date an observation represents. You can query this field using date-times - or intervals, adhering to RFC 3339, or using ISO 8601 duration objects. - Intervals may be bounded or half-bounded (double-dots at start or end). - Only features that have a time that intersects the value of datetime are - selected. If a feature has multiple temporal properties, it is the - decision of the server whether only a single temporal property is used - to determine the extent or all relevant temporal properties. - Examples: - - * A date-time: "2018-02-12T23:20:50Z" - * A bounded interval: "2018-02-12T00:00:00Z/2018-03-18T12:31:12Z" - * Half-bounded intervals: "2018-02-12T00:00:00Z/.." or - "../2018-03-18T12:31:12Z" - * Duration objects: "P1M" for data from the past month or - "PT36H" for the last 36 hours - - bbox : list of numbers, optional - Only features that have a geometry that intersects the bounding box are - selected. The bounding box is provided as four or six numbers, - depending on whether the coordinate reference system includes a vertical - axis (height or depth). Coordinates are assumed to be in crs 4326. The - expected format is ``[xmin, ymin, xmax, ymax]``, i.e. - ``[Western-most longitude, Southern-most latitude, Eastern-most - longitude, Northern-most latitude]``. - limit : int, optional - The optional limit parameter is used to control the subset of the - selected features that should be returned in each page. The maximum - allowable limit is 50000. It may be beneficial to set this number lower - if your internet connection is spotty. The default (None) will set the - limit to the maximum allowable limit for the service. - This is a per-page size, not a cap on the total result: a query - matching more rows than ``limit`` still returns every matching row - across multiple pages. Use ``max_rows`` to cap the total instead. - filter, filter_lang : optional - Server-side CQL filter passed through as the OGC ``filter`` / - ``filter-lang`` query parameters. See - :mod:`dataretrieval.ogc.filters` for syntax, auto-chunking, - and the lexicographic-comparison pitfall. - convert_type : boolean, optional - If True, converts columns to appropriate types. - max_rows : int, optional - Cap the total number of rows returned, stopping pagination early - instead of downloading the whole result. Unlike ``limit`` (the - per-page size), this bounds the total result across every page. - The default (None) follows pagination to completion. - **queryables : string or iterable of strings, optional - Any other queryable property of this collection, passed through as a - server-side filter. Call :func:`get_queryables` to see the queryables a - collection supports. - - Returns - ------- - df : ``pandas.DataFrame`` or ``geopandas.GeoDataFrame`` - Formatted data returned from the API query. - md: :obj:`dataretrieval.utils.BaseMetadata` - A custom metadata object - - Raises - ------ - ChunkInterrupted - A transient failure (429 / 5xx / timeout) interrupted the request - after the built-in retries. Completed work is preserved; resume - with ``exc.call.resume()`` (see :doc:`/userguide/errors`). - - Examples - -------- - .. code:: - - >>> # Get field measurements from a single groundwater site - >>> # and parameter code, and do not return geometry - >>> df, md = dataretrieval.waterdata.get_field_measurements( - ... monitoring_location_id="USGS-375907091432201", - ... parameter_code="72019", - ... skip_geometry=True, - ... ) - - >>> # Half-bounded time range: every measurement at this site since - >>> # 1980 (open-ended end). Use ``"../"`` for the inverse - >>> # (everything up to a date). - >>> df, md = dataretrieval.waterdata.get_field_measurements( - ... monitoring_location_id="USGS-425957088141001", - ... time="1980-01-01/..", - ... ) - - >>> # Get field measurements from multiple sites and - >>> # parameter codes from the last 20 years - >>> df, md = dataretrieval.waterdata.get_field_measurements( - ... monitoring_location_id=[ - ... "USGS-451605097071701", - ... "USGS-263819081585801", - ... ], - ... parameter_code=["62611", "72019"], - ... time="P20Y", - ... ) - """ - service = "field-measurements" - - # Build argument dictionary, omitting None values - args = _get_args(locals(), exclude={"max_rows"}) - - return get_ogc_data(args, service, max_rows=max_rows) - - -def get_field_measurements_metadata( - monitoring_location_id: str | Iterable[str] | None = None, - parameter_code: str | Iterable[str] | None = None, - parameter_name: str | Iterable[str] | None = None, - parameter_description: str | Iterable[str] | None = None, - begin: str | Iterable[str] | None = None, - end: str | Iterable[str] | None = None, - last_modified: str | Iterable[str] | None = None, - properties: str | Iterable[str] | None = None, - skip_geometry: bool | None = None, - bbox: list[float] | None = None, - limit: int | None = None, - filter: str | None = None, - filter_lang: FILTER_LANG | None = None, - convert_type: bool = True, - max_rows: int | None = None, - **queryables: Any, -) -> tuple[pd.DataFrame, BaseMetadata]: - """Get field-measurement metadata: one row per (location, parameter) series. - - Each row describes a single field-measurement series — what parameter is - measured at the location, the period of record (``begin`` / ``end``), the - units, and so on — without returning the underlying observations - themselves. Use :func:`get_field_measurements` to fetch the values. - - This is the discrete-measurement analogue to - :func:`get_time_series_metadata` (which describes daily and continuous - series). It's primarily useful for inventory queries: "what - field-measurement parameters does this site have, and over what date - range?" - - See the OpenAPI reference for the full list of supported fields: - https://api.waterdata.usgs.gov/ogcapi/v0/openapi?f=html#/field-measurements-metadata - - Parameters - ---------- - monitoring_location_id : string or iterable of strings, optional - A unique identifier representing a single monitoring location, in - ``AGENCY-ID`` form (e.g. ``"USGS-02238500"``). - parameter_code : string or iterable of strings, optional - 5-digit parameter code. See - https://help.waterdata.usgs.gov/codes-and-parameters/parameters. - parameter_name : string or iterable of strings, optional - A human-understandable name corresponding to ``parameter_code``. - parameter_description : string or iterable of strings, optional - A human-readable description of what is being measured. - begin, end, last_modified : string, optional - Datetime fields that accept either an RFC 3339 datetime, an - interval (``"start/end"``, optionally half-bounded with ``..``), - or an ISO 8601 duration (e.g. ``"P1M"``, ``"PT36H"``). See - :func:`get_time_series_metadata` for the full grammar. - properties : string or iterable of strings, optional - Subset of columns to return. Defaults to every available property. - skip_geometry : boolean, optional - Skip per-feature geometries; the returned object will be a plain - ``DataFrame`` with no spatial information. - bbox : list of numbers, optional - Only features whose geometry intersects the bounding box are - selected. Format: ``[xmin, ymin, xmax, ymax]`` in CRS 4326 - (longitude / latitude, west-south-east-north). - limit : int, optional - Page size; the maximum allowable value is 50000. Default - (``None``) requests the maximum allowable limit. This is a - per-page size, not a cap on the total result: a query matching more - rows than ``limit`` still returns every matching row across - multiple pages. Use ``max_rows`` to cap the total instead. - filter, filter_lang : optional - Server-side CQL filter passed through as the OGC ``filter`` / - ``filter-lang`` query parameters. See - :mod:`dataretrieval.ogc.filters` for syntax, auto-chunking, - and the lexicographic-comparison pitfall. - convert_type : boolean, optional - If True, converts columns to appropriate types. - max_rows : int, optional - Cap the total number of rows returned, stopping pagination early - instead of downloading the whole result. Unlike ``limit`` (the - per-page size), this bounds the total result across every page. - The default (None) follows pagination to completion. - **queryables : string or iterable of strings, optional - Any other queryable property of this collection, passed through as a - server-side filter. Call :func:`get_queryables` to see the queryables a - collection supports. - - Returns - ------- - df : ``pandas.DataFrame`` or ``geopandas.GeoDataFrame`` - Formatted data returned from the API query. - md : :obj:`dataretrieval.utils.BaseMetadata` - A custom metadata object pertaining to the query. - - Raises - ------ - ChunkInterrupted - A transient failure (429 / 5xx / timeout) interrupted the request - after the built-in retries. Completed work is preserved; resume - with ``exc.call.resume()`` (see :doc:`/userguide/errors`). - - Examples - -------- - .. code:: - - >>> # All field-measurement series at a surface-water site - >>> df, md = dataretrieval.waterdata.get_field_measurements_metadata( - ... monitoring_location_id="USGS-02238500" - ... ) - - >>> # Same, for a groundwater well - >>> df, md = dataretrieval.waterdata.get_field_measurements_metadata( - ... monitoring_location_id="USGS-375907091432201" - ... ) - - >>> # Multi-site, narrowed to two parameter codes - >>> df, md = dataretrieval.waterdata.get_field_measurements_metadata( - ... monitoring_location_id=[ - ... "USGS-451605097071701", - ... "USGS-263819081585801", - ... ], - ... parameter_code=["62611", "72019"], - ... ) - - >>> # Series modified in the last year — useful for incremental ETL - >>> df, md = dataretrieval.waterdata.get_field_measurements_metadata( - ... monitoring_location_id="USGS-375907091432201", - ... parameter_code="72019", - ... last_modified="P1Y", - ... ) - - """ - service = "field-measurements-metadata" - - args = _get_args(locals(), exclude={"max_rows"}) - - return get_ogc_data(args, service, max_rows=max_rows) - - -def get_peaks( - monitoring_location_id: str | Iterable[str] | None = None, - parameter_code: str | Iterable[str] | None = None, - time_series_id: str | Iterable[str] | None = None, - unit_of_measure: str | Iterable[str] | None = None, - time: str | Iterable[str] | None = None, - last_modified: str | Iterable[str] | None = None, - water_year: int | list[int] | None = None, - year: int | list[int] | None = None, - month: int | list[int] | None = None, - day: int | list[int] | None = None, - peak_since: int | list[int] | None = None, - properties: str | Iterable[str] | None = None, - skip_geometry: bool | None = None, - bbox: list[float] | None = None, - limit: int | None = None, - filter: str | None = None, - filter_lang: FILTER_LANG | None = None, - convert_type: bool = True, - max_rows: int | None = None, - **queryables: Any, -) -> tuple[pd.DataFrame, BaseMetadata]: - """Get the annual peak streamflow / stage record for a monitoring location. - - Peaks are the largest values observed at a site each water year and are - the standard input to flood-frequency analysis (e.g. log-Pearson Type III - fits). The endpoint returns one row per (monitoring location, parameter, - water year), with the peak ``value`` and the ``time`` it occurred. - - The collection covers both stage (parameter ``"00065"``, ``ft``) and - discharge (parameter ``"00060"``, ``ft^3/s``); a typical streamgage has a - series for each. Reference docs: - https://api.waterdata.usgs.gov/ogcapi/v0/openapi?f=html#/peaks - - Parameters - ---------- - monitoring_location_id : string or iterable of strings, optional - A unique identifier representing a single monitoring location, in - ``AGENCY-ID`` form (e.g. ``"USGS-02238500"``). - parameter_code : string or iterable of strings, optional - 5-digit parameter code. Most peaks records are ``"00060"`` (discharge) - or ``"00065"`` (stage / gage height). Full list at - https://help.waterdata.usgs.gov/codes-and-parameters/parameters. - time_series_id : string or iterable of strings, optional - ID of the time series the peak belongs to. - unit_of_measure : string or iterable of strings, optional - Human-readable units (e.g. ``"ft^3/s"``, ``"ft"``). - time : string, optional - Datetime, interval, or duration filter on the peak's date. - See :func:`get_time_series_metadata` for the full grammar. - last_modified : string, optional - Same datetime grammar as ``time``; filters on the database - last-modified timestamp (useful for incremental ETL polling). - water_year, year, month, day : int or list of ints, optional - Calendar / water-year filters on the peak event. The water year ends - September 30 (e.g. WY2024 = Oct 1, 2023 – Sep 30, 2024). - peak_since : int or list of ints, optional - Filter on the year since which the peak value has stood as the - record (the API serves this field as an integer; many rows are - ``null``). - properties : string or iterable of strings, optional - Subset of columns to return. Defaults to every available property. - skip_geometry : boolean, optional - Skip per-feature geometries; the returned object will be a plain - ``DataFrame`` with no spatial information. - bbox : list of numbers, optional - Only features whose geometry intersects the bounding box are - selected. Format: ``[xmin, ymin, xmax, ymax]`` in CRS 4326 - (longitude / latitude, west-south-east-north). - limit : int, optional - Page size; the maximum allowable value is 50000. Default - (``None``) requests the maximum allowable limit. This is a - per-page size, not a cap on the total result: a query matching more - rows than ``limit`` still returns every matching row across - multiple pages. Use ``max_rows`` to cap the total instead. - filter, filter_lang : optional - Server-side CQL filter passed through as the OGC ``filter`` / - ``filter-lang`` query parameters. See - :mod:`dataretrieval.ogc.filters` for syntax, auto-chunking, - and the lexicographic-comparison pitfall. - convert_type : boolean, optional - If True, converts columns to appropriate types. - max_rows : int, optional - Cap the total number of rows returned, stopping pagination early - instead of downloading the whole result. Unlike ``limit`` (the - per-page size), this bounds the total result across every page. - The default (None) follows pagination to completion. - **queryables : string or iterable of strings, optional - Any other queryable property of this collection, passed through as a - server-side filter. Call :func:`get_queryables` to see the queryables a - collection supports. - - Returns - ------- - df : ``pandas.DataFrame`` or ``geopandas.GeoDataFrame`` - Formatted data returned from the API query. - md : :obj:`dataretrieval.utils.BaseMetadata` - A custom metadata object pertaining to the query. - - Raises - ------ - ChunkInterrupted - A transient failure (429 / 5xx / timeout) interrupted the request - after the built-in retries. Completed work is preserved; resume - with ``exc.call.resume()`` (see :doc:`/userguide/errors`). - - Examples - -------- - .. code:: - - >>> # Full annual peak record at one site (both stage and discharge) - >>> df, md = dataretrieval.waterdata.get_peaks( - ... monitoring_location_id="USGS-02238500" - ... ) - - >>> # Discharge peaks only - >>> df, md = dataretrieval.waterdata.get_peaks( - ... monitoring_location_id="USGS-02238500", - ... parameter_code="00060", - ... ) - - >>> # Multi-site peaks for a parameter, narrowed to a water-year range - >>> df, md = dataretrieval.waterdata.get_peaks( - ... monitoring_location_id=[ - ... "USGS-07069000", - ... "USGS-07064000", - ... "USGS-07068000", - ... ], - ... parameter_code="00060", - ... water_year=[2020, 2021, 2022, 2023], - ... ) - - """ - service = "peaks" - - args = _get_args(locals(), exclude={"max_rows"}) - - return get_ogc_data(args, service, max_rows=max_rows) - - -def get_reference_table( - collection: str, - limit: int | None = None, - query: dict[str, Any] | None = None, - max_rows: int | None = None, -) -> tuple[pd.DataFrame, BaseMetadata]: - """Get metadata reference tables for the USGS Water Data API. - - Reference tables provide the range of allowable values for parameter - arguments in the waterdata module. - - Parameters - ---------- - collection : string - One of the following options: "agency-codes", "altitude-datums", - "aquifer-codes", "aquifer-types", "coordinate-accuracy-codes", - "coordinate-datum-codes", "coordinate-method-codes", "counties", - "hydrologic-unit-codes", "medium-codes", "national-aquifer-codes", - "parameter-codes", "reliability-codes", "site-types", "states", - "statistic-codes", "topographic-codes", "time-zone-codes" - limit : int, optional - The optional limit parameter is used to control the subset of the - selected features that should be returned in each page. The maximum - allowable limit is 50000. It may be beneficial to set this number lower - if your internet connection is spotty. The default (None) will set the - limit to the maximum allowable limit for the service. - query: dictionary, optional - The optional query parameter can be used to pass a dictionary of - query parameters to the collection API call. - max_rows : int, optional - Cap the total number of rows returned, stopping pagination early - instead of downloading the whole table. Useful for cheaply - previewing large tables (e.g. ``hydrologic-unit-codes`` has ~125k - rows). Unlike ``limit`` (the per-page size), this bounds the total - result. The default (None) downloads every page. - - Returns - ------- - df : ``pandas.DataFrame`` or ``geopandas.GeoDataFrame`` - Formatted data returned from the API query. The primary metadata - of each reference table will show up in the first column, where - the name of the column is the singular form of the collection name, - separated by underscores (e.g. the "medium-codes" reference table - has a column called "medium_code", which contains all possible - medium code values). - md: :obj:`dataretrieval.utils.BaseMetadata` - A custom metadata object including the URL request and query time. - - Raises - ------ - ChunkInterrupted - A transient failure (429 / 5xx / timeout) interrupted the request - after the built-in retries. Completed work is preserved; resume - with ``exc.call.resume()`` (see :doc:`/userguide/errors`). - - Examples - -------- - .. code:: - - >>> # Get table of USGS parameter codes - >>> ref, md = dataretrieval.waterdata.get_reference_table( - ... collection="parameter-codes" - ... ) - - >>> # Get table of selected USGS parameter codes - >>> ref, md = dataretrieval.waterdata.get_reference_table( - ... collection="parameter-codes", - ... query={"id": "00001,00002"}, - ... ) - """ - valid_code_services = get_args(METADATA_COLLECTIONS) - if collection not in valid_code_services: - raise ValueError( - f"Invalid code service: '{collection}'. " - f"Valid options are: {valid_code_services}." - ) - - # Give the ID column the collection name, singularized and underscored. - if collection == "counties": - output_id = "county" - elif collection.endswith("s"): - output_id = collection[:-1].replace("-", "_") - else: - output_id = collection.replace("-", "_") - - query_args = dict(query) if query else {} - if limit is not None: - query_args["limit"] = limit - return get_ogc_data( - args=query_args, output_id=output_id, service=collection, max_rows=max_rows - ) - - -def get_queryables(collection: str) -> tuple[pd.DataFrame, BaseMetadata]: - """List the queryable properties of a Water Data API collection. - - Every OGC collection (``daily``, ``continuous``, ``monitoring-locations``, - ...) advertises the set of properties that can be filtered on -- exposed as - the typed keyword arguments of the matching ``get_*`` function, and usable - directly in a CQL2 ``filter``. This returns that set, so the available - filters can be discovered programmatically and monitored for upstream - additions. - - Parameters - ---------- - collection : string - The collection id, e.g. ``"daily"``, ``"continuous"``, - ``"monitoring-locations"``, or ``"time-series-metadata"``. See - :data:`dataretrieval.waterdata.types.WATERDATA_SERVICES` for the data - collections; reference collections (e.g. ``"parameter-codes"``) work - too. - - Returns - ------- - df : ``pandas.DataFrame`` - One row per queryable, sorted by name, with columns ``queryable`` (the - property name), ``type``, ``title``, and ``description``. - md : :class:`dataretrieval.utils.BaseMetadata` - Metadata describing the request (URL, query time, response headers). - - Raises - ------ - DataRetrievalError - On an HTTP error response (e.g. an unknown ``collection`` yields a 404), - the typed subclass for the status. - - Examples - -------- - .. doctest:: - :skipif: True # network - - >>> from dataretrieval import waterdata - >>> df, md = waterdata.get_queryables("daily") - >>> df.set_index("queryable").loc["state_name", "type"] - 'string' - """ - # The OGC queryables document is a JSON Schema whose ``properties`` map each - # filterable property name to a ``{title, type, description}`` definition. - body, response = _check_ogc_requests(endpoint=collection, req_type="queryables") - properties: dict[str, Any] = body.get("properties", {}) - df = pd.DataFrame( - [ - { - "queryable": name, - "type": prop.get("type"), - "title": prop.get("title"), - "description": (prop.get("description") or "").strip(), - } - for name, prop in sorted(properties.items()) - ], - columns=["queryable", "type", "title", "description"], - ) - return df, BaseMetadata(response) - - -def get_codes(code_service: CODE_SERVICES) -> tuple[pd.DataFrame, BaseMetadata]: - """Return codes from a Samples code service. - - Parameters - ---------- - code_service : string - One of the following options: "states", "counties", "countries", - "sitetype", "samplemedia", "characteristicgroup", "characteristics", - or "observedproperty" - - Returns - ------- - df : ``pandas.DataFrame`` - The requested code table. - md : :obj:`dataretrieval.utils.BaseMetadata` - Metadata for the query (URL, query time, response headers). - """ - valid_code_services = get_args(CODE_SERVICES) - if code_service not in valid_code_services: - raise ValueError( - f"Invalid code service: '{code_service}'. " - f"Valid options are: {valid_code_services}." - ) - - url = f"{SAMPLES_URL}/codeservice/{code_service}?mimeType=application%2Fjson" - - response = _get(url, headers=_default_headers(url), **HTTPX_DEFAULTS) - - _raise_for_non_200(response) - - data_dict = json.loads(response.text) - data_list = data_dict["data"] - - df = pd.DataFrame(data_list) - - return df, BaseMetadata(response) - - -def _get_samples_csv( - url: str, params: dict[str, Any], ssl_check: bool -) -> tuple[pd.DataFrame, httpx.Response]: - """Issue a Samples CSV request and parse the body into a DataFrame. - - Shared tail for the Samples getters: sends the GET with the standard - headers (including ``X-Api-Key``), raises a typed error on a non-200 - (consistent with the OGC/stats path) instead of a bare - ``HTTPStatusError``, and reads the CSV. The caller wraps the response - as metadata and applies any per-getter post-step. - """ - logger.debug("Request: %s", httpx.URL(url).copy_merge_params(params)) - response = _get( - url, - params=params, - verify=ssl_check, - headers=_default_headers(url), - **HTTPX_DEFAULTS, - ) - _raise_for_non_200(response) - df = pd.read_csv(StringIO(response.text), delimiter=",") - return df, response - - -# Map the public snake_case ``get_samples`` parameters to the camelCase query -# parameter names the Samples API expects on the wire. ``characteristic`` is -# already snake_case-compatible (single word) and is sent unchanged. The -# remaining snake_case params are bookkeeping (``service``/``profile``/ -# ``ssl_check``) and never reach the request. -_SAMPLES_PARAM_TO_API = { - "activity_media_name": "activityMediaName", - "activity_start_date_lower": "activityStartDateLower", - "activity_start_date_upper": "activityStartDateUpper", - "activity_type_code": "activityTypeCode", - "characteristic_group": "characteristicGroup", - "characteristic_user_supplied": "characteristicUserSupplied", - "bbox": "boundingBox", - "country_code": "countryFips", - "state_code": "stateFips", - "county_code": "countyFips", - "site_type_code": "siteTypeCode", - "site_type_name": "siteTypeName", - "usgs_pcode": "usgsPCode", - "hydrologic_unit": "hydrologicUnit", - "monitoring_location_id": "monitoringLocationIdentifier", - "organization_id": "organizationIdentifier", - "point_location_latitude": "pointLocationLatitude", - "point_location_longitude": "pointLocationLongitude", - "point_location_within_miles": "pointLocationWithinMiles", - "project_id": "projectIdentifier", - "record_identifier_user_supplied": "recordIdentifierUserSupplied", -} - -# Deprecated camelCase keyword names (the Samples-API spelling) accepted for -# backward compatibility, mapped to the new snake_case parameter names. Derived -# from ``_SAMPLES_PARAM_TO_API`` so the two never drift apart. -_SAMPLES_LEGACY_KWARGS = { - api_name: py_name for py_name, api_name in _SAMPLES_PARAM_TO_API.items() -} - - -@_accept_legacy_kwargs(_SAMPLES_LEGACY_KWARGS) -def get_samples( - ssl_check: bool = True, - service: SERVICES = "results", - profile: PROFILES = "fullphyschem", - activity_media_name: str | Iterable[str] | None = None, - activity_start_date_lower: str | None = None, - activity_start_date_upper: str | None = None, - activity_type_code: str | Iterable[str] | None = None, - characteristic_group: str | Iterable[str] | None = None, - characteristic: str | Iterable[str] | None = None, - characteristic_user_supplied: str | Iterable[str] | None = None, - bbox: list[float] | None = None, - country_code: str | Iterable[str] | None = None, - state_code: str | Iterable[str] | None = None, - county_code: str | Iterable[str] | None = None, - site_type_code: str | Iterable[str] | None = None, - site_type_name: str | Iterable[str] | None = None, - usgs_pcode: str | Iterable[str] | None = None, - hydrologic_unit: str | Iterable[str] | None = None, - monitoring_location_id: str | Iterable[str] | None = None, - organization_id: str | Iterable[str] | None = None, - point_location_latitude: float | None = None, - point_location_longitude: float | None = None, - point_location_within_miles: float | None = None, - project_id: str | Iterable[str] | None = None, - record_identifier_user_supplied: str | Iterable[str] | None = None, -) -> tuple[pd.DataFrame, BaseMetadata]: - """Search Samples database for USGS water quality data. - This is a wrapper function for the Samples database API. All potential - filters are provided as arguments to the function, but please do not - populate all possible filters; leave as many as feasible with their default - value (None). This is important because overcomplicated web service queries - can bog down the database's ability to return an applicable dataset before - it times out. - - The web GUI for the Samples database can be found here: - https://waterdata.usgs.gov/download-samples/#dataProfile=site - - If you would like more details on feasible query parameters (complete with - examples), please visit the Samples database swagger docs, here: - https://api.waterdata.usgs.gov/samples-data/docs#/ - - Parameters - ---------- - ssl_check : bool, optional - Check the SSL certificate. - service : string - One of the available Samples services: "results", "locations", "activities", - "projects", or "organizations". Defaults to "results". - profile : string - One of the available profiles associated with a service. Options for each - service are: - results - "fullphyschem", "basicphyschem", - "fullbio", "basicbio", "narrow", - "resultdetectionquantitationlimit", - "labsampleprep", "count" - locations - "site", "count" - activities - "sampact", "actmetric", - "actgroup", "count" - projects - "project", "projectmonitoringlocationweight" - organizations - "organization", "count" - activity_media_name : string or iterable of strings, optional - Name or code indicating environmental medium in which sample was taken. - Call ``get_codes("samplemedia")`` for the valid inputs. - Example: "Water". (Samples API: ``activityMediaName``) - activity_start_date_lower : string, optional - The start date if using a date range. Takes the format YYYY-MM-DD. - The logic is inclusive, i.e. it will also return results that - match the date. If left as None, will pull all data on or before - ``activity_start_date_upper``, if populated. - (Samples API: ``activityStartDateLower``) - activity_start_date_upper : string, optional - The end date if using a date range. Takes the format YYYY-MM-DD. - The logic is inclusive, i.e. it will also return results that - match the date. If left as None, will pull all data after - ``activity_start_date_lower`` up to the most recent available results. - (Samples API: ``activityStartDateUpper``) - activity_type_code : string or iterable of strings, optional - Text code that describes type of field activity performed. - Example: "Sample-Routine, regular". (Samples API: ``activityTypeCode``) - characteristic_group : string or iterable of strings, optional - Characteristic group is a broad category of characteristics - describing one or more results. Call ``get_codes("characteristicgroup")`` - for the valid inputs. - Example: "Organics, PFAS" (Samples API: ``characteristicGroup``) - characteristic : string or iterable of strings, optional - Characteristic is a specific category describing one or more results. - Call ``get_codes("characteristics")`` for the valid inputs. - Example: "Suspended Sediment Discharge" (Samples API: ``characteristic``) - characteristic_user_supplied : string or iterable of strings, optional - A user supplied characteristic name describing one or more results. - (Samples API: ``characteristicUserSupplied``) - bbox : list of four floats, optional - Filters on the associated monitoring location's point location - by checking if it is located within the specified geographic area. - The logic is inclusive, i.e. it will include locations that overlap - with the edge of the bounding box. Values are separated by commas, - expressed in decimal degrees, NAD83, and longitudes west of Greenwich - are negative. The format is a list consisting of: - - * Western-most longitude - * Southern-most latitude - * Eastern-most longitude - * Northern-most latitude - - Example: [-92.8,44.2,-88.9,46.0] (Samples API: ``boundingBox``) - country_code : string or iterable of strings, optional - Example: "US" (United States) (Samples API: ``countryFips``) - state_code : string or iterable of strings, optional - Call ``get_codes("states")`` for the valid inputs. - Example: "US:15" (United States: Hawaii) (Samples API: ``stateFips``) - county_code : string or iterable of strings, optional - Call ``get_codes("counties")`` for the valid inputs. - Example: "US:15:001" (United States: Hawaii, Hawaii County) - (Samples API: ``countyFips``) - site_type_code : string or iterable of strings, optional - An abbreviation for a certain site type. Call ``get_codes("sitetype")`` - for the valid inputs. - Example: "GW" (Groundwater site) (Samples API: ``siteTypeCode``) - site_type_name : string or iterable of strings, optional - A full name for a certain site type. Call ``get_codes("sitetype")`` - for the valid inputs. - Example: "Well" (Samples API: ``siteTypeName``) - usgs_pcode : string or iterable of strings, optional - 5-digit number used in the US Geological Survey computerized - data system, National Water Information System (NWIS), to - uniquely identify a specific constituent (the ``parameterCode`` column - of ``get_codes("characteristics")``). - Example: "00060" (Discharge, cubic feet per second) - (Samples API: ``usgsPCode``) - hydrologic_unit : string or iterable of strings, optional - Max 12-digit number used to describe a hydrologic unit. - Example: "070900020502" (Samples API: ``hydrologicUnit``) - monitoring_location_id : string or iterable of strings, optional - A monitoring location identifier has two parts: the agency code - and the location number, separated by a dash (-). - Example: "USGS-040851385" - (Samples API: ``monitoringLocationIdentifier``) - organization_id : string or iterable of strings, optional - Designator used to uniquely identify a specific organization. - Currently only accepting the organization "USGS". - (Samples API: ``organizationIdentifier``) - point_location_latitude : float, optional - Latitude for a point/radius query (decimal degrees). Must be used - with ``point_location_longitude`` and ``point_location_within_miles``. - (Samples API: ``pointLocationLatitude``) - point_location_longitude : float, optional - Longitude for a point/radius query (decimal degrees). Must be used - with ``point_location_latitude`` and ``point_location_within_miles``. - (Samples API: ``pointLocationLongitude``) - point_location_within_miles : float, optional - Radius for a point/radius query. Must be used with - ``point_location_latitude`` and ``point_location_longitude``. - (Samples API: ``pointLocationWithinMiles``) - project_id : string or iterable of strings, optional - Designator used to uniquely identify a data collection project. Project - identifiers are specific to an organization (e.g. USGS). - Example: "ZH003QW03" (Samples API: ``projectIdentifier``) - record_identifier_user_supplied : string or iterable of strings, optional - Internal AQS record identifier that returns 1 entry. Only available - for the "results" service. - (Samples API: ``recordIdentifierUserSupplied``) - - Returns - ------- - df : ``pandas.DataFrame`` - Formatted data returned from the API query. For each - ``Date`` / ``Time`` / ``TimeZone`` triplet in - the response (e.g. ``Activity_StartDate``, ``Activity_StartTime``, - ``Activity_StartTimeZone``), an additional ``DateTime`` column - is appended holding a UTC ``Timestamp`` derived from the three. The - original Date/Time/TimeZone columns are left intact; rows whose - timezone abbreviation is not recognized resolve to ``NaT``. Rows are - sorted by ``Activity_StartDateTime`` when present (the API's default - order is unstable). - md : :obj:`dataretrieval.utils.BaseMetadata` - Custom ``dataretrieval`` metadata object pertaining to the query. - - Examples - -------- - .. code:: - - >>> # Get PFAS results within a bounding box - >>> df, md = dataretrieval.waterdata.get_samples( - ... bbox=[-90.2, 42.6, -88.7, 43.2], - ... characteristic_group="Organics, PFAS", - ... ) - - >>> # Get all activities for the Commonwealth of Virginia over a date range - >>> df, md = dataretrieval.waterdata.get_samples( - ... service="activities", - ... profile="sampact", - ... activity_start_date_lower="2023-10-01", - ... activity_start_date_upper="2024-01-01", - ... state_code="US:51", - ... ) - - >>> # Get all pH samples for two sites in Utah - >>> df, md = dataretrieval.waterdata.get_samples( - ... monitoring_location_id=[ - ... "USGS-393147111462301", - ... "USGS-393343111454101", - ... ], - ... usgs_pcode="00400", - ... ) - - """ - - _check_profiles(service, profile) - - # Build argument dictionary, omitting None values. Parameters are the - # public snake_case names here; translate them to the camelCase names the - # Samples API expects just before building the request. - args = _get_args(locals(), exclude={"ssl_check", "profile"}) - params = {_SAMPLES_PARAM_TO_API.get(key, key): value for key, value in args.items()} - - params.update({"mimeType": "text/csv"}) - - if "boundingBox" in params: - params["boundingBox"] = to_str(params["boundingBox"]) - - url = f"{SAMPLES_URL}/{service}/{profile}" - - df, response = _get_samples_csv(url, params, ssl_check) - df = _attach_datetime_columns(df) - - return df, BaseMetadata(response) - - -@_accept_legacy_kwargs({"monitoringLocationIdentifier": "monitoring_location_id"}) -def get_samples_summary( - monitoring_location_id: str, - ssl_check: bool = True, -) -> tuple[pd.DataFrame, BaseMetadata]: - """Get a summary of discrete water-quality samples at a single monitoring location. - - Wraps the Samples database summary service described at - https://api.waterdata.usgs.gov/samples-data/docs. The service returns one - row per (characteristic group, characteristic, user-supplied characteristic) - combination with result and activity counts and the first / most recent - activity dates — useful for taking inventory of what discrete-sample data - exists at a site before pulling the underlying observations with - :func:`get_samples`. - - The summary service is single-site only: it accepts exactly one monitoring - location per request. - - Parameters - ---------- - monitoring_location_id : string - A monitoring location identifier has two parts, separated by a dash - (``-``): the agency code and the location number. Examples: - ``"USGS-040851385"``, ``"AZ014-320821110580701"``, - ``"CAX01-15304600"``. Bare location numbers without an agency prefix - are accepted by the service but return an empty result, so a prefix - is effectively required. (Samples API: ``monitoringLocationIdentifier``) - ssl_check : bool, optional - Check the SSL certificate. Default is True. - - Returns - ------- - df : ``pandas.DataFrame`` - Formatted data returned from the API query. - md : :obj:`dataretrieval.utils.BaseMetadata` - Custom ``dataretrieval`` metadata object pertaining to the query. - - Examples - -------- - .. code:: - - >>> # What discrete-sample data is available at this site? - >>> df, md = dataretrieval.waterdata.get_samples_summary( - ... monitoring_location_id="USGS-04074950" - ... ) - - """ - if not isinstance(monitoring_location_id, str): - raise TypeError( - "monitoring_location_id must be a string; the Samples " - "summary service accepts exactly one monitoring location per " - f"request, got {type(monitoring_location_id).__name__}." - ) - - url = f"{SAMPLES_URL}/summary/{quote(monitoring_location_id, safe='')}" - params = {"mimeType": "text/csv"} - - df, response = _get_samples_csv(url, params, ssl_check) - - return df, BaseMetadata(response) - - -def get_stats_por( - approval_status: str | None = None, - computation_type: str | Iterable[str] | None = None, - country_code: str | Iterable[str] | None = None, - state: str | Iterable[str] | None = None, - state_code: str | Iterable[str] | None = None, - county_code: str | Iterable[str] | None = None, - start_date: str | None = None, - end_date: str | None = None, - monitoring_location_id: str | Iterable[str] | None = None, - page_size: int = 1000, - parent_time_series_id: str | Iterable[str] | None = None, - site_type_code: str | Iterable[str] | None = None, - site_type_name: str | Iterable[str] | None = None, - parameter_code: str | Iterable[str] | None = None, - normal_type: str | None = None, - expand_percentiles: bool = True, -) -> tuple[pd.DataFrame, BaseMetadata]: - """Get day-of-year and month-of-year water data statistics from the - USGS Water Data API. - This service (called the "observationNormals" endpoint on api.waterdata.usgs.gov) - provides endpoints for access to computations on the historical record regarding - water conditions, including minimum, maximum, mean, median, and percentiles for - day of year and month of year. For more information regarding the calculation of - statistics and other details, please visit the Statistics documentation page: - https://waterdata.usgs.gov/statistics-documentation/. - - Note: This API is under active beta development and subject to - change. Improved handling of significant figures will be - addressed in a future release. - - Parameters - ---------- - approval_status: string, optional - Whether to include approved and/or provisional observations. - At this time, only approved observations are returned. - computation_type: string, optional - Desired statistical computation method. Available values are: - arithmetic_mean, maximum, median, minimum, percentile. - country_code: string, optional - Country query parameter. API defaults to "US". - state: string or iterable of strings, optional - State/territory filter (the recommended parameter). Accepts a full name - ("Wisconsin"), a two-letter postal code ("WI"), or a two-digit - ANSI/FIPS code ("55"). - state_code: string, optional - State query parameter. Takes the format "US:XX", where XX is - the two-digit state code. API defaults to "US:42" (Pennsylvania). - county_code: string, optional - County query parameter. Takes the format "US:XX:YYY", where XX is - the two-digit state code and YYY is the three-digit county code. - API defaults to "US:42:103" (Pennsylvania, Pike County). - start_date: string or datetime, optional - Start day for the query in the month-day format (MM-DD). - end_date: string or datetime, optional - End day for the query in the month-day format (MM-DD). - monitoring_location_id : string or iterable of strings, optional - A unique identifier representing a single monitoring location. This - corresponds to the id field in the monitoring-locations endpoint. - Monitoring location IDs are created by combining the agency code of the - agency responsible for the monitoring location (e.g. USGS) with the ID - number of the monitoring location (e.g. 02238500), separated by a hyphen - (e.g. USGS-02238500). - page_size : int, optional - The number of results to return per page, where one result represents a - monitoring location. The default is 1000. - parent_time_series_id: string, optional - The parent_time_series_id returns statistics tied to a - particular database entry. - site_type_code: string, optional - Site type code query parameter. - A list of valid site type codes is available at: - https://api.waterdata.usgs.gov/ogcapi/v0/collections/site-types/items. - Example: "GW" (Groundwater site) - site_type_name: string, optional - Site type name query parameter. - parameter_code : string or iterable of strings, optional - Parameter codes are 5-digit codes used to identify the constituent - measured and the units of measure. A complete list of parameter codes - and associated groupings can be found at - https://help.waterdata.usgs.gov/codes-and-parameters/parameters. - normal_type : string, optional - Filter the returned normals to a single period. If unspecified - (default), all matching data are returned. Available values: - "DOY" (day-of-year) and "MOY" (month-of-year). - expand_percentiles : boolean - Percentile data for a given day of year or month of year by default - are returned from the service as lists of string values and percentile - thresholds in the "values" and "percentiles" columns, respectively. - When `expand_percentiles` is set to True (default), each value and - percentile threshold specific to a computation id are returned as - individual rows in the dataframe, with the value reported in the - "value" column and the corresponding percentile reported in a - "percentile" column (and the "values" and "percentiles" columns - are removed). Missing percentile values expressed as 'nan' in the - list of string values are removed from the dataframe to save space. - Setting `expand_percentiles` to False retains the "values" and - "percentiles" columns produced by the service. Including - both 'percentiles' and one or more other statistics ('median', - 'minimum', 'maximum', or 'arithmetic_mean') in the `computation_type` - argument will return both the "values" column, containing the list - of percentile threshold values, and a "value" column, containing - the singular summary value for the other statistics. - - Returns - ------- - df : ``pandas.DataFrame`` or ``geopandas.GeoDataFrame`` - Formatted data returned from the API query. - md : :obj:`dataretrieval.utils.BaseMetadata` - A custom metadata object. - - Examples - -------- - .. code:: - - >>> # Get daily, monthly, and annual percentiles for streamflow at - >>> # a monitoring location of interest - >>> df, md = dataretrieval.waterdata.get_stats_por( - ... monitoring_location_id="USGS-05114000", - ... parameter_code="00060", - ... computation_type="percentile", - ... ) - - >>> # Get all daily and monthly statistics for the month of January - >>> # over the entire period of record for streamflow and gage height - >>> # at a monitoring location of interest - >>> df, md = dataretrieval.waterdata.get_stats_por( - ... monitoring_location_id="USGS-05114000", - ... parameter_code=["00060", "00065"], - ... start_date="01-01", - ... end_date="01-31", - ... ) - """ - # Build argument dictionary, omitting None values - params = _get_args( - _with_state(locals(), to="fips_us", into="state_code"), - exclude={"expand_percentiles"}, - ) - - return stats.get_data( - args=params, service="observationNormals", expand_percentiles=expand_percentiles - ) - - -def get_stats_date_range( - approval_status: str | None = None, - computation_type: str | Iterable[str] | None = None, - country_code: str | Iterable[str] | None = None, - state: str | Iterable[str] | None = None, - state_code: str | Iterable[str] | None = None, - county_code: str | Iterable[str] | None = None, - start_date: str | None = None, - end_date: str | None = None, - monitoring_location_id: str | Iterable[str] | None = None, - page_size: int = 1000, - parent_time_series_id: str | Iterable[str] | None = None, - site_type_code: str | Iterable[str] | None = None, - site_type_name: str | Iterable[str] | None = None, - parameter_code: str | Iterable[str] | None = None, - interval_type: str | Iterable[str] | None = None, - expand_percentiles: bool = True, -) -> tuple[pd.DataFrame, BaseMetadata]: - """Get monthly and annual water data statistics from the USGS Water Data API. - This service (called the "observationIntervals" endpoint on api.waterdata.usgs.gov) - provides endpoints for access to computations on the historical record regarding - water conditions, including minimum, maximum, mean, median, and percentiles for - month-year, and water/calendar years. For more information regarding the calculation - of statistics and other details, please visit the Statistics documentation page: - https://waterdata.usgs.gov/statistics-documentation/. - - Note: This API is under active beta development and subject to - change. Improved handling of significant figures will be - addressed in a future release. - - Parameters - ---------- - approval_status: string, optional - Whether to include approved and/or provisional observations. - At this time, only approved observations are returned. - computation_type: string, optional - Desired statistical computation method. Available values are: - arithmetic_mean, maximum, median, minimum, percentile. - country_code: string, optional - Country query parameter. API defaults to "US". - state: string or iterable of strings, optional - State/territory filter (the recommended parameter). Accepts a full name - ("Wisconsin"), a two-letter postal code ("WI"), or a two-digit - ANSI/FIPS code ("55"). - state_code: string, optional - State query parameter. Takes the format "US:XX", where XX is - the two-digit state code. API defaults to "US:42" (Pennsylvania). - county_code: string, optional - County query parameter. Takes the format "US:XX:YYY", where XX is - the two-digit state code and YYY is the three-digit county code. - API defaults to "US:42:103" (Pennsylvania, Pike County). - start_date: string or datetime, optional - Start date for the query in the year-month-day format - (YYYY-MM-DD). - end_date: string or datetime, optional - End date for the query in the year-month-day format - (YYYY-MM-DD). - monitoring_location_id : string or iterable of strings, optional - A unique identifier representing a single monitoring location. This - corresponds to the id field in the monitoring-locations endpoint. - Monitoring location IDs are created by combining the agency code of the - agency responsible for the monitoring location (e.g. USGS) with the ID - number of the monitoring location (e.g. 02238500), separated by a hyphen - (e.g. USGS-02238500). - page_size : int, optional - The number of results to return per page, where one result represents a - monitoring location. The default is 1000. - parent_time_series_id: string, optional - The parent_time_series_id returns statistics tied to a - particular database entry. - site_type_code: string, optional - Site type code query parameter. - You can see a list of valid site type codes here: - https://api.waterdata.usgs.gov/ogcapi/v0/collections/site-types/items. - Example: "GW" (Groundwater site) - site_type_name: string, optional - Site type name query parameter. - You can see a list of valid site type names here: - https://api.waterdata.usgs.gov/ogcapi/v0/collections/site-types/items. - Example: "Well" - parameter_code : string or iterable of strings, optional - Parameter codes are 5-digit codes used to identify the constituent - measured and the units of measure. A complete list of parameter codes - and associated groupings can be found at - https://help.waterdata.usgs.gov/codes-and-parameters/parameters. - interval_type : string or iterable of strings, optional - Filter the returned intervals to one or more periods. If unspecified - (default), all matching data are returned. Available values: - "M" (month), "CY" (calendar year), and "WY" (water year). - expand_percentiles : boolean - Percentile data for a given day of year or month of year by default - are returned from the service as lists of string values and percentile - thresholds in the "values" and "percentiles" columns, respectively. - When `expand_percentiles` is set to True (default), each value and - percentile threshold specific to a computation id are returned as - individual rows in the dataframe, with the value reported in the - "value" column and the corresponding percentile reported in a - "percentile" column (and the "values" and "percentiles" columns - are removed). Missing percentile values expressed as 'nan' in the - list of string values are removed from the dataframe to save space. - Setting `expand_percentiles` to False retains the "values" and - "percentiles" columns produced by the service. Including - both 'percentiles' and one or more other statistics ('median', - 'minimum', 'maximum', or 'arithmetic_mean') in the `computation_type` - argument will return both the "values" column, containing the list - of percentile threshold values, and a "value" column, containing - the singular summary value for the other statistics. - - Returns - ------- - df : ``pandas.DataFrame`` or ``geopandas.GeoDataFrame`` - Formatted data returned from the API query. - md : :obj:`dataretrieval.utils.BaseMetadata` - A custom metadata object. - - Examples - -------- - .. code:: - - >>> # Get monthly and yearly medians for streamflow at streams in Rhode Island - >>> # from calendar year 2024. - >>> df, md = dataretrieval.waterdata.get_stats_date_range( - ... state="RI", # Rhode Island (postal code, name, or FIPS all work) - ... parameter_code="00060", - ... site_type_code="ST", - ... start_date="2024-01-01", - ... end_date="2024-12-31", - ... computation_type="median", - ... ) - - >>> # Get monthly and yearly minimum and maximums for gage height at - >>> # a monitoring location of interest - >>> df, md = dataretrieval.waterdata.get_stats_date_range( - ... monitoring_location_id="USGS-05114000", - ... parameter_code="00065", - ... computation_type=["minimum", "maximum"], - ... ) - """ - # Build argument dictionary, omitting None values - params = _get_args( - _with_state(locals(), to="fips_us", into="state_code"), - exclude={"expand_percentiles"}, - ) - - return stats.get_data( - args=params, - service="observationIntervals", - expand_percentiles=expand_percentiles, - ) - - -def get_channel( - monitoring_location_id: str | Iterable[str] | None = None, - field_visit_id: str | Iterable[str] | None = None, - measurement_number: str | Iterable[str] | None = None, - time: str | Iterable[str] | None = None, - channel_name: str | Iterable[str] | None = None, - channel_flow: str | Iterable[str] | None = None, - channel_flow_unit: str | Iterable[str] | None = None, - channel_width: str | Iterable[str] | None = None, - channel_width_unit: str | Iterable[str] | None = None, - channel_area: str | Iterable[str] | None = None, - channel_area_unit: str | Iterable[str] | None = None, - channel_velocity: str | Iterable[str] | None = None, - channel_velocity_unit: str | Iterable[str] | None = None, - channel_location_distance: str | Iterable[str] | None = None, - channel_location_distance_unit: str | Iterable[str] | None = None, - channel_stability: str | Iterable[str] | None = None, - channel_material: str | Iterable[str] | None = None, - channel_evenness: str | Iterable[str] | None = None, - horizontal_velocity_description: str | Iterable[str] | None = None, - vertical_velocity_description: str | Iterable[str] | None = None, - longitudinal_velocity_description: str | Iterable[str] | None = None, - measurement_type: str | Iterable[str] | None = None, - last_modified: str | Iterable[str] | None = None, - channel_measurement_type: str | Iterable[str] | None = None, - properties: str | Iterable[str] | None = None, - skip_geometry: bool | None = None, - bbox: list[float] | None = None, - limit: int | None = None, - filter: str | None = None, - filter_lang: FILTER_LANG | None = None, - convert_type: bool = True, - max_rows: int | None = None, - **queryables: Any, -) -> tuple[pd.DataFrame, BaseMetadata]: - """ - Channel measurements taken as part of streamflow field measurements. - - Parameters - ---------- - monitoring_location_id : string or iterable of strings, optional - A unique identifier representing a single monitoring location. This - corresponds to the id field in the monitoring-locations endpoint. - Monitoring location IDs are created by combining the agency code of - the agency responsible for the monitoring location (e.g. USGS) with - the ID number of the monitoring location (e.g. 02238500), separated - by a hyphen (e.g. USGS-02238500). - field_visit_id : string or iterable of strings, optional - A universally unique identifier (UUID) for the field visit. - Multiple measurements - may be made during a single field visit. - measurement_number : string or iterable of strings, optional - Measurement number. - time : string or iterable of strings, optional - The date an observation represents. You can query this field using - date-times or intervals, adhering to RFC 3339, or using ISO 8601 - duration objects. Intervals may be bounded or half-bounded (double-dots - at start or end). Only features that have a time that intersects the - value of datetime are selected. If a feature has multiple temporal - properties, it is the decision of the server whether only a single - temporal property is used to determine the extent or all relevant - temporal properties. - Examples: - - * A date-time: "2018-02-12T23:20:50Z" - * A bounded interval: "2018-02-12T00:00:00Z/2018-03-18T12:31:12Z" - * Half-bounded intervals: "2018-02-12T00:00:00Z/.." or - "../2018-03-18T12:31:12Z" - * Duration objects: "P1M" for data from the past month or "PT36H" - for the last 36 hours - - channel_name : string or iterable of strings, optional - The channel name. - channel_flow : string or iterable of strings, optional - The channel discharge (flow). - channel_flow_unit : string or iterable of strings, optional - The units for channel discharge. - channel_width : string or iterable of strings, optional - The channel width. - channel_width_unit : string or iterable of strings, optional - The units for channel width. - channel_area : string or iterable of strings, optional - The channel area. - channel_area_unit : string or iterable of strings, optional - The units for channel area. - channel_velocity : string or iterable of strings, optional - The mean channel velocity. - channel_velocity_unit : string or iterable of strings, optional - The units for channel velocity. - channel_location_distance : string or iterable of strings, optional - The channel location distance. - channel_location_distance_unit : string or iterable of strings, optional - The units for channel location distance. - channel_stability : string or iterable of strings, optional - The stability of the channel material. - channel_material : string or iterable of strings, optional - The channel material. - channel_evenness : string or iterable of strings, optional - The channel evenness from bank to bank. - horizontal_velocity_description : string or iterable of strings, optional - The horizontal velocity description. - vertical_velocity_description : string or iterable of strings, optional - The vertical velocity description. - longitudinal_velocity_description : string or iterable of strings, optional - The longitudinal velocity description. - measurement_type : string or iterable of strings, optional - The type of channel measurement. - last_modified : string, optional - The last time a record was refreshed in our database. This may happen - due to regular operational processes and does not necessarily indicate - that anything about the measurement has changed. You can query this field - using date-times or intervals, adhering to RFC 3339, or using ISO 8601 - duration objects. Intervals may be bounded or half-bounded (double-dots - at start or end). Only features that have a last_modified that - intersects the value of datetime are selected. - Examples: - - * A date-time: "2018-02-12T23:20:50Z" - * A bounded interval: "2018-02-12T00:00:00Z/2018-03-18T12:31:12Z" - * Half-bounded intervals: "2018-02-12T00:00:00Z/.." or - "../2018-03-18T12:31:12Z" - * Duration objects: "P1M" for data from the past month or - "PT36H" for the last 36 hours - - skip_geometry : boolean, optional - This option can be used to skip response geometries for each feature. - The returning object will be a data frame with no spatial information. - Note that the USGS Water Data APIs use camelCase "skipGeometry" in - CQL2 queries. - channel_measurement_type : string or iterable of strings, optional - The channel measurement type. - properties : string or iterable of strings, optional - A list of requested columns to be returned from the query. Available - options are: geometry, channel_measurements_id, monitoring_location_id, - field_visit_id, measurement_number, time, channel_name, channel_flow, - channel_flow_unit, channel_width, channel_width_unit, channel_area, - channel_area_unit, channel_velocity, channel_velocity_unit, - channel_location_distance, channel_location_distance_unit, channel_stability, - channel_material, channel_evenness, horizontal_velocity_description, - vertical_velocity_description, longitudinal_velocity_description, - measurement_type, last_modified, channel_measurement_type. The default - (None) will return all columns of the data. - bbox : list of numbers, optional - Only features that have a geometry that intersects the bounding box are - selected. The bounding box is provided as four or six numbers, - depending on whether the coordinate reference system includes a vertical - axis (height or depth). Coordinates are assumed to be in crs 4326. The - expected format is ``[xmin, ymin, xmax, ymax]``, i.e. - ``[Western-most longitude, Southern-most latitude, Eastern-most - longitude, Northern-most latitude]``. - limit : int, optional - The optional limit parameter is used to control the subset of the - selected features that should be returned in each page. The maximum - allowable limit is 50000. It may be beneficial to set this number lower - if your internet connection is spotty. The default (None) will set the - limit to the maximum allowable limit for the service. - This is a per-page size, not a cap on the total result: a query - matching more rows than ``limit`` still returns every matching row - across multiple pages. Use ``max_rows`` to cap the total instead. - filter, filter_lang : optional - Server-side CQL filter passed through as the OGC ``filter`` / - ``filter-lang`` query parameters. See - :mod:`dataretrieval.ogc.filters` for syntax, auto-chunking, - and the lexicographic-comparison pitfall. - convert_type : boolean, optional - If True, converts columns to appropriate types. - max_rows : int, optional - Cap the total number of rows returned, stopping pagination early - instead of downloading the whole result. Unlike ``limit`` (the - per-page size), this bounds the total result across every page. - The default (None) follows pagination to completion. - **queryables : string or iterable of strings, optional - Any other queryable property of this collection, passed through as a - server-side filter. Call :func:`get_queryables` to see the queryables a - collection supports. - - Returns - ------- - df : ``pandas.DataFrame`` or ``geopandas.GeoDataFrame`` - Formatted data returned from the API query. - md: :obj:`dataretrieval.utils.BaseMetadata` - A custom metadata object - - Raises - ------ - ChunkInterrupted - A transient failure (429 / 5xx / timeout) interrupted the request - after the built-in retries. Completed work is preserved; resume - with ``exc.call.resume()`` (see :doc:`/userguide/errors`). - - Examples - -------- - .. code:: - - >>> # Get channel data from a - >>> # single site from a single year - >>> df, md = dataretrieval.waterdata.get_channel( - ... monitoring_location_id="USGS-02238500", - ... ) - """ - service = "channel-measurements" - - args = _get_args(locals(), exclude={"max_rows"}) - - return get_ogc_data(args, service, max_rows=max_rows) - - -def get_cql( - service: WATERDATA_SERVICES, - cql: str | dict[str, Any], - *, - properties: str | Iterable[str] | None = None, - bbox: list[float] | None = None, - limit: int | None = None, - skip_geometry: bool | None = None, - convert_type: bool = True, -) -> tuple[pd.DataFrame, BaseMetadata]: - """Query a Water Data OGC API collection with an arbitrary CQL2 filter. - - Sends ``cql`` as a CQL2 filter against ``service`` and returns the matching - features, shaped like the typed getters (``get_daily``, ``get_continuous``, - …): the wire ``id`` renamed to the service's id column, columns ordered and - sorted, and dtypes coerced. Use it when you need a predicate the typed - getters can't express — a top-level ``or``, ``like`` with ``%`` wildcards, - comparison operators, nested boolean trees, or a geometry predicate beyond a - bounding box; prefer a typed getter when one covers the query. - - The request is a single POST with the ``cql`` body sent verbatim, so there - are no multi-value arguments to chunk: narrow a query whose URL or body - would exceed the server's size cap rather than relying on automatic - chunking. - - The CQL2 grammar is documented at - https://api.waterdata.usgs.gov/docs/ogcapi/complex-queries/. - - Parameters - ---------- - service : str - OGC collection name. Must be one of - :data:`dataretrieval.waterdata.types.WATERDATA_SERVICES` - (e.g. ``"daily"``, ``"monitoring-locations"``). - cql : str or dict - CQL2 query. A ``dict`` is JSON-serialized for transport; a ``str`` is - sent through unchanged. The query goes into the HTTP POST body with - ``Content-Type: application/query-cql-json``. - properties : str or iterable of str, optional - Server-side property whitelist (passed as ``properties=`` on the URL). - Reduces payload size. ``"id"`` resolves to the service's ``output_id`` - (e.g. ``daily_id``) the same way it does in the typed wrappers. - bbox : list of float, optional - Bounding box ``[xmin, ymin, xmax, ymax]`` in CRS 4326. Combines with the - CQL filter as an additional spatial predicate. - limit : int, optional - Page size, clamped server-side to 50,000. - skip_geometry : bool, optional - If True, the server omits geometry from each feature - (``skipGeometry=true``). - convert_type : bool, default True - Coerce date/datetime/numeric columns to typed dtypes after the - DataFrame is built. - - Returns - ------- - df : pandas.DataFrame or geopandas.GeoDataFrame - Result of the query. GeoDataFrame when ``geopandas`` is installed and - geometry is present. - md : :class:`dataretrieval.utils.BaseMetadata` - Request metadata (URL, query time, response headers). - - Examples - -------- - .. code:: - - >>> # Daily values for two parameter codes at two sites - >>> # (compound AND-of-INs). - >>> from dataretrieval import waterdata - >>> cql = { - ... "op": "and", - ... "args": [ - ... { - ... "op": "in", - ... "args": [ - ... {"property": "parameter_code"}, - ... ["00060", "00065"], - ... ], - ... }, - ... { - ... "op": "in", - ... "args": [ - ... {"property": "monitoring_location_id"}, - ... ["USGS-07367300", "USGS-03277200"], - ... ], - ... }, - ... ], - ... } - >>> df, md = waterdata.get_cql(service="daily", cql=cql) - - >>> # Monitoring locations whose HUC starts with "02070010" - >>> # (LIKE with the CQL2 ``%`` wildcard). - >>> df, md = waterdata.get_cql( - ... service="monitoring-locations", - ... cql='{"op": "like", "args": [' - ... '{"property": "hydrologic_unit_code"},' - ... ' "02070010%"]}', - ... ) - """ - if service not in _OUTPUT_ID_BY_SERVICE: - raise ValueError( - f"Unknown service {service!r}. Valid services: " - f"{sorted(_OUTPUT_ID_BY_SERVICE)}." - ) - output_id = _OUTPUT_ID_BY_SERVICE[service] - - # ``dict`` is the pythonic input — serialize on the way out. ``str`` is sent - # verbatim so callers who already have a CQL2 doc (e.g. imported from a - # config file) don't need to re-parse it. - body = json.dumps(cql, separators=(",", ":")) if isinstance(cql, dict) else cql - - properties_list = _as_str_list(properties, "properties") - - # Drop id aliases (``daily_id``/``id``) and ``geometry`` from the wire - # ``properties`` (the feature ``id`` is always returned and renamed - # downstream), matching the typed getters. - wire_properties = _switch_properties_id(properties_list, output_id, service) - - req = _construct_cql_request( - service, - body, - properties=wire_properties, - bbox=bbox, - limit=limit, - skip_geometry=skip_geometry, - ) - - df, response = fetch_ogc_request(req, service=service) - - return _finalize_ogc( - df, - response, - properties=properties_list, - output_id=output_id, - convert_type=convert_type, - service=service, - ) +from dataretrieval.waterdata.utils import get_ogc_data as _get_ogc_data + +__all__ = [ + "get_channel", + "get_codes", + "get_combined_metadata", + "get_continuous", + "get_cql", + "get_daily", + "get_field_measurements", + "get_field_measurements_metadata", + "get_latest_continuous", + "get_latest_daily", + "get_monitoring_locations", + "get_peaks", + "get_queryables", + "get_reference_table", + "get_samples", + "get_samples_summary", + "get_stats_date_range", + "get_stats_por", + "get_time_series_metadata", +] + +# Preserve the documented legacy implementation path for introspection and +# Sphinx while the function objects live in cohesive family modules. +for _name in __all__: + globals()[_name].__module__ = __name__ +del _name + +# Private compatibility names used by existing callers and patch targets. +_SAMPLES_PARAM_TO_API = _samples._SAMPLES_PARAM_TO_API +_SAMPLES_LEGACY_KWARGS = _samples._SAMPLES_LEGACY_KWARGS +get_ogc_data = _get_ogc_data diff --git a/dataretrieval/waterdata/cql.py b/dataretrieval/waterdata/cql.py new file mode 100644 index 00000000..1134ca86 --- /dev/null +++ b/dataretrieval/waterdata/cql.py @@ -0,0 +1,166 @@ +"""Generalized CQL2 request adapter for Water Data collections.""" + +from __future__ import annotations + +import json +from collections.abc import Iterable +from typing import Any + +import pandas as pd + +from dataretrieval.ogc import fetch_ogc_request +from dataretrieval.ogc.requests import ( + _as_str_list, + _construct_cql_request, + _switch_properties_id, +) +from dataretrieval.utils import BaseMetadata +from dataretrieval.waterdata.types import ( + WATERDATA_SERVICES, +) +from dataretrieval.waterdata.utils import ( + _OUTPUT_ID_BY_SERVICE, + _finalize_ogc, +) + + +def get_cql( + service: WATERDATA_SERVICES, + cql: str | dict[str, Any], + *, + properties: str | Iterable[str] | None = None, + bbox: list[float] | None = None, + limit: int | None = None, + skip_geometry: bool | None = None, + convert_type: bool = True, +) -> tuple[pd.DataFrame, BaseMetadata]: + """Query a Water Data OGC API collection with an arbitrary CQL2 filter. + + Sends ``cql`` as a CQL2 filter against ``service`` and returns the matching + features, shaped like the typed getters (``get_daily``, ``get_continuous``, + …): the wire ``id`` renamed to the service's id column, columns ordered and + sorted, and dtypes coerced. Use it when you need a predicate the typed + getters can't express — a top-level ``or``, ``like`` with ``%`` wildcards, + comparison operators, nested boolean trees, or a geometry predicate beyond a + bounding box; prefer a typed getter when one covers the query. + + The request is a single POST with the ``cql`` body sent verbatim, so there + are no multi-value arguments to chunk: narrow a query whose URL or body + would exceed the server's size cap rather than relying on automatic + chunking. + + The CQL2 grammar is documented at + https://api.waterdata.usgs.gov/docs/ogcapi/complex-queries/. + + Parameters + ---------- + service : str + OGC collection name. Must be one of + :data:`dataretrieval.waterdata.types.WATERDATA_SERVICES` + (e.g. ``"daily"``, ``"monitoring-locations"``). + cql : str or dict + CQL2 query. A ``dict`` is JSON-serialized for transport; a ``str`` is + sent through unchanged. The query goes into the HTTP POST body with + ``Content-Type: application/query-cql-json``. + properties : str or iterable of str, optional + Server-side property whitelist (passed as ``properties=`` on the URL). + Reduces payload size. ``"id"`` resolves to the service's ``output_id`` + (e.g. ``daily_id``) the same way it does in the typed wrappers. + bbox : list of float, optional + Bounding box ``[xmin, ymin, xmax, ymax]`` in CRS 4326. Combines with the + CQL filter as an additional spatial predicate. + limit : int, optional + Page size, clamped server-side to 50,000. + skip_geometry : bool, optional + If True, the server omits geometry from each feature + (``skipGeometry=true``). + convert_type : bool, default True + Coerce date/datetime/numeric columns to typed dtypes after the + DataFrame is built. + + Returns + ------- + df : pandas.DataFrame or geopandas.GeoDataFrame + Result of the query. GeoDataFrame when ``geopandas`` is installed and + geometry is present. + md : :class:`dataretrieval.utils.BaseMetadata` + Request metadata (URL, query time, response headers). + + Examples + -------- + .. code:: + + >>> # Daily values for two parameter codes at two sites + >>> # (compound AND-of-INs). + >>> from dataretrieval import waterdata + >>> cql = { + ... "op": "and", + ... "args": [ + ... { + ... "op": "in", + ... "args": [ + ... {"property": "parameter_code"}, + ... ["00060", "00065"], + ... ], + ... }, + ... { + ... "op": "in", + ... "args": [ + ... {"property": "monitoring_location_id"}, + ... ["USGS-07367300", "USGS-03277200"], + ... ], + ... }, + ... ], + ... } + >>> df, md = waterdata.get_cql(service="daily", cql=cql) + + >>> # Monitoring locations whose HUC starts with "02070010" + >>> # (LIKE with the CQL2 ``%`` wildcard). + >>> df, md = waterdata.get_cql( + ... service="monitoring-locations", + ... cql='{"op": "like", "args": [' + ... '{"property": "hydrologic_unit_code"},' + ... ' "02070010%"]}', + ... ) + """ + if service not in _OUTPUT_ID_BY_SERVICE: + raise ValueError( + f"Unknown service {service!r}. Valid services: " + f"{sorted(_OUTPUT_ID_BY_SERVICE)}." + ) + output_id = _OUTPUT_ID_BY_SERVICE[service] + + # ``dict`` is the pythonic input — serialize on the way out. ``str`` is sent + # verbatim so callers who already have a CQL2 doc (e.g. imported from a + # config file) don't need to re-parse it. + body = json.dumps(cql, separators=(",", ":")) if isinstance(cql, dict) else cql + + properties_list = _as_str_list(properties, "properties") + + # Drop id aliases (``daily_id``/``id``) and ``geometry`` from the wire + # ``properties`` (the feature ``id`` is always returned and renamed + # downstream), matching the typed getters. + wire_properties = _switch_properties_id(properties_list, output_id, service) + + req = _construct_cql_request( + service, + body, + properties=wire_properties, + bbox=bbox, + limit=limit, + skip_geometry=skip_geometry, + ) + + df, response = fetch_ogc_request(req, service=service) + + return _finalize_ogc( + df, + response, + properties=properties_list, + output_id=output_id, + convert_type=convert_type, + service=service, + ) + + +__all__ = ["get_cql"] diff --git a/dataretrieval/waterdata/measurements.py b/dataretrieval/waterdata/measurements.py new file mode 100644 index 00000000..ac8a1d54 --- /dev/null +++ b/dataretrieval/waterdata/measurements.py @@ -0,0 +1,576 @@ +"""Discrete field, peak, and channel measurement getters.""" + +from __future__ import annotations + +from collections.abc import Iterable +from typing import Any + +import pandas as pd + +from dataretrieval.ogc.filters import FILTER_LANG +from dataretrieval.utils import BaseMetadata +from dataretrieval.waterdata.utils import ( + _get_args, + get_ogc_data, +) + + +def get_field_measurements( + monitoring_location_id: str | Iterable[str] | None = None, + parameter_code: str | Iterable[str] | None = None, + observing_procedure_code: str | Iterable[str] | None = None, + properties: str | Iterable[str] | None = None, + field_visit_id: str | Iterable[str] | None = None, + approval_status: str | Iterable[str] | None = None, + unit_of_measure: str | Iterable[str] | None = None, + qualifier: str | Iterable[str] | None = None, + value: str | Iterable[str] | None = None, + last_modified: str | Iterable[str] | None = None, + observing_procedure: str | Iterable[str] | None = None, + vertical_datum: str | Iterable[str] | None = None, + measuring_agency: str | Iterable[str] | None = None, + skip_geometry: bool | None = None, + time: str | Iterable[str] | None = None, + bbox: list[float] | None = None, + limit: int | None = None, + filter: str | None = None, + filter_lang: FILTER_LANG | None = None, + convert_type: bool = True, + max_rows: int | None = None, + **queryables: Any, +) -> tuple[pd.DataFrame, BaseMetadata]: + """Field measurements are physically measured values collected during a + visit to the monitoring location. Field measurements consist of measurements + of gage height and discharge, and readings of groundwater levels, and are + primarily used as calibration readings for the automated sensors collecting + continuous data. They are collected at a low frequency, and delivery of the + data in WDFN may be delayed due to data processing time. + + Parameters + ---------- + monitoring_location_id : string or iterable of strings, optional + A unique identifier representing a single monitoring location. This + corresponds to the id field in the monitoring-locations endpoint. + Monitoring location IDs are created by combining the agency code of the + agency responsible for the monitoring location (e.g. USGS) with the ID + number of the monitoring location (e.g. 02238500), separated by a hyphen + (e.g. USGS-02238500). + parameter_code : string or iterable of strings, optional + Parameter codes are 5-digit codes used to identify the constituent + measured and the units of measure. A complete list of parameter codes + and associated groupings can be found at + https://help.waterdata.usgs.gov/codes-and-parameters/parameters. + observing_procedure_code : string or iterable of strings, optional + A short code corresponding to the observing procedure for the field + measurement. + properties : string or iterable of strings, optional + A list of requested columns to be returned from the query. See the + field-measurements schema in the OpenAPI reference for the available + columns (e.g. geometry, id, monitoring_location_id, parameter_code, + value, unit_of_measure, approval_status, qualifier, last_modified): + https://api.waterdata.usgs.gov/ogcapi/v0/openapi?f=html#/field-measurements + field_visit_id : string or iterable of strings, optional + A universally unique identifier (UUID) for the field visit. + Multiple measurements may be made during a single field visit. + approval_status : string or iterable of strings, optional + Some of the data that you have obtained from this U.S. Geological Survey + database may not have received Director's approval. Any such data values + are qualified as provisional and are subject to revision. Provisional + data are released on the condition that neither the USGS nor the United + States Government may be held liable for any damages resulting from its + use. This field reflects the approval status of each record, and is either + "Approved", meaning processing review has been completed and the data is + approved for publication, or "Provisional" and subject to revision. For + more information about provisional data, go to: + https://waterdata.usgs.gov/provisional-data-statement/. + unit_of_measure : string or iterable of strings, optional + A human-readable description of the units of measurement associated + with an observation. + qualifier : string or iterable of strings, optional + This field indicates any qualifiers associated with an observation, for + instance if a sensor may have been impacted by ice or if values were + estimated. + value : string or iterable of strings, optional + The value of the observation. Values are transmitted as strings in + the JSON response format in order to preserve precision. + last_modified : string, optional + The last time a record was refreshed in our database. This may happen + due to regular operational processes and does not necessarily indicate + that anything about the measurement has changed. You can query this field + using date-times or intervals, adhering to RFC 3339, or using ISO 8601 + duration objects. Intervals may be bounded or half-bounded (double-dots + at start or end). Only features that have a last_modified that + intersects the value of datetime are selected. + Examples: + + * A date-time: "2018-02-12T23:20:50Z" + * A bounded interval: "2018-02-12T00:00:00Z/2018-03-18T12:31:12Z" + * Half-bounded intervals: "2018-02-12T00:00:00Z/.." or + "../2018-03-18T12:31:12Z" + * Duration objects: "P1M" for data from the past month or + "PT36H" for the last 36 hours + + observing_procedure : string or iterable of strings, optional + Water measurement or water-quality observing procedure descriptions. + vertical_datum : string or iterable of strings, optional + The datum used to determine altitude and vertical position at the + monitoring location. + measuring_agency : string or iterable of strings, optional + The agency performing the measurement. + skip_geometry : boolean, optional + This option can be used to skip response geometries for each feature. + The returning object will be a data frame with no spatial information. + Note that the USGS Water Data APIs use camelCase "skipGeometry" in + CQL2 queries. + time : string, optional + The date an observation represents. You can query this field using date-times + or intervals, adhering to RFC 3339, or using ISO 8601 duration objects. + Intervals may be bounded or half-bounded (double-dots at start or end). + Only features that have a time that intersects the value of datetime are + selected. If a feature has multiple temporal properties, it is the + decision of the server whether only a single temporal property is used + to determine the extent or all relevant temporal properties. + Examples: + + * A date-time: "2018-02-12T23:20:50Z" + * A bounded interval: "2018-02-12T00:00:00Z/2018-03-18T12:31:12Z" + * Half-bounded intervals: "2018-02-12T00:00:00Z/.." or + "../2018-03-18T12:31:12Z" + * Duration objects: "P1M" for data from the past month or + "PT36H" for the last 36 hours + + bbox : list of numbers, optional + Only features that have a geometry that intersects the bounding box are + selected. The bounding box is provided as four or six numbers, + depending on whether the coordinate reference system includes a vertical + axis (height or depth). Coordinates are assumed to be in crs 4326. The + expected format is ``[xmin, ymin, xmax, ymax]``, i.e. + ``[Western-most longitude, Southern-most latitude, Eastern-most + longitude, Northern-most latitude]``. + limit : int, optional + The optional limit parameter is used to control the subset of the + selected features that should be returned in each page. The maximum + allowable limit is 50000. It may be beneficial to set this number lower + if your internet connection is spotty. The default (None) will set the + limit to the maximum allowable limit for the service. + This is a per-page size, not a cap on the total result: a query + matching more rows than ``limit`` still returns every matching row + across multiple pages. Use ``max_rows`` to cap the total instead. + filter, filter_lang : optional + Server-side CQL filter passed through as the OGC ``filter`` / + ``filter-lang`` query parameters. See + :mod:`dataretrieval.ogc.filters` for syntax, auto-chunking, + and the lexicographic-comparison pitfall. + convert_type : boolean, optional + If True, converts columns to appropriate types. + max_rows : int, optional + Cap the total number of rows returned, stopping pagination early + instead of downloading the whole result. Unlike ``limit`` (the + per-page size), this bounds the total result across every page. + The default (None) follows pagination to completion. + **queryables : string or iterable of strings, optional + Any other queryable property of this collection, passed through as a + server-side filter. Call :func:`get_queryables` to see the queryables a + collection supports. + + Returns + ------- + df : ``pandas.DataFrame`` or ``geopandas.GeoDataFrame`` + Formatted data returned from the API query. + md: :obj:`dataretrieval.utils.BaseMetadata` + A custom metadata object + + Raises + ------ + ChunkInterrupted + A transient failure (429 / 5xx / timeout) interrupted the request + after the built-in retries. Completed work is preserved; resume + with ``exc.call.resume()`` (see :doc:`/userguide/errors`). + + Examples + -------- + .. code:: + + >>> # Get field measurements from a single groundwater site + >>> # and parameter code, and do not return geometry + >>> df, md = dataretrieval.waterdata.get_field_measurements( + ... monitoring_location_id="USGS-375907091432201", + ... parameter_code="72019", + ... skip_geometry=True, + ... ) + + >>> # Half-bounded time range: every measurement at this site since + >>> # 1980 (open-ended end). Use ``"../"`` for the inverse + >>> # (everything up to a date). + >>> df, md = dataretrieval.waterdata.get_field_measurements( + ... monitoring_location_id="USGS-425957088141001", + ... time="1980-01-01/..", + ... ) + + >>> # Get field measurements from multiple sites and + >>> # parameter codes from the last 20 years + >>> df, md = dataretrieval.waterdata.get_field_measurements( + ... monitoring_location_id=[ + ... "USGS-451605097071701", + ... "USGS-263819081585801", + ... ], + ... parameter_code=["62611", "72019"], + ... time="P20Y", + ... ) + """ + service = "field-measurements" + + # Build argument dictionary, omitting None values + args = _get_args(locals(), exclude={"max_rows"}) + + return get_ogc_data(args, service, max_rows=max_rows) + + +def get_peaks( + monitoring_location_id: str | Iterable[str] | None = None, + parameter_code: str | Iterable[str] | None = None, + time_series_id: str | Iterable[str] | None = None, + unit_of_measure: str | Iterable[str] | None = None, + time: str | Iterable[str] | None = None, + last_modified: str | Iterable[str] | None = None, + water_year: int | list[int] | None = None, + year: int | list[int] | None = None, + month: int | list[int] | None = None, + day: int | list[int] | None = None, + peak_since: int | list[int] | None = None, + properties: str | Iterable[str] | None = None, + skip_geometry: bool | None = None, + bbox: list[float] | None = None, + limit: int | None = None, + filter: str | None = None, + filter_lang: FILTER_LANG | None = None, + convert_type: bool = True, + max_rows: int | None = None, + **queryables: Any, +) -> tuple[pd.DataFrame, BaseMetadata]: + """Get the annual peak streamflow / stage record for a monitoring location. + + Peaks are the largest values observed at a site each water year and are + the standard input to flood-frequency analysis (e.g. log-Pearson Type III + fits). The endpoint returns one row per (monitoring location, parameter, + water year), with the peak ``value`` and the ``time`` it occurred. + + The collection covers both stage (parameter ``"00065"``, ``ft``) and + discharge (parameter ``"00060"``, ``ft^3/s``); a typical streamgage has a + series for each. Reference docs: + https://api.waterdata.usgs.gov/ogcapi/v0/openapi?f=html#/peaks + + Parameters + ---------- + monitoring_location_id : string or iterable of strings, optional + A unique identifier representing a single monitoring location, in + ``AGENCY-ID`` form (e.g. ``"USGS-02238500"``). + parameter_code : string or iterable of strings, optional + 5-digit parameter code. Most peaks records are ``"00060"`` (discharge) + or ``"00065"`` (stage / gage height). Full list at + https://help.waterdata.usgs.gov/codes-and-parameters/parameters. + time_series_id : string or iterable of strings, optional + ID of the time series the peak belongs to. + unit_of_measure : string or iterable of strings, optional + Human-readable units (e.g. ``"ft^3/s"``, ``"ft"``). + time : string, optional + Datetime, interval, or duration filter on the peak's date. + See :func:`get_time_series_metadata` for the full grammar. + last_modified : string, optional + Same datetime grammar as ``time``; filters on the database + last-modified timestamp (useful for incremental ETL polling). + water_year, year, month, day : int or list of ints, optional + Calendar / water-year filters on the peak event. The water year ends + September 30 (e.g. WY2024 = Oct 1, 2023 – Sep 30, 2024). + peak_since : int or list of ints, optional + Filter on the year since which the peak value has stood as the + record (the API serves this field as an integer; many rows are + ``null``). + properties : string or iterable of strings, optional + Subset of columns to return. Defaults to every available property. + skip_geometry : boolean, optional + Skip per-feature geometries; the returned object will be a plain + ``DataFrame`` with no spatial information. + bbox : list of numbers, optional + Only features whose geometry intersects the bounding box are + selected. Format: ``[xmin, ymin, xmax, ymax]`` in CRS 4326 + (longitude / latitude, west-south-east-north). + limit : int, optional + Page size; the maximum allowable value is 50000. Default + (``None``) requests the maximum allowable limit. This is a + per-page size, not a cap on the total result: a query matching more + rows than ``limit`` still returns every matching row across + multiple pages. Use ``max_rows`` to cap the total instead. + filter, filter_lang : optional + Server-side CQL filter passed through as the OGC ``filter`` / + ``filter-lang`` query parameters. See + :mod:`dataretrieval.ogc.filters` for syntax, auto-chunking, + and the lexicographic-comparison pitfall. + convert_type : boolean, optional + If True, converts columns to appropriate types. + max_rows : int, optional + Cap the total number of rows returned, stopping pagination early + instead of downloading the whole result. Unlike ``limit`` (the + per-page size), this bounds the total result across every page. + The default (None) follows pagination to completion. + **queryables : string or iterable of strings, optional + Any other queryable property of this collection, passed through as a + server-side filter. Call :func:`get_queryables` to see the queryables a + collection supports. + + Returns + ------- + df : ``pandas.DataFrame`` or ``geopandas.GeoDataFrame`` + Formatted data returned from the API query. + md : :obj:`dataretrieval.utils.BaseMetadata` + A custom metadata object pertaining to the query. + + Raises + ------ + ChunkInterrupted + A transient failure (429 / 5xx / timeout) interrupted the request + after the built-in retries. Completed work is preserved; resume + with ``exc.call.resume()`` (see :doc:`/userguide/errors`). + + Examples + -------- + .. code:: + + >>> # Full annual peak record at one site (both stage and discharge) + >>> df, md = dataretrieval.waterdata.get_peaks( + ... monitoring_location_id="USGS-02238500" + ... ) + + >>> # Discharge peaks only + >>> df, md = dataretrieval.waterdata.get_peaks( + ... monitoring_location_id="USGS-02238500", + ... parameter_code="00060", + ... ) + + >>> # Multi-site peaks for a parameter, narrowed to a water-year range + >>> df, md = dataretrieval.waterdata.get_peaks( + ... monitoring_location_id=[ + ... "USGS-07069000", + ... "USGS-07064000", + ... "USGS-07068000", + ... ], + ... parameter_code="00060", + ... water_year=[2020, 2021, 2022, 2023], + ... ) + + """ + service = "peaks" + + args = _get_args(locals(), exclude={"max_rows"}) + + return get_ogc_data(args, service, max_rows=max_rows) + + +def get_channel( + monitoring_location_id: str | Iterable[str] | None = None, + field_visit_id: str | Iterable[str] | None = None, + measurement_number: str | Iterable[str] | None = None, + time: str | Iterable[str] | None = None, + channel_name: str | Iterable[str] | None = None, + channel_flow: str | Iterable[str] | None = None, + channel_flow_unit: str | Iterable[str] | None = None, + channel_width: str | Iterable[str] | None = None, + channel_width_unit: str | Iterable[str] | None = None, + channel_area: str | Iterable[str] | None = None, + channel_area_unit: str | Iterable[str] | None = None, + channel_velocity: str | Iterable[str] | None = None, + channel_velocity_unit: str | Iterable[str] | None = None, + channel_location_distance: str | Iterable[str] | None = None, + channel_location_distance_unit: str | Iterable[str] | None = None, + channel_stability: str | Iterable[str] | None = None, + channel_material: str | Iterable[str] | None = None, + channel_evenness: str | Iterable[str] | None = None, + horizontal_velocity_description: str | Iterable[str] | None = None, + vertical_velocity_description: str | Iterable[str] | None = None, + longitudinal_velocity_description: str | Iterable[str] | None = None, + measurement_type: str | Iterable[str] | None = None, + last_modified: str | Iterable[str] | None = None, + channel_measurement_type: str | Iterable[str] | None = None, + properties: str | Iterable[str] | None = None, + skip_geometry: bool | None = None, + bbox: list[float] | None = None, + limit: int | None = None, + filter: str | None = None, + filter_lang: FILTER_LANG | None = None, + convert_type: bool = True, + max_rows: int | None = None, + **queryables: Any, +) -> tuple[pd.DataFrame, BaseMetadata]: + """ + Channel measurements taken as part of streamflow field measurements. + + Parameters + ---------- + monitoring_location_id : string or iterable of strings, optional + A unique identifier representing a single monitoring location. This + corresponds to the id field in the monitoring-locations endpoint. + Monitoring location IDs are created by combining the agency code of + the agency responsible for the monitoring location (e.g. USGS) with + the ID number of the monitoring location (e.g. 02238500), separated + by a hyphen (e.g. USGS-02238500). + field_visit_id : string or iterable of strings, optional + A universally unique identifier (UUID) for the field visit. + Multiple measurements + may be made during a single field visit. + measurement_number : string or iterable of strings, optional + Measurement number. + time : string or iterable of strings, optional + The date an observation represents. You can query this field using + date-times or intervals, adhering to RFC 3339, or using ISO 8601 + duration objects. Intervals may be bounded or half-bounded (double-dots + at start or end). Only features that have a time that intersects the + value of datetime are selected. If a feature has multiple temporal + properties, it is the decision of the server whether only a single + temporal property is used to determine the extent or all relevant + temporal properties. + Examples: + + * A date-time: "2018-02-12T23:20:50Z" + * A bounded interval: "2018-02-12T00:00:00Z/2018-03-18T12:31:12Z" + * Half-bounded intervals: "2018-02-12T00:00:00Z/.." or + "../2018-03-18T12:31:12Z" + * Duration objects: "P1M" for data from the past month or "PT36H" + for the last 36 hours + + channel_name : string or iterable of strings, optional + The channel name. + channel_flow : string or iterable of strings, optional + The channel discharge (flow). + channel_flow_unit : string or iterable of strings, optional + The units for channel discharge. + channel_width : string or iterable of strings, optional + The channel width. + channel_width_unit : string or iterable of strings, optional + The units for channel width. + channel_area : string or iterable of strings, optional + The channel area. + channel_area_unit : string or iterable of strings, optional + The units for channel area. + channel_velocity : string or iterable of strings, optional + The mean channel velocity. + channel_velocity_unit : string or iterable of strings, optional + The units for channel velocity. + channel_location_distance : string or iterable of strings, optional + The channel location distance. + channel_location_distance_unit : string or iterable of strings, optional + The units for channel location distance. + channel_stability : string or iterable of strings, optional + The stability of the channel material. + channel_material : string or iterable of strings, optional + The channel material. + channel_evenness : string or iterable of strings, optional + The channel evenness from bank to bank. + horizontal_velocity_description : string or iterable of strings, optional + The horizontal velocity description. + vertical_velocity_description : string or iterable of strings, optional + The vertical velocity description. + longitudinal_velocity_description : string or iterable of strings, optional + The longitudinal velocity description. + measurement_type : string or iterable of strings, optional + The type of channel measurement. + last_modified : string, optional + The last time a record was refreshed in our database. This may happen + due to regular operational processes and does not necessarily indicate + that anything about the measurement has changed. You can query this field + using date-times or intervals, adhering to RFC 3339, or using ISO 8601 + duration objects. Intervals may be bounded or half-bounded (double-dots + at start or end). Only features that have a last_modified that + intersects the value of datetime are selected. + Examples: + + * A date-time: "2018-02-12T23:20:50Z" + * A bounded interval: "2018-02-12T00:00:00Z/2018-03-18T12:31:12Z" + * Half-bounded intervals: "2018-02-12T00:00:00Z/.." or + "../2018-03-18T12:31:12Z" + * Duration objects: "P1M" for data from the past month or + "PT36H" for the last 36 hours + + skip_geometry : boolean, optional + This option can be used to skip response geometries for each feature. + The returning object will be a data frame with no spatial information. + Note that the USGS Water Data APIs use camelCase "skipGeometry" in + CQL2 queries. + channel_measurement_type : string or iterable of strings, optional + The channel measurement type. + properties : string or iterable of strings, optional + A list of requested columns to be returned from the query. Available + options are: geometry, channel_measurements_id, monitoring_location_id, + field_visit_id, measurement_number, time, channel_name, channel_flow, + channel_flow_unit, channel_width, channel_width_unit, channel_area, + channel_area_unit, channel_velocity, channel_velocity_unit, + channel_location_distance, channel_location_distance_unit, channel_stability, + channel_material, channel_evenness, horizontal_velocity_description, + vertical_velocity_description, longitudinal_velocity_description, + measurement_type, last_modified, channel_measurement_type. The default + (None) will return all columns of the data. + bbox : list of numbers, optional + Only features that have a geometry that intersects the bounding box are + selected. The bounding box is provided as four or six numbers, + depending on whether the coordinate reference system includes a vertical + axis (height or depth). Coordinates are assumed to be in crs 4326. The + expected format is ``[xmin, ymin, xmax, ymax]``, i.e. + ``[Western-most longitude, Southern-most latitude, Eastern-most + longitude, Northern-most latitude]``. + limit : int, optional + The optional limit parameter is used to control the subset of the + selected features that should be returned in each page. The maximum + allowable limit is 50000. It may be beneficial to set this number lower + if your internet connection is spotty. The default (None) will set the + limit to the maximum allowable limit for the service. + This is a per-page size, not a cap on the total result: a query + matching more rows than ``limit`` still returns every matching row + across multiple pages. Use ``max_rows`` to cap the total instead. + filter, filter_lang : optional + Server-side CQL filter passed through as the OGC ``filter`` / + ``filter-lang`` query parameters. See + :mod:`dataretrieval.ogc.filters` for syntax, auto-chunking, + and the lexicographic-comparison pitfall. + convert_type : boolean, optional + If True, converts columns to appropriate types. + max_rows : int, optional + Cap the total number of rows returned, stopping pagination early + instead of downloading the whole result. Unlike ``limit`` (the + per-page size), this bounds the total result across every page. + The default (None) follows pagination to completion. + **queryables : string or iterable of strings, optional + Any other queryable property of this collection, passed through as a + server-side filter. Call :func:`get_queryables` to see the queryables a + collection supports. + + Returns + ------- + df : ``pandas.DataFrame`` or ``geopandas.GeoDataFrame`` + Formatted data returned from the API query. + md: :obj:`dataretrieval.utils.BaseMetadata` + A custom metadata object + + Raises + ------ + ChunkInterrupted + A transient failure (429 / 5xx / timeout) interrupted the request + after the built-in retries. Completed work is preserved; resume + with ``exc.call.resume()`` (see :doc:`/userguide/errors`). + + Examples + -------- + .. code:: + + >>> # Get channel data from a + >>> # single site from a single year + >>> df, md = dataretrieval.waterdata.get_channel( + ... monitoring_location_id="USGS-02238500", + ... ) + """ + service = "channel-measurements" + + args = _get_args(locals(), exclude={"max_rows"}) + + return get_ogc_data(args, service, max_rows=max_rows) + + +__all__ = ["get_field_measurements", "get_peaks", "get_channel"] diff --git a/dataretrieval/waterdata/metadata.py b/dataretrieval/waterdata/metadata.py new file mode 100644 index 00000000..4c288f1a --- /dev/null +++ b/dataretrieval/waterdata/metadata.py @@ -0,0 +1,994 @@ +"""Monitoring-location and data-inventory metadata getters.""" + +from __future__ import annotations + +from collections.abc import Iterable +from typing import Any + +import pandas as pd + +from dataretrieval.ogc.filters import FILTER_LANG +from dataretrieval.utils import BaseMetadata +from dataretrieval.waterdata.utils import ( + _get_args, + _with_state, + get_ogc_data, +) + + +def get_monitoring_locations( + monitoring_location_id: str | Iterable[str] | None = None, + agency_code: str | Iterable[str] | None = None, + agency_name: str | Iterable[str] | None = None, + monitoring_location_number: str | Iterable[str] | None = None, + monitoring_location_name: str | Iterable[str] | None = None, + district_code: str | Iterable[str] | None = None, + country_code: str | Iterable[str] | None = None, + country_name: str | Iterable[str] | None = None, + state: str | Iterable[str] | None = None, + state_code: str | Iterable[str] | None = None, + state_name: str | Iterable[str] | None = None, + county_code: str | Iterable[str] | None = None, + county_name: str | Iterable[str] | None = None, + minor_civil_division_code: str | Iterable[str] | None = None, + site_type_code: str | Iterable[str] | None = None, + site_type: str | Iterable[str] | None = None, + hydrologic_unit_code: str | Iterable[str] | None = None, + basin_code: str | Iterable[str] | None = None, + altitude: str | Iterable[str] | None = None, + altitude_accuracy: str | Iterable[str] | None = None, + altitude_method_code: str | Iterable[str] | None = None, + altitude_method_name: str | Iterable[str] | None = None, + vertical_datum: str | Iterable[str] | None = None, + vertical_datum_name: str | Iterable[str] | None = None, + horizontal_positional_accuracy_code: str | Iterable[str] | None = None, + horizontal_positional_accuracy: str | Iterable[str] | None = None, + horizontal_position_method_code: str | Iterable[str] | None = None, + horizontal_position_method_name: str | Iterable[str] | None = None, + original_horizontal_datum: str | Iterable[str] | None = None, + original_horizontal_datum_name: str | Iterable[str] | None = None, + drainage_area: str | Iterable[str] | None = None, + contributing_drainage_area: str | Iterable[str] | None = None, + time_zone_abbreviation: str | Iterable[str] | None = None, + uses_daylight_savings: str | Iterable[str] | None = None, + construction_date: str | Iterable[str] | None = None, + aquifer_code: str | Iterable[str] | None = None, + national_aquifer_code: str | Iterable[str] | None = None, + aquifer_type_code: str | Iterable[str] | None = None, + well_constructed_depth: str | Iterable[str] | None = None, + hole_constructed_depth: str | Iterable[str] | None = None, + depth_source_code: str | Iterable[str] | None = None, + properties: str | Iterable[str] | None = None, + skip_geometry: bool | None = None, + bbox: list[float] | None = None, + limit: int | None = None, + filter: str | None = None, + filter_lang: FILTER_LANG | None = None, + convert_type: bool = True, + max_rows: int | None = None, + **queryables: Any, +) -> tuple[pd.DataFrame, BaseMetadata]: + """Location information is basic information about the monitoring location + including the name, identifier, agency responsible for data collection, and + the date the location was established. It also includes information about + the type of location, such as stream, lake, or groundwater, and geographic + information about the location, such as state, county, latitude and + longitude, and hydrologic unit code (HUC). + + Parameters + ---------- + monitoring_location_id : string or iterable of strings, optional + A unique identifier representing a single monitoring location. This + corresponds to the id field in the monitoring-locations endpoint. + Monitoring location IDs are created by combining the agency code of + the agency responsible for the monitoring location (e.g. USGS) with + the ID number of the monitoring location (e.g. 02238500), separated + by a hyphen (e.g. USGS-02238500). + agency_code : string or iterable of strings, optional + The agency that is reporting the data. Agency codes are fixed values + assigned by the National Water Information System (NWIS). + agency_name : string or iterable of strings, optional + The name of the agency that is reporting the data. + monitoring_location_number : string or iterable of strings, optional + Each monitoring location in the USGS data base has a unique 8- to + 15-digit identification number. Monitoring location numbers are + assigned based on this logic: + https://help.waterdata.usgs.gov/faq/sites/do-station-numbers-have-any-particular-meaning. + monitoring_location_name : string or iterable of strings, optional + This is the official name of the monitoring location in the database. + For well information this can be a district-assigned local number. + district_code : string or iterable of strings, optional + The Water Science Centers (WSCs) across the United States use the FIPS + state code as the district code. In some cases, monitoring locations and + samples may be managed by a water science center that is adjacent to the + state in which the monitoring location actually resides. For example a + monitoring location may have a district code of 30 which translates to + Montana, but the state code could be 56 for Wyoming because that is where + the monitoring location actually is located. + country_code : string or iterable of strings, optional + The code for the country in which the monitoring location is located. + country_name : string or iterable of strings, optional + The name of the country in which the monitoring location is located. + state : string or iterable of strings, optional + State/territory filter (the recommended parameter). Accepts a full name + (``"Wisconsin"``), a two-letter postal code (``"WI"``), or a two-digit + ANSI/FIPS code (``"55"``). + state_code : string or iterable of strings, optional + State code. A two-digit ANSI code (formerly FIPS code) as defined by + the American National Standards Institute, to define States and + equivalents. A three-digit ANSI code is used to define counties and + county equivalents. A `lookup table + `_ + is available. The only countries with + political subdivisions other than the US are Mexico and Canada. The Mexican + states have US state codes ranging from 81-86 and Canadian provinces have + state codes ranging from 90-98. + state_name : string or iterable of strings, optional + The name of the state or state equivalent in which the monitoring location + is located. + county_code : string or iterable of strings, optional + The code for the county or county equivalent (parish, borough, etc.) in which + the monitoring location is located. A `list of codes + `__ is available. + county_name : string or iterable of strings, optional + The name of the county or county equivalent (parish, borough, etc.) in which + the monitoring location is located. A `list of codes + `__ is available. + minor_civil_division_code : string or iterable of strings, optional + Codes for primary governmental or administrative divisions of the county or + county equivalent in which the monitoring location is located. + site_type_code : string or iterable of strings, optional + A code describing the hydrologic setting of the monitoring location. + site_type : string or iterable of strings, optional + A description of the hydrologic setting of the monitoring location. + hydrologic_unit_code : string or iterable of strings, optional + The United States is divided and sub-divided into successively smaller + hydrologic units which are classified into four levels: regions, + sub-regions, accounting units, and cataloging units. The hydrologic + units are arranged within each other, from the smallest (cataloging + units) to the largest (regions). Each hydrologic unit is identified by a + unique hydrologic unit code (HUC) consisting of two to eight digits + based on the four levels of classification in the hydrologic unit + system. + basin_code : string or iterable of strings, optional + The Basin Code or "drainage basin code" is a two-digit code that further + subdivides the 8-digit hydrologic-unit code. The drainage basin code is + defined by the USGS State Office where the monitoring location is + located. + altitude : string or iterable of strings, optional + Altitude of the monitoring location referenced to the specified Vertical + Datum. + altitude_accuracy : string or iterable of strings, optional + Accuracy of the altitude, in feet. An accuracy of +/- 0.1 foot would be + entered as “.1”. Many altitudes are interpolated from the contours on + topographic maps; accuracies determined in this way are generally + entered as one-half of the contour interval. + altitude_method_code : string or iterable of strings, optional + Codes representing the method used to measure altitude. + altitude_method_name : string or iterable of strings, optional + The name of the method used to measure altitude. + vertical_datum : string or iterable of strings, optional + The datum used to determine altitude and vertical position at the + monitoring location. + vertical_datum_name : string or iterable of strings, optional + The datum used to determine altitude and vertical position at the + monitoring location. + horizontal_positional_accuracy_code : string or iterable of strings, optional + Indicates the accuracy of the latitude longitude values. + horizontal_positional_accuracy : string or iterable of strings, optional + Indicates the accuracy of the latitude longitude values. + horizontal_position_method_code : string or iterable of strings, optional + Indicates the method used to determine latitude longitude values. + horizontal_position_method_name : string or iterable of strings, optional + Indicates the method used to determine latitude longitude values. + original_horizontal_datum : string or iterable of strings, optional + Coordinates are published in EPSG:4326 / WGS84 / World Geodetic System + 1984. This field indicates the original datum used to determine + coordinates before they were converted. + original_horizontal_datum_name : string or iterable of strings, optional + Coordinates are published in EPSG:4326 / WGS84 / World Geodetic System + 1984. This field indicates the original datum used to determine coordinates + before they were converted. + drainage_area : string or iterable of strings, optional + The area enclosed by a topographic divide from which direct surface runoff + from precipitation normally drains by gravity into the stream above that + point. + contributing_drainage_area : string or iterable of strings, optional + The contributing drainage area of a lake, stream, wetland, or estuary + monitoring location, in square miles. This item should be present only + if the contributing area is different from the total drainage area. This + situation can occur when part of the drainage area consists of very + porous soil or depressions that either allow all runoff to enter the + groundwater or trap the water in ponds so that rainfall does not + contribute to runoff. A transbasin diversion can also affect the total + drainage area. + time_zone_abbreviation : string or iterable of strings, optional + A short code describing the time zone used by a monitoring location. + uses_daylight_savings : string or iterable of strings, optional + A flag indicating whether or not a monitoring location uses daylight savings. + construction_date : string or iterable of strings, optional + Date the well was completed. + aquifer_code : string or iterable of strings, optional + Local aquifers in the USGS water resources data base are identified by a + geohydrologic unit code (a three-digit number related to the age of the + formation, followed by a 4 or 5 character abbreviation for the geologic + unit or aquifer name). Additional information is available + `at this link `_. + national_aquifer_code : string or iterable of strings, optional + National aquifers are the principal aquifers or aquifer systems in the United + States, defined as regionally extensive aquifers or aquifer systems that have + the potential to be used as a source of potable water. Not all groundwater + monitoring locations can be associated with a National Aquifer. Such + monitoring locations will not be retrieved using this search criteria. A `list + of National aquifer codes and names `_ + is available. + aquifer_type_code : string or iterable of strings, optional + Groundwater occurs in aquifers under two different conditions. Where water + only partly fills an aquifer, the upper surface is free to rise and decline. + These aquifers are referred to as unconfined (or water-table) aquifers. Where + water completely fills an aquifer that is overlain by a confining bed, the + aquifer is referred to as a confined (or artesian) aquifer. When a confined + aquifer is penetrated by a well, the water level in the well will rise above + the top of the aquifer (but not necessarily above land surface). Additional + information is available `at this link `_. + well_constructed_depth : string or iterable of strings, optional + The depth of the finished well, in feet below land surface datum. Note: Not + all groundwater monitoring locations have information on Well Depth. Such + monitoring locations will not be retrieved using this search criteria. + hole_constructed_depth : string or iterable of strings, optional + The total depth to which the hole is drilled, in feet below land surface datum. + Note: Not all groundwater monitoring locations have information on Hole Depth. + Such monitoring locations will not be retrieved using this search criteria. + depth_source_code : string or iterable of strings, optional + A code indicating the source of water-level data. A `list of + codes `_ + is available. + properties : string or iterable of strings, optional + A list of requested columns to be returned from the query. Available + options are: geometry, id, agency_code, agency_name, + monitoring_location_number, monitoring_location_name, district_code, + country_code, country_name, state_code, state_name, county_code, + county_name, minor_civil_division_code, site_type_code, site_type, + hydrologic_unit_code, basin_code, altitude, altitude_accuracy, + altitude_method_code, altitude_method_name, vertical_datum, + vertical_datum_name, horizontal_positional_accuracy_code, + horizontal_positional_accuracy, horizontal_position_method_code, + horizontal_position_method_name, original_horizontal_datum, + original_horizontal_datum_name, drainage_area, + contributing_drainage_area, time_zone_abbreviation, + uses_daylight_savings, construction_date, aquifer_code, + national_aquifer_code, aquifer_type_code, well_constructed_depth, + hole_constructed_depth, depth_source_code. + bbox : list of numbers, optional + Only features that have a geometry that intersects the bounding box are + selected. The bounding box is provided as four or six numbers, + depending on whether the coordinate reference system includes a vertical + axis (height or depth). Coordinates are assumed to be in crs 4326. The + expected format is ``[xmin, ymin, xmax, ymax]``, i.e. + ``[Western-most longitude, Southern-most latitude, Eastern-most + longitude, Northern-most latitude]``. + limit : int, optional + The optional limit parameter is used to control the subset of the + selected features that should be returned in each page. The maximum + allowable limit is 50000. It may be beneficial to set this number lower + if your internet connection is spotty. The default (None) will set the + limit to the maximum allowable limit for the service. + This is a per-page size, not a cap on the total result: a query + matching more rows than ``limit`` still returns every matching row + across multiple pages. Use ``max_rows`` to cap the total instead. + skip_geometry : boolean, optional + This option can be used to skip response geometries for each feature. + The returning object will be a data frame with no spatial information. + Note that the USGS Water Data APIs use camelCase "skipGeometry" in + CQL2 queries. + filter, filter_lang : optional + Server-side CQL filter passed through as the OGC ``filter`` / + ``filter-lang`` query parameters. See + :mod:`dataretrieval.ogc.filters` for syntax, auto-chunking, + and the lexicographic-comparison pitfall. + convert_type : boolean, optional + If True, converts columns to appropriate types. + max_rows : int, optional + Cap the total number of rows returned, stopping pagination early + instead of downloading the whole result. Unlike ``limit`` (the + per-page size), this bounds the total result across every page. + The default (None) follows pagination to completion. + **queryables : string or iterable of strings, optional + Any other queryable property of this collection, passed through as a + server-side filter. Call :func:`get_queryables` to see the queryables a + collection supports. + + Returns + ------- + df : ``pandas.DataFrame`` or ``geopandas.GeoDataFrame`` + Formatted data returned from the API query. + md: :obj:`dataretrieval.utils.BaseMetadata` + A custom metadata object + + Raises + ------ + ChunkInterrupted + A transient failure (429 / 5xx / timeout) interrupted the request + after the built-in retries. Completed work is preserved; resume + with ``exc.call.resume()`` (see :doc:`/userguide/errors`). + + Examples + -------- + .. code:: + + >>> # Get monitoring locations within a bounding box + >>> # and leave out geometry + >>> df, md = dataretrieval.waterdata.get_monitoring_locations( + ... bbox=[-90.2, 42.6, -88.7, 43.2], skip_geometry=True + ... ) + + >>> # Get monitoring location info for specific sites + >>> # and only specific properties + >>> df, md = dataretrieval.waterdata.get_monitoring_locations( + ... monitoring_location_id=["USGS-05114000", "USGS-09423350"], + ... properties=["monitoring_location_id", "state_name", "country_name"], + ... ) + """ + service = "monitoring-locations" + + # Build argument dictionary, omitting None values (resolving the unified + # `state` argument into the OGC `state_name` queryable). + args = _get_args( + _with_state(locals(), to="name", into="state_name"), exclude={"max_rows"} + ) + + return get_ogc_data(args, service, max_rows=max_rows) + + +def get_time_series_metadata( + monitoring_location_id: str | Iterable[str] | None = None, + parameter_code: str | Iterable[str] | None = None, + parameter_name: str | Iterable[str] | None = None, + properties: str | Iterable[str] | None = None, + statistic_id: str | Iterable[str] | None = None, + hydrologic_unit_code: str | Iterable[str] | None = None, + state: str | Iterable[str] | None = None, + state_name: str | Iterable[str] | None = None, + last_modified: str | Iterable[str] | None = None, + begin: str | Iterable[str] | None = None, + end: str | Iterable[str] | None = None, + begin_utc: str | Iterable[str] | None = None, + end_utc: str | Iterable[str] | None = None, + unit_of_measure: str | Iterable[str] | None = None, + computation_period_identifier: str | Iterable[str] | None = None, + computation_identifier: str | Iterable[str] | None = None, + thresholds: float | list[float] | None = None, + sublocation_identifier: str | Iterable[str] | None = None, + primary: str | Iterable[str] | None = None, + parent_time_series_id: str | Iterable[str] | None = None, + time_series_id: str | Iterable[str] | None = None, + web_description: str | Iterable[str] | None = None, + skip_geometry: bool | None = None, + bbox: list[float] | None = None, + limit: int | None = None, + filter: str | None = None, + filter_lang: FILTER_LANG | None = None, + convert_type: bool = True, + max_rows: int | None = None, + **queryables: Any, +) -> tuple[pd.DataFrame, BaseMetadata]: + """Daily data and continuous measurements are grouped into time series, + which represent a collection of observations of a single parameter, + potentially aggregated using a standard statistic, at a single monitoring + location. This endpoint provides metadata about those time series, + including their operational thresholds, units of measurement, and when + the earliest and most recent observations in a time series occurred. + + Parameters + ---------- + monitoring_location_id : string or iterable of strings, optional + A unique identifier representing a single monitoring location. This + corresponds to the id field in the monitoring-locations endpoint. + Monitoring location IDs are created by combining the agency code of + the agency responsible for the monitoring location (e.g. USGS) with + the ID number of the monitoring location (e.g. 02238500), separated + by a hyphen (e.g. USGS-02238500). + parameter_code : string or iterable of strings, optional + Parameter codes are 5-digit codes used to identify the constituent + measured and the units of measure. A complete list of parameter + codes and associated groupings can be found at + https://help.waterdata.usgs.gov/codes-and-parameters/parameters. + parameter_name : string or iterable of strings, optional + A human-understandable name corresponding to parameter_code. + properties : string or iterable of strings, optional + A list of requested columns to be returned from the query. + Available options are: begin, begin_utc, computation_identifier, + computation_period_identifier, end, end_utc, geometry, + hydrologic_unit_code, id, last_modified, monitoring_location_id, + parameter_code, parameter_description, parameter_name, + parent_time_series_id, primary, state_name, statistic_id, + sublocation_identifier, thresholds, unit_of_measure, web_description + statistic_id : string or iterable of strings, optional + A code corresponding to the statistic an observation represents. + Example codes include 00001 (max), 00002 (min), and 00003 (mean). + A complete list of codes and their descriptions can be found at + https://help.waterdata.usgs.gov/code/stat_cd_nm_query?stat_nm_cd=%25&fmt=html. + hydrologic_unit_code : string or iterable of strings, optional + The United States is divided and sub-divided into successively smaller + hydrologic units which are classified into four levels: regions, + sub-regions, accounting units, and cataloging units. The hydrologic + units are arranged within each other, from the smallest (cataloging units) + to the largest (regions). Each hydrologic unit is identified by a unique + hydrologic unit code (HUC) consisting of two to eight digits based on the + four levels of classification in the hydrologic unit system. + state : string or iterable of strings, optional + State/territory filter (the recommended parameter). Accepts a full name + (``"Wisconsin"``), a two-letter postal code (``"WI"``), or a two-digit + ANSI/FIPS code (``"55"``). + state_name : string or iterable of strings, optional + The name of the state or state equivalent in which the monitoring location + is located. + last_modified : string, optional + The last time a record was refreshed in our database. This may happen + due to regular operational processes and does not necessarily indicate + that anything about the measurement has changed. You can query this field + using date-times or intervals, adhering to RFC 3339, or using ISO 8601 + duration objects. Intervals may be bounded or half-bounded (double-dots + at start or end). Only features that have a last_modified that + intersects the value of datetime are selected. + Examples: + + * A date-time: "2018-02-12T23:20:50Z" + * A bounded interval: "2018-02-12T00:00:00Z/2018-03-18T12:31:12Z" + * Half-bounded intervals: "2018-02-12T00:00:00Z/.." or + "../2018-03-18T12:31:12Z" + * Duration objects: "P1M" for data from the past month or "PT36H" + for the last 36 hours + + begin : string or iterable of strings, optional + This field contains the same information as "begin_utc", but in the + local time of the monitoring location. It is retained for backwards + compatibility, but will be removed in V1 of these APIs. + end : string or iterable of strings, optional + This field contains the same information as "end_utc", but in the + local time of the monitoring location. It is retained for backwards + compatibility, but will be removed in V1 of these APIs. + begin_utc : string or iterable of strings, optional + The datetime of the earliest observation in the time series. Together + with end, this field represents the period of record of a time series. + Note that some time series may have large gaps in their collection + record. This field is currently in the local time of the monitoring + location. We intend to update this in version v0 to use UTC with a time + zone. You can query this field using date-times or intervals, adhering + to RFC 3339, or using ISO 8601 duration objects. Intervals may be + bounded or half-bounded (double-dots at start or end). Only features + that have a begin that intersects the value of datetime are selected. + Examples: + + * A date-time: "2018-02-12T23:20:50Z" + * A bounded interval: "2018-02-12T00:00:00Z/2018-03-18T12:31:12Z" + * Half-bounded intervals: "2018-02-12T00:00:00Z/.." or + "../2018-03-18T12:31:12Z" + * Duration objects: "P1M" for data from the past month or + "PT36H" for the last 36 hours + + end_utc : string or iterable of strings, optional + The datetime of the most recent observation in the time series. Data returned by + this endpoint updates at most once per day, and potentially less frequently than + that, and as such there may be more recent observations within a time series + than the time series end value reflects. Together with begin, this field + represents the period of record of a time series. It is additionally used to + determine whether a time series is "active". We intend to update this in + version v0 to use UTC with a time zone. + You can query this field using date-times or intervals, + adhering to RFC 3339, or using ISO 8601 duration objects. Intervals + may be bounded or half-bounded (double-dots at start or end). Only + features that have an end that intersects the value of datetime are + selected. + Examples: + + * A date-time: "2018-02-12T23:20:50Z" + * A bounded interval: "2018-02-12T00:00:00Z/2018-03-18T12:31:12Z" + * Half-bounded intervals: "2018-02-12T00:00:00Z/.." or + "../2018-03-18T12:31:12Z" + * Duration objects: "P1M" for data from the past month or + "PT36H" for the last 36 hours + + unit_of_measure : string or iterable of strings, optional + A human-readable description of the units of measurement associated + with an observation. + computation_period_identifier : string or iterable of strings, optional + Indicates the period of data used for any statistical computations. + computation_identifier : string or iterable of strings, optional + Indicates whether the data from this time series represent a specific + statistical computation. + thresholds : number or list of numbers, optional + Thresholds represent known numeric limits for a time series, for example + the historic maximum value for a parameter or a level below which a + sensor is non-operative. These thresholds are sometimes used to + automatically determine if an observation is erroneous due to sensor + error, and therefore shouldn't be included in the time series. + sublocation_identifier : string or iterable of strings, optional + primary : string or iterable of strings, optional + parent_time_series_id : string or iterable of strings, optional + time_series_id : string or iterable of strings, optional + A unique identifier representing a single time series. This + corresponds to the id field in the time-series-metadata endpoint. + web_description : string or iterable of strings, optional + A description of what this time series represents, as used by WDFN and + other USGS data dissemination products. + skip_geometry : boolean, optional + This option can be used to skip response geometries for each feature. + The returning object will be a data frame with no spatial information. + Note that the USGS Water Data APIs use camelCase "skipGeometry" in + CQL2 queries. + bbox : list of numbers, optional + Only features that have a geometry that intersects the bounding box are + selected. The bounding box is provided as four or six numbers, + depending on whether the coordinate reference system includes a vertical + axis (height or depth). Coordinates are assumed to be in crs 4326. The + expected format is ``[xmin, ymin, xmax, ymax]``, i.e. + ``[Western-most longitude, Southern-most latitude, Eastern-most + longitude, Northern-most latitude]``. + limit : int, optional + The optional limit parameter is used to control the subset of the + selected features that should be returned in each page. The maximum + allowable limit is 50000. It may be beneficial to set this number lower + if your internet connection is spotty. The default (None) will set the + limit to the maximum allowable limit for the service. + This is a per-page size, not a cap on the total result: a query + matching more rows than ``limit`` still returns every matching row + across multiple pages. Use ``max_rows`` to cap the total instead. + filter, filter_lang : optional + Server-side CQL filter passed through as the OGC ``filter`` / + ``filter-lang`` query parameters. See + :mod:`dataretrieval.ogc.filters` for syntax, auto-chunking, + and the lexicographic-comparison pitfall. + convert_type : boolean, optional + If True, converts columns to appropriate types. + max_rows : int, optional + Cap the total number of rows returned, stopping pagination early + instead of downloading the whole result. Unlike ``limit`` (the + per-page size), this bounds the total result across every page. + The default (None) follows pagination to completion. + **queryables : string or iterable of strings, optional + Any other queryable property of this collection, passed through as a + server-side filter. Call :func:`get_queryables` to see the queryables a + collection supports. + + Returns + ------- + df : ``pandas.DataFrame`` or ``geopandas.GeoDataFrame`` + Formatted data returned from the API query. + md: :obj:`dataretrieval.utils.BaseMetadata` + A custom metadata object + + Raises + ------ + ChunkInterrupted + A transient failure (429 / 5xx / timeout) interrupted the request + after the built-in retries. Completed work is preserved; resume + with ``exc.call.resume()`` (see :doc:`/userguide/errors`). + + Examples + -------- + .. code:: + + >>> # Get timeseries metadata information from a single site + >>> # over a yearlong period + >>> df, md = dataretrieval.waterdata.get_time_series_metadata( + ... monitoring_location_id="USGS-02238500" + ... ) + + >>> # Get timeseries metadata information from multiple sites + >>> # that begin after January 1, 1990. + >>> df, md = dataretrieval.waterdata.get_time_series_metadata( + ... monitoring_location_id=["USGS-05114000", "USGS-09423350"], + ... begin="1990-01-01/..", + ... ) + """ + service = "time-series-metadata" + + # Build argument dictionary, omitting None values (resolving the unified + # `state` argument into the OGC `state_name` queryable). + args = _get_args( + _with_state(locals(), to="name", into="state_name"), exclude={"max_rows"} + ) + + return get_ogc_data(args, service, max_rows=max_rows) + + +def get_combined_metadata( + monitoring_location_id: str | Iterable[str] | None = None, + parameter_code: str | Iterable[str] | None = None, + parameter_name: str | Iterable[str] | None = None, + parameter_description: str | Iterable[str] | None = None, + unit_of_measure: str | Iterable[str] | None = None, + statistic_id: str | Iterable[str] | None = None, + data_type: str | Iterable[str] | None = None, + computation_identifier: str | Iterable[str] | None = None, + thresholds: float | list[float] | None = None, + sublocation_identifier: str | Iterable[str] | None = None, + primary: str | Iterable[str] | None = None, + parent_time_series_id: str | Iterable[str] | None = None, + web_description: str | Iterable[str] | None = None, + last_modified: str | Iterable[str] | None = None, + begin: str | Iterable[str] | None = None, + end: str | Iterable[str] | None = None, + agency_code: str | Iterable[str] | None = None, + agency_name: str | Iterable[str] | None = None, + monitoring_location_number: str | Iterable[str] | None = None, + monitoring_location_name: str | Iterable[str] | None = None, + district_code: str | Iterable[str] | None = None, + country_code: str | Iterable[str] | None = None, + country_name: str | Iterable[str] | None = None, + state: str | Iterable[str] | None = None, + state_code: str | Iterable[str] | None = None, + state_name: str | Iterable[str] | None = None, + county_code: str | Iterable[str] | None = None, + county_name: str | Iterable[str] | None = None, + minor_civil_division_code: str | Iterable[str] | None = None, + site_type_code: str | Iterable[str] | None = None, + site_type: str | Iterable[str] | None = None, + hydrologic_unit_code: str | Iterable[str] | None = None, + basin_code: str | Iterable[str] | None = None, + altitude: str | Iterable[str] | None = None, + altitude_accuracy: str | Iterable[str] | None = None, + altitude_method_code: str | Iterable[str] | None = None, + altitude_method_name: str | Iterable[str] | None = None, + vertical_datum: str | Iterable[str] | None = None, + vertical_datum_name: str | Iterable[str] | None = None, + horizontal_positional_accuracy_code: str | Iterable[str] | None = None, + horizontal_positional_accuracy: str | Iterable[str] | None = None, + horizontal_position_method_code: str | Iterable[str] | None = None, + horizontal_position_method_name: str | Iterable[str] | None = None, + original_horizontal_datum: str | Iterable[str] | None = None, + original_horizontal_datum_name: str | Iterable[str] | None = None, + drainage_area: str | Iterable[str] | None = None, + contributing_drainage_area: str | Iterable[str] | None = None, + time_zone_abbreviation: str | Iterable[str] | None = None, + uses_daylight_savings: str | Iterable[str] | None = None, + construction_date: str | Iterable[str] | None = None, + aquifer_code: str | Iterable[str] | None = None, + national_aquifer_code: str | Iterable[str] | None = None, + aquifer_type_code: str | Iterable[str] | None = None, + well_constructed_depth: str | Iterable[str] | None = None, + hole_constructed_depth: str | Iterable[str] | None = None, + depth_source_code: str | Iterable[str] | None = None, + properties: str | Iterable[str] | None = None, + skip_geometry: bool | None = None, + bbox: list[float] | None = None, + limit: int | None = None, + filter: str | None = None, + filter_lang: FILTER_LANG | None = None, + convert_type: bool = True, + max_rows: int | None = None, + **queryables: Any, +) -> tuple[pd.DataFrame, BaseMetadata]: + """Get combined monitoring-location and time-series metadata. + + The ``combined-metadata`` collection joins the monitoring-locations + catalog with the time-series-metadata catalog so that one row is + returned per (location, parameter, statistic) inventory entry, + carrying every column from both source endpoints. This makes it the + most flexible "what data is available" endpoint in the Water Data + API: any monitoring-location attribute (state, HUC, site type, + drainage area, well-construction depth, …) can be combined with any + time-series attribute (parameter code, statistic, data type, period + of record, …) in a single query. + + See the OpenAPI reference for the full list of supported fields: + https://api.waterdata.usgs.gov/ogcapi/v0/openapi?f=html#/combined-metadata + + All ~35 location-catalog kwargs are accepted (``agency_code``, + ``state_name``, ``drainage_area``, ``aquifer_code``, …) but only + the most-used ones are documented below; see + :func:`get_monitoring_locations` for per-field descriptions. + + Parameters + ---------- + monitoring_location_id : string or iterable of strings, optional + A unique identifier representing a single monitoring location. + Created by combining the agency code (e.g. ``USGS``) with the ID + number (e.g. ``02238500``), separated by a hyphen + (e.g. ``"USGS-02238500"``). + parameter_code : string or iterable of strings, optional + 5-digit codes used to identify the constituent measured and the + units of measure. See + https://help.waterdata.usgs.gov/codes-and-parameters/parameters. + parameter_name : string or iterable of strings, optional + A human-understandable name corresponding to ``parameter_code``. + parameter_description : string or iterable of strings, optional + A human-readable description of what is being measured. + unit_of_measure : string or iterable of strings, optional + A human-readable description of the units of measurement + associated with an observation. + statistic_id : string or iterable of strings, optional + A code corresponding to the statistic an observation represents + (e.g. ``00001`` max, ``00002`` min, ``00003`` mean). Full list at + https://help.waterdata.usgs.gov/code/stat_cd_nm_query?stat_nm_cd=%25&fmt=html. + data_type : string or iterable of strings, optional + The type of data the time series represents, e.g. + ``"Continuous values"``, ``"Daily values"``, + ``"Field measurements"``. + computation_identifier : string or iterable of strings, optional + Indicates whether the data from this time series represent a + specific statistical computation. + thresholds : number or list of numbers, optional + Numeric limits known for a time series (e.g. historic maximum, + below-which-the-sensor-is-non-operative). + sublocation_identifier : string or iterable of strings, optional + primary : string or iterable of strings, optional + A flag identifying whether the time series is "primary". Primary + time series are standard observations that have undergone Bureau + review and approval. Non-primary (provisional) time series have a + missing ``primary`` value, are produced for timely best-science + use, and are retained by this system for only 120 days. + parent_time_series_id : string or iterable of strings, optional + web_description : string or iterable of strings, optional + A description of what this time series represents, as used by + WDFN and other USGS data dissemination products. + last_modified, begin, end : string, optional + Datetime fields that accept either an RFC 3339 datetime, an + interval (``"start/end"``, optionally half-bounded with ``..``), + or an ISO 8601 duration (e.g. ``"P1M"``, ``"PT36H"``). See + :func:`get_time_series_metadata` for the full grammar. + state : string or iterable of strings, optional + State/territory filter (the recommended parameter). Accepts a full + name (``"Wisconsin"``), a two-letter postal code (``"WI"``), or a + two-digit ANSI/FIPS code (``"55"``). + state_name, county_name, hydrologic_unit_code, site_type, \ +site_type_code : string or iterable of strings, optional + Common location-catalog filters carried over from the + ``monitoring-locations`` collection. The function also accepts + the full list of location-catalog kwargs (agency, district, + altitude, vertical/horizontal datum, drainage area, aquifer, + well construction, …); see :func:`get_monitoring_locations` for + descriptions of each. + properties : string or iterable of strings, optional + Subset of columns to return. Defaults to every available + property. + skip_geometry : boolean, optional + Skip per-feature geometries; the returned object will be a plain + ``DataFrame`` with no spatial information. The Water Data APIs + use camelCase ``skipGeometry`` in CQL2 queries. + bbox : list of numbers, optional + Only features whose geometry intersects the bounding box are + selected. Format: ``[xmin, ymin, xmax, ymax]`` in CRS 4326 + (longitude/latitude, west-south-east-north). + limit : int, optional + Page size; the maximum allowable value is 50000. Default + (``None``) requests the maximum allowable limit. This is a + per-page size, not a cap on the total result: a query matching more + rows than ``limit`` still returns every matching row across + multiple pages. Use ``max_rows`` to cap the total instead. + filter, filter_lang : optional + Server-side CQL filter passed through as the OGC ``filter`` / + ``filter-lang`` query parameters. See + :mod:`dataretrieval.ogc.filters` for syntax, auto-chunking, + and the lexicographic-comparison pitfall. + convert_type : boolean, optional + If True, converts columns to appropriate types. + max_rows : int, optional + Cap the total number of rows returned, stopping pagination early + instead of downloading the whole result. Unlike ``limit`` (the + per-page size), this bounds the total result across every page. + The default (None) follows pagination to completion. + **queryables : string or iterable of strings, optional + Any other queryable property of this collection, passed through as a + server-side filter. Call :func:`get_queryables` to see the queryables a + collection supports. + + Returns + ------- + df : ``pandas.DataFrame`` or ``geopandas.GeoDataFrame`` + Formatted data returned from the API query. + md : :obj:`dataretrieval.utils.BaseMetadata` + A custom metadata object pertaining to the query. + + Raises + ------ + ChunkInterrupted + A transient failure (429 / 5xx / timeout) interrupted the request + after the built-in retries. Completed work is preserved; resume + with ``exc.call.resume()`` (see :doc:`/userguide/errors`). + + Examples + -------- + .. code:: + + >>> # All time series and field measurements at a single surface-water site + >>> df, md = dataretrieval.waterdata.get_combined_metadata( + ... monitoring_location_id="USGS-05407000" + ... ) + + >>> # Same, for a groundwater well — water-level and aquifer columns + >>> # are populated where the surface-water example has nulls + >>> df, md = dataretrieval.waterdata.get_combined_metadata( + ... monitoring_location_id="USGS-375907091432201" + ... ) + + >>> # Every series in a single county, useful for area-of-interest workflows + >>> df, md = dataretrieval.waterdata.get_combined_metadata( + ... state="Wisconsin", county_name="Dane County" + ... ) + + >>> # Inventory across multiple HUCs, restricted to streams and springs + >>> df, md = dataretrieval.waterdata.get_combined_metadata( + ... hydrologic_unit_code=["11010008", "11010009"], + ... site_type=["Stream", "Spring"], + ... ) + + >>> # Discharge time series at three sites with at least one + >>> # observation in the past month + >>> df, md = dataretrieval.waterdata.get_combined_metadata( + ... monitoring_location_id=[ + ... "USGS-07069000", + ... "USGS-07064000", + ... "USGS-07068000", + ... ], + ... end="P1M", + ... parameter_code="00060", + ... ) + + >>> # Two-step "what's available?" → "fetch it" workflow: + >>> # 1. inventory the sites in two HUCs + >>> hucs, _ = dataretrieval.waterdata.get_combined_metadata( + ... hydrologic_unit_code=["11010008", "11010009"], + ... site_type="Stream", + ... ) + >>> # 2. pull continuous discharge at every distinct site found + >>> sites = hucs["monitoring_location_id"].unique().tolist() + >>> df, md = dataretrieval.waterdata.get_continuous( + ... monitoring_location_id=sites, + ... parameter_code="00060", + ... time="P1D", + ... ) + + """ + service = "combined-metadata" + + # Resolve the unified `state` argument into the OGC `state_name` queryable. + args = _get_args( + _with_state(locals(), to="name", into="state_name"), exclude={"max_rows"} + ) + + return get_ogc_data(args, service, max_rows=max_rows) + + +def get_field_measurements_metadata( + monitoring_location_id: str | Iterable[str] | None = None, + parameter_code: str | Iterable[str] | None = None, + parameter_name: str | Iterable[str] | None = None, + parameter_description: str | Iterable[str] | None = None, + begin: str | Iterable[str] | None = None, + end: str | Iterable[str] | None = None, + last_modified: str | Iterable[str] | None = None, + properties: str | Iterable[str] | None = None, + skip_geometry: bool | None = None, + bbox: list[float] | None = None, + limit: int | None = None, + filter: str | None = None, + filter_lang: FILTER_LANG | None = None, + convert_type: bool = True, + max_rows: int | None = None, + **queryables: Any, +) -> tuple[pd.DataFrame, BaseMetadata]: + """Get field-measurement metadata: one row per (location, parameter) series. + + Each row describes a single field-measurement series — what parameter is + measured at the location, the period of record (``begin`` / ``end``), the + units, and so on — without returning the underlying observations + themselves. Use :func:`get_field_measurements` to fetch the values. + + This is the discrete-measurement analogue to + :func:`get_time_series_metadata` (which describes daily and continuous + series). It's primarily useful for inventory queries: "what + field-measurement parameters does this site have, and over what date + range?" + + See the OpenAPI reference for the full list of supported fields: + https://api.waterdata.usgs.gov/ogcapi/v0/openapi?f=html#/field-measurements-metadata + + Parameters + ---------- + monitoring_location_id : string or iterable of strings, optional + A unique identifier representing a single monitoring location, in + ``AGENCY-ID`` form (e.g. ``"USGS-02238500"``). + parameter_code : string or iterable of strings, optional + 5-digit parameter code. See + https://help.waterdata.usgs.gov/codes-and-parameters/parameters. + parameter_name : string or iterable of strings, optional + A human-understandable name corresponding to ``parameter_code``. + parameter_description : string or iterable of strings, optional + A human-readable description of what is being measured. + begin, end, last_modified : string, optional + Datetime fields that accept either an RFC 3339 datetime, an + interval (``"start/end"``, optionally half-bounded with ``..``), + or an ISO 8601 duration (e.g. ``"P1M"``, ``"PT36H"``). See + :func:`get_time_series_metadata` for the full grammar. + properties : string or iterable of strings, optional + Subset of columns to return. Defaults to every available property. + skip_geometry : boolean, optional + Skip per-feature geometries; the returned object will be a plain + ``DataFrame`` with no spatial information. + bbox : list of numbers, optional + Only features whose geometry intersects the bounding box are + selected. Format: ``[xmin, ymin, xmax, ymax]`` in CRS 4326 + (longitude / latitude, west-south-east-north). + limit : int, optional + Page size; the maximum allowable value is 50000. Default + (``None``) requests the maximum allowable limit. This is a + per-page size, not a cap on the total result: a query matching more + rows than ``limit`` still returns every matching row across + multiple pages. Use ``max_rows`` to cap the total instead. + filter, filter_lang : optional + Server-side CQL filter passed through as the OGC ``filter`` / + ``filter-lang`` query parameters. See + :mod:`dataretrieval.ogc.filters` for syntax, auto-chunking, + and the lexicographic-comparison pitfall. + convert_type : boolean, optional + If True, converts columns to appropriate types. + max_rows : int, optional + Cap the total number of rows returned, stopping pagination early + instead of downloading the whole result. Unlike ``limit`` (the + per-page size), this bounds the total result across every page. + The default (None) follows pagination to completion. + **queryables : string or iterable of strings, optional + Any other queryable property of this collection, passed through as a + server-side filter. Call :func:`get_queryables` to see the queryables a + collection supports. + + Returns + ------- + df : ``pandas.DataFrame`` or ``geopandas.GeoDataFrame`` + Formatted data returned from the API query. + md : :obj:`dataretrieval.utils.BaseMetadata` + A custom metadata object pertaining to the query. + + Raises + ------ + ChunkInterrupted + A transient failure (429 / 5xx / timeout) interrupted the request + after the built-in retries. Completed work is preserved; resume + with ``exc.call.resume()`` (see :doc:`/userguide/errors`). + + Examples + -------- + .. code:: + + >>> # All field-measurement series at a surface-water site + >>> df, md = dataretrieval.waterdata.get_field_measurements_metadata( + ... monitoring_location_id="USGS-02238500" + ... ) + + >>> # Same, for a groundwater well + >>> df, md = dataretrieval.waterdata.get_field_measurements_metadata( + ... monitoring_location_id="USGS-375907091432201" + ... ) + + >>> # Multi-site, narrowed to two parameter codes + >>> df, md = dataretrieval.waterdata.get_field_measurements_metadata( + ... monitoring_location_id=[ + ... "USGS-451605097071701", + ... "USGS-263819081585801", + ... ], + ... parameter_code=["62611", "72019"], + ... ) + + >>> # Series modified in the last year — useful for incremental ETL + >>> df, md = dataretrieval.waterdata.get_field_measurements_metadata( + ... monitoring_location_id="USGS-375907091432201", + ... parameter_code="72019", + ... last_modified="P1Y", + ... ) + + """ + service = "field-measurements-metadata" + + args = _get_args(locals(), exclude={"max_rows"}) + + return get_ogc_data(args, service, max_rows=max_rows) + + +__all__ = [ + "get_monitoring_locations", + "get_time_series_metadata", + "get_combined_metadata", + "get_field_measurements_metadata", +] diff --git a/dataretrieval/waterdata/nearest.py b/dataretrieval/waterdata/nearest.py index edf1a912..3ee0c063 100644 --- a/dataretrieval/waterdata/nearest.py +++ b/dataretrieval/waterdata/nearest.py @@ -11,7 +11,10 @@ import pandas as pd from dataretrieval.utils import BaseMetadata -from dataretrieval.waterdata.api import get_continuous +from dataretrieval.waterdata.time_series import get_continuous + +__all__ = ["get_nearest_continuous"] + OnTie = Literal["first", "last", "mean"] _VALID_ON_TIE: tuple[OnTie, ...] = get_args(OnTie) diff --git a/dataretrieval/waterdata/ratings.py b/dataretrieval/waterdata/ratings.py index 6cdc3896..df13dedf 100644 --- a/dataretrieval/waterdata/ratings.py +++ b/dataretrieval/waterdata/ratings.py @@ -36,6 +36,9 @@ from .utils import BASE_URL +__all__ = ["get_ratings"] + + logger = logging.getLogger(__name__) STAC_URL = f"{BASE_URL}/stac/v0" diff --git a/dataretrieval/waterdata/reference.py b/dataretrieval/waterdata/reference.py new file mode 100644 index 00000000..ce95a155 --- /dev/null +++ b/dataretrieval/waterdata/reference.py @@ -0,0 +1,174 @@ +"""Reference-table and queryables discovery getters.""" + +from __future__ import annotations + +from typing import Any, get_args + +import pandas as pd + +from dataretrieval.ogc.schema import _check_ogc_requests +from dataretrieval.utils import BaseMetadata +from dataretrieval.waterdata.types import ( + METADATA_COLLECTIONS, +) +from dataretrieval.waterdata.utils import ( + get_ogc_data, +) + + +def get_reference_table( + collection: str, + limit: int | None = None, + query: dict[str, Any] | None = None, + max_rows: int | None = None, +) -> tuple[pd.DataFrame, BaseMetadata]: + """Get metadata reference tables for the USGS Water Data API. + + Reference tables provide the range of allowable values for parameter + arguments in the waterdata module. + + Parameters + ---------- + collection : string + One of the following options: "agency-codes", "altitude-datums", + "aquifer-codes", "aquifer-types", "coordinate-accuracy-codes", + "coordinate-datum-codes", "coordinate-method-codes", "counties", + "hydrologic-unit-codes", "medium-codes", "national-aquifer-codes", + "parameter-codes", "reliability-codes", "site-types", "states", + "statistic-codes", "topographic-codes", "time-zone-codes" + limit : int, optional + The optional limit parameter is used to control the subset of the + selected features that should be returned in each page. The maximum + allowable limit is 50000. It may be beneficial to set this number lower + if your internet connection is spotty. The default (None) will set the + limit to the maximum allowable limit for the service. + query: dictionary, optional + The optional query parameter can be used to pass a dictionary of + query parameters to the collection API call. + max_rows : int, optional + Cap the total number of rows returned, stopping pagination early + instead of downloading the whole table. Useful for cheaply + previewing large tables (e.g. ``hydrologic-unit-codes`` has ~125k + rows). Unlike ``limit`` (the per-page size), this bounds the total + result. The default (None) downloads every page. + + Returns + ------- + df : ``pandas.DataFrame`` or ``geopandas.GeoDataFrame`` + Formatted data returned from the API query. The primary metadata + of each reference table will show up in the first column, where + the name of the column is the singular form of the collection name, + separated by underscores (e.g. the "medium-codes" reference table + has a column called "medium_code", which contains all possible + medium code values). + md: :obj:`dataretrieval.utils.BaseMetadata` + A custom metadata object including the URL request and query time. + + Raises + ------ + ChunkInterrupted + A transient failure (429 / 5xx / timeout) interrupted the request + after the built-in retries. Completed work is preserved; resume + with ``exc.call.resume()`` (see :doc:`/userguide/errors`). + + Examples + -------- + .. code:: + + >>> # Get table of USGS parameter codes + >>> ref, md = dataretrieval.waterdata.get_reference_table( + ... collection="parameter-codes" + ... ) + + >>> # Get table of selected USGS parameter codes + >>> ref, md = dataretrieval.waterdata.get_reference_table( + ... collection="parameter-codes", + ... query={"id": "00001,00002"}, + ... ) + """ + valid_code_services = get_args(METADATA_COLLECTIONS) + if collection not in valid_code_services: + raise ValueError( + f"Invalid code service: '{collection}'. " + f"Valid options are: {valid_code_services}." + ) + + # Give the ID column the collection name, singularized and underscored. + if collection == "counties": + output_id = "county" + elif collection.endswith("s"): + output_id = collection[:-1].replace("-", "_") + else: + output_id = collection.replace("-", "_") + + query_args = dict(query) if query else {} + if limit is not None: + query_args["limit"] = limit + return get_ogc_data( + args=query_args, output_id=output_id, service=collection, max_rows=max_rows + ) + + +def get_queryables(collection: str) -> tuple[pd.DataFrame, BaseMetadata]: + """List the queryable properties of a Water Data API collection. + + Every OGC collection (``daily``, ``continuous``, ``monitoring-locations``, + ...) advertises the set of properties that can be filtered on -- exposed as + the typed keyword arguments of the matching ``get_*`` function, and usable + directly in a CQL2 ``filter``. This returns that set, so the available + filters can be discovered programmatically and monitored for upstream + additions. + + Parameters + ---------- + collection : string + The collection id, e.g. ``"daily"``, ``"continuous"``, + ``"monitoring-locations"``, or ``"time-series-metadata"``. See + :data:`dataretrieval.waterdata.types.WATERDATA_SERVICES` for the data + collections; reference collections (e.g. ``"parameter-codes"``) work + too. + + Returns + ------- + df : ``pandas.DataFrame`` + One row per queryable, sorted by name, with columns ``queryable`` (the + property name), ``type``, ``title``, and ``description``. + md : :class:`dataretrieval.utils.BaseMetadata` + Metadata describing the request (URL, query time, response headers). + + Raises + ------ + DataRetrievalError + On an HTTP error response (e.g. an unknown ``collection`` yields a 404), + the typed subclass for the status. + + Examples + -------- + .. doctest:: + :skipif: True # network + + >>> from dataretrieval import waterdata + >>> df, md = waterdata.get_queryables("daily") + >>> df.set_index("queryable").loc["state_name", "type"] + 'string' + """ + # The OGC queryables document is a JSON Schema whose ``properties`` map each + # filterable property name to a ``{title, type, description}`` definition. + body, response = _check_ogc_requests(endpoint=collection, req_type="queryables") + properties: dict[str, Any] = body.get("properties", {}) + df = pd.DataFrame( + [ + { + "queryable": name, + "type": prop.get("type"), + "title": prop.get("title"), + "description": (prop.get("description") or "").strip(), + } + for name, prop in sorted(properties.items()) + ], + columns=["queryable", "type", "title", "description"], + ) + return df, BaseMetadata(response) + + +__all__ = ["get_reference_table", "get_queryables"] diff --git a/dataretrieval/waterdata/samples.py b/dataretrieval/waterdata/samples.py new file mode 100644 index 00000000..aaba15cf --- /dev/null +++ b/dataretrieval/waterdata/samples.py @@ -0,0 +1,432 @@ +"""Aquarius Samples API getters and wire-parameter policy.""" + +from __future__ import annotations + +import json +import logging +from collections.abc import Iterable +from io import StringIO +from typing import Any, get_args +from urllib.parse import quote + +import httpx +import pandas as pd + +from dataretrieval.ogc.errors import _raise_for_non_200 +from dataretrieval.transport.http import ( + HTTPX_DEFAULTS, +) +from dataretrieval.transport.http import ( + default_headers as _default_headers, +) +from dataretrieval.transport.http import ( + get as _get, +) +from dataretrieval.utils import BaseMetadata, _attach_datetime_columns, to_str +from dataretrieval.waterdata.types import ( + CODE_SERVICES, + PROFILES, + SERVICES, +) +from dataretrieval.waterdata.utils import ( + SAMPLES_URL, + _accept_legacy_kwargs, + _check_profiles, + _get_args, +) + +logger = logging.getLogger(__name__) + + +def get_codes(code_service: CODE_SERVICES) -> tuple[pd.DataFrame, BaseMetadata]: + """Return codes from a Samples code service. + + Parameters + ---------- + code_service : string + One of the following options: "states", "counties", "countries", + "sitetype", "samplemedia", "characteristicgroup", "characteristics", + or "observedproperty" + + Returns + ------- + df : ``pandas.DataFrame`` + The requested code table. + md : :obj:`dataretrieval.utils.BaseMetadata` + Metadata for the query (URL, query time, response headers). + """ + valid_code_services = get_args(CODE_SERVICES) + if code_service not in valid_code_services: + raise ValueError( + f"Invalid code service: '{code_service}'. " + f"Valid options are: {valid_code_services}." + ) + + url = f"{SAMPLES_URL}/codeservice/{code_service}?mimeType=application%2Fjson" + + response = _get(url, headers=_default_headers(url), **HTTPX_DEFAULTS) + + _raise_for_non_200(response) + + data_dict = json.loads(response.text) + data_list = data_dict["data"] + + df = pd.DataFrame(data_list) + + return df, BaseMetadata(response) + + +def _get_samples_csv( + url: str, params: dict[str, Any], ssl_check: bool +) -> tuple[pd.DataFrame, httpx.Response]: + """Issue a Samples CSV request and parse the body into a DataFrame. + + Shared tail for the Samples getters: sends the GET with the standard + headers (including ``X-Api-Key``), raises a typed error on a non-200 + (consistent with the OGC/stats path) instead of a bare + ``HTTPStatusError``, and reads the CSV. The caller wraps the response + as metadata and applies any per-getter post-step. + """ + logger.debug("Request: %s", httpx.URL(url).copy_merge_params(params)) + response = _get( + url, + params=params, + verify=ssl_check, + headers=_default_headers(url), + **HTTPX_DEFAULTS, + ) + _raise_for_non_200(response) + df = pd.read_csv(StringIO(response.text), delimiter=",") + return df, response + + +# Map the public snake_case ``get_samples`` parameters to the camelCase query +# parameter names the Samples API expects on the wire. ``characteristic`` is +# already snake_case-compatible (single word) and is sent unchanged. The +# remaining snake_case params are bookkeeping (``service``/``profile``/ +# ``ssl_check``) and never reach the request. +_SAMPLES_PARAM_TO_API = { + "activity_media_name": "activityMediaName", + "activity_start_date_lower": "activityStartDateLower", + "activity_start_date_upper": "activityStartDateUpper", + "activity_type_code": "activityTypeCode", + "characteristic_group": "characteristicGroup", + "characteristic_user_supplied": "characteristicUserSupplied", + "bbox": "boundingBox", + "country_code": "countryFips", + "state_code": "stateFips", + "county_code": "countyFips", + "site_type_code": "siteTypeCode", + "site_type_name": "siteTypeName", + "usgs_pcode": "usgsPCode", + "hydrologic_unit": "hydrologicUnit", + "monitoring_location_id": "monitoringLocationIdentifier", + "organization_id": "organizationIdentifier", + "point_location_latitude": "pointLocationLatitude", + "point_location_longitude": "pointLocationLongitude", + "point_location_within_miles": "pointLocationWithinMiles", + "project_id": "projectIdentifier", + "record_identifier_user_supplied": "recordIdentifierUserSupplied", +} + +# Deprecated camelCase keyword names (the Samples-API spelling) accepted for +# backward compatibility, mapped to the new snake_case parameter names. Derived +# from ``_SAMPLES_PARAM_TO_API`` so the two never drift apart. +_SAMPLES_LEGACY_KWARGS = { + api_name: py_name for py_name, api_name in _SAMPLES_PARAM_TO_API.items() +} + + +@_accept_legacy_kwargs(_SAMPLES_LEGACY_KWARGS) +def get_samples( + ssl_check: bool = True, + service: SERVICES = "results", + profile: PROFILES = "fullphyschem", + activity_media_name: str | Iterable[str] | None = None, + activity_start_date_lower: str | None = None, + activity_start_date_upper: str | None = None, + activity_type_code: str | Iterable[str] | None = None, + characteristic_group: str | Iterable[str] | None = None, + characteristic: str | Iterable[str] | None = None, + characteristic_user_supplied: str | Iterable[str] | None = None, + bbox: list[float] | None = None, + country_code: str | Iterable[str] | None = None, + state_code: str | Iterable[str] | None = None, + county_code: str | Iterable[str] | None = None, + site_type_code: str | Iterable[str] | None = None, + site_type_name: str | Iterable[str] | None = None, + usgs_pcode: str | Iterable[str] | None = None, + hydrologic_unit: str | Iterable[str] | None = None, + monitoring_location_id: str | Iterable[str] | None = None, + organization_id: str | Iterable[str] | None = None, + point_location_latitude: float | None = None, + point_location_longitude: float | None = None, + point_location_within_miles: float | None = None, + project_id: str | Iterable[str] | None = None, + record_identifier_user_supplied: str | Iterable[str] | None = None, +) -> tuple[pd.DataFrame, BaseMetadata]: + """Search Samples database for USGS water quality data. + This is a wrapper function for the Samples database API. All potential + filters are provided as arguments to the function, but please do not + populate all possible filters; leave as many as feasible with their default + value (None). This is important because overcomplicated web service queries + can bog down the database's ability to return an applicable dataset before + it times out. + + The web GUI for the Samples database can be found here: + https://waterdata.usgs.gov/download-samples/#dataProfile=site + + If you would like more details on feasible query parameters (complete with + examples), please visit the Samples database swagger docs, here: + https://api.waterdata.usgs.gov/samples-data/docs#/ + + Parameters + ---------- + ssl_check : bool, optional + Check the SSL certificate. + service : string + One of the available Samples services: "results", "locations", "activities", + "projects", or "organizations". Defaults to "results". + profile : string + One of the available profiles associated with a service. Options for each + service are: + results - "fullphyschem", "basicphyschem", + "fullbio", "basicbio", "narrow", + "resultdetectionquantitationlimit", + "labsampleprep", "count" + locations - "site", "count" + activities - "sampact", "actmetric", + "actgroup", "count" + projects - "project", "projectmonitoringlocationweight" + organizations - "organization", "count" + activity_media_name : string or iterable of strings, optional + Name or code indicating environmental medium in which sample was taken. + Call ``get_codes("samplemedia")`` for the valid inputs. + Example: "Water". (Samples API: ``activityMediaName``) + activity_start_date_lower : string, optional + The start date if using a date range. Takes the format YYYY-MM-DD. + The logic is inclusive, i.e. it will also return results that + match the date. If left as None, will pull all data on or before + ``activity_start_date_upper``, if populated. + (Samples API: ``activityStartDateLower``) + activity_start_date_upper : string, optional + The end date if using a date range. Takes the format YYYY-MM-DD. + The logic is inclusive, i.e. it will also return results that + match the date. If left as None, will pull all data after + ``activity_start_date_lower`` up to the most recent available results. + (Samples API: ``activityStartDateUpper``) + activity_type_code : string or iterable of strings, optional + Text code that describes type of field activity performed. + Example: "Sample-Routine, regular". (Samples API: ``activityTypeCode``) + characteristic_group : string or iterable of strings, optional + Characteristic group is a broad category of characteristics + describing one or more results. Call ``get_codes("characteristicgroup")`` + for the valid inputs. + Example: "Organics, PFAS" (Samples API: ``characteristicGroup``) + characteristic : string or iterable of strings, optional + Characteristic is a specific category describing one or more results. + Call ``get_codes("characteristics")`` for the valid inputs. + Example: "Suspended Sediment Discharge" (Samples API: ``characteristic``) + characteristic_user_supplied : string or iterable of strings, optional + A user supplied characteristic name describing one or more results. + (Samples API: ``characteristicUserSupplied``) + bbox : list of four floats, optional + Filters on the associated monitoring location's point location + by checking if it is located within the specified geographic area. + The logic is inclusive, i.e. it will include locations that overlap + with the edge of the bounding box. Values are separated by commas, + expressed in decimal degrees, NAD83, and longitudes west of Greenwich + are negative. The format is a list consisting of: + + * Western-most longitude + * Southern-most latitude + * Eastern-most longitude + * Northern-most latitude + + Example: [-92.8,44.2,-88.9,46.0] (Samples API: ``boundingBox``) + country_code : string or iterable of strings, optional + Example: "US" (United States) (Samples API: ``countryFips``) + state_code : string or iterable of strings, optional + Call ``get_codes("states")`` for the valid inputs. + Example: "US:15" (United States: Hawaii) (Samples API: ``stateFips``) + county_code : string or iterable of strings, optional + Call ``get_codes("counties")`` for the valid inputs. + Example: "US:15:001" (United States: Hawaii, Hawaii County) + (Samples API: ``countyFips``) + site_type_code : string or iterable of strings, optional + An abbreviation for a certain site type. Call ``get_codes("sitetype")`` + for the valid inputs. + Example: "GW" (Groundwater site) (Samples API: ``siteTypeCode``) + site_type_name : string or iterable of strings, optional + A full name for a certain site type. Call ``get_codes("sitetype")`` + for the valid inputs. + Example: "Well" (Samples API: ``siteTypeName``) + usgs_pcode : string or iterable of strings, optional + 5-digit number used in the US Geological Survey computerized + data system, National Water Information System (NWIS), to + uniquely identify a specific constituent (the ``parameterCode`` column + of ``get_codes("characteristics")``). + Example: "00060" (Discharge, cubic feet per second) + (Samples API: ``usgsPCode``) + hydrologic_unit : string or iterable of strings, optional + Max 12-digit number used to describe a hydrologic unit. + Example: "070900020502" (Samples API: ``hydrologicUnit``) + monitoring_location_id : string or iterable of strings, optional + A monitoring location identifier has two parts: the agency code + and the location number, separated by a dash (-). + Example: "USGS-040851385" + (Samples API: ``monitoringLocationIdentifier``) + organization_id : string or iterable of strings, optional + Designator used to uniquely identify a specific organization. + Currently only accepting the organization "USGS". + (Samples API: ``organizationIdentifier``) + point_location_latitude : float, optional + Latitude for a point/radius query (decimal degrees). Must be used + with ``point_location_longitude`` and ``point_location_within_miles``. + (Samples API: ``pointLocationLatitude``) + point_location_longitude : float, optional + Longitude for a point/radius query (decimal degrees). Must be used + with ``point_location_latitude`` and ``point_location_within_miles``. + (Samples API: ``pointLocationLongitude``) + point_location_within_miles : float, optional + Radius for a point/radius query. Must be used with + ``point_location_latitude`` and ``point_location_longitude``. + (Samples API: ``pointLocationWithinMiles``) + project_id : string or iterable of strings, optional + Designator used to uniquely identify a data collection project. Project + identifiers are specific to an organization (e.g. USGS). + Example: "ZH003QW03" (Samples API: ``projectIdentifier``) + record_identifier_user_supplied : string or iterable of strings, optional + Internal AQS record identifier that returns 1 entry. Only available + for the "results" service. + (Samples API: ``recordIdentifierUserSupplied``) + + Returns + ------- + df : ``pandas.DataFrame`` + Formatted data returned from the API query. For each + ``Date`` / ``Time`` / ``TimeZone`` triplet in + the response (e.g. ``Activity_StartDate``, ``Activity_StartTime``, + ``Activity_StartTimeZone``), an additional ``DateTime`` column + is appended holding a UTC ``Timestamp`` derived from the three. The + original Date/Time/TimeZone columns are left intact; rows whose + timezone abbreviation is not recognized resolve to ``NaT``. Rows are + sorted by ``Activity_StartDateTime`` when present (the API's default + order is unstable). + md : :obj:`dataretrieval.utils.BaseMetadata` + Custom ``dataretrieval`` metadata object pertaining to the query. + + Examples + -------- + .. code:: + + >>> # Get PFAS results within a bounding box + >>> df, md = dataretrieval.waterdata.get_samples( + ... bbox=[-90.2, 42.6, -88.7, 43.2], + ... characteristic_group="Organics, PFAS", + ... ) + + >>> # Get all activities for the Commonwealth of Virginia over a date range + >>> df, md = dataretrieval.waterdata.get_samples( + ... service="activities", + ... profile="sampact", + ... activity_start_date_lower="2023-10-01", + ... activity_start_date_upper="2024-01-01", + ... state_code="US:51", + ... ) + + >>> # Get all pH samples for two sites in Utah + >>> df, md = dataretrieval.waterdata.get_samples( + ... monitoring_location_id=[ + ... "USGS-393147111462301", + ... "USGS-393343111454101", + ... ], + ... usgs_pcode="00400", + ... ) + + """ + + _check_profiles(service, profile) + + # Build argument dictionary, omitting None values. Parameters are the + # public snake_case names here; translate them to the camelCase names the + # Samples API expects just before building the request. + args = _get_args(locals(), exclude={"ssl_check", "profile"}) + params = {_SAMPLES_PARAM_TO_API.get(key, key): value for key, value in args.items()} + + params.update({"mimeType": "text/csv"}) + + if "boundingBox" in params: + params["boundingBox"] = to_str(params["boundingBox"]) + + url = f"{SAMPLES_URL}/{service}/{profile}" + + df, response = _get_samples_csv(url, params, ssl_check) + df = _attach_datetime_columns(df) + + return df, BaseMetadata(response) + + +@_accept_legacy_kwargs({"monitoringLocationIdentifier": "monitoring_location_id"}) +def get_samples_summary( + monitoring_location_id: str, + ssl_check: bool = True, +) -> tuple[pd.DataFrame, BaseMetadata]: + """Get a summary of discrete water-quality samples at a single monitoring location. + + Wraps the Samples database summary service described at + https://api.waterdata.usgs.gov/samples-data/docs. The service returns one + row per (characteristic group, characteristic, user-supplied characteristic) + combination with result and activity counts and the first / most recent + activity dates — useful for taking inventory of what discrete-sample data + exists at a site before pulling the underlying observations with + :func:`get_samples`. + + The summary service is single-site only: it accepts exactly one monitoring + location per request. + + Parameters + ---------- + monitoring_location_id : string + A monitoring location identifier has two parts, separated by a dash + (``-``): the agency code and the location number. Examples: + ``"USGS-040851385"``, ``"AZ014-320821110580701"``, + ``"CAX01-15304600"``. Bare location numbers without an agency prefix + are accepted by the service but return an empty result, so a prefix + is effectively required. (Samples API: ``monitoringLocationIdentifier``) + ssl_check : bool, optional + Check the SSL certificate. Default is True. + + Returns + ------- + df : ``pandas.DataFrame`` + Formatted data returned from the API query. + md : :obj:`dataretrieval.utils.BaseMetadata` + Custom ``dataretrieval`` metadata object pertaining to the query. + + Examples + -------- + .. code:: + + >>> # What discrete-sample data is available at this site? + >>> df, md = dataretrieval.waterdata.get_samples_summary( + ... monitoring_location_id="USGS-04074950" + ... ) + + """ + if not isinstance(monitoring_location_id, str): + raise TypeError( + "monitoring_location_id must be a string; the Samples " + "summary service accepts exactly one monitoring location per " + f"request, got {type(monitoring_location_id).__name__}." + ) + + url = f"{SAMPLES_URL}/summary/{quote(monitoring_location_id, safe='')}" + params = {"mimeType": "text/csv"} + + df, response = _get_samples_csv(url, params, ssl_check) + + return df, BaseMetadata(response) + + +__all__ = ["get_codes", "get_samples", "get_samples_summary"] diff --git a/dataretrieval/waterdata/stats.py b/dataretrieval/waterdata/stats.py index ba4097fc..0a352365 100644 --- a/dataretrieval/waterdata/stats.py +++ b/dataretrieval/waterdata/stats.py @@ -30,6 +30,9 @@ from dataretrieval.utils import BaseMetadata from dataretrieval.waterdata.utils import BASE_URL +__all__ = ["get_data"] + + # ``_handle_nesting``'s geopandas branch calls ``gpd.GeoDataFrame.from_features`` # directly, so this module needs its own bound ``gpd`` name. Import it under the # same guard the engine uses; when geopandas is absent ``gpd`` is left unbound diff --git a/dataretrieval/waterdata/time_series.py b/dataretrieval/waterdata/time_series.py new file mode 100644 index 00000000..b5fb78a9 --- /dev/null +++ b/dataretrieval/waterdata/time_series.py @@ -0,0 +1,1197 @@ +"""Time-series observation and statistics getters.""" + +from __future__ import annotations + +from collections.abc import Iterable +from typing import Any + +import pandas as pd + +from dataretrieval.ogc.filters import FILTER_LANG +from dataretrieval.utils import BaseMetadata +from dataretrieval.waterdata import stats +from dataretrieval.waterdata.utils import ( + _get_args, + _with_state, + get_ogc_data, +) + + +def get_daily( + monitoring_location_id: str | Iterable[str] | None = None, + parameter_code: str | Iterable[str] | None = None, + statistic_id: str | Iterable[str] | None = None, + properties: str | Iterable[str] | None = None, + time_series_id: str | Iterable[str] | None = None, + daily_id: str | Iterable[str] | None = None, + approval_status: str | Iterable[str] | None = None, + unit_of_measure: str | Iterable[str] | None = None, + qualifier: str | Iterable[str] | None = None, + value: str | Iterable[str] | None = None, + last_modified: str | Iterable[str] | None = None, + skip_geometry: bool | None = None, + time: str | Iterable[str] | None = None, + bbox: list[float] | None = None, + limit: int | None = None, + filter: str | None = None, + filter_lang: FILTER_LANG | None = None, + convert_type: bool = True, + max_rows: int | None = None, + **queryables: Any, +) -> tuple[pd.DataFrame, BaseMetadata]: + """Daily data provide one data value to represent water conditions for the + day. + + Throughout much of the history of the USGS, the primary water data available + was daily data collected manually at the monitoring location once each day. + With improved availability of computer storage and automated transmission of + data, the daily data published today are generally a statistical summary or + metric of the continuous data collected each day, such as the daily mean, + minimum, or maximum value. Daily data are automatically calculated from the + continuous data of the same parameter code and are described by parameter + code and a statistic code. These data have also been referred to as “daily + values” or “DV”. + + Parameters + ---------- + monitoring_location_id : string or iterable of strings, optional + A unique identifier representing a single monitoring location. This + corresponds to the id field in the monitoring-locations endpoint. + Monitoring location IDs are created by combining the agency code of + the agency responsible for the monitoring location (e.g. USGS) with + the ID number of the monitoring location (e.g. 02238500), separated + by a hyphen (e.g. USGS-02238500). + parameter_code : string or iterable of strings, optional + Parameter codes are 5-digit codes used to identify the constituent + measured and the units of measure. A complete list of parameter + codes and associated groupings can be found at + https://help.waterdata.usgs.gov/codes-and-parameters/parameters. + statistic_id : string or iterable of strings, optional + A code corresponding to the statistic an observation represents. + Example codes include 00001 (max), 00002 (min), and 00003 (mean). + A complete list of codes and their descriptions can be found at + https://help.waterdata.usgs.gov/code/stat_cd_nm_query?stat_nm_cd=%25&fmt=html. + properties : string or iterable of strings, optional + A list of requested columns to be returned from the query. + Available options are: geometry, id, time_series_id, + monitoring_location_id, parameter_code, statistic_id, time, value, + unit_of_measure, approval_status, qualifier, last_modified + time_series_id : string or iterable of strings, optional + A unique identifier representing a single time series. This + corresponds to the id field in the time-series-metadata endpoint. + daily_id : string or iterable of strings, optional + A universally unique identifier (UUID) representing a single version of + a record. It is not stable over time. Every time the record is refreshed + in our database (which may happen as part of normal operations and does + not imply any change to the data itself) a new ID will be generated. To + uniquely identify a single observation over time, compare the time and + time_series_id fields; each time series will only have a single + observation at a given time. + approval_status : string or iterable of strings, optional + Some of the data that you have obtained from this U.S. Geological Survey + database may not have received Director's approval. Any such data values + are qualified as provisional and are subject to revision. Provisional + data are released on the condition that neither the USGS nor the United + States Government may be held liable for any damages resulting from its + use. This field reflects the approval status of each record, and is either + "Approved", meaning processing review has been completed and the data is + approved for publication, or "Provisional" and subject to revision. For + more information about provisional data, go to: + https://waterdata.usgs.gov/provisional-data-statement/. + unit_of_measure : string or iterable of strings, optional + A human-readable description of the units of measurement associated + with an observation. + qualifier : string or iterable of strings, optional + This field indicates any qualifiers associated with an observation, for + instance if a sensor may have been impacted by ice or if values were + estimated. + value : string or iterable of strings, optional + The value of the observation. Values are transmitted as strings in + the JSON response format in order to preserve precision. + last_modified : string, optional + The last time a record was refreshed in our database. This may happen + due to regular operational processes and does not necessarily indicate + that anything about the measurement has changed. You can query this field + using date-times or intervals, adhering to RFC 3339, or using ISO 8601 + duration objects. Intervals may be bounded or half-bounded (double-dots + at start or end). + Examples: + + * A date-time: "2018-02-12T23:20:50Z" + * A bounded interval: "2018-02-12T00:00:00Z/2018-03-18T12:31:12Z" + * Half-bounded intervals: "2018-02-12T00:00:00Z/.." or + "../2018-03-18T12:31:12Z" + * Duration objects: "P1M" for data from the past month or + "PT36H" for the last 36 hours + + Only features that have a last_modified that intersects the value of + datetime are selected. + skip_geometry : boolean, optional + This option can be used to skip response geometries for each feature. + The returning object will be a data frame with no spatial information. + Note that the USGS Water Data APIs use camelCase "skipGeometry" in + CQL2 queries. + time : string, optional + The date an observation represents. You can query this field using + date-times or intervals, adhering to RFC 3339, or using ISO 8601 + duration objects. Intervals may be bounded or half-bounded (double-dots + at start or end). Only features that have a time that intersects the + value of datetime are selected. If a feature has multiple temporal + properties, it is the decision of the server whether only a single + temporal property is used to determine the extent or all relevant + temporal properties. + Examples: + + * A date-time: "2018-02-12T23:20:50Z" + * A bounded interval: "2018-02-12T00:00:00Z/2018-03-18T12:31:12Z" + * Half-bounded intervals: "2018-02-12T00:00:00Z/.." or + "../2018-03-18T12:31:12Z" + * Duration objects: "P1M" for data from the past month or + "PT36H" for the last 36 hours + + bbox : list of numbers, optional + Only features that have a geometry that intersects the bounding box are + selected. The bounding box is provided as four or six numbers, + depending on whether the coordinate reference system includes a vertical + axis (height or depth). Coordinates are assumed to be in crs 4326. The + expected format is ``[xmin, ymin, xmax, ymax]``, i.e. + ``[Western-most longitude, Southern-most latitude, Eastern-most + longitude, Northern-most latitude]``. + limit : int, optional + The optional limit parameter is used to control the subset of the + selected features that should be returned in each page. The maximum + allowable limit is 50000. It may be beneficial to set this number lower + if your internet connection is spotty. The default (None) will set the + limit to the maximum allowable limit for the service. + This is a per-page size, not a cap on the total result: a query + matching more rows than ``limit`` still returns every matching row + across multiple pages. Use ``max_rows`` to cap the total instead. + filter, filter_lang : optional + Server-side CQL filter passed through as the OGC ``filter`` / + ``filter-lang`` query parameters. See + :mod:`dataretrieval.ogc.filters` for syntax, auto-chunking, + and the lexicographic-comparison pitfall. + convert_type : boolean, optional + If True, converts columns to appropriate types. + max_rows : int, optional + Cap the total number of rows returned, stopping pagination early + instead of downloading the whole result. Unlike ``limit`` (the + per-page size), this bounds the total result across every page. + The default (None) follows pagination to completion. + **queryables : string or iterable of strings, optional + Any other queryable property of this collection, passed through as a + server-side filter. Call :func:`get_queryables` to see the queryables a + collection supports. + + Returns + ------- + df : ``pandas.DataFrame`` or ``geopandas.GeoDataFrame`` + Formatted data returned from the API query. + md: :obj:`dataretrieval.utils.BaseMetadata` + A custom metadata object + + Raises + ------ + ChunkInterrupted + A transient failure (429 / 5xx / timeout) interrupted the request + after the built-in retries. Completed work is preserved; resume + with ``exc.call.resume()`` (see :doc:`/userguide/errors`). + + Examples + -------- + .. code:: + + >>> # Get daily flow data from a single site + >>> # over a yearlong period + >>> df, md = dataretrieval.waterdata.get_daily( + ... monitoring_location_id="USGS-02238500", + ... parameter_code="00060", + ... time="2021-01-01T00:00:00Z/2022-01-01T00:00:00Z", + ... ) + + >>> # Quick "show me the last week" idiom (ISO 8601 duration) + >>> df, md = dataretrieval.waterdata.get_daily( + ... monitoring_location_id="USGS-02238500", + ... parameter_code="00060", + ... time="P7D", + ... ) + + >>> # Get approved daily flow data from multiple sites + >>> df, md = dataretrieval.waterdata.get_daily( + ... monitoring_location_id=["USGS-05114000", "USGS-09423350"], + ... approval_status="Approved", + ... time="2024-01-01/..", + ... ) + + >>> # Pull only rows whose underlying record was refreshed in the + >>> # last 7 days — handy for incremental ETL polling + >>> df, md = dataretrieval.waterdata.get_daily( + ... monitoring_location_id="USGS-02238500", + ... parameter_code="00060", + ... last_modified="P7D", + ... ) + + >>> # Chain queries: pull all stream sites in a state, then their + >>> # daily discharge for the last week. The site list can be hundreds + >>> # of values long — the request is transparently chunked across + >>> # multiple sub-requests so the URL stays under the server's byte + >>> # limit. Combined output looks like a single query. + >>> sites_df, _ = dataretrieval.waterdata.get_monitoring_locations( + ... state="Ohio", + ... site_type="Stream", + ... ) + >>> df, md = dataretrieval.waterdata.get_daily( + ... monitoring_location_id=sites_df["monitoring_location_id"].tolist(), + ... parameter_code="00060", + ... time="P7D", + ... ) + """ + service = "daily" + + # Build argument dictionary, omitting None values + args = _get_args(locals(), exclude={"max_rows"}) + + return get_ogc_data(args, service, max_rows=max_rows) + + +def get_continuous( + monitoring_location_id: str | Iterable[str] | None = None, + parameter_code: str | Iterable[str] | None = None, + statistic_id: str | Iterable[str] | None = None, + properties: str | Iterable[str] | None = None, + time_series_id: str | Iterable[str] | None = None, + continuous_id: str | Iterable[str] | None = None, + approval_status: str | Iterable[str] | None = None, + unit_of_measure: str | Iterable[str] | None = None, + qualifier: str | Iterable[str] | None = None, + value: str | Iterable[str] | None = None, + last_modified: str | Iterable[str] | None = None, + time: str | Iterable[str] | None = None, + limit: int | None = None, + filter: str | None = None, + filter_lang: FILTER_LANG | None = None, + convert_type: bool = True, + max_rows: int | None = None, + **queryables: Any, +) -> tuple[pd.DataFrame, BaseMetadata]: + """ + Continuous data provide instantaneous water conditions. + + This is an early version of the continuous endpoint that is feature-complete + and is being made available for limited use. Geometries are not included + with the continuous endpoint. If the "time" input is left blank, the service + will return the most recent year of measurements. Users may request no more + than three years of data with each function call. + + Continuous data are collected at a high frequency, typically 15-minute + intervals. Depending on the specific monitoring location, the data may be + transmitted automatically via telemetry and be available on WDFN within + minutes of collection, while other times the delivery of data may be delayed + if the monitoring location does not have the capacity to automatically + transmit data. Continuous data are described by parameter name and + parameter code (pcode). These data might also be referred to as + "instantaneous values" or "IV". + + Parameters + ---------- + monitoring_location_id : string or iterable of strings, optional + A unique identifier representing a single monitoring location. This + corresponds to the id field in the monitoring-locations endpoint. + Monitoring location IDs are created by combining the agency code of + the agency responsible for the monitoring location (e.g. USGS) with + the ID number of the monitoring location (e.g. 02238500), separated + by a hyphen (e.g. USGS-02238500). + parameter_code : string or iterable of strings, optional + Parameter codes are 5-digit codes used to identify the constituent + measured and the units of measure. A complete list of parameter + codes and associated groupings can be found at + https://help.waterdata.usgs.gov/codes-and-parameters/parameters. + statistic_id : string or iterable of strings, optional + A code corresponding to the statistic an observation represents. + Continuous data are nearly always associated with statistic id + 00011. Using a different code (such as 00003 for mean) will + typically return no results. A complete list of codes and their + descriptions can be found at + https://help.waterdata.usgs.gov/code/stat_cd_nm_query?stat_nm_cd=%25&fmt=html. + properties : string or iterable of strings, optional + A list of requested columns to be returned from the query. + Available options are: geometry, id, time_series_id, + monitoring_location_id, parameter_code, statistic_id, time, value, + unit_of_measure, approval_status, qualifier, last_modified + time_series_id : string or iterable of strings, optional + A unique identifier representing a single time series. This + corresponds to the id field in the time-series-metadata endpoint. + continuous_id : string or iterable of strings, optional + A universally unique identifier (UUID) representing a single version of + a record. It is not stable over time. Every time the record is refreshed + in our database (which may happen as part of normal operations and does + not imply any change to the data itself) a new ID will be generated. To + uniquely identify a single observation over time, compare the time and + time_series_id fields; each time series will only have a single + observation at a given time. + approval_status : string or iterable of strings, optional + Some of the data that you have obtained from this U.S. Geological Survey + database may not have received Director's approval. Any such data values + are qualified as provisional and are subject to revision. Provisional + data are released on the condition that neither the USGS nor the United + States Government may be held liable for any damages resulting from its + use. This field reflects the approval status of each record, and is either + "Approved", meaning processing review has been completed and the data is + approved for publication, or "Provisional" and subject to revision. For + more information about provisional data, go to: + https://waterdata.usgs.gov/provisional-data-statement/. + unit_of_measure : string or iterable of strings, optional + A human-readable description of the units of measurement associated + with an observation. + qualifier : string or iterable of strings, optional + This field indicates any qualifiers associated with an observation, for + instance if a sensor may have been impacted by ice or if values were + estimated. + value : string or iterable of strings, optional + The value of the observation. Values are transmitted as strings in + the JSON response format in order to preserve precision. + last_modified : string, optional + The last time a record was refreshed in our database. This may happen + due to regular operational processes and does not necessarily indicate + that anything about the measurement has changed. You can query this field + using date-times or intervals, adhering to RFC 3339, or using ISO 8601 + duration objects. Intervals may be bounded or half-bounded (double-dots + at start or end). + Examples: + + * A date-time: "2018-02-12T23:20:50Z" + * A bounded interval: "2018-02-12T00:00:00Z/2018-03-18T12:31:12Z" + * Half-bounded intervals: "2018-02-12T00:00:00Z/.." or + "../2018-03-18T12:31:12Z" + * Duration objects: "P1M" for data from the past month or + "PT36H" for the last 36 hours + + Only features that have a last_modified that intersects the value of + datetime are selected. + time : string, optional + The date an observation represents. You can query this field using + date-times or intervals, adhering to RFC 3339, or using ISO 8601 + duration objects. Intervals may be bounded or half-bounded (double-dots + at start or end). Only features that have a time that intersects the + value of datetime are selected. If a feature has multiple temporal + properties, it is the decision of the server whether only a single + temporal property is used to determine the extent or all relevant + temporal properties. + Examples: + + * A date-time: "2018-02-12T23:20:50Z" + * A bounded interval: "2018-02-12T00:00:00Z/2018-03-18T12:31:12Z" + * Half-bounded intervals: "2018-02-12T00:00:00Z/.." or + "../2018-03-18T12:31:12Z" + * Duration objects: "P1M" for data from the past month or + "PT36H" for the last 36 hours + + limit : int, optional + The optional limit parameter is used to control the subset of the + selected features that should be returned in each page. The maximum + allowable limit is 10000. It may be beneficial to set this number lower + if your internet connection is spotty. The default (None) will set the + limit to the maximum allowable limit for the service. + This is a per-page size, not a cap on the total result: a query + matching more rows than ``limit`` still returns every matching row + across multiple pages. Use ``max_rows`` to cap the total instead. + filter, filter_lang : optional + Server-side CQL filter passed through as the OGC ``filter`` / + ``filter-lang`` query parameters. See + :mod:`dataretrieval.ogc.filters` for syntax, auto-chunking, + and the lexicographic-comparison pitfall. + convert_type : boolean, optional + If True, converts columns to appropriate types. + max_rows : int, optional + Cap the total number of rows returned, stopping pagination early + instead of downloading the whole result. Unlike ``limit`` (the + per-page size), this bounds the total result across every page. + The default (None) follows pagination to completion. + **queryables : string or iterable of strings, optional + Any other queryable property of this collection, passed through as a + server-side filter. Call :func:`get_queryables` to see the queryables a + collection supports. + + Returns + ------- + df : ``pandas.DataFrame`` or ``geopandas.GeoDataFrame`` + Formatted data returned from the API query. + md: :obj:`dataretrieval.utils.BaseMetadata` + A custom metadata object + + Raises + ------ + ChunkInterrupted + A transient failure (429 / 5xx / timeout) interrupted the request + after the built-in retries. Completed work is preserved; resume + with ``exc.call.resume()`` (see :doc:`/userguide/errors`). + + Examples + -------- + .. code:: + + >>> # Get instantaneous gage height data from a + >>> # single site from a single year + >>> df, md = dataretrieval.waterdata.get_continuous( + ... monitoring_location_id="USGS-02238500", + ... parameter_code="00065", + ... time="2021-01-01T00:00:00Z/2022-01-01T00:00:00Z", + ... ) + + >>> # Pull several disjoint time windows in one call via a CQL + >>> # ``filter``. See ``dataretrieval.ogc.filters`` for the + >>> # full grammar, auto-chunking, and pitfalls. + >>> df, md = dataretrieval.waterdata.get_continuous( + ... monitoring_location_id="USGS-02238500", + ... parameter_code="00060", + ... filter=( + ... "(time >= '2023-06-01T12:00:00Z' " + ... "AND time <= '2023-06-01T13:00:00Z') " + ... "OR (time >= '2023-06-15T12:00:00Z' " + ... "AND time <= '2023-06-15T13:00:00Z')" + ... ), + ... filter_lang="cql-text", + ... ) + """ + service = "continuous" + + # Build argument dictionary, omitting None values + args = _get_args(locals(), exclude={"max_rows"}) + + return get_ogc_data(args, service, max_rows=max_rows) + + +def get_latest_continuous( + monitoring_location_id: str | Iterable[str] | None = None, + parameter_code: str | Iterable[str] | None = None, + statistic_id: str | Iterable[str] | None = None, + properties: str | Iterable[str] | None = None, + time_series_id: str | Iterable[str] | None = None, + latest_continuous_id: str | Iterable[str] | None = None, + approval_status: str | Iterable[str] | None = None, + unit_of_measure: str | Iterable[str] | None = None, + qualifier: str | Iterable[str] | None = None, + value: str | Iterable[str] | None = None, + last_modified: str | Iterable[str] | None = None, + skip_geometry: bool | None = None, + time: str | Iterable[str] | None = None, + bbox: list[float] | None = None, + limit: int | None = None, + filter: str | None = None, + filter_lang: FILTER_LANG | None = None, + convert_type: bool = True, + max_rows: int | None = None, + **queryables: Any, +) -> tuple[pd.DataFrame, BaseMetadata]: + """This endpoint provides the most recent observation for each time series + of continuous data. Continuous data are collected via automated sensors + installed at a monitoring location. They are collected at a high frequency + and often at a fixed 15-minute interval. Depending on the specific monitoring + location, the data may be transmitted automatically via telemetry and be + available on WDFN within minutes of collection, while other times the delivery + of data may be delayed if the monitoring location does not have the capacity to + automatically transmit data. Continuous data are described by parameter name + and parameter code. These data might also be referred to as "instantaneous + values" or "IV". + + Parameters + ---------- + monitoring_location_id : string or iterable of strings, optional + A unique identifier representing a single monitoring location. This + corresponds to the id field in the monitoring-locations endpoint. + Monitoring location IDs are created by combining the agency code of the + agency responsible for the monitoring location (e.g. USGS) with the ID + number of the monitoring location (e.g. 02238500), separated by a hyphen + (e.g. USGS-02238500). + parameter_code : string or iterable of strings, optional + Parameter codes are 5-digit codes used to identify the constituent + measured and the units of measure. A complete list of parameter codes + and associated groupings can be found at + https://help.waterdata.usgs.gov/codes-and-parameters/parameters. + statistic_id : string or iterable of strings, optional + A code corresponding to the statistic an observation represents. + Example codes include 00001 (max), 00002 (min), and 00003 (mean). + A complete list of codes and their descriptions can be found at + https://help.waterdata.usgs.gov/code/stat_cd_nm_query?stat_nm_cd=%25&fmt=html. + properties : string or iterable of strings, optional + A list of requested columns to be returned from the query. Available + options are: geometry, id, time_series_id, monitoring_location_id, + parameter_code, statistic_id, time, value, unit_of_measure, + approval_status, qualifier, last_modified + time_series_id : string or iterable of strings, optional + A unique identifier representing a single time series. This + corresponds to the id field in the time-series-metadata endpoint. + latest_continuous_id : string or iterable of strings, optional + A universally unique identifier (UUID) representing a single version of + a record. It is not stable over time. Every time the record is refreshed + in our database (which may happen as part of normal operations and does + not imply any change to the data itself) a new ID will be generated. To + uniquely identify a single observation over time, compare the time and + time_series_id fields; each time series will only have a single + observation at a given time. + approval_status : string or iterable of strings, optional + Some of the data that you have obtained from this U.S. Geological Survey + database may not have received Director's approval. Any such data values + are qualified as provisional and are subject to revision. Provisional + data are released on the condition that neither the USGS nor the United + States Government may be held liable for any damages resulting from its + use. This field reflects the approval status of each record, and is either + "Approved", meaning processing review has been completed and the data is + approved for publication, or "Provisional" and subject to revision. For + more information about provisional data, go to: + https://waterdata.usgs.gov/provisional-data-statement/. + unit_of_measure : string or iterable of strings, optional + A human-readable description of the units of measurement associated + with an observation. + qualifier : string or iterable of strings, optional + This field indicates any qualifiers associated with an observation, for + instance if a sensor may have been impacted by ice or if values were + estimated. + value : string or iterable of strings, optional + The value of the observation. Values are transmitted as strings in + the JSON response format in order to preserve precision. + last_modified : string, optional + The last time a record was refreshed in our database. This may happen + due to regular operational processes and does not necessarily indicate + that anything about the measurement has changed. You can query this field + using date-times or intervals, adhering to RFC 3339, or using ISO 8601 + duration objects. Intervals may be bounded or half-bounded (double-dots + at start or end). Only features that have a last_modified that + intersects the value of datetime are selected. + Examples: + + * A date-time: "2018-02-12T23:20:50Z" + * A bounded interval: "2018-02-12T00:00:00Z/2018-03-18T12:31:12Z" + * Half-bounded intervals: "2018-02-12T00:00:00Z/.." or + "../2018-03-18T12:31:12Z" + * Duration objects: "P1M" for data from the past month or + "PT36H" for the last 36 hours + + skip_geometry : boolean, optional + This option can be used to skip response geometries for each feature. + The returning object will be a data frame with no spatial information. + Note that the USGS Water Data APIs use camelCase "skipGeometry" in + CQL2 queries. + time : string, optional + The date an observation represents. You can query this field using + date-times or intervals, adhering to RFC 3339, or using ISO 8601 + duration objects. Intervals may be bounded or half-bounded (double-dots + at start or end). Only features that have a time that intersects the + value of datetime are selected. If a feature has multiple temporal + properties, it is the decision of the server whether only a single + temporal property is used to determine the extent or all relevant + temporal properties. + Examples: + + * A date-time: "2018-02-12T23:20:50Z" + * A bounded interval: "2018-02-12T00:00:00Z/2018-03-18T12:31:12Z" + * Half-bounded intervals: "2018-02-12T00:00:00Z/.." or + "../2018-03-18T12:31:12Z" + * Duration objects: "P1M" for data from the past month or + "PT36H" for the last 36 hours + + bbox : list of numbers, optional + Only features that have a geometry that intersects the bounding box are + selected. The bounding box is provided as four or six numbers, + depending on whether the coordinate reference system includes a vertical + axis (height or depth). Coordinates are assumed to be in crs 4326. The + expected format is ``[xmin, ymin, xmax, ymax]``, i.e. + ``[Western-most longitude, Southern-most latitude, Eastern-most + longitude, Northern-most latitude]``. + limit : int, optional + The optional limit parameter is used to control the subset of the + selected features that should be returned in each page. The maximum + allowable limit is 50000. It may be beneficial to set this number lower + if your internet connection is spotty. The default (None) will set the + limit to the maximum allowable limit for the service. + This is a per-page size, not a cap on the total result: a query + matching more rows than ``limit`` still returns every matching row + across multiple pages. Use ``max_rows`` to cap the total instead. + filter, filter_lang : optional + Server-side CQL filter passed through as the OGC ``filter`` / + ``filter-lang`` query parameters. See + :mod:`dataretrieval.ogc.filters` for syntax, auto-chunking, + and the lexicographic-comparison pitfall. + convert_type : boolean, optional + If True, converts columns to appropriate types. + max_rows : int, optional + Cap the total number of rows returned, stopping pagination early + instead of downloading the whole result. Unlike ``limit`` (the + per-page size), this bounds the total result across every page. + The default (None) follows pagination to completion. + **queryables : string or iterable of strings, optional + Any other queryable property of this collection, passed through as a + server-side filter. Call :func:`get_queryables` to see the queryables a + collection supports. + + Returns + ------- + df : ``pandas.DataFrame`` or ``geopandas.GeoDataFrame`` + Formatted data returned from the API query. + md: :obj:`dataretrieval.utils.BaseMetadata` + A custom metadata object + + Raises + ------ + ChunkInterrupted + A transient failure (429 / 5xx / timeout) interrupted the request + after the built-in retries. Completed work is preserved; resume + with ``exc.call.resume()`` (see :doc:`/userguide/errors`). + + Examples + -------- + .. code:: + + >>> # Get latest flow data from a single site + >>> df, md = dataretrieval.waterdata.get_latest_continuous( + ... monitoring_location_id="USGS-02238500", parameter_code="00060" + ... ) + + >>> # Restrict to the last 7 days; sites with no observation in that + >>> # window are dropped instead of returned with stale values + >>> df, md = dataretrieval.waterdata.get_latest_continuous( + ... monitoring_location_id="USGS-02238500", + ... parameter_code="00060", + ... time="P7D", + ... ) + + >>> # Pull only rows whose underlying record was refreshed in the + >>> # last 7 days, across multiple sites and parameters + >>> df, md = dataretrieval.waterdata.get_latest_continuous( + ... monitoring_location_id=["USGS-451605097071701", "USGS-14181500"], + ... parameter_code=["00060", "72019"], + ... last_modified="P7D", + ... ) + + >>> # Get latest continuous measurements for multiple sites + >>> df, md = dataretrieval.waterdata.get_latest_continuous( + ... monitoring_location_id=["USGS-05114000", "USGS-09423350"] + ... ) + """ + service = "latest-continuous" + + # Build argument dictionary, omitting None values + args = _get_args(locals(), exclude={"max_rows"}) + + return get_ogc_data(args, service, max_rows=max_rows) + + +def get_latest_daily( + monitoring_location_id: str | Iterable[str] | None = None, + parameter_code: str | Iterable[str] | None = None, + statistic_id: str | Iterable[str] | None = None, + properties: str | Iterable[str] | None = None, + time_series_id: str | Iterable[str] | None = None, + latest_daily_id: str | Iterable[str] | None = None, + approval_status: str | Iterable[str] | None = None, + unit_of_measure: str | Iterable[str] | None = None, + qualifier: str | Iterable[str] | None = None, + value: str | Iterable[str] | None = None, + last_modified: str | Iterable[str] | None = None, + skip_geometry: bool | None = None, + time: str | Iterable[str] | None = None, + bbox: list[float] | None = None, + limit: int | None = None, + filter: str | None = None, + filter_lang: FILTER_LANG | None = None, + convert_type: bool = True, + max_rows: int | None = None, + **queryables: Any, +) -> tuple[pd.DataFrame, BaseMetadata]: + """Daily data provide one data value to represent water conditions for the + day. + + Throughout much of the history of the USGS, the primary water data available + was daily data collected manually at the monitoring location once each day. + With improved availability of computer storage and automated transmission of + data, the daily data published today are generally a statistical summary or + metric of the continuous data collected each day, such as the daily mean, + minimum, or maximum value. Daily data are automatically calculated from the + continuous data of the same parameter code and are described by parameter + code and a statistic code. These data have also been referred to as “daily + values” or “DV”. + + Parameters + ---------- + monitoring_location_id : string or iterable of strings, optional + A unique identifier representing a single monitoring location. This + corresponds to the id field in the monitoring-locations endpoint. + Monitoring location IDs are created by combining the agency code of the + agency responsible for the monitoring location (e.g. USGS) with the ID + number of the monitoring location (e.g. 02238500), separated by a hyphen + (e.g. USGS-02238500). + parameter_code : string or iterable of strings, optional + Parameter codes are 5-digit codes used to identify the constituent + measured and the units of measure. A complete list of parameter codes + and associated groupings can be found at + https://help.waterdata.usgs.gov/codes-and-parameters/parameters. + statistic_id : string or iterable of strings, optional + A code corresponding to the statistic an observation represents. + Example codes include 00001 (max), 00002 (min), and 00003 (mean). + A complete list of codes and their descriptions can be found at + https://help.waterdata.usgs.gov/code/stat_cd_nm_query?stat_nm_cd=%25&fmt=html. + properties : string or iterable of strings, optional + A list of requested columns to be returned from the query. Available + options are: geometry, id, time_series_id, monitoring_location_id, + parameter_code, statistic_id, time, value, unit_of_measure, + approval_status, qualifier, last_modified + time_series_id : string or iterable of strings, optional + A unique identifier representing a single time series. This + corresponds to the id field in the time-series-metadata endpoint. + latest_daily_id : string or iterable of strings, optional + A universally unique identifier (UUID) representing a single version of + a record. It is not stable over time. Every time the record is refreshed + in our database (which may happen as part of normal operations and does + not imply any change to the data itself) a new ID will be generated. To + uniquely identify a single observation over time, compare the time and + time_series_id fields; each time series will only have a single + observation at a given time. + approval_status : string or iterable of strings, optional + Some of the data that you have obtained from this U.S. Geological Survey + database may not have received Director's approval. Any such data values + are qualified as provisional and are subject to revision. Provisional + data are released on the condition that neither the USGS nor the United + States Government may be held liable for any damages resulting from its + use. This field reflects the approval status of each record, and is either + "Approved", meaning processing review has been completed and the data is + approved for publication, or "Provisional" and subject to revision. For + more information about provisional data, go to: + https://waterdata.usgs.gov/provisional-data-statement/. + unit_of_measure : string or iterable of strings, optional + A human-readable description of the units of measurement associated + with an observation. + qualifier : string or iterable of strings, optional + This field indicates any qualifiers associated with an observation, for + instance if a sensor may have been impacted by ice or if values were + estimated. + value : string or iterable of strings, optional + The value of the observation. Values are transmitted as strings in + the JSON response format in order to preserve precision. + last_modified : string, optional + The last time a record was refreshed in our database. This may happen + due to regular operational processes and does not necessarily indicate + that anything about the measurement has changed. You can query this field + using date-times or intervals, adhering to RFC 3339, or using ISO 8601 + duration objects. Intervals may be bounded or half-bounded (double-dots + at start or end). Only features that have a last_modified that + intersects the value of datetime are selected. + Examples: + + * A date-time: "2018-02-12T23:20:50Z" + * A bounded interval: "2018-02-12T00:00:00Z/2018-03-18T12:31:12Z" + * Half-bounded intervals: "2018-02-12T00:00:00Z/.." or + "../2018-03-18T12:31:12Z" + * Duration objects: "P1M" for data from the past month or + "PT36H" for the last 36 hours + + skip_geometry : boolean, optional + This option can be used to skip response geometries for each feature. + The returning object will be a data frame with no spatial information. + Note that the USGS Water Data APIs use camelCase "skipGeometry" in + CQL2 queries. + time : string, optional + The date an observation represents. You can query this field using + date-times or intervals, adhering to RFC 3339, or using ISO 8601 + duration objects. Intervals may be bounded or half-bounded (double-dots + at start or end). Only features that have a time that intersects the + value of datetime are selected. If a feature has multiple temporal + properties, it is the decision of the server whether only a single + temporal property is used to determine the extent or all relevant + temporal properties. + Examples: + + * A date-time: "2018-02-12T23:20:50Z" + * A bounded interval: "2018-02-12T00:00:00Z/2018-03-18T12:31:12Z" + * Half-bounded intervals: "2018-02-12T00:00:00Z/.." or + "../2018-03-18T12:31:12Z" + * Duration objects: "P1M" for data from the past month or + "PT36H" for the last 36 hours + + bbox : list of numbers, optional + Only features that have a geometry that intersects the bounding box are + selected. The bounding box is provided as four or six numbers, + depending on whether the coordinate reference system includes a vertical + axis (height or depth). Coordinates are assumed to be in crs 4326. The + expected format is ``[xmin, ymin, xmax, ymax]``, i.e. + ``[Western-most longitude, Southern-most latitude, Eastern-most + longitude, Northern-most latitude]``. + limit : int, optional + The optional limit parameter is used to control the subset of the + selected features that should be returned in each page. The maximum + allowable limit is 50000. It may be beneficial to set this number lower + if your internet connection is spotty. The default (None) will set the + limit to the maximum allowable limit for the service. + This is a per-page size, not a cap on the total result: a query + matching more rows than ``limit`` still returns every matching row + across multiple pages. Use ``max_rows`` to cap the total instead. + filter, filter_lang : optional + Server-side CQL filter passed through as the OGC ``filter`` / + ``filter-lang`` query parameters. See + :mod:`dataretrieval.ogc.filters` for syntax, auto-chunking, + and the lexicographic-comparison pitfall. + convert_type : boolean, optional + If True, converts columns to appropriate types. + max_rows : int, optional + Cap the total number of rows returned, stopping pagination early + instead of downloading the whole result. Unlike ``limit`` (the + per-page size), this bounds the total result across every page. + The default (None) follows pagination to completion. + **queryables : string or iterable of strings, optional + Any other queryable property of this collection, passed through as a + server-side filter. Call :func:`get_queryables` to see the queryables a + collection supports. + + Returns + ------- + df : ``pandas.DataFrame`` or ``geopandas.GeoDataFrame`` + Formatted data returned from the API query. + md: :obj:`dataretrieval.utils.BaseMetadata` + A custom metadata object + + Raises + ------ + ChunkInterrupted + A transient failure (429 / 5xx / timeout) interrupted the request + after the built-in retries. Completed work is preserved; resume + with ``exc.call.resume()`` (see :doc:`/userguide/errors`). + + Examples + -------- + .. code:: + + >>> # Get most recent daily flow data from a single site + >>> df, md = dataretrieval.waterdata.get_latest_daily( + ... monitoring_location_id="USGS-02238500", parameter_code="00060" + ... ) + + >>> # Restrict to rows whose underlying record was refreshed in the + >>> # last 7 days + >>> df, md = dataretrieval.waterdata.get_latest_daily( + ... monitoring_location_id="USGS-02238500", + ... parameter_code="00060", + ... last_modified="P7D", + ... ) + + >>> # Multi-site, multi-parameter — discharge and water temperature + >>> # at two sites in a single round-trip + >>> df, md = dataretrieval.waterdata.get_latest_daily( + ... monitoring_location_id=["USGS-01491000", "USGS-01645000"], + ... parameter_code=["00060", "00010"], + ... ) + + >>> # Get most recent daily measurements for two sites + >>> df, md = dataretrieval.waterdata.get_latest_daily( + ... monitoring_location_id=["USGS-05114000", "USGS-09423350"] + ... ) + """ + service = "latest-daily" + + # Build argument dictionary, omitting None values + args = _get_args(locals(), exclude={"max_rows"}) + + return get_ogc_data(args, service, max_rows=max_rows) + + +def get_stats_por( + approval_status: str | None = None, + computation_type: str | Iterable[str] | None = None, + country_code: str | Iterable[str] | None = None, + state: str | Iterable[str] | None = None, + state_code: str | Iterable[str] | None = None, + county_code: str | Iterable[str] | None = None, + start_date: str | None = None, + end_date: str | None = None, + monitoring_location_id: str | Iterable[str] | None = None, + page_size: int = 1000, + parent_time_series_id: str | Iterable[str] | None = None, + site_type_code: str | Iterable[str] | None = None, + site_type_name: str | Iterable[str] | None = None, + parameter_code: str | Iterable[str] | None = None, + normal_type: str | None = None, + expand_percentiles: bool = True, +) -> tuple[pd.DataFrame, BaseMetadata]: + """Get day-of-year and month-of-year water data statistics from the + USGS Water Data API. + This service (called the "observationNormals" endpoint on api.waterdata.usgs.gov) + provides endpoints for access to computations on the historical record regarding + water conditions, including minimum, maximum, mean, median, and percentiles for + day of year and month of year. For more information regarding the calculation of + statistics and other details, please visit the Statistics documentation page: + https://waterdata.usgs.gov/statistics-documentation/. + + Note: This API is under active beta development and subject to + change. Improved handling of significant figures will be + addressed in a future release. + + Parameters + ---------- + approval_status: string, optional + Whether to include approved and/or provisional observations. + At this time, only approved observations are returned. + computation_type: string, optional + Desired statistical computation method. Available values are: + arithmetic_mean, maximum, median, minimum, percentile. + country_code: string, optional + Country query parameter. API defaults to "US". + state: string or iterable of strings, optional + State/territory filter (the recommended parameter). Accepts a full name + ("Wisconsin"), a two-letter postal code ("WI"), or a two-digit + ANSI/FIPS code ("55"). + state_code: string, optional + State query parameter. Takes the format "US:XX", where XX is + the two-digit state code. API defaults to "US:42" (Pennsylvania). + county_code: string, optional + County query parameter. Takes the format "US:XX:YYY", where XX is + the two-digit state code and YYY is the three-digit county code. + API defaults to "US:42:103" (Pennsylvania, Pike County). + start_date: string or datetime, optional + Start day for the query in the month-day format (MM-DD). + end_date: string or datetime, optional + End day for the query in the month-day format (MM-DD). + monitoring_location_id : string or iterable of strings, optional + A unique identifier representing a single monitoring location. This + corresponds to the id field in the monitoring-locations endpoint. + Monitoring location IDs are created by combining the agency code of the + agency responsible for the monitoring location (e.g. USGS) with the ID + number of the monitoring location (e.g. 02238500), separated by a hyphen + (e.g. USGS-02238500). + page_size : int, optional + The number of results to return per page, where one result represents a + monitoring location. The default is 1000. + parent_time_series_id: string, optional + The parent_time_series_id returns statistics tied to a + particular database entry. + site_type_code: string, optional + Site type code query parameter. + A list of valid site type codes is available at: + https://api.waterdata.usgs.gov/ogcapi/v0/collections/site-types/items. + Example: "GW" (Groundwater site) + site_type_name: string, optional + Site type name query parameter. + parameter_code : string or iterable of strings, optional + Parameter codes are 5-digit codes used to identify the constituent + measured and the units of measure. A complete list of parameter codes + and associated groupings can be found at + https://help.waterdata.usgs.gov/codes-and-parameters/parameters. + normal_type : string, optional + Filter the returned normals to a single period. If unspecified + (default), all matching data are returned. Available values: + "DOY" (day-of-year) and "MOY" (month-of-year). + expand_percentiles : boolean + Percentile data for a given day of year or month of year by default + are returned from the service as lists of string values and percentile + thresholds in the "values" and "percentiles" columns, respectively. + When `expand_percentiles` is set to True (default), each value and + percentile threshold specific to a computation id are returned as + individual rows in the dataframe, with the value reported in the + "value" column and the corresponding percentile reported in a + "percentile" column (and the "values" and "percentiles" columns + are removed). Missing percentile values expressed as 'nan' in the + list of string values are removed from the dataframe to save space. + Setting `expand_percentiles` to False retains the "values" and + "percentiles" columns produced by the service. Including + both 'percentiles' and one or more other statistics ('median', + 'minimum', 'maximum', or 'arithmetic_mean') in the `computation_type` + argument will return both the "values" column, containing the list + of percentile threshold values, and a "value" column, containing + the singular summary value for the other statistics. + + Returns + ------- + df : ``pandas.DataFrame`` or ``geopandas.GeoDataFrame`` + Formatted data returned from the API query. + md : :obj:`dataretrieval.utils.BaseMetadata` + A custom metadata object. + + Examples + -------- + .. code:: + + >>> # Get daily, monthly, and annual percentiles for streamflow at + >>> # a monitoring location of interest + >>> df, md = dataretrieval.waterdata.get_stats_por( + ... monitoring_location_id="USGS-05114000", + ... parameter_code="00060", + ... computation_type="percentile", + ... ) + + >>> # Get all daily and monthly statistics for the month of January + >>> # over the entire period of record for streamflow and gage height + >>> # at a monitoring location of interest + >>> df, md = dataretrieval.waterdata.get_stats_por( + ... monitoring_location_id="USGS-05114000", + ... parameter_code=["00060", "00065"], + ... start_date="01-01", + ... end_date="01-31", + ... ) + """ + # Build argument dictionary, omitting None values + params = _get_args( + _with_state(locals(), to="fips_us", into="state_code"), + exclude={"expand_percentiles"}, + ) + + return stats.get_data( + args=params, service="observationNormals", expand_percentiles=expand_percentiles + ) + + +def get_stats_date_range( + approval_status: str | None = None, + computation_type: str | Iterable[str] | None = None, + country_code: str | Iterable[str] | None = None, + state: str | Iterable[str] | None = None, + state_code: str | Iterable[str] | None = None, + county_code: str | Iterable[str] | None = None, + start_date: str | None = None, + end_date: str | None = None, + monitoring_location_id: str | Iterable[str] | None = None, + page_size: int = 1000, + parent_time_series_id: str | Iterable[str] | None = None, + site_type_code: str | Iterable[str] | None = None, + site_type_name: str | Iterable[str] | None = None, + parameter_code: str | Iterable[str] | None = None, + interval_type: str | Iterable[str] | None = None, + expand_percentiles: bool = True, +) -> tuple[pd.DataFrame, BaseMetadata]: + """Get monthly and annual water data statistics from the USGS Water Data API. + This service (called the "observationIntervals" endpoint on api.waterdata.usgs.gov) + provides endpoints for access to computations on the historical record regarding + water conditions, including minimum, maximum, mean, median, and percentiles for + month-year, and water/calendar years. For more information regarding the calculation + of statistics and other details, please visit the Statistics documentation page: + https://waterdata.usgs.gov/statistics-documentation/. + + Note: This API is under active beta development and subject to + change. Improved handling of significant figures will be + addressed in a future release. + + Parameters + ---------- + approval_status: string, optional + Whether to include approved and/or provisional observations. + At this time, only approved observations are returned. + computation_type: string, optional + Desired statistical computation method. Available values are: + arithmetic_mean, maximum, median, minimum, percentile. + country_code: string, optional + Country query parameter. API defaults to "US". + state: string or iterable of strings, optional + State/territory filter (the recommended parameter). Accepts a full name + ("Wisconsin"), a two-letter postal code ("WI"), or a two-digit + ANSI/FIPS code ("55"). + state_code: string, optional + State query parameter. Takes the format "US:XX", where XX is + the two-digit state code. API defaults to "US:42" (Pennsylvania). + county_code: string, optional + County query parameter. Takes the format "US:XX:YYY", where XX is + the two-digit state code and YYY is the three-digit county code. + API defaults to "US:42:103" (Pennsylvania, Pike County). + start_date: string or datetime, optional + Start date for the query in the year-month-day format + (YYYY-MM-DD). + end_date: string or datetime, optional + End date for the query in the year-month-day format + (YYYY-MM-DD). + monitoring_location_id : string or iterable of strings, optional + A unique identifier representing a single monitoring location. This + corresponds to the id field in the monitoring-locations endpoint. + Monitoring location IDs are created by combining the agency code of the + agency responsible for the monitoring location (e.g. USGS) with the ID + number of the monitoring location (e.g. 02238500), separated by a hyphen + (e.g. USGS-02238500). + page_size : int, optional + The number of results to return per page, where one result represents a + monitoring location. The default is 1000. + parent_time_series_id: string, optional + The parent_time_series_id returns statistics tied to a + particular database entry. + site_type_code: string, optional + Site type code query parameter. + You can see a list of valid site type codes here: + https://api.waterdata.usgs.gov/ogcapi/v0/collections/site-types/items. + Example: "GW" (Groundwater site) + site_type_name: string, optional + Site type name query parameter. + You can see a list of valid site type names here: + https://api.waterdata.usgs.gov/ogcapi/v0/collections/site-types/items. + Example: "Well" + parameter_code : string or iterable of strings, optional + Parameter codes are 5-digit codes used to identify the constituent + measured and the units of measure. A complete list of parameter codes + and associated groupings can be found at + https://help.waterdata.usgs.gov/codes-and-parameters/parameters. + interval_type : string or iterable of strings, optional + Filter the returned intervals to one or more periods. If unspecified + (default), all matching data are returned. Available values: + "M" (month), "CY" (calendar year), and "WY" (water year). + expand_percentiles : boolean + Percentile data for a given day of year or month of year by default + are returned from the service as lists of string values and percentile + thresholds in the "values" and "percentiles" columns, respectively. + When `expand_percentiles` is set to True (default), each value and + percentile threshold specific to a computation id are returned as + individual rows in the dataframe, with the value reported in the + "value" column and the corresponding percentile reported in a + "percentile" column (and the "values" and "percentiles" columns + are removed). Missing percentile values expressed as 'nan' in the + list of string values are removed from the dataframe to save space. + Setting `expand_percentiles` to False retains the "values" and + "percentiles" columns produced by the service. Including + both 'percentiles' and one or more other statistics ('median', + 'minimum', 'maximum', or 'arithmetic_mean') in the `computation_type` + argument will return both the "values" column, containing the list + of percentile threshold values, and a "value" column, containing + the singular summary value for the other statistics. + + Returns + ------- + df : ``pandas.DataFrame`` or ``geopandas.GeoDataFrame`` + Formatted data returned from the API query. + md : :obj:`dataretrieval.utils.BaseMetadata` + A custom metadata object. + + Examples + -------- + .. code:: + + >>> # Get monthly and yearly medians for streamflow at streams in Rhode Island + >>> # from calendar year 2024. + >>> df, md = dataretrieval.waterdata.get_stats_date_range( + ... state="RI", # Rhode Island (postal code, name, or FIPS all work) + ... parameter_code="00060", + ... site_type_code="ST", + ... start_date="2024-01-01", + ... end_date="2024-12-31", + ... computation_type="median", + ... ) + + >>> # Get monthly and yearly minimum and maximums for gage height at + >>> # a monitoring location of interest + >>> df, md = dataretrieval.waterdata.get_stats_date_range( + ... monitoring_location_id="USGS-05114000", + ... parameter_code="00065", + ... computation_type=["minimum", "maximum"], + ... ) + """ + # Build argument dictionary, omitting None values + params = _get_args( + _with_state(locals(), to="fips_us", into="state_code"), + exclude={"expand_percentiles"}, + ) + + return stats.get_data( + args=params, + service="observationIntervals", + expand_percentiles=expand_percentiles, + ) + + +__all__ = [ + "get_daily", + "get_continuous", + "get_latest_continuous", + "get_latest_daily", + "get_stats_por", + "get_stats_date_range", +] diff --git a/dataretrieval/waterdata/types.py b/dataretrieval/waterdata/types.py index 20753d3f..022627d0 100644 --- a/dataretrieval/waterdata/types.py +++ b/dataretrieval/waterdata/types.py @@ -1,5 +1,15 @@ from typing import Literal +__all__ = [ + "CODE_SERVICES", + "METADATA_COLLECTIONS", + "SERVICES", + "WATERDATA_SERVICES", + "PROFILES", + "PROFILE_LOOKUP", +] + + CODE_SERVICES = Literal[ "characteristicgroup", "characteristics", diff --git a/dataretrieval/wateruse.py b/dataretrieval/wateruse.py index a3c788ee..b78bdd51 100644 --- a/dataretrieval/wateruse.py +++ b/dataretrieval/wateruse.py @@ -61,6 +61,15 @@ from dataretrieval.transport.sync import run_sync from dataretrieval.utils import BaseMetadata, _raise_for_status, to_str +__all__ = [ + "get_wateruse", + "WATERUSE_URL", + "MODELS", + "TIME_RESOLUTIONS", + "MAX_CONCURRENT_REQUESTS", +] + + WATERUSE_URL = "https://api.water.usgs.gov/nwaa-data/data" _WATERUSE_HOST = httpx.URL(WATERUSE_URL).host # Hosts a ``rel="next"`` cursor may name for this same service; each is diff --git a/dataretrieval/wqp.py b/dataretrieval/wqp.py index a0b5e642..a64d4dbf 100644 --- a/dataretrieval/wqp.py +++ b/dataretrieval/wqp.py @@ -19,6 +19,22 @@ from .utils import BaseMetadata, _attach_datetime_columns, _query_with_retry +__all__ = [ + "get_results", + "what_sites", + "what_organizations", + "what_projects", + "what_activities", + "what_detection_limits", + "what_habitat_metrics", + "what_project_weights", + "what_activity_metrics", + "wqp_url", + "wqx3_url", + "WQP_Metadata", +] + + if TYPE_CHECKING: import httpx from pandas import DataFrame diff --git a/docs/source/architecture/decisions/0007-adapter-facades.rst b/docs/source/architecture/decisions/0007-adapter-facades.rst new file mode 100644 index 00000000..2244f09c --- /dev/null +++ b/docs/source/architecture/decisions/0007-adapter-facades.rst @@ -0,0 +1,64 @@ +ADR 0007: Organize service adapters behind stable facades +========================================================= + +Status +------ + +Accepted + +Context +------- + +A service facade can remain stable while its implementation grows for unrelated +upstream collections. Keeping every Water Data getter in one module coupled +changes to time series, monitoring metadata, field measurements, reference +catalogs, Samples, statistics, and generalized CQL queries. Active service +modules also relied on Python's implicit wildcard-export behavior, making their +intended public surfaces difficult to distinguish from imported helpers. + +Decision +-------- + +``dataretrieval.waterdata.api`` is a compatibility facade with no collection +logic. Implementation functions are grouped by collection family in +``time_series``, ``metadata``, ``measurements``, ``reference``, ``samples``, and +``cql``. Existing focused modules continue to own ratings, nearest-value +selection, Statistics API execution, shared Water Data policy, and type +vocabularies. + +The facade re-exports the established functions and preserves their signatures, +identity at ``dataretrieval.waterdata``, legacy ``__module__`` value, and private +Samples constants used by compatibility tests. Collection-family modules do not +import one another; shared behavior belongs in Water Data policy, OGC, or +transport modules. + +Active service and focused implementation modules declare explicit ``__all__`` +exports. Deprecated NWIS remains outside this modernization. Service adapters do +not import another adapter's implementation to obtain transport behavior. + +Return contracts remain service-specific. Tabular services generally return a +``(DataFrame, metadata)`` pair, while NLDI returns geospatial values directly, +StreamStats exposes response/domain objects, and ratings return parsed tables or +raw catalog features. Uniformity is not a reason to break these established +contracts. + +Consequences +------------ + +- Collection changes have a smaller implementation and test blast radius. +- Existing package and ``waterdata.api`` import paths remain stable. +- Explicit exports make accidental public-surface growth reviewable. +- More modules require a maintained facade and executable signature/export + snapshots. +- Tests are described as public-contract, adapter-contract, component, or + cross-component layers without forcing a disruptive move of established + files. + +Compliance +---------- + +``tests/contracts/public_api_test.py`` freezes Water Data imports, signatures, +facade identity, and compatibility names. ``tests/architecture_test.py`` +requires a logic-free facade, exact active-service exports, isolated collection +families, no lateral adapter reach-through, and separate OGC request +construction and schema execution. diff --git a/docs/source/architecture/decisions/index.rst b/docs/source/architecture/decisions/index.rst index 92a9d3f1..94f03476 100644 --- a/docs/source/architecture/decisions/index.rst +++ b/docs/source/architecture/decisions/index.rst @@ -23,4 +23,5 @@ records sequentially. 0004-error-retry-resume 0005-legacy-nwis 0006-service-neutral-transport + 0007-adapter-facades template diff --git a/docs/source/architecture/index.rst b/docs/source/architecture/index.rst index 09672a6e..9760caa3 100644 --- a/docs/source/architecture/index.rst +++ b/docs/source/architecture/index.rst @@ -67,11 +67,13 @@ Public service facades ^^^^^^^^^^^^^^^^^^^^^^ ``dataretrieval.waterdata`` - Modern USGS Water Data API facade. Its generic adapter uses the four-symbol - OGC facade; internal Water Data modules import protocol helpers from their - canonical OGC modules. Water-Data-specific utilities own service policy and - wrappers without re-exporting private OGC helpers. Statistics, ratings, and - nearest-value operations live in separate modules. + Modern USGS Water Data API facade. ``waterdata.api`` is a logic-free + compatibility facade over collection-family modules: ``time_series``, + ``metadata``, ``measurements``, ``reference``, ``samples``, and ``cql``. + Focused modules own ratings, nearest-value selection, statistics execution, + shared service policy, and type vocabularies. Internal modules import + protocol helpers from their canonical OGC modules rather than re-exporting + them through Water Data utilities. ``dataretrieval.ngwmn`` NGWMN facade. Its only OGC dependency is the public OGC facade, which it @@ -99,15 +101,16 @@ Shared components (``__init__.py``) exposes the service-adapter seam: ``OgcDialect``, ``prepare_request_args``, ``get_ogc_data``, and ``fetch_ogc_request``. Internally, ``policy`` defines the dialect type and endpoint constants - (depends only on stdlib); ``requests`` owns request construction, argument - normalization, and queryables/schema lookup; ``engine`` supplies OGC cursor - and response strategies to transport pagination; ``planning`` determines - chunk boundaries; ``chunking`` executes plans and retains resumable state; - ``interruptions`` defines the resumable failure contract; ``retry`` - classifies failures into OGC interruption types; and ``shaping``, ``dates``, - ``filters``, and ``errors`` isolate their named protocol concerns. The full - runtime OGC graph, including the facade, is acyclic — enforced by - ``tests/architecture_test.py``. + (depends only on stdlib); ``context`` owns ambient base URL, dialect, and row + cap state; ``requests`` owns argument normalization and HTTP request + construction; ``schema`` executes queryables/schema requests; ``engine`` + supplies OGC cursor and response strategies to transport pagination; + ``planning`` determines chunk boundaries; ``chunking`` executes plans and + retains resumable state; ``interruptions`` defines the resumable failure + contract; ``retry`` classifies failures into OGC interruption types; and + ``shaping``, ``dates``, ``filters``, and ``errors`` isolate their named + protocol concerns. The full runtime OGC graph, including the facade, is + acyclic — enforced by ``tests/architecture_test.py``. ``dataretrieval.transport`` Internal service-neutral execution layer. Owns guarded client lifecycle and @@ -159,6 +162,34 @@ Underscore-prefixed symbols are implementation details even where existing internal adapters currently import them; those imports are known variances, not new extension points. +Service return contracts +------------------------ + +The library preserves meaningful upstream differences rather than forcing every +service into one return shape: + +- Water Data, NGWMN, and Water Use tabular getters return ``(DataFrame, + BaseMetadata)``. Geometry-bearing Water Data and NGWMN results may use a + ``GeoDataFrame`` in the first position when geopandas is installed. + ``BaseMetadata`` carries request URL, elapsed query time, response headers, + and comments where the upstream format provides them. +- WQP getters return ``(DataFrame, WQP_Metadata)``; the service-specific + metadata extends ``BaseMetadata`` with WQP query parameters and site lookup. +- ``waterdata.get_ratings`` returns a mapping of feature IDs to parsed rating + ``DataFrame`` objects by default, or the raw STAC feature list when downloads + are disabled. +- NLDI navigation functions return ``GeoDataFrame`` objects directly, or raw + GeoJSON-like dictionaries when ``as_json=True``; they do not add a metadata + tuple. +- StreamStats functions return raw ``httpx.Response`` objects or the + service-specific ``Watershed`` domain object, depending on the requested + format. +- Deprecated NWIS functions retain their established DataFrame and legacy + metadata contracts through the published deprecation window. + +Changing one of these shapes is a public compatibility change and requires the +project's deprecation process; consistency alone is not sufficient reason. + Interaction view ---------------- @@ -236,7 +267,10 @@ This view records categories and representative locations of debt. The fitness functions in ``tests/architecture_test.py`` are authoritative for exact current dependency allowlists. -- ``waterdata/api.py`` and ``ogc/engine.py`` contain multiple reasons to change. +- ``ogc/engine.py`` retains compatibility wrappers alongside OGC orchestration. +- ``utils.py`` combines metadata, shaping, ambient configuration, legacy + request composition, and transport compatibility imports. + These are documented so guardrails distinguish accepted current dependencies from new erosion. They should be removed through small, test-protected changes, not a rewrite. diff --git a/tests/architecture_test.py b/tests/architecture_test.py index c4d4c5b7..51fabb50 100644 --- a/tests/architecture_test.py +++ b/tests/architecture_test.py @@ -489,3 +489,204 @@ def visit(module: str, path: tuple[str, ...]) -> None: for module in graph: visit(module, ()) + + +# --- Adapter structure and public export boundaries --- + +_EXPECTED_MODULE_EXPORTS = { + "ngwmn.py": { + "get_sites", + "get_water_level", + "get_lithology", + "get_well_construction", + "get_providers", + }, + "nldi.py": { + "get_flowlines", + "get_basin", + "get_features", + "get_features_by_data_source", + "search", + }, + "streamstats.py": { + "download_workspace", + "get_sample_watershed", + "get_watershed", + "Watershed", + }, + "wateruse.py": { + "get_wateruse", + "WATERUSE_URL", + "MODELS", + "TIME_RESOLUTIONS", + "MAX_CONCURRENT_REQUESTS", + }, + "wqp.py": { + "get_results", + "what_sites", + "what_organizations", + "what_projects", + "what_activities", + "what_detection_limits", + "what_habitat_metrics", + "what_project_weights", + "what_activity_metrics", + "wqp_url", + "wqx3_url", + "WQP_Metadata", + }, + "waterdata/api.py": { + "get_channel", + "get_codes", + "get_combined_metadata", + "get_continuous", + "get_cql", + "get_daily", + "get_field_measurements", + "get_field_measurements_metadata", + "get_latest_continuous", + "get_latest_daily", + "get_monitoring_locations", + "get_peaks", + "get_queryables", + "get_reference_table", + "get_samples", + "get_samples_summary", + "get_stats_date_range", + "get_stats_por", + "get_time_series_metadata", + }, + "waterdata/time_series.py": { + "get_daily", + "get_continuous", + "get_latest_continuous", + "get_latest_daily", + "get_stats_por", + "get_stats_date_range", + }, + "waterdata/metadata.py": { + "get_monitoring_locations", + "get_time_series_metadata", + "get_combined_metadata", + "get_field_measurements_metadata", + }, + "waterdata/measurements.py": { + "get_field_measurements", + "get_peaks", + "get_channel", + }, + "waterdata/reference.py": {"get_reference_table", "get_queryables"}, + "waterdata/samples.py": {"get_codes", "get_samples", "get_samples_summary"}, + "waterdata/cql.py": {"get_cql"}, + "waterdata/ratings.py": {"get_ratings"}, + "waterdata/nearest.py": {"get_nearest_continuous"}, + "waterdata/stats.py": {"get_data"}, + "waterdata/types.py": { + "CODE_SERVICES", + "METADATA_COLLECTIONS", + "SERVICES", + "WATERDATA_SERVICES", + "PROFILES", + "PROFILE_LOOKUP", + }, +} + + +def _literal_exports(path: Path) -> set[str]: + tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path)) + for node in tree.body: + if isinstance(node, ast.Assign) and any( + isinstance(target, ast.Name) and target.id == "__all__" + for target in node.targets + ): + return set(ast.literal_eval(node.value)) + raise AssertionError(f"{path.relative_to(PACKAGE_ROOT.parent)} has no __all__") + + +def test_active_service_exports_are_explicit_and_stable() -> None: + for relative, expected in _EXPECTED_MODULE_EXPORTS.items(): + assert _literal_exports(PACKAGE_ROOT / relative) == expected, relative + + +def test_waterdata_api_is_a_logic_free_compatibility_facade() -> None: + path = PACKAGE_ROOT / "waterdata" / "api.py" + tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path)) + definitions = [ + node.name + for node in tree.body + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)) + ] + assert not definitions, f"waterdata.api contains implementation: {definitions}" + + +def test_waterdata_collection_families_do_not_import_each_other() -> None: + families = { + "dataretrieval.waterdata.time_series", + "dataretrieval.waterdata.metadata", + "dataretrieval.waterdata.measurements", + "dataretrieval.waterdata.reference", + "dataretrieval.waterdata.samples", + "dataretrieval.waterdata.cql", + } + violations = [] + graph = _package_import_graph() + for module in families: + for dependency in graph[module]: + if dependency in families: + violations.append(f"{module} -> {dependency}") + assert not violations, "Lateral collection-family imports:\n" + "\n".join( + violations + ) + + +def test_service_adapters_do_not_reach_through_each_other() -> None: + adapters = { + "dataretrieval.ngwmn", + "dataretrieval.nldi", + "dataretrieval.streamstats", + "dataretrieval.waterdata", + "dataretrieval.wateruse", + "dataretrieval.wqp", + } + violations: list[str] = [] + for module, imports in _package_import_graph().items(): + owner = next( + ( + adapter + for adapter in adapters + if module == adapter or module.startswith(adapter + ".") + ), + None, + ) + if owner is None: + continue + for dependency in imports: + target = next( + ( + adapter + for adapter in adapters + if dependency == adapter or dependency.startswith(adapter + ".") + ), + None, + ) + if target is not None and target != owner: + violations.append(f"{module} -> {dependency}") + assert not violations, "Adapter-to-adapter imports:\n" + "\n".join( + sorted(set(violations)) + ) + + +def test_ogc_request_construction_does_not_execute_http() -> None: + path = PACKAGE_ROOT / "ogc" / "requests.py" + tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path)) + transport_names = { + alias.name + for node in tree.body + if isinstance(node, ast.ImportFrom) + and node.module == "dataretrieval.transport.http" + for alias in node.names + } + assert transport_names == {"default_headers"} + assert "dataretrieval.ogc.schema" in _runtime_imports( + PACKAGE_ROOT / "ogc" / "shaping.py" + ) diff --git a/tests/contracts/README.md b/tests/contracts/README.md new file mode 100644 index 00000000..d05bed14 --- /dev/null +++ b/tests/contracts/README.md @@ -0,0 +1,19 @@ +# Test layers + +The suite uses four dependency-oriented layers without moving established tests: + +- **Public contract** (`tests/contracts/`): imports, exports, signatures, return + annotations, metadata/error promises, and compatibility paths. These tests use + public modules and no live services. +- **Adapter contract** (`waterdata_test.py`, `ngwmn_test.py`, `wateruse_test.py`, + `wqp_test.py`, `nldi_test.py`, `streamstats_test.py`): service request wiring, + response parsing, and documented protocol behavior. +- **Component** (`transport_test.py`, `waterdata_chunking_test.py`, + `waterdata_queryables_test.py`, `rdb_test.py`): one internal responsibility in + isolation. +- **Cross-component** (`architecture_test.py`, `headers_host_scoping_test.py`, + `waterdata_progress_test.py`): dependency fitness functions and behavior that + spans adapters, OGC, transport, or security boundaries. + +Live API cases remain in their existing adapter files and are not part of the +public-contract layer. diff --git a/tests/contracts/public_api_test.py b/tests/contracts/public_api_test.py new file mode 100644 index 00000000..d1d26d31 --- /dev/null +++ b/tests/contracts/public_api_test.py @@ -0,0 +1,309 @@ +"""Public import, export, and signature contracts for Water Data.""" +# ruff: noqa: E501 + +from __future__ import annotations + +import inspect + +from dataretrieval import waterdata +from dataretrieval.waterdata import api + +_EXPECTED_WATERDATA_ALL = [ + "CODE_SERVICES", + "FILTER_LANG", + "PROFILES", + "PROFILE_LOOKUP", + "SERVICES", + "WATERDATA_SERVICES", + "parallel_chunks", + "get_channel", + "get_codes", + "get_combined_metadata", + "get_continuous", + "get_cql", + "get_daily", + "get_field_measurements", + "get_field_measurements_metadata", + "get_latest_continuous", + "get_latest_daily", + "get_monitoring_locations", + "get_nearest_continuous", + "get_peaks", + "get_queryables", + "get_ratings", + "get_reference_table", + "get_samples", + "get_samples_summary", + "get_stats_date_range", + "get_stats_por", + "get_time_series_metadata", +] + +_EXPECTED_API_NAMES = [ + "get_channel", + "get_codes", + "get_combined_metadata", + "get_continuous", + "get_cql", + "get_daily", + "get_field_measurements", + "get_field_measurements_metadata", + "get_latest_continuous", + "get_latest_daily", + "get_monitoring_locations", + "get_peaks", + "get_queryables", + "get_reference_table", + "get_samples", + "get_samples_summary", + "get_stats_date_range", + "get_stats_por", + "get_time_series_metadata", +] + +_EXPECTED_SIGNATURES = { + "get_channel": "(monitoring_location_id: 'str | Iterable[str] | None' = None, field_visit_id: 'str | Iterable[str] | " + "None' = None, measurement_number: 'str | Iterable[str] | None' = None, time: 'str | Iterable[str] | " + "None' = None, channel_name: 'str | Iterable[str] | None' = None, channel_flow: 'str | Iterable[str] | " + "None' = None, channel_flow_unit: 'str | Iterable[str] | None' = None, channel_width: 'str | " + "Iterable[str] | None' = None, channel_width_unit: 'str | Iterable[str] | None' = None, channel_area: " + "'str | Iterable[str] | None' = None, channel_area_unit: 'str | Iterable[str] | None' = None, " + "channel_velocity: 'str | Iterable[str] | None' = None, channel_velocity_unit: 'str | Iterable[str] | " + "None' = None, channel_location_distance: 'str | Iterable[str] | None' = None, " + "channel_location_distance_unit: 'str | Iterable[str] | None' = None, channel_stability: 'str | " + "Iterable[str] | None' = None, channel_material: 'str | Iterable[str] | None' = None, " + "channel_evenness: 'str | Iterable[str] | None' = None, horizontal_velocity_description: 'str | " + "Iterable[str] | None' = None, vertical_velocity_description: 'str | Iterable[str] | None' = None, " + "longitudinal_velocity_description: 'str | Iterable[str] | None' = None, measurement_type: 'str | " + "Iterable[str] | None' = None, last_modified: 'str | Iterable[str] | None' = None, " + "channel_measurement_type: 'str | Iterable[str] | None' = None, properties: 'str | Iterable[str] | " + "None' = None, skip_geometry: 'bool | None' = None, bbox: 'list[float] | None' = None, limit: 'int | " + "None' = None, filter: 'str | None' = None, filter_lang: 'FILTER_LANG | None' = None, convert_type: " + "'bool' = True, max_rows: 'int | None' = None, **queryables: 'Any') -> 'tuple[pd.DataFrame, " + "BaseMetadata]'", + "get_codes": "(code_service: 'CODE_SERVICES') -> 'tuple[pd.DataFrame, BaseMetadata]'", + "get_combined_metadata": "(monitoring_location_id: 'str | Iterable[str] | None' = None, parameter_code: 'str | " + "Iterable[str] | None' = None, parameter_name: 'str | Iterable[str] | None' = None, " + "parameter_description: 'str | Iterable[str] | None' = None, unit_of_measure: 'str | " + "Iterable[str] | None' = None, statistic_id: 'str | Iterable[str] | None' = None, data_type: " + "'str | Iterable[str] | None' = None, computation_identifier: 'str | Iterable[str] | None' = " + "None, thresholds: 'float | list[float] | None' = None, sublocation_identifier: 'str | " + "Iterable[str] | None' = None, primary: 'str | Iterable[str] | None' = None, " + "parent_time_series_id: 'str | Iterable[str] | None' = None, web_description: 'str | " + "Iterable[str] | None' = None, last_modified: 'str | Iterable[str] | None' = None, begin: " + "'str | Iterable[str] | None' = None, end: 'str | Iterable[str] | None' = None, agency_code: " + "'str | Iterable[str] | None' = None, agency_name: 'str | Iterable[str] | None' = None, " + "monitoring_location_number: 'str | Iterable[str] | None' = None, monitoring_location_name: " + "'str | Iterable[str] | None' = None, district_code: 'str | Iterable[str] | None' = None, " + "country_code: 'str | Iterable[str] | None' = None, country_name: 'str | Iterable[str] | " + "None' = None, state: 'str | Iterable[str] | None' = None, state_code: 'str | Iterable[str] " + "| None' = None, state_name: 'str | Iterable[str] | None' = None, county_code: 'str | " + "Iterable[str] | None' = None, county_name: 'str | Iterable[str] | None' = None, " + "minor_civil_division_code: 'str | Iterable[str] | None' = None, site_type_code: 'str | " + "Iterable[str] | None' = None, site_type: 'str | Iterable[str] | None' = None, " + "hydrologic_unit_code: 'str | Iterable[str] | None' = None, basin_code: 'str | Iterable[str] " + "| None' = None, altitude: 'str | Iterable[str] | None' = None, altitude_accuracy: 'str | " + "Iterable[str] | None' = None, altitude_method_code: 'str | Iterable[str] | None' = None, " + "altitude_method_name: 'str | Iterable[str] | None' = None, vertical_datum: 'str | " + "Iterable[str] | None' = None, vertical_datum_name: 'str | Iterable[str] | None' = None, " + "horizontal_positional_accuracy_code: 'str | Iterable[str] | None' = None, " + "horizontal_positional_accuracy: 'str | Iterable[str] | None' = None, " + "horizontal_position_method_code: 'str | Iterable[str] | None' = None, " + "horizontal_position_method_name: 'str | Iterable[str] | None' = None, " + "original_horizontal_datum: 'str | Iterable[str] | None' = None, " + "original_horizontal_datum_name: 'str | Iterable[str] | None' = None, drainage_area: 'str | " + "Iterable[str] | None' = None, contributing_drainage_area: 'str | Iterable[str] | None' = " + "None, time_zone_abbreviation: 'str | Iterable[str] | None' = None, uses_daylight_savings: " + "'str | Iterable[str] | None' = None, construction_date: 'str | Iterable[str] | None' = " + "None, aquifer_code: 'str | Iterable[str] | None' = None, national_aquifer_code: 'str | " + "Iterable[str] | None' = None, aquifer_type_code: 'str | Iterable[str] | None' = None, " + "well_constructed_depth: 'str | Iterable[str] | None' = None, hole_constructed_depth: 'str | " + "Iterable[str] | None' = None, depth_source_code: 'str | Iterable[str] | None' = None, " + "properties: 'str | Iterable[str] | None' = None, skip_geometry: 'bool | None' = None, bbox: " + "'list[float] | None' = None, limit: 'int | None' = None, filter: 'str | None' = None, " + "filter_lang: 'FILTER_LANG | None' = None, convert_type: 'bool' = True, max_rows: 'int | " + "None' = None, **queryables: 'Any') -> 'tuple[pd.DataFrame, BaseMetadata]'", + "get_continuous": "(monitoring_location_id: 'str | Iterable[str] | None' = None, parameter_code: 'str | Iterable[str] " + "| None' = None, statistic_id: 'str | Iterable[str] | None' = None, properties: 'str | " + "Iterable[str] | None' = None, time_series_id: 'str | Iterable[str] | None' = None, continuous_id: " + "'str | Iterable[str] | None' = None, approval_status: 'str | Iterable[str] | None' = None, " + "unit_of_measure: 'str | Iterable[str] | None' = None, qualifier: 'str | Iterable[str] | None' = " + "None, value: 'str | Iterable[str] | None' = None, last_modified: 'str | Iterable[str] | None' = " + "None, time: 'str | Iterable[str] | None' = None, limit: 'int | None' = None, filter: 'str | None' " + "= None, filter_lang: 'FILTER_LANG | None' = None, convert_type: 'bool' = True, max_rows: 'int | " + "None' = None, **queryables: 'Any') -> 'tuple[pd.DataFrame, BaseMetadata]'", + "get_cql": "(service: 'WATERDATA_SERVICES', cql: 'str | dict[str, Any]', *, properties: 'str | Iterable[str] | None' " + "= None, bbox: 'list[float] | None' = None, limit: 'int | None' = None, skip_geometry: 'bool | None' = " + "None, convert_type: 'bool' = True) -> 'tuple[pd.DataFrame, BaseMetadata]'", + "get_daily": "(monitoring_location_id: 'str | Iterable[str] | None' = None, parameter_code: 'str | Iterable[str] | " + "None' = None, statistic_id: 'str | Iterable[str] | None' = None, properties: 'str | Iterable[str] | " + "None' = None, time_series_id: 'str | Iterable[str] | None' = None, daily_id: 'str | Iterable[str] | " + "None' = None, approval_status: 'str | Iterable[str] | None' = None, unit_of_measure: 'str | " + "Iterable[str] | None' = None, qualifier: 'str | Iterable[str] | None' = None, value: 'str | " + "Iterable[str] | None' = None, last_modified: 'str | Iterable[str] | None' = None, skip_geometry: 'bool " + "| None' = None, time: 'str | Iterable[str] | None' = None, bbox: 'list[float] | None' = None, limit: " + "'int | None' = None, filter: 'str | None' = None, filter_lang: 'FILTER_LANG | None' = None, " + "convert_type: 'bool' = True, max_rows: 'int | None' = None, **queryables: 'Any') -> " + "'tuple[pd.DataFrame, BaseMetadata]'", + "get_field_measurements": "(monitoring_location_id: 'str | Iterable[str] | None' = None, parameter_code: 'str | " + "Iterable[str] | None' = None, observing_procedure_code: 'str | Iterable[str] | None' = " + "None, properties: 'str | Iterable[str] | None' = None, field_visit_id: 'str | " + "Iterable[str] | None' = None, approval_status: 'str | Iterable[str] | None' = None, " + "unit_of_measure: 'str | Iterable[str] | None' = None, qualifier: 'str | Iterable[str] | " + "None' = None, value: 'str | Iterable[str] | None' = None, last_modified: 'str | " + "Iterable[str] | None' = None, observing_procedure: 'str | Iterable[str] | None' = None, " + "vertical_datum: 'str | Iterable[str] | None' = None, measuring_agency: 'str | " + "Iterable[str] | None' = None, skip_geometry: 'bool | None' = None, time: 'str | " + "Iterable[str] | None' = None, bbox: 'list[float] | None' = None, limit: 'int | None' = " + "None, filter: 'str | None' = None, filter_lang: 'FILTER_LANG | None' = None, convert_type: " + "'bool' = True, max_rows: 'int | None' = None, **queryables: 'Any') -> 'tuple[pd.DataFrame, " + "BaseMetadata]'", + "get_field_measurements_metadata": "(monitoring_location_id: 'str | Iterable[str] | None' = None, parameter_code: " + "'str | Iterable[str] | None' = None, parameter_name: 'str | Iterable[str] | None' " + "= None, parameter_description: 'str | Iterable[str] | None' = None, begin: 'str | " + "Iterable[str] | None' = None, end: 'str | Iterable[str] | None' = None, " + "last_modified: 'str | Iterable[str] | None' = None, properties: 'str | " + "Iterable[str] | None' = None, skip_geometry: 'bool | None' = None, bbox: " + "'list[float] | None' = None, limit: 'int | None' = None, filter: 'str | None' = " + "None, filter_lang: 'FILTER_LANG | None' = None, convert_type: 'bool' = True, " + "max_rows: 'int | None' = None, **queryables: 'Any') -> 'tuple[pd.DataFrame, " + "BaseMetadata]'", + "get_latest_continuous": "(monitoring_location_id: 'str | Iterable[str] | None' = None, parameter_code: 'str | " + "Iterable[str] | None' = None, statistic_id: 'str | Iterable[str] | None' = None, " + "properties: 'str | Iterable[str] | None' = None, time_series_id: 'str | Iterable[str] | " + "None' = None, latest_continuous_id: 'str | Iterable[str] | None' = None, approval_status: " + "'str | Iterable[str] | None' = None, unit_of_measure: 'str | Iterable[str] | None' = None, " + "qualifier: 'str | Iterable[str] | None' = None, value: 'str | Iterable[str] | None' = None, " + "last_modified: 'str | Iterable[str] | None' = None, skip_geometry: 'bool | None' = None, " + "time: 'str | Iterable[str] | None' = None, bbox: 'list[float] | None' = None, limit: 'int | " + "None' = None, filter: 'str | None' = None, filter_lang: 'FILTER_LANG | None' = None, " + "convert_type: 'bool' = True, max_rows: 'int | None' = None, **queryables: 'Any') -> " + "'tuple[pd.DataFrame, BaseMetadata]'", + "get_latest_daily": "(monitoring_location_id: 'str | Iterable[str] | None' = None, parameter_code: 'str | " + "Iterable[str] | None' = None, statistic_id: 'str | Iterable[str] | None' = None, properties: " + "'str | Iterable[str] | None' = None, time_series_id: 'str | Iterable[str] | None' = None, " + "latest_daily_id: 'str | Iterable[str] | None' = None, approval_status: 'str | Iterable[str] | " + "None' = None, unit_of_measure: 'str | Iterable[str] | None' = None, qualifier: 'str | " + "Iterable[str] | None' = None, value: 'str | Iterable[str] | None' = None, last_modified: 'str | " + "Iterable[str] | None' = None, skip_geometry: 'bool | None' = None, time: 'str | Iterable[str] | " + "None' = None, bbox: 'list[float] | None' = None, limit: 'int | None' = None, filter: 'str | " + "None' = None, filter_lang: 'FILTER_LANG | None' = None, convert_type: 'bool' = True, max_rows: " + "'int | None' = None, **queryables: 'Any') -> 'tuple[pd.DataFrame, BaseMetadata]'", + "get_monitoring_locations": "(monitoring_location_id: 'str | Iterable[str] | None' = None, agency_code: 'str | " + "Iterable[str] | None' = None, agency_name: 'str | Iterable[str] | None' = None, " + "monitoring_location_number: 'str | Iterable[str] | None' = None, " + "monitoring_location_name: 'str | Iterable[str] | None' = None, district_code: 'str | " + "Iterable[str] | None' = None, country_code: 'str | Iterable[str] | None' = None, " + "country_name: 'str | Iterable[str] | None' = None, state: 'str | Iterable[str] | None' = " + "None, state_code: 'str | Iterable[str] | None' = None, state_name: 'str | Iterable[str] " + "| None' = None, county_code: 'str | Iterable[str] | None' = None, county_name: 'str | " + "Iterable[str] | None' = None, minor_civil_division_code: 'str | Iterable[str] | None' = " + "None, site_type_code: 'str | Iterable[str] | None' = None, site_type: 'str | " + "Iterable[str] | None' = None, hydrologic_unit_code: 'str | Iterable[str] | None' = None, " + "basin_code: 'str | Iterable[str] | None' = None, altitude: 'str | Iterable[str] | None' " + "= None, altitude_accuracy: 'str | Iterable[str] | None' = None, altitude_method_code: " + "'str | Iterable[str] | None' = None, altitude_method_name: 'str | Iterable[str] | None' " + "= None, vertical_datum: 'str | Iterable[str] | None' = None, vertical_datum_name: 'str | " + "Iterable[str] | None' = None, horizontal_positional_accuracy_code: 'str | Iterable[str] " + "| None' = None, horizontal_positional_accuracy: 'str | Iterable[str] | None' = None, " + "horizontal_position_method_code: 'str | Iterable[str] | None' = None, " + "horizontal_position_method_name: 'str | Iterable[str] | None' = None, " + "original_horizontal_datum: 'str | Iterable[str] | None' = None, " + "original_horizontal_datum_name: 'str | Iterable[str] | None' = None, drainage_area: 'str " + "| Iterable[str] | None' = None, contributing_drainage_area: 'str | Iterable[str] | None' " + "= None, time_zone_abbreviation: 'str | Iterable[str] | None' = None, " + "uses_daylight_savings: 'str | Iterable[str] | None' = None, construction_date: 'str | " + "Iterable[str] | None' = None, aquifer_code: 'str | Iterable[str] | None' = None, " + "national_aquifer_code: 'str | Iterable[str] | None' = None, aquifer_type_code: 'str | " + "Iterable[str] | None' = None, well_constructed_depth: 'str | Iterable[str] | None' = " + "None, hole_constructed_depth: 'str | Iterable[str] | None' = None, depth_source_code: " + "'str | Iterable[str] | None' = None, properties: 'str | Iterable[str] | None' = None, " + "skip_geometry: 'bool | None' = None, bbox: 'list[float] | None' = None, limit: 'int | " + "None' = None, filter: 'str | None' = None, filter_lang: 'FILTER_LANG | None' = None, " + "convert_type: 'bool' = True, max_rows: 'int | None' = None, **queryables: 'Any') -> " + "'tuple[pd.DataFrame, BaseMetadata]'", + "get_peaks": "(monitoring_location_id: 'str | Iterable[str] | None' = None, parameter_code: 'str | Iterable[str] | " + "None' = None, time_series_id: 'str | Iterable[str] | None' = None, unit_of_measure: 'str | " + "Iterable[str] | None' = None, time: 'str | Iterable[str] | None' = None, last_modified: 'str | " + "Iterable[str] | None' = None, water_year: 'int | list[int] | None' = None, year: 'int | list[int] | " + "None' = None, month: 'int | list[int] | None' = None, day: 'int | list[int] | None' = None, peak_since: " + "'int | list[int] | None' = None, properties: 'str | Iterable[str] | None' = None, skip_geometry: 'bool " + "| None' = None, bbox: 'list[float] | None' = None, limit: 'int | None' = None, filter: 'str | None' = " + "None, filter_lang: 'FILTER_LANG | None' = None, convert_type: 'bool' = True, max_rows: 'int | None' = " + "None, **queryables: 'Any') -> 'tuple[pd.DataFrame, BaseMetadata]'", + "get_queryables": "(collection: 'str') -> 'tuple[pd.DataFrame, BaseMetadata]'", + "get_reference_table": "(collection: 'str', limit: 'int | None' = None, query: 'dict[str, Any] | None' = None, " + "max_rows: 'int | None' = None) -> 'tuple[pd.DataFrame, BaseMetadata]'", + "get_samples": "(ssl_check: 'bool' = True, service: 'SERVICES' = 'results', profile: 'PROFILES' = 'fullphyschem', " + "activity_media_name: 'str | Iterable[str] | None' = None, activity_start_date_lower: 'str | None' = " + "None, activity_start_date_upper: 'str | None' = None, activity_type_code: 'str | Iterable[str] | " + "None' = None, characteristic_group: 'str | Iterable[str] | None' = None, characteristic: 'str | " + "Iterable[str] | None' = None, characteristic_user_supplied: 'str | Iterable[str] | None' = None, " + "bbox: 'list[float] | None' = None, country_code: 'str | Iterable[str] | None' = None, state_code: " + "'str | Iterable[str] | None' = None, county_code: 'str | Iterable[str] | None' = None, " + "site_type_code: 'str | Iterable[str] | None' = None, site_type_name: 'str | Iterable[str] | None' = " + "None, usgs_pcode: 'str | Iterable[str] | None' = None, hydrologic_unit: 'str | Iterable[str] | None' " + "= None, monitoring_location_id: 'str | Iterable[str] | None' = None, organization_id: 'str | " + "Iterable[str] | None' = None, point_location_latitude: 'float | None' = None, " + "point_location_longitude: 'float | None' = None, point_location_within_miles: 'float | None' = None, " + "project_id: 'str | Iterable[str] | None' = None, record_identifier_user_supplied: 'str | " + "Iterable[str] | None' = None) -> 'tuple[pd.DataFrame, BaseMetadata]'", + "get_samples_summary": "(monitoring_location_id: 'str', ssl_check: 'bool' = True) -> 'tuple[pd.DataFrame, " + "BaseMetadata]'", + "get_stats_date_range": "(approval_status: 'str | None' = None, computation_type: 'str | Iterable[str] | None' = " + "None, country_code: 'str | Iterable[str] | None' = None, state: 'str | Iterable[str] | None' " + "= None, state_code: 'str | Iterable[str] | None' = None, county_code: 'str | Iterable[str] | " + "None' = None, start_date: 'str | None' = None, end_date: 'str | None' = None, " + "monitoring_location_id: 'str | Iterable[str] | None' = None, page_size: 'int' = 1000, " + "parent_time_series_id: 'str | Iterable[str] | None' = None, site_type_code: 'str | " + "Iterable[str] | None' = None, site_type_name: 'str | Iterable[str] | None' = None, " + "parameter_code: 'str | Iterable[str] | None' = None, interval_type: 'str | Iterable[str] | " + "None' = None, expand_percentiles: 'bool' = True) -> 'tuple[pd.DataFrame, BaseMetadata]'", + "get_stats_por": "(approval_status: 'str | None' = None, computation_type: 'str | Iterable[str] | None' = None, " + "country_code: 'str | Iterable[str] | None' = None, state: 'str | Iterable[str] | None' = None, " + "state_code: 'str | Iterable[str] | None' = None, county_code: 'str | Iterable[str] | None' = None, " + "start_date: 'str | None' = None, end_date: 'str | None' = None, monitoring_location_id: 'str | " + "Iterable[str] | None' = None, page_size: 'int' = 1000, parent_time_series_id: 'str | Iterable[str] " + "| None' = None, site_type_code: 'str | Iterable[str] | None' = None, site_type_name: 'str | " + "Iterable[str] | None' = None, parameter_code: 'str | Iterable[str] | None' = None, normal_type: " + "'str | None' = None, expand_percentiles: 'bool' = True) -> 'tuple[pd.DataFrame, BaseMetadata]'", + "get_time_series_metadata": "(monitoring_location_id: 'str | Iterable[str] | None' = None, parameter_code: 'str | " + "Iterable[str] | None' = None, parameter_name: 'str | Iterable[str] | None' = None, " + "properties: 'str | Iterable[str] | None' = None, statistic_id: 'str | Iterable[str] | " + "None' = None, hydrologic_unit_code: 'str | Iterable[str] | None' = None, state: 'str | " + "Iterable[str] | None' = None, state_name: 'str | Iterable[str] | None' = None, " + "last_modified: 'str | Iterable[str] | None' = None, begin: 'str | Iterable[str] | None' " + "= None, end: 'str | Iterable[str] | None' = None, begin_utc: 'str | Iterable[str] | " + "None' = None, end_utc: 'str | Iterable[str] | None' = None, unit_of_measure: 'str | " + "Iterable[str] | None' = None, computation_period_identifier: 'str | Iterable[str] | " + "None' = None, computation_identifier: 'str | Iterable[str] | None' = None, thresholds: " + "'float | list[float] | None' = None, sublocation_identifier: 'str | Iterable[str] | " + "None' = None, primary: 'str | Iterable[str] | None' = None, parent_time_series_id: 'str " + "| Iterable[str] | None' = None, time_series_id: 'str | Iterable[str] | None' = None, " + "web_description: 'str | Iterable[str] | None' = None, skip_geometry: 'bool | None' = " + "None, bbox: 'list[float] | None' = None, limit: 'int | None' = None, filter: 'str | " + "None' = None, filter_lang: 'FILTER_LANG | None' = None, convert_type: 'bool' = True, " + "max_rows: 'int | None' = None, **queryables: 'Any') -> 'tuple[pd.DataFrame, " + "BaseMetadata]'", +} + + +def test_waterdata_exports_are_stable() -> None: + assert waterdata.__all__ == _EXPECTED_WATERDATA_ALL + assert api.__all__ == _EXPECTED_API_NAMES + assert all(hasattr(waterdata, name) for name in waterdata.__all__) + + +def test_api_facade_preserves_function_contracts() -> None: + for name, expected_signature in _EXPECTED_SIGNATURES.items(): + package_function = getattr(waterdata, name) + facade_function = getattr(api, name) + assert package_function is facade_function + assert str(inspect.signature(facade_function)) == expected_signature + assert facade_function.__module__ == "dataretrieval.waterdata.api" + + +def test_api_private_samples_compatibility_names_remain() -> None: + assert isinstance(api._SAMPLES_PARAM_TO_API, dict) + assert isinstance(api._SAMPLES_LEGACY_KWARGS, dict) + assert callable(api.get_ogc_data) diff --git a/tests/waterdata_test.py b/tests/waterdata_test.py index df317244..542d03ff 100644 --- a/tests/waterdata_test.py +++ b/tests/waterdata_test.py @@ -567,7 +567,7 @@ def test_get_daily_max_rows_is_excluded_from_request_and_forwarded(): # met, then truncate the combined frame to exactly N) is covered without a # network round-trip by the ``_row_cap`` / ``_finalize_ogc`` tests in # tests/waterdata_utils_test.py. - with mock.patch("dataretrieval.waterdata.api.get_ogc_data") as fake: + with mock.patch("dataretrieval.waterdata.time_series.get_ogc_data") as fake: fake.return_value = (pd.DataFrame(), mock.MagicMock(spec=[])) get_daily( monitoring_location_id="USGS-05427718", @@ -1086,7 +1086,7 @@ def test_get_daily_parameter_code_as_series(self): URL (or POST body). Post-fix, ``_normalize_str_iterable`` materializes it to ``list`` at the function boundary. """ - with mock.patch("dataretrieval.waterdata.api.get_ogc_data") as fake: + with mock.patch("dataretrieval.waterdata.time_series.get_ogc_data") as fake: fake.return_value = (pd.DataFrame(), mock.MagicMock(spec=[])) get_daily( monitoring_location_id="USGS-05427718", diff --git a/tests/waterdata_utils_test.py b/tests/waterdata_utils_test.py index 453a3de2..30995deb 100644 --- a/tests/waterdata_utils_test.py +++ b/tests/waterdata_utils_test.py @@ -18,6 +18,7 @@ ServiceUnavailable, TransientError, ) +from dataretrieval.ogc.context import _row_cap from dataretrieval.ogc.dates import _format_api_dates from dataretrieval.ogc.engine import ( _next_req_url, @@ -28,7 +29,7 @@ _parse_retry_after, _raise_for_non_200, ) -from dataretrieval.ogc.requests import _check_ogc_requests, _row_cap +from dataretrieval.ogc.schema import _check_ogc_requests from dataretrieval.ogc.shaping import ( _arrange_cols, _get_resp_data, @@ -1087,7 +1088,7 @@ def test_ogc_getter_resolves_state_at_getter_layer(monkeypatch): """The OGC getters resolve the unified ``state`` into ``state_name`` themselves (any encoding), so the shared ``get_ogc_data`` wrapper stays state-agnostic.""" - import dataretrieval.waterdata.api as _api + import dataretrieval.waterdata.metadata as _metadata captured: dict = {} @@ -1095,8 +1096,8 @@ def fake_get_ogc_data(args, service, *a, **k): captured.update(args=args, service=service) return pd.DataFrame(), mock.Mock() - monkeypatch.setattr(_api, "get_ogc_data", fake_get_ogc_data) - _api.get_monitoring_locations(state="55") # FIPS in -> full name out + monkeypatch.setattr(_metadata, "get_ogc_data", fake_get_ogc_data) + _metadata.get_monitoring_locations(state="55") # FIPS in -> full name out assert captured["args"].get("state_name") == "Wisconsin" assert "state" not in captured["args"] From affb802898e732f384943a019910f3b50a3abadc Mon Sep 17 00:00:00 2001 From: thodson-usgs Date: Thu, 6 Aug 2026 14:14:06 -0500 Subject: [PATCH 2/4] refactor(waterdata): simplify the adapter split's seams and guards Cleanup pass over the collection-family split. No behavior change to any getter; the moved function bodies are untouched. Remove a dead re-export chain. requests.py forwarded _row_cap and _check_ogc_requests purely to preserve old private paths, and used neither. _check_ogc_requests had no consumer at all -- engine.py imported it only to re-export it, and every real caller already went to ogc.schema directly. Forwarding it also gave request construction an edge to the one OGC module that executes HTTP, so the guard asserting the opposite passed while the rule was broken. Engine now takes the ambient row cap from its owner, ogc.context. Drop the __module__ rewrite loop. It mutated function objects the family modules own, so a traceback pointed at api.py, which contains no code. Its stated Sphinx rationale is disproved in this same package: get_ratings and get_nearest_continuous sit in the same __all__, keep their real __module__, and document fine. Drop api.get_ogc_data. Nothing imported it, and it did not work as the patch target its comment claimed -- family modules bind the name at import, so patching the facade silently no-ops. Derive what was frozen three times. The facade's 19 exports were written out in api.py, architecture_test, and public_api_test; the latter two are now derived, which also turns a change-detector into a real invariant: the facade must export exactly the union of the family modules, so a family gaining an export the facade forgets now fails. Likewise the family and adapter sets are derived rather than enumerated, so a seventh family is covered on arrival instead of being silently exempt. Cache the two AST walks in architecture_test (240ms -> 87ms of parsing; the file runs in 0.21s, was 0.71s), delete eight wheel-smoke assertions that cannot fail because the imports four lines above already cover them, and drop three decorative empty __all__ lists from internal modules. Co-Authored-By: Claude Opus 5 --- .github/workflows/python-package.yml | 8 -- dataretrieval/ogc/context.py | 2 - dataretrieval/ogc/engine.py | 3 +- dataretrieval/ogc/requests.py | 12 +- dataretrieval/ogc/schema.py | 3 - dataretrieval/transport/__init__.py | 2 - dataretrieval/waterdata/api.py | 8 -- tests/architecture_test.py | 178 +++++++++++++++------------ tests/contracts/public_api_test.py | 28 +---- 9 files changed, 105 insertions(+), 139 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 6537f9c3..c56bdf39 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -61,15 +61,7 @@ jobs: installed = Path(dataretrieval.__file__).resolve() assert not installed.is_relative_to(checkout), (installed, checkout) assert importlib.util.find_spec("dataretrieval.waterdata.api") is not None - assert importlib.util.find_spec("dataretrieval.waterdata.time_series") is not None - assert importlib.util.find_spec("dataretrieval.waterdata.metadata") is not None - assert importlib.util.find_spec("dataretrieval.waterdata.measurements") is not None - assert importlib.util.find_spec("dataretrieval.waterdata.reference") is not None - assert importlib.util.find_spec("dataretrieval.waterdata.samples") is not None - assert importlib.util.find_spec("dataretrieval.waterdata.cql") is not None assert importlib.util.find_spec("dataretrieval.ogc.engine") is not None - assert importlib.util.find_spec("dataretrieval.ogc.context") is not None - assert importlib.util.find_spec("dataretrieval.ogc.schema") is not None assert files("dataretrieval").joinpath("py.typed").is_file() assert waterdata.get_daily assert ngwmn.get_sites diff --git a/dataretrieval/ogc/context.py b/dataretrieval/ogc/context.py index 8b607509..0dea357a 100644 --- a/dataretrieval/ogc/context.py +++ b/dataretrieval/ogc/context.py @@ -11,5 +11,3 @@ # Per-call request and response dialect. _dialect: Ambient[OgcDialect] = Ambient("ogc_dialect", DEFAULT_DIALECT) - -__all__: list[str] = [] diff --git a/dataretrieval/ogc/engine.py b/dataretrieval/ogc/engine.py index 2ab300d7..975259ed 100644 --- a/dataretrieval/ogc/engine.py +++ b/dataretrieval/ogc/engine.py @@ -39,6 +39,7 @@ import dataretrieval.progress as _progress from dataretrieval.credentials import without_embedded_credentials from dataretrieval.ogc.chunking import get_active_client +from dataretrieval.ogc.context import _row_cap from dataretrieval.ogc.errors import _raise_for_non_200 from dataretrieval.ogc.policy import ( BASE_URL, # noqa: F401 — compatibility alias @@ -52,7 +53,6 @@ _NO_NORMALIZE_PARAMS, _as_str_list, _check_monitoring_location_id, - _check_ogc_requests, _construct_api_requests, _construct_cql_request, _cql2_param, @@ -61,7 +61,6 @@ _normalize_str_iterable, _ogc_base_url, _ogc_query_params, - _row_cap, _switch_arg_id, _switch_properties_id, prepare_request_args, diff --git a/dataretrieval/ogc/requests.py b/dataretrieval/ogc/requests.py index 2efc7a02..9f5b5f42 100644 --- a/dataretrieval/ogc/requests.py +++ b/dataretrieval/ogc/requests.py @@ -1,8 +1,10 @@ """OGC argument normalization and HTTP request construction. Ambient request state lives in :mod:`dataretrieval.ogc.context`; queryables and -schema execution live in :mod:`dataretrieval.ogc.schema`. The schema helper is -imported here only to preserve its previous private path. +schema execution live in :mod:`dataretrieval.ogc.schema`. Neither is re-exported +from here -- importing the schema helper only to forward it would give this +module an edge to the one part of OGC that executes HTTP, which is exactly what +request *construction* is supposed to be free of. """ from __future__ import annotations @@ -16,15 +18,9 @@ from dataretrieval.ogc.context import _dialect as _dialect from dataretrieval.ogc.context import _ogc_base_url as _ogc_base_url -from dataretrieval.ogc.context import _row_cap as _context_row_cap from dataretrieval.ogc.dates import _DATE_RANGE_PARAMS, _format_api_dates -from dataretrieval.ogc.schema import _check_ogc_requests as _schema_check_ogc_requests from dataretrieval.transport.http import default_headers as _default_headers -# Previous private paths remain available while ownership lives in context/schema. -_row_cap = _context_row_cap -_check_ogc_requests = _schema_check_ogc_requests - # --------------------------------------------------------------------------- # Monitoring location ID validation # --------------------------------------------------------------------------- diff --git a/dataretrieval/ogc/schema.py b/dataretrieval/ogc/schema.py index 1b92ae64..4bab6956 100644 --- a/dataretrieval/ogc/schema.py +++ b/dataretrieval/ogc/schema.py @@ -23,6 +23,3 @@ def _check_ogc_requests( response = _get(url, headers=_default_headers(url), **HTTPX_DEFAULTS) _raise_for_non_200(response) return cast("dict[str, Any]", response.json()), response - - -__all__: list[str] = [] diff --git a/dataretrieval/transport/__init__.py b/dataretrieval/transport/__init__.py index 6f8e1aa7..e437aa13 100644 --- a/dataretrieval/transport/__init__.py +++ b/dataretrieval/transport/__init__.py @@ -5,5 +5,3 @@ Service and protocol adapters consume these components; this package is not a public framework API. """ - -__all__: list[str] = [] diff --git a/dataretrieval/waterdata/api.py b/dataretrieval/waterdata/api.py index 71fe61af..d4afbcab 100644 --- a/dataretrieval/waterdata/api.py +++ b/dataretrieval/waterdata/api.py @@ -29,7 +29,6 @@ get_stats_date_range, get_stats_por, ) -from dataretrieval.waterdata.utils import get_ogc_data as _get_ogc_data __all__ = [ "get_channel", @@ -53,13 +52,6 @@ "get_time_series_metadata", ] -# Preserve the documented legacy implementation path for introspection and -# Sphinx while the function objects live in cohesive family modules. -for _name in __all__: - globals()[_name].__module__ = __name__ -del _name - # Private compatibility names used by existing callers and patch targets. _SAMPLES_PARAM_TO_API = _samples._SAMPLES_PARAM_TO_API _SAMPLES_LEGACY_KWARGS = _samples._SAMPLES_LEGACY_KWARGS -get_ogc_data = _get_ogc_data diff --git a/tests/architecture_test.py b/tests/architecture_test.py index 51fabb50..cc2a0466 100644 --- a/tests/architecture_test.py +++ b/tests/architecture_test.py @@ -3,6 +3,7 @@ from __future__ import annotations import ast +import functools import sys from pathlib import Path @@ -29,7 +30,6 @@ "_NO_NORMALIZE_PARAMS", "_as_str_list", "_check_monitoring_location_id", - "_check_ogc_requests", "_construct_api_requests", "_construct_cql_request", "_cql2_param", @@ -38,7 +38,6 @@ "_normalize_str_iterable", "_ogc_base_url", "_ogc_query_params", - "_row_cap", "_switch_arg_id", "_switch_properties_id", "prepare_request_args", @@ -103,6 +102,7 @@ def visit_ImportFrom(self, node: ast.ImportFrom) -> None: self.modules.update(_resolve_from(self.current_module, self.path, node)) +@functools.cache def _runtime_imports(path: Path) -> set[str]: module = _module_name(path) visitor = _RuntimeImportVisitor(module, path) @@ -110,6 +110,9 @@ def _runtime_imports(path: Path) -> set[str]: return visitor.modules +# Both are pure over an unchanging tree and are called from a dozen places; +# without caching the suite re-parses every package file on each call. +@functools.cache def _package_import_graph() -> dict[str, set[str]]: return { _module_name(path): _runtime_imports(path) @@ -117,6 +120,17 @@ def _package_import_graph() -> dict[str, set[str]]: } +def _literal_exports(path: Path) -> set[str]: + tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path)) + for node in tree.body: + if isinstance(node, ast.Assign) and any( + isinstance(target, ast.Name) and target.id == "__all__" + for target in node.targets + ): + return set(ast.literal_eval(node.value)) + raise AssertionError(f"{path.relative_to(PACKAGE_ROOT.parent)} has no __all__") + + def test_exceptions_has_no_runtime_third_party_dependency() -> None: """The shared error-policy leaf must remain cheap and cycle-safe.""" imports = _runtime_imports(PACKAGE_ROOT / "exceptions.py") @@ -291,24 +305,12 @@ def test_waterdata_utils_is_not_an_ogc_reexport_hub() -> None: "dataretrieval.ogc.shaping", }, f"Water Data utils crossed its intended OGC seam: {ogc_deps}" - tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path)) - exports: set[str] | None = None - for node in tree.body: - if not isinstance(node, ast.Assign): - continue - if any( - isinstance(target, ast.Name) and target.id == "__all__" - for target in node.targets - ): - exports = set(ast.literal_eval(node.value)) - break - assert exports is not None, "waterdata.utils must declare its intentional exports" + exports = _literal_exports(path) old_reexports = { "GEOPANDAS", "_as_str_list", "_check_monitoring_location_id", - "_check_ogc_requests", "_construct_api_requests", "_construct_cql_request", "_default_headers", @@ -535,27 +537,6 @@ def visit(module: str, path: tuple[str, ...]) -> None: "wqx3_url", "WQP_Metadata", }, - "waterdata/api.py": { - "get_channel", - "get_codes", - "get_combined_metadata", - "get_continuous", - "get_cql", - "get_daily", - "get_field_measurements", - "get_field_measurements_metadata", - "get_latest_continuous", - "get_latest_daily", - "get_monitoring_locations", - "get_peaks", - "get_queryables", - "get_reference_table", - "get_samples", - "get_samples_summary", - "get_stats_date_range", - "get_stats_por", - "get_time_series_metadata", - }, "waterdata/time_series.py": { "get_daily", "get_continuous", @@ -592,15 +573,15 @@ def visit(module: str, path: tuple[str, ...]) -> None: } -def _literal_exports(path: Path) -> set[str]: - tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path)) - for node in tree.body: - if isinstance(node, ast.Assign) and any( - isinstance(target, ast.Name) and target.id == "__all__" - for target in node.targets - ): - return set(ast.literal_eval(node.value)) - raise AssertionError(f"{path.relative_to(PACKAGE_ROOT.parent)} has no __all__") +# The six collection-family modules the ``waterdata.api`` facade re-exports. +_WATERDATA_FAMILIES = ( + "waterdata/time_series.py", + "waterdata/metadata.py", + "waterdata/measurements.py", + "waterdata/reference.py", + "waterdata/samples.py", + "waterdata/cql.py", +) def test_active_service_exports_are_explicit_and_stable() -> None: @@ -608,6 +589,17 @@ def test_active_service_exports_are_explicit_and_stable() -> None: assert _literal_exports(PACKAGE_ROOT / relative) == expected, relative +def test_api_facade_exports_exactly_the_family_union() -> None: + """The facade re-exports every family getter and invents none of its own. + + Derived rather than frozen: a hardcoded copy of the union is 19 more strings + to edit per new getter, and it would still pass if a family gained an export + the facade forgot to re-export -- the one thing worth catching here. + """ + families = set().union(*(_EXPECTED_MODULE_EXPORTS[f] for f in _WATERDATA_FAMILIES)) + assert _literal_exports(PACKAGE_ROOT / "waterdata/api.py") == families + + def test_waterdata_api_is_a_logic_free_compatibility_facade() -> None: path = PACKAGE_ROOT / "waterdata" / "api.py" tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path)) @@ -620,19 +612,27 @@ def test_waterdata_api_is_a_logic_free_compatibility_facade() -> None: def test_waterdata_collection_families_do_not_import_each_other() -> None: + """Families share through Water Data policy, OGC, and transport -- not laterally. + + A "family" is a module the ``waterdata.api`` facade re-exports from, so the + set comes from :data:`_WATERDATA_FAMILIES` rather than being restated. That + list is self-enforcing: a seventh family the facade re-exports must be added + to ``_EXPECTED_MODULE_EXPORTS`` or ``test_api_facade_exports_exactly_the_ + family_union`` fails, and it lands here on the same edit. + + Composed getters like ``nearest`` are deliberately outside it. They are not + peers of a family; ``get_nearest_continuous`` builds on ``get_continuous``, + which is ordinary layering rather than a lateral reach. + """ + graph = _package_import_graph() families = { - "dataretrieval.waterdata.time_series", - "dataretrieval.waterdata.metadata", - "dataretrieval.waterdata.measurements", - "dataretrieval.waterdata.reference", - "dataretrieval.waterdata.samples", - "dataretrieval.waterdata.cql", + "dataretrieval." + relative.removesuffix(".py").replace("/", ".") + for relative in _WATERDATA_FAMILIES } violations = [] - graph = _package_import_graph() - for module in families: + for module in sorted(families): for dependency in graph[module]: - if dependency in families: + if dependency in families and dependency != module: violations.append(f"{module} -> {dependency}") assert not violations, "Lateral collection-family imports:\n" + "\n".join( violations @@ -640,36 +640,25 @@ def test_waterdata_collection_families_do_not_import_each_other() -> None: def test_service_adapters_do_not_reach_through_each_other() -> None: - adapters = { - "dataretrieval.ngwmn", - "dataretrieval.nldi", - "dataretrieval.streamstats", - "dataretrieval.waterdata", - "dataretrieval.wateruse", - "dataretrieval.wqp", - } - violations: list[str] = [] - for module, imports in _package_import_graph().items(): - owner = next( - ( - adapter - for adapter in adapters - if module == adapter or module.startswith(adapter + ".") - ), + # Every active service; NWIS is excluded because its deprecation is governed + # by its own fitness function. + adapters = set(_SERVICE_PREFIXES) - {"dataretrieval.nwis"} + + def owner(name: str) -> str | None: + """The adapter ``name`` belongs to, or None if it is shared code.""" + return next( + (a for a in adapters if name == a or name.startswith(a + ".")), None, ) - if owner is None: + + violations: list[str] = [] + for module, imports in _package_import_graph().items(): + source = owner(module) + if source is None: continue for dependency in imports: - target = next( - ( - adapter - for adapter in adapters - if dependency == adapter or dependency.startswith(adapter + ".") - ), - None, - ) - if target is not None and target != owner: + target = owner(dependency) + if target is not None and target != source: violations.append(f"{module} -> {dependency}") assert not violations, "Adapter-to-adapter imports:\n" + "\n".join( sorted(set(violations)) @@ -677,6 +666,19 @@ def test_service_adapters_do_not_reach_through_each_other() -> None: def test_ogc_request_construction_does_not_execute_http() -> None: + """Building a request may borrow header policy, never the executing calls. + + Two assertions, because the first alone was once true while the rule was + broken: ``requests.py`` imported ``ogc.schema`` purely to forward a name, + and ``ogc.schema`` calls ``transport.http.get`` -- so constructing a request + dragged in the executing path with this test still green. + + The edge is named explicitly rather than checked over the transitive graph. + A closure from ``ogc.requests`` reaches the whole package, because + ``transport.retry`` imports ``dataretrieval`` for the progress reporter and + the package ``__init__`` imports every service; a rule stated that way would + be either vacuous or a list of exceptions. + """ path = PACKAGE_ROOT / "ogc" / "requests.py" tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path)) transport_names = { @@ -686,7 +688,21 @@ def test_ogc_request_construction_does_not_execute_http() -> None: and node.module == "dataretrieval.transport.http" for alias in node.names } - assert transport_names == {"default_headers"} + assert transport_names == {"default_headers"}, ( + f"ogc.requests imports executing transport helpers: {sorted(transport_names)}" + ) + assert "dataretrieval.ogc.schema" not in _runtime_imports(path), ( + "ogc.requests imports ogc.schema, which executes HTTP; import the schema " + "helper from ogc.schema at its point of use instead of forwarding it here" + ) + + +def test_empty_result_shaping_consults_the_schema_endpoint() -> None: + """``_deal_with_empty`` names columns from the collection schema. + + That is a real network call on an empty result, so the dependency is worth + pinning deliberately rather than leaving it to be removed as dead weight. + """ assert "dataretrieval.ogc.schema" in _runtime_imports( PACKAGE_ROOT / "ogc" / "shaping.py" ) diff --git a/tests/contracts/public_api_test.py b/tests/contracts/public_api_test.py index d1d26d31..4ff6741a 100644 --- a/tests/contracts/public_api_test.py +++ b/tests/contracts/public_api_test.py @@ -39,28 +39,6 @@ "get_time_series_metadata", ] -_EXPECTED_API_NAMES = [ - "get_channel", - "get_codes", - "get_combined_metadata", - "get_continuous", - "get_cql", - "get_daily", - "get_field_measurements", - "get_field_measurements_metadata", - "get_latest_continuous", - "get_latest_daily", - "get_monitoring_locations", - "get_peaks", - "get_queryables", - "get_reference_table", - "get_samples", - "get_samples_summary", - "get_stats_date_range", - "get_stats_por", - "get_time_series_metadata", -] - _EXPECTED_SIGNATURES = { "get_channel": "(monitoring_location_id: 'str | Iterable[str] | None' = None, field_visit_id: 'str | Iterable[str] | " "None' = None, measurement_number: 'str | Iterable[str] | None' = None, time: 'str | Iterable[str] | " @@ -290,7 +268,9 @@ def test_waterdata_exports_are_stable() -> None: assert waterdata.__all__ == _EXPECTED_WATERDATA_ALL - assert api.__all__ == _EXPECTED_API_NAMES + # Derived from the signature snapshot below: one list to keep current, + # not two that can disagree about what the facade exports. + assert api.__all__ == list(_EXPECTED_SIGNATURES) assert all(hasattr(waterdata, name) for name in waterdata.__all__) @@ -300,10 +280,8 @@ def test_api_facade_preserves_function_contracts() -> None: facade_function = getattr(api, name) assert package_function is facade_function assert str(inspect.signature(facade_function)) == expected_signature - assert facade_function.__module__ == "dataretrieval.waterdata.api" def test_api_private_samples_compatibility_names_remain() -> None: assert isinstance(api._SAMPLES_PARAM_TO_API, dict) assert isinstance(api._SAMPLES_LEGACY_KWARGS, dict) - assert callable(api.get_ogc_data) From 022284397316775db94156da8f8dbcc6c5ae2db3 Mon Sep 17 00:00:00 2001 From: thodson-usgs Date: Thu, 6 Aug 2026 14:57:08 -0500 Subject: [PATCH 3/4] test(architecture): check statement kinds in the facade, not just definitions The guard certifying waterdata/api.py "logic-free" scanned its body for def and class only. A module-level for loop lived there unremarked, rewriting every re-exported getter's __module__ -- code owned by the family modules, mutated from a file the test called free of implementation. It was removed in the preceding cleanup; this is what would have caught it. A facade's whole legitimate vocabulary is a docstring, imports, and assignments. Anything else is a statement that runs at import. Verified by reintroducing the loop: the guard fires. Co-Authored-By: Claude Opus 5 --- tests/architecture_test.py | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/tests/architecture_test.py b/tests/architecture_test.py index cc2a0466..f59ae45d 100644 --- a/tests/architecture_test.py +++ b/tests/architecture_test.py @@ -601,14 +601,26 @@ def test_api_facade_exports_exactly_the_family_union() -> None: def test_waterdata_api_is_a_logic_free_compatibility_facade() -> None: + """The facade re-exports; it does not run anything. + + Statement *kinds* are checked, not just ``def``/``class``. Scanning for + definitions alone let a module-level ``for`` loop live here that rewrote + every re-exported getter's ``__module__`` -- code owned by the family + modules, mutated from a file certified "logic-free". A docstring, imports, + and plain assignments are the whole legitimate vocabulary of a facade. + """ path = PACKAGE_ROOT / "waterdata" / "api.py" tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path)) - definitions = [ - node.name + allowed = (ast.Import, ast.ImportFrom, ast.Assign, ast.AnnAssign) + offenders = [ + f"line {node.lineno}: {type(node).__name__}" for node in tree.body - if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)) + if not isinstance(node, allowed) + and not ( + isinstance(node, ast.Expr) and isinstance(node.value, ast.Constant) + ) # the module docstring ] - assert not definitions, f"waterdata.api contains implementation: {definitions}" + assert not offenders, f"waterdata.api contains implementation: {offenders}" def test_waterdata_collection_families_do_not_import_each_other() -> None: From 804d809f278bef091c520c0387df7cfc1ad6f475 Mon Sep 17 00:00:00 2001 From: thodson-usgs Date: Thu, 6 Aug 2026 15:07:06 -0500 Subject: [PATCH 4/4] docs(waterdata): lead each getter with what it returns, not what the data is Most getter docstrings opened by defining the dataset -- "Daily data provide one data value to represent water conditions for the day" -- which tells a reader what daily data is but not what calling the function gives them. Sphinx summaries and IDE tooltips both show that first line alone, so the one place a summary is guaranteed to be read was spending itself on background. Each now opens with the call's result and keeps the domain context as the paragraph after it. The USGS background is the valuable part of these docstrings and is preserved; it just no longer occupies the summary slot. get_latest_daily had get_daily's summary verbatim, so the one thing that distinguishes it -- returning only the most recent value -- was documented nowhere. It and get_latest_continuous now say so and point at the full-history getter, which is the choice a reader is actually making. get_samples loses "This is a wrapper function for the Samples database API": callers cannot act on that. Its advice against over-filtering stays, since that one has a consequence users feel. The six family modules had one-line docstrings naming what they contain, which the filename already did. Each now says what unites the family and which neighbor to reach for instead -- discovery in metadata before observations in time_series, cql as the escape hatch when a typed getter cannot express the query. That navigation is the payoff of splitting the module, and it was the part left unwritten. Parameter sections are deliberately untouched. They are the public contract for getters taking up to nineteen arguments, and they are reference material a caller reads while writing the call -- not implementation detail. Also corrects ADR 0007, which still claimed the facade preserves the legacy __module__ value. It no longer does -- that loop was removed -- and each function now reports the family module defining it. Co-Authored-By: Claude Opus 5 --- dataretrieval/ogc/context.py | 9 ++- dataretrieval/ogc/schema.py | 7 ++- dataretrieval/waterdata/api.py | 9 ++- dataretrieval/waterdata/cql.py | 8 ++- dataretrieval/waterdata/measurements.py | 14 ++++- dataretrieval/waterdata/metadata.py | 18 +++++- dataretrieval/waterdata/reference.py | 8 ++- dataretrieval/waterdata/samples.py | 21 ++++--- dataretrieval/waterdata/time_series.py | 56 ++++++++++++------- .../decisions/0007-adapter-facades.rst | 6 +- 10 files changed, 115 insertions(+), 41 deletions(-) diff --git a/dataretrieval/ogc/context.py b/dataretrieval/ogc/context.py index 0dea357a..c97f64d6 100644 --- a/dataretrieval/ogc/context.py +++ b/dataretrieval/ogc/context.py @@ -1,4 +1,11 @@ -"""Ambient per-call OGC request context.""" +"""Per-call OGC request state, scoped rather than passed. + +The base URL, dialect, and row cap apply to a whole call but are read deep +inside request construction, several frames below whoever set them. Threading +them through every signature would put protocol plumbing in the getters, so they +travel as context variables -- which also makes them safe under the concurrent +fan-out, where a thread-global would not be. +""" from dataretrieval.ogc.policy import DEFAULT_DIALECT, OGC_API_URL, OgcDialect from dataretrieval.utils import Ambient diff --git a/dataretrieval/ogc/schema.py b/dataretrieval/ogc/schema.py index 4bab6956..a32114e6 100644 --- a/dataretrieval/ogc/schema.py +++ b/dataretrieval/ogc/schema.py @@ -1,4 +1,9 @@ -"""OGC queryables and schema retrieval.""" +"""Asking an OGC service to describe itself. + +Queryables and collection schemas: which properties a collection accepts, and +what columns it returns. Separate from request construction because answering +these questions means *issuing* a request, and building one must not. +""" from __future__ import annotations diff --git a/dataretrieval/waterdata/api.py b/dataretrieval/waterdata/api.py index d4afbcab..c6acbb9b 100644 --- a/dataretrieval/waterdata/api.py +++ b/dataretrieval/waterdata/api.py @@ -1,4 +1,11 @@ -"""Backward-compatible facade for Water Data collection-family adapters.""" +"""Compatibility facade: the import path the collection getters used to have. + +Every getter here is defined in a collection-family module and re-exported +unchanged. The path is kept because it is published, and this file exists only +to preserve it -- it holds no logic, and a test enforces that. + +Import from :mod:`dataretrieval.waterdata` instead. +""" from __future__ import annotations diff --git a/dataretrieval/waterdata/cql.py b/dataretrieval/waterdata/cql.py index 1134ca86..c4bee8ac 100644 --- a/dataretrieval/waterdata/cql.py +++ b/dataretrieval/waterdata/cql.py @@ -1,4 +1,10 @@ -"""Generalized CQL2 request adapter for Water Data collections.""" +"""One getter for queries the typed getters cannot express. + +The other families expose a fixed argument per filter, which covers the common +cases and keeps them discoverable. This is the escape hatch: an arbitrary CQL2 +filter against any collection, for the query nobody anticipated. Prefer a typed +getter when one fits -- it validates more and reads better. +""" from __future__ import annotations diff --git a/dataretrieval/waterdata/measurements.py b/dataretrieval/waterdata/measurements.py index ac8a1d54..18640b0c 100644 --- a/dataretrieval/waterdata/measurements.py +++ b/dataretrieval/waterdata/measurements.py @@ -1,4 +1,9 @@ -"""Discrete field, peak, and channel measurement getters.""" +"""Getters for values measured in person rather than by a sensor. + +Field measurements, annual peaks, and channel geometry. These are collected +during site visits, at low frequency and with delivery lag, which is why they +are grouped apart from the continuous record they help calibrate. +""" from __future__ import annotations @@ -39,7 +44,9 @@ def get_field_measurements( max_rows: int | None = None, **queryables: Any, ) -> tuple[pd.DataFrame, BaseMetadata]: - """Field measurements are physically measured values collected during a + """Get discrete measurements collected in person during a site visit. + + Field measurements are physically measured values collected during a visit to the monitoring location. Field measurements consist of measurements of gage height and discharge, and readings of groundwater levels, and are primarily used as calibration readings for the automated sensors collecting @@ -401,7 +408,8 @@ def get_channel( max_rows: int | None = None, **queryables: Any, ) -> tuple[pd.DataFrame, BaseMetadata]: - """ + """Get channel-geometry measurements recorded during streamflow field visits. + Channel measurements taken as part of streamflow field measurements. Parameters diff --git a/dataretrieval/waterdata/metadata.py b/dataretrieval/waterdata/metadata.py index 4c288f1a..0f10f2ae 100644 --- a/dataretrieval/waterdata/metadata.py +++ b/dataretrieval/waterdata/metadata.py @@ -1,4 +1,10 @@ -"""Monitoring-location and data-inventory metadata getters.""" +"""Getters that answer "what data exists?" rather than returning it. + +The monitoring-location catalog, the time-series inventory, and the joins over +them. These are the discovery step: narrow down which locations and parameters +are worth requesting before pulling observations from +:mod:`~dataretrieval.waterdata.time_series`. +""" from __future__ import annotations @@ -68,7 +74,9 @@ def get_monitoring_locations( max_rows: int | None = None, **queryables: Any, ) -> tuple[pd.DataFrame, BaseMetadata]: - """Location information is basic information about the monitoring location + """Get the catalog of monitoring locations and their attributes. + + Location information is basic information about the monitoring location including the name, identifier, agency responsible for data collection, and the date the location was established. It also includes information about the type of location, such as stream, lake, or groundwater, and geographic @@ -372,7 +380,11 @@ def get_time_series_metadata( max_rows: int | None = None, **queryables: Any, ) -> tuple[pd.DataFrame, BaseMetadata]: - """Daily data and continuous measurements are grouped into time series, + """Get metadata describing the time series available at a location. + + Use this to discover what a location measures before requesting the + observations themselves. Daily data and continuous measurements are + grouped into time series, which represent a collection of observations of a single parameter, potentially aggregated using a standard statistic, at a single monitoring location. This endpoint provides metadata about those time series, diff --git a/dataretrieval/waterdata/reference.py b/dataretrieval/waterdata/reference.py index ce95a155..b19b31bc 100644 --- a/dataretrieval/waterdata/reference.py +++ b/dataretrieval/waterdata/reference.py @@ -1,4 +1,10 @@ -"""Reference-table and queryables discovery getters.""" +"""Getters for the API's own vocabularies. + +Reference tables and per-collection queryables -- the parameter codes, statistic +codes, and filterable properties the other getters accept. These describe the +service rather than the water, so they are the one family whose results are +mostly stable between calls. +""" from __future__ import annotations diff --git a/dataretrieval/waterdata/samples.py b/dataretrieval/waterdata/samples.py index aaba15cf..bda8d363 100644 --- a/dataretrieval/waterdata/samples.py +++ b/dataretrieval/waterdata/samples.py @@ -1,4 +1,10 @@ -"""Aquarius Samples API getters and wire-parameter policy.""" +"""Getters for the Aquarius Samples API, and its wire-parameter policy. + +Discrete water-quality results, which come from a different upstream service +than the rest of Water Data -- with its own parameter spellings and its own +error envelope. The translation between this package's argument names and that +service's wire names lives here, next to the getters that need it. +""" from __future__ import annotations @@ -165,13 +171,12 @@ def get_samples( project_id: str | Iterable[str] | None = None, record_identifier_user_supplied: str | Iterable[str] | None = None, ) -> tuple[pd.DataFrame, BaseMetadata]: - """Search Samples database for USGS water quality data. - This is a wrapper function for the Samples database API. All potential - filters are provided as arguments to the function, but please do not - populate all possible filters; leave as many as feasible with their default - value (None). This is important because overcomplicated web service queries - can bog down the database's ability to return an applicable dataset before - it times out. + """Search the USGS Samples database for discrete water-quality results. + + Every available filter is exposed as an argument, but leave as many as + feasible at their default of ``None``. An overcomplicated query can bog + down the database's ability to assemble a result before it times out, so + filtering narrowly is faster than filtering exhaustively. The web GUI for the Samples database can be found here: https://waterdata.usgs.gov/download-samples/#dataProfile=site diff --git a/dataretrieval/waterdata/time_series.py b/dataretrieval/waterdata/time_series.py index b5fb78a9..819e73a6 100644 --- a/dataretrieval/waterdata/time_series.py +++ b/dataretrieval/waterdata/time_series.py @@ -1,4 +1,13 @@ -"""Time-series observation and statistics getters.""" +"""Getters for observations that form a time series. + +Daily and continuous values, their most-recent counterparts, and the +period-of-record statistics computed over them. What unites them is shape: a +monitoring location and a parameter, repeated over time. + +Metadata *about* these series -- what a location measures, over what period -- +lives in :mod:`~dataretrieval.waterdata.metadata`, so a caller can discover what +exists before asking for the observations. +""" from __future__ import annotations @@ -39,7 +48,9 @@ def get_daily( max_rows: int | None = None, **queryables: Any, ) -> tuple[pd.DataFrame, BaseMetadata]: - """Daily data provide one data value to represent water conditions for the + """Get daily values: one value per monitoring location, parameter, and day. + + Daily data provide one data value to represent water conditions for the day. Throughout much of the history of the USGS, the primary water data available @@ -274,7 +285,8 @@ def get_continuous( max_rows: int | None = None, **queryables: Any, ) -> tuple[pd.DataFrame, BaseMetadata]: - """ + """Get continuous sensor observations, typically at a 15-minute interval. + Continuous data provide instantaneous water conditions. This is an early version of the continuous endpoint that is feature-complete @@ -483,8 +495,10 @@ def get_latest_continuous( max_rows: int | None = None, **queryables: Any, ) -> tuple[pd.DataFrame, BaseMetadata]: - """This endpoint provides the most recent observation for each time series - of continuous data. Continuous data are collected via automated sensors + """Get only the most recent observation of each continuous time series. + + Use this for a current-conditions view; use :func:`get_continuous` for a + history. Continuous data are collected via automated sensors installed at a monitoring location. They are collected at a high frequency and often at a fixed 15-minute interval. Depending on the specific monitoring location, the data may be transmitted automatically via telemetry and be @@ -698,8 +712,11 @@ def get_latest_daily( max_rows: int | None = None, **queryables: Any, ) -> tuple[pd.DataFrame, BaseMetadata]: - """Daily data provide one data value to represent water conditions for the - day. + """Get only the most recent daily value of each time series. + + Use this for a current-conditions view; use :func:`get_daily` for a + history. Daily data provide one data value to represent water conditions + for the day. Throughout much of the history of the USGS, the primary water data available was daily data collected manually at the monitoring location once each day. @@ -910,13 +927,12 @@ def get_stats_por( normal_type: str | None = None, expand_percentiles: bool = True, ) -> tuple[pd.DataFrame, BaseMetadata]: - """Get day-of-year and month-of-year water data statistics from the - USGS Water Data API. - This service (called the "observationNormals" endpoint on api.waterdata.usgs.gov) - provides endpoints for access to computations on the historical record regarding - water conditions, including minimum, maximum, mean, median, and percentiles for - day of year and month of year. For more information regarding the calculation of - statistics and other details, please visit the Statistics documentation page: + """Get day-of-year and month-of-year statistics over the historical record. + + Answers "how does today compare to a normal day here?" -- minimum, maximum, + mean, median, and percentiles computed per day of year and month of year + (the ``observationNormals`` endpoint). For more on how these statistics are + calculated, see the Statistics documentation page: https://waterdata.usgs.gov/statistics-documentation/. Note: This API is under active beta development and subject to @@ -1054,12 +1070,12 @@ def get_stats_date_range( interval_type: str | Iterable[str] | None = None, expand_percentiles: bool = True, ) -> tuple[pd.DataFrame, BaseMetadata]: - """Get monthly and annual water data statistics from the USGS Water Data API. - This service (called the "observationIntervals" endpoint on api.waterdata.usgs.gov) - provides endpoints for access to computations on the historical record regarding - water conditions, including minimum, maximum, mean, median, and percentiles for - month-year, and water/calendar years. For more information regarding the calculation - of statistics and other details, please visit the Statistics documentation page: + """Get statistics summarizing whole months and years of the record. + + Answers "how did this month or year compare to others?" -- minimum, maximum, + mean, median, and percentiles per month-year and per water or calendar year + (the ``observationIntervals`` endpoint). For more on how these statistics are + calculated, see the Statistics documentation page: https://waterdata.usgs.gov/statistics-documentation/. Note: This API is under active beta development and subject to diff --git a/docs/source/architecture/decisions/0007-adapter-facades.rst b/docs/source/architecture/decisions/0007-adapter-facades.rst index 2244f09c..d4cc90cf 100644 --- a/docs/source/architecture/decisions/0007-adapter-facades.rst +++ b/docs/source/architecture/decisions/0007-adapter-facades.rst @@ -27,8 +27,10 @@ selection, Statistics API execution, shared Water Data policy, and type vocabularies. The facade re-exports the established functions and preserves their signatures, -identity at ``dataretrieval.waterdata``, legacy ``__module__`` value, and private -Samples constants used by compatibility tests. Collection-family modules do not +their identity at ``dataretrieval.waterdata``, and the private Samples constants +compatibility tests rely on. It does not rewrite their ``__module__``: each +function reports the family module that defines it, so a traceback names a file +that contains code. Collection-family modules do not import one another; shared behavior belongs in Water Data policy, OGC, or transport modules.