From 873e12642b9fd86926a01b719b15b61681bdb3ba Mon Sep 17 00:00:00 2001 From: thodson-usgs Date: Sat, 15 Aug 2026 11:14:55 -0500 Subject: [PATCH] feat(config)!: resolve settings through a layered chain Every tunable setting -- the Water Data API key, fan-out concurrency, retries, the stall timeout, the progress line -- now resolves through one ordered chain instead of the environment alone: a `dataretrieval.configure(...)` block, then the setting's `API_USGS_*` variable, then `~/.dataretrieval/config.toml` (or the path in `DATARETRIEVAL_CONFIG`), then the built-in default. Precedence applies per setting, so a file that sets only `concurrency` leaves an environment `API_USGS_PAT` in effect. The public surface is `dataretrieval.configuration`, over a private `_configuration_core` holding the model, grammar, and file caches: - `configure()` takes configuration objects positionally, at most one per adapter. Each adapter owns its class in the module that reads the settings (`WaterdataConfiguration`, `NgwmnConfiguration`, `NwdcConfiguration`, `WqpConfiguration`, `NldiConfiguration`, `StreamstatsConfiguration`) and accepts only the settings it reads, so `[streamstats] parallel_chunks = 8` is an error rather than a line that quietly does nothing. The block is delivered through a ContextVar (the shared `Ambient` leaf), so a credential set inside it cannot leak across threads or asyncio tasks (#352). - The file gains named profiles beside each adapter's default profile: `[waterdata]` is always in effect, `[waterdata.bulk]` only when a caller selects it with `WaterdataConfiguration.load("bulk")`. A selected profile still inherits per setting, and a profile named in code outranks the environment -- the one deliberate inversion of the ladder. - An adapter's configuration may carry a `base_url` that redirects that adapter's requests for the duration of the block; the file and the environment refuse it. Water Data acquires endpoints through `waterdata.endpoints`, so one value moves the OGC collections, Samples, Statistics, and the STAC catalog together. - `show_configuration()` reports each setting's effective value and exact source -- naming the profile behind a value, listing the profiles a file defines, and naming unimported adapters -- without ever printing the key. - One parser per setting owns its grammar and one roster its type policy, so a value means the same thing whichever source wrote it. Breaking / behavior changes (details in NEWS.md): - `RetryPolicy.from_env()` is now `RetryPolicy.from_configuration()` and resolves through the whole chain; `transport/env.py` is retired. - A credential-shaped keyword in a getter's `**kwargs` passthrough (`api_key=`, `token=`, ...) raises `TypeError` naming `configure()` instead of putting a secret in a URL. - `API_USGS_STALL_TIMEOUT` now resolves through the chain like every other setting; it was previously read straight from `os.environ` and invisible to blocks, the file, and `show_configuration()`. `dataretrieval.wateruse` is renamed `dataretrieval.nwdc`, after the service, like every other adapter. The old name remains as a forwarding alias emitting a dated `DeprecationWarning` (removal on or after 2027-08-11, recorded in `_deprecation.REMOVALS`). Rationale in ADRs 0009-0011; vocabulary in CONTEXT.md; user guide at docs/source/userguide/configuration.rst. Co-Authored-By: Claude Fable 5 --- .gitignore | 8 + .importlinter | 30 +- AGENTS.md | 2 +- CONTEXT.md | 72 +- CONTRIBUTING.md | 7 +- NEWS.md | 4 + README.md | 69 +- dataretrieval/__init__.py | 30 +- dataretrieval/_configuration_core.py | 1405 ++++++++++++ dataretrieval/_deprecation.py | 1 + dataretrieval/_querying.py | 19 +- dataretrieval/configuration.py | 767 +++++++ dataretrieval/credentials.py | 102 +- dataretrieval/exceptions.py | 40 +- dataretrieval/ngwmn.py | 65 +- dataretrieval/nldi.py | 81 +- dataretrieval/nwdc.py | 505 ++++ dataretrieval/nwis.py | 6 +- dataretrieval/ogc/chunking.py | 68 +- dataretrieval/ogc/engine.py | 14 +- dataretrieval/progress.py | 9 +- dataretrieval/streamstats.py | 71 +- dataretrieval/transport/env.py | 57 - dataretrieval/transport/fanout.py | 107 +- dataretrieval/transport/http.py | 18 +- dataretrieval/transport/pagination.py | 12 +- dataretrieval/transport/retry.py | 47 +- dataretrieval/waterdata/__init__.py | 2 + dataretrieval/waterdata/configuration.py | 69 + dataretrieval/waterdata/endpoints.py | 64 +- dataretrieval/waterdata/ratings.py | 10 +- dataretrieval/waterdata/reference.py | 12 +- dataretrieval/waterdata/samples.py | 8 +- dataretrieval/waterdata/stats.py | 5 +- dataretrieval/waterdata/utils.py | 35 +- dataretrieval/wateruse.py | 475 +--- dataretrieval/wqp.py | 90 +- demos/USGS_WaterUse_Examples.ipynb | 6 +- .../decisions/0008-fan-out-execution.rst | 4 +- .../decisions/0009-layered-configuration.rst | 194 ++ .../0010-adapter-scoped-settings.rst | 286 +++ .../decisions/0011-configuration-profiles.rst | 241 ++ docs/source/architecture/decisions/index.rst | 3 + docs/source/architecture/index.rst | 46 +- docs/source/reference/config.rst | 18 + docs/source/reference/index.rst | 3 +- docs/source/reference/nwdc.rst | 12 + docs/source/reference/wateruse.rst | 7 - docs/source/userguide/configuration.rst | 598 +++++ docs/source/userguide/errors.rst | 2 +- docs/source/userguide/index.rst | 1 + pyproject.toml | 4 + tests/architecture_test.py | 53 +- tests/configuration_test.py | 2026 +++++++++++++++++ tests/conftest.py | 10 +- tests/contracts/README.md | 2 +- tests/contracts/public_api_test.py | 1 + tests/ngwmn_test.py | 34 +- tests/nldi_test.py | 35 +- tests/{wateruse_test.py => nwdc_test.py} | 160 +- tests/transport_test.py | 6 +- tests/waterdata_chunking_test.py | 118 +- tests/waterdata_utils_test.py | 22 + tests/wqp_test.py | 49 + 64 files changed, 7508 insertions(+), 819 deletions(-) create mode 100644 dataretrieval/_configuration_core.py create mode 100644 dataretrieval/configuration.py create mode 100644 dataretrieval/nwdc.py delete mode 100644 dataretrieval/transport/env.py create mode 100644 dataretrieval/waterdata/configuration.py create mode 100644 docs/source/architecture/decisions/0009-layered-configuration.rst create mode 100644 docs/source/architecture/decisions/0010-adapter-scoped-settings.rst create mode 100644 docs/source/architecture/decisions/0011-configuration-profiles.rst create mode 100644 docs/source/reference/config.rst create mode 100644 docs/source/reference/nwdc.rst delete mode 100644 docs/source/reference/wateruse.rst create mode 100644 docs/source/userguide/configuration.rst create mode 100644 tests/configuration_test.py rename tests/{wateruse_test.py => nwdc_test.py} (83%) diff --git a/.gitignore b/.gitignore index 3dfbb55fa..d3b9beeaf 100644 --- a/.gitignore +++ b/.gitignore @@ -118,3 +118,11 @@ ENV/ # pyscn analysis reports (rebuildable: `pyscn analyze dataretrieval`) .pyscn/ + +# Working design note for the layered-configuration work; the durable +# record is ADR 0009 + docs/source/userguide/configuration.rst. +CONFIG-PLAN.md + +# Resolver lock for local dev; the package ships a range-based pyproject and +# is not deployed from a pinned set. +uv.lock diff --git a/.importlinter b/.importlinter index 0b993b9cf..18812ecf6 100644 --- a/.importlinter +++ b/.importlinter @@ -21,7 +21,11 @@ type = layers containers = dataretrieval layers = - ngwmn | nldi | nwis | streamstats | waterdata | wateruse | wqp +; The deprecated ``wateruse`` alias re-exports ``nwdc``, so it sits above the +; adapters rather than beside them. A compatibility facade may depend on the +; adapter it forwards to; nothing may depend on the facade. + wateruse + ngwmn | nldi | nwdc | nwis | streamstats | waterdata | wqp ogc utils _querying @@ -30,19 +34,32 @@ layers = ; Response-format conventions sit above the pure leaves because they read the ; code tables, and below every adapter that shapes a response with them. _wqx - _ambient | _response_metadata | codes | combining | interruptions | rdb + _response_metadata | codes | combining | interruptions | rdb credentials + configuration + _configuration_core exceptions -; Pure advisory and argument-validation mechanisms sit together at the floor: -; neither imports first-party code, so every layer can use them without -; reaching sideways or creating an artificial dependency between the two. - _deprecation | _validation +; Pure dependency-free mechanisms sit together at the floor: none imports +; first-party code, so every layer can use them without reaching sideways or +; creating artificial dependencies among them. + _ambient | _deprecation | _validation ; Every top-level module must be placed in the stack deliberately. A new ; top-level module fails this contract until someone decides where it sits. exhaustive = True exhaustive_ignores = _version +[importlinter:contract:configuration-core] +name = Only the configuration facade may import its private core +; Keep the extracted foundation private: adapters and other package modules +; continue to depend on dataretrieval.configuration, preserving all existing +; import paths and preventing parallel configuration interfaces. +type = protected +protected_modules = + dataretrieval._configuration_core +allowed_importers = + dataretrieval.configuration + [importlinter:contract:ogc-consumers] name = Only NGWMN and Water Data consume the OGC subsystem (ADR 0003) type = protected @@ -123,6 +140,7 @@ source_modules = dataretrieval.streamstats dataretrieval.transport dataretrieval.utils + dataretrieval.nwdc dataretrieval.waterdata dataretrieval.wateruse dataretrieval.wqp diff --git a/AGENTS.md b/AGENTS.md index 455c108cd..7c821ecfd 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -6,7 +6,7 @@ - Exclude `.claude/worktrees/` from searches and edits; it contains stale worktrees that pollute results. ## Example Notebooks -- `demos/*.ipynb` — top-level Water Data tour: `USGS_WaterData_Introduction_Examples.ipynb` is the entry point; `_ContinuousData_`, `_DailyStatistics_`, `_DiscreteSamples_`, `_ReferenceLists_` cover individual collections; `WaterData_demo.ipynb`, `peak_streamflow_trends.ipynb`, `USGS_WaterUse_Examples.ipynb` (NWDC water-use data via `wateruse.get_wateruse`), and `R Python Vignette equivalents.ipynb` are standalone walkthroughs. +- `demos/*.ipynb` — top-level Water Data tour: `USGS_WaterData_Introduction_Examples.ipynb` is the entry point; `_ContinuousData_`, `_DailyStatistics_`, `_DiscreteSamples_`, `_ReferenceLists_` cover individual collections; `WaterData_demo.ipynb`, `peak_streamflow_trends.ipynb`, `USGS_WaterUse_Examples.ipynb` (NWDC water-use data via `nwdc.get_wateruse`), and `R Python Vignette equivalents.ipynb` are standalone walkthroughs. - `demos/hydroshare/*.ipynb` — per-service HydroShare examples (NLDI, NWIS WaterUse, and Water Data DailyValues / GroundwaterLevels / Measurements / ParameterCodes / Peaks / Ratings / Samples / SiteInfo / SiteInventory / Statistics / UnitValues). Mirror these when adding examples for a new collection. - `demos/nwqn_data_pull/` — non-notebook example: a lithops/Docker batch pipeline (`retrieve_nwqn_samples.py`, `retrieve_nwqn_streamflow.py`) with its own `README.md`. - Any `Untitled*.ipynb`, `*_test.ipynb`, or notebooks not listed here are untracked local scratch; ignore them. diff --git a/CONTEXT.md b/CONTEXT.md index 8b9e20329..78f87fa0a 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -71,8 +71,12 @@ statistics. The package's primary target. **NGWMN** — The National Ground-Water Monitoring Network, a distinct OGC API covering sites, water levels, lithology, well construction, and providers. -**NWDC** — The National Water Availability Assessment Data Companion, providing -modeled national-scale water-use data. +**NWDC** — The National Water Availability Assessment Data Companion. Serves +ten modeled national-scale datasets, of which the water-use models are five; +the rest are hydrologic, atmospheric-forcing, and assessment outputs. The +package reaches it through the `nwdc` adapter, named for the service like every +other adapter. Legacy: that module was `wateruse`, which named one subset of +what the service offers. **WQP** — The Water Quality Portal, a multi-agency water-quality clearinghouse. @@ -105,6 +109,70 @@ term. Legacy: the deprecated NWIS getters and the WQP profiles call this a **Metadata** — The second half of every getter's return: the request URL, the elapsed time, and the response headers. Describes the *retrieval*, not the data. +## Configuration + +**Configuration profile** — A named set of settings for one adapter, stored in +the configuration file or built in code. **Configuration** is the short form. +A profile is an *input* to resolution, never its result. + +**Default profile** — The profile an adapter uses when no other is selected: +the `[]` table's own keys. Always in effect. A **named profile** +(`[.bulk]`) is in effect only when a caller selects it, so adding one +to a file never changes an existing script. + +**Effective configuration** — The resolved set of settings a call will use: +what the chain produces after every profile, variable and default has been +applied. Distinct from a configuration profile, which is one contribution to +it. **Configure** is the verb for applying one. + +**Setting** — One named tunable the caller may adjust: the API key, the +concurrency cap, the retry count, the progress line, the fan-out baseline. A +setting means the same thing wherever it applies, but it does not apply +everywhere: `concurrency` and `parallel_chunks` are meaningless to an adapter +that issues one request, and `ssl_check` is meaningful to only three. Which +settings an adapter accepts is part of that adapter's vocabulary. + +**Package-wide setting** — A setting that applies to every adapter: the retry +count, the progress line, the stall timeout. Set once, honored everywhere. + +**Adapter-scoped setting** — A setting named under one adapter, applying to +that adapter and no other. It overrides the package-wide value for that adapter +alone; it does not replace the package-wide tier. An adapter rejects a setting +it has no use for, rather than accepting and ignoring it. + +The scope is the *adapter*, not the service and not the host, because the +adapter is what owns the conventions being tuned. The API key is the +counter-example that fixes the distinction: it belongs to the gateway fronting +a host, so Water Data and NGWMN — two adapters, one host — necessarily share +one key and one quota pool. Credentials are host-scoped; tunables are +adapter-scoped. + +**Source** — Where a setting's value came from. Sources are ordered, and the +order is resolved per setting rather than per source: a value supplied for one +setting does not displace another setting's value from a lower source. + +**Selection** — Naming which profile an adapter should use. Done in code; a +profile is never selected by the environment or implied by the file, so the +set of profiles in a file is inert until something asks for one. + +**Built-in default** — The value a setting takes when no source supplies one. +Package-wide. + +**Adapter default** — The value a *particular adapter* prefers when no source +supplies one, because that adapter warrants a different figure — NWDC asks for +4 concurrent requests where the OGC getters take 32. Supplied by the adapter in +code, not by the user. It replaces the built-in default for calls through that +adapter and nothing else. A value from any source outranks it: an adapter able +to override an explicit setting would make that setting a lie. + +Distinct from an **adapter-scoped setting**, which is the *user* naming a value +for one adapter. Both narrow to a single adapter; only one of them is something +the caller wrote. + +All three are called "the default" in casual use, and they are not the same +value. Where the distinction matters — reporting what a call will actually use +— say which one is meant. + ## Boundaries **Adapter** — A module owning one service's conventions: its URLs, parameters, diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c6ce3b8ff..4416e8d76 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -115,9 +115,10 @@ about the upstream service rather than about this package. **Before adding a small helper, check whether a leaf already generalizes it.** This package keeps its general mechanisms in dependency-free leaves -- -`_ambient.Ambient` for scoped context values, `transport.retry._read_env_number` -for `API_USGS_*` settings, `transport.links.resolve_next_url` for pagination -cursors. Each of those has been re-implemented at least once by someone who +`_ambient.Ambient` for scoped context values, `config` for every setting +(`API_USGS_*`, the config file, and `configure()` blocks all resolve through +it, and it is the only module that reads the environment for one), +`transport.links.resolve_next_url` for pagination cursors. Each of those has been re-implemented at least once by someone who did not know it was there, and the copies drift: the same question gets a different cycle guard, a different error message, a different edge case. None of the automated checks catch it, because two eight-line helpers are below the diff --git a/NEWS.md b/NEWS.md index b4abd5ec5..137ba5bf2 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,5 +1,9 @@ **08/13/2026:** Warning categories now say what they mean. Two advisories about *upstream data* were emitted as `DeprecationWarning` and as an uncategorized `warnings.warn` respectively; both are now `DataCurrencyWarning` (a `UserWarning` subclass, exported as `dataretrieval.DataCurrencyWarning`). **Behavior change:** WQP's legacy-WQX notice moves from `DeprecationWarning` to `DataCurrencyWarning`. Because `legacy=True` is the default on every WQP getter and the notice is unconditional, a downstream project running `-W error::DeprecationWarning` previously could not call any WQP getter with default arguments; it now can. The flip side is visibility — `DeprecationWarning` is silent by default outside `__main__`, so this notice will now print to stderr for library and notebook callers who never saw it. Silence it with `warnings.filterwarnings("ignore", category=dataretrieval.DataCurrencyWarning)`, or set `legacy=False` where a WQX3.0 profile exists. The NWIS qw-endpoint retirement notice gains the same category (it previously had none, arriving as a bare `UserWarning` that deprecation filters ignored). `DeprecationWarning` now means only that a name in this package is going away, always with a replacement and, where published, a removal date; those horizons are declared once in `dataretrieval._deprecation.REMOVALS` rather than spelled at each call site. +**08/11/2026:** Settings resolve through a layered chain instead of the environment alone, and a *configuration profile* is a named set of settings for **one adapter**. The new `dataretrieval.configuration` module resolves every setting in one order, highest first: a configuration passed to an active `dataretrieval.configure(...)` block, a profile that block selected, the setting's `API_USGS_*` environment variable, the adapter's `[]` table in `~/.dataretrieval/config.toml` (or `DATARETRIEVAL_CONFIG`), the file's top-level keys, the adapter's own built-in preference, then the package default. Precedence applies **per setting**, so a file that sets only `concurrency` leaves an environment `API_USGS_PAT` in effect, and a `[ngwmn]` table still inherits every top-level key it does not name. `configure()` takes configuration objects positionally, at most one per adapter and nothing else: `configure(Configuration(api_key=vault.read("usgs/pat")), WaterdataConfiguration.load("bulk"), NgwmnConfiguration(concurrency=4))`. The adapter a configuration targets is a property of its class, so a caller never restates it, and each adapter owns its class in the module that *reads* those settings (`waterdata.WaterdataConfiguration`, `ngwmn.NgwmnConfiguration`, `nwdc.NwdcConfiguration`, `wqp.WqpConfiguration`, `nldi.NldiConfiguration`, `streamstats.StreamstatsConfiguration`) — an adapter accepts only the settings it reads, so `[streamstats] parallel_chunks = 8` is an error rather than a line that quietly does nothing. The block is delivered through a `ContextVar`, so a credential set inside it cannot leak across threads or asyncio tasks, which is what makes it safe for a server or notebook handling several users' keys and is the thing assigning to `os.environ` could never do (issue #352). The file gains named profiles beside each adapter's default profile: `[waterdata]` is always in effect, `[waterdata.bulk]` only when a caller selects it with `WaterdataConfiguration.load("bulk")`, and a selected profile still inherits the default profile and the package-wide keys per setting. A profile named in code outranks the setting's environment variable — the one place the ladder inverts the environment-above-file rule, because losing a deliberate selection to a stale shell export is what a caller would file a bug about. An adapter's configuration may also carry a `base_url`, which redirects that adapter's requests for the duration of the block — a staging instance, a mirror, a recording proxy — and no other adapter's; for Water Data one value moves the OGC collections, the Samples database, the statistics service and the STAC catalog together. It is settable in a `configure()` block only: a `base_url` key in the file and an exported `API_USGS_BASE_URL` each raise rather than being read, since a redirect a config file or a shell profile can set is one no reader of the script can see. The API key does not follow a redirect — it is scoped to the single host that honors it — and is deliberately not per-adapter: it authenticates to the gateway fronting a host, and Water Data and NGWMN share that host, one key, and one hourly quota. `dataretrieval.show_configuration()` reports each setting's effective value and where it came from, naming the profile behind each value (`configure() block [waterdata.bulk]` rather than a bare block), listing the profiles a file defines whether or not this run selected any, and naming any adapter this process has not imported rather than omitting it — without ever printing the key. One parser per setting owns its grammar, so a value means the same thing whichever source wrote it. **Breaking change:** `RetryPolicy.from_env()` is now `RetryPolicy.from_configuration()` and resolves through the whole chain rather than the environment alone. **Behavior change:** a credential-shaped keyword passed to a getter's `**kwargs` query passthrough — Water Data's `**queryables` and every WQP getter's search filters — now raises `TypeError` naming `configure(Configuration(api_key=...))` instead of putting a secret in a URL that clients, proxies and logs retain. The names refused are `api_key=`, `token=`, `x_api_key=`, `password=`, `auth=`, `pat=` and similar spellings; a filter the server actually defines is unaffected. **Bug fix:** `API_USGS_STALL_TIMEOUT` was read straight from `os.environ`, so it could not be set by a `configure()` block or the config file and never appeared in `show_configuration()`; it now resolves through the chain like every other setting. Rationale in ADRs 0009, 0010 and 0011; terms in `CONTEXT.md`. + +**08/11/2026:** `dataretrieval.wateruse` is now `dataretrieval.nwdc`. Every other adapter is named for the service it retrieves from — `ngwmn`, `nldi`, `wqp`, `streamstats`, `nwis` — and this one was named for one subset of what its service offers. The National Water Availability Assessment Data Companion serves ten modeled datasets; the water-use models are five of them, the rest being hydrologic, atmospheric-forcing, and assessment outputs (`GET https://api.water.usgs.gov/nwaa-data/models`). **Deprecation:** `dataretrieval.wateruse` still works and re-exports `dataretrieval.nwdc` unchanged, emitting a `DeprecationWarning` on import; it will be removed on or after 2027-08-11. The alias forwards rather than copies, so `wateruse.get_wateruse is nwdc.get_wateruse` — monkeypatching or identity comparison through either spelling behaves the same. `import dataretrieval` stays silent: the package imports `nwdc` directly, so only code naming `wateruse` itself sees the warning. Function and constant names are unchanged (`get_wateruse`, `MODELS`, `WATERUSE_URL`, `DEFAULT_CONCURRENT_REQUESTS`). Terms are defined in `CONTEXT.md`. + **08/09/2026:** `waterdata.get_cql` takes `collection` rather than `service`. OGC API - Features (17-069r4) normatively names this value the `collectionId`: Requirement 20 fixes the path template `/collections/{collectionId}/items`, and Requirement 18 defines `collectionId` as each `id` in the collections response -- which is literally how the package builds the URL, and what the live API returns. *Service* names the API itself (Water Data, NGWMN). **Deprecation:** `service=` still works and resolves to `collection`, with a `DeprecationWarning`; it will be removed on or after 2027-08-09. Positional callers (`get_cql("daily", cql)`) are unaffected. The `WATERDATA_SERVICES` type alias is now `WATERDATA_COLLECTIONS`, with `WATERDATA_SERVICES` retained as a permanent alias for the same object. Terms are defined in `CONTEXT.md`. **08/09/2026:** Every retrieval path now runs through one executor. `waterdata.get_cql` (via the OGC `fetch_ogc_request`) and `waterdata.get_stats_por` / `get_stats_date_range` (via the Statistics page walk) previously bypassed `dataretrieval.transport.fanout.FanOut` through a private sync bridge, which meant they were the only getters in the package with **no retry**: a mid-page-walk 429 or 503 failed the whole call while every typed getter and Water Use rode it out. Both now run as a one-item fan-out and the 25-line `transport/sync.py` is gone. **Behavior change:** those three getters now retry transient failures (`API_USGS_RETRIES`, default 4) and, when the retries are exhausted, raise the resumable `ServiceInterrupted` / `QuotaExhausted` rather than `ServiceUnavailable` / `RateLimited` / `NetworkError` — all remain `DataRetrievalError`, so broad handlers are unaffected, but narrow handlers around those calls must widen, and `.call.resume()` is now available on the interruption. A failure that retrying cannot fix (bad scheme, a hostname that does not resolve) still surfaces as `NetworkError` immediately. The progress line moved with it: `FanOut.resume()` opens the reporter it ticks into, so a driver can no longer run the shared executor and silently print nothing, and a `.call.resume()` fired long after the interruption now reports progress instead of running mute. Internal tidying with no public effect: the WQX3 / legacy-WQP CSV datetime shaping moved out of `dataretrieval.utils` (whose docstring reserves it for non-service-specific shaping) into the `dataretrieval._wqx` leaf; the five Water Data endpoint URLs are declared once in `dataretrieval.waterdata.endpoints` instead of being derived in three modules; the OGC queryables document is parsed by `dataretrieval.ogc.schema` so every OGC adapter can offer the table, with `waterdata.get_queryables` unchanged as its documented wrapper; and `ogc/engine.py` imports each symbol from the module that defines it. diff --git a/README.md b/README.md index 41b3299c6..b45eb5e10 100644 --- a/README.md +++ b/README.md @@ -42,16 +42,57 @@ pip install git+https://github.com/DOI-USGS/dataretrieval-python.git Access USGS water-monitoring data. -**Important:** We strongly encourage you to obtain an API key for higher -rate limits. [Register for an API key](https://api.waterdata.usgs.gov/signup/) -and set it as an environment variable: +**Important:** Users are strongly encouraged to obtain an API key for higher +rate limits. [Register for an API key](https://api.waterdata.usgs.gov/signup/), +then supply it in whichever of these ways suits you. They are listed from +highest to lowest precedence, so an explicit block or deployment environment +can override a file without editing it: ```python -import os +# 1. a configure() block - for one call, an interactive prompt, or when +# different threads/tasks need different credentials. +from getpass import getpass -os.environ["API_USGS_PAT"] = "your_api_key_here" +import dataretrieval +from dataretrieval import Configuration, waterdata + +with dataretrieval.configure(Configuration(api_key=getpass("USGS API key: "))): + df, metadata = waterdata.get_daily(monitoring_location_id="USGS-01646500") +``` + +```bash +# 2. an environment variable (the R dataRetrieval package uses the same +# variable, so one export serves both) +export API_USGS_PAT="your_api_key_here" +``` + +```toml +# 3. ~/.dataretrieval/config.toml - keeps the key out of your shell +# environment, where every process you start inherits it. +# Restrict it afterwards: chmod 600 ~/.dataretrieval/config.toml +api_key = "your_api_key_here" ``` +`dataretrieval.show_configuration()` reports what is in effect and where each setting +came from, without printing the key. Concurrency, retries, and the progress +line are configured the same way, and can be narrowed to one service: a +`configure()` block takes at most one configuration per adapter, each either +built in code or loaded by name from a profile in the file. + +```python +from dataretrieval.ngwmn import NgwmnConfiguration +from dataretrieval.waterdata import WaterdataConfiguration + +with dataretrieval.configure( + WaterdataConfiguration.load("overnight"), # a profile in config.toml + NgwmnConfiguration(concurrency=2), # built here +): + ... +``` + +See the +[configuration guide](https://doi-usgs.github.io/dataretrieval-python/userguide/configuration.html). + The following example retrieves daily streamflow data for a specific monitoring location. The `/` in the `time` argument separates the start and end of the desired range: @@ -127,7 +168,7 @@ from dataretrieval import waterdata # enough to span many pages, so it profits from a finer split. sites, _ = waterdata.get_monitoring_locations(state="Ohio", site_type_code="ST") -with waterdata.parallel_chunks(32): # fan out into 32 sub-requests +with waterdata.parallel_chunks(32): # request up to 32 optional chunks df, md = waterdata.get_daily( monitoring_location_id=sites["monitoring_location_id"], parameter_code="00060", # discharge @@ -147,11 +188,11 @@ Benchmark — a fixed 271-site subset of Ohio stream gages the effect of parallelism). Each `n` ran against its own cold 1-year time window, so no run is served from the server's data-window cache: -| `n` | parallelism | pages | wall-clock | speedup | -| ---- | ----------- | ----- | ----------------------- | ------- | -| off | 1 | ~30 | 9.5 s / 9.1 s (2 runs) | 1× | -| `8` | 8 | ~32 | 2.2 s / 1.9 s | ~4.5× | -| `32` | 32 | 54 | 1.2 s | ~8× | +| `n` | optional fan-out | pages | wall-clock | speedup | +| ---- | ---------------- | ----- | ---------------------- | ------- | +| off | 1 | ~30 | 9.5 s / 9.1 s (2 runs) | 1× | +| `8` | 8 | ~32 | 2.2 s / 1.9 s | ~4.5× | +| `32` | 32 | 54 | 1.2 s | ~8× | The gain comes from overlapping each sub-request's per-page latency and server-side work. The exact multiplier therefore scales with how many pages the @@ -252,11 +293,11 @@ Retrieve modeled water-use estimates from the National Water Availability Assessment Data Companion: ```python -from dataretrieval import wateruse +from dataretrieval import nwdc # Monthly public-supply withdrawals for Rhode Island, split into # groundwater and surface-water sources (returns a DataFrame and metadata). -df, metadata = wateruse.get_wateruse( +df, metadata = nwdc.get_wateruse( model="wu-public-supply-wd", variable=["pswdtot", "pswdgw", "pswdsw"], state="RI", # name/postal/FIPS; pass a list to fan out over several areas @@ -312,7 +353,7 @@ print(statewide.head()) - `get_features`: Find monitoring sites, dams, and other features along the network - `get_features_by_data_source`: Features from a specific data source -### Water Use (NWDC) — `dataretrieval.wateruse` +### NWDC (National Water Availability Assessment Data Companion) — `dataretrieval.nwdc` - `get_wateruse`: Modeled water-use estimates — public-supply, irrigation, and thermoelectric withdrawals and consumptive use — on a national 12-digit hydrologic-unit (HUC12) grid, summarizable to counties, states, or coarser hydrologic units ## More Examples diff --git a/dataretrieval/__init__.py b/dataretrieval/__init__.py index 2ed3d415f..41b529cad 100644 --- a/dataretrieval/__init__.py +++ b/dataretrieval/__init__.py @@ -11,12 +11,21 @@ df, meta = nwis.get_dv(sites="05427718") Available service modules: ``waterdata``, ``wqp`` (Water Quality Portal), -``wateruse`` (NWDC water-use data), ``nldi``, ``streamstats``, and the +``nwdc`` (National Water Availability Assessment Data Companion, incl. +water use), ``nldi``, ``streamstats``, and the deprecated ``nwis``. ``nldi`` requires geopandas (``pip install dataretrieval[nldi]``) and is imported on demand: ``from dataretrieval import nldi``. +Settings -- the Water Data API key, fan-out concurrency, retries, the progress +line -- resolve through :mod:`dataretrieval.configuration`: a +``with dataretrieval.configure(Configuration(...))`` block, then the +``API_USGS_*`` environment variables, then ``~/.dataretrieval/config.toml``. +A setting for one service goes on that adapter's own configuration, such as +``waterdata.WaterdataConfiguration``. ``dataretrieval.show_configuration()`` +reports what is in effect and where each value came from. + A failed request raises a subclass of :class:`dataretrieval.DataRetrievalError` (the taxonomy lives in ``dataretrieval.exceptions``); connection-level failures (timeouts, DNS) are wrapped as :class:`dataretrieval.NetworkError`. A fanned-out @@ -32,6 +41,15 @@ except PackageNotFoundError: __version__ = "version-unknown" +# Layered configuration: a ``with configure(...)`` block, the environment, then +# the config file. The canonical home is ``dataretrieval.configuration``; +# the callable is named ``configure`` so it doesn't shadow that module. +# +# The module itself is deliberately absent from ``__all__`` below: it and the +# ``Configuration`` class differ only by case, and keeping the module out of the +# package's exports means ``from dataretrieval import configuration, +# Configuration`` never arises (ADR 0011). +from dataretrieval.configuration import Configuration, configure, show_configuration from dataretrieval.exceptions import ( ConfigurationError, DataCurrencyWarning, @@ -70,27 +88,31 @@ from . import ( exceptions, ngwmn, + nwdc, nwis, streamstats, utils, waterdata, - wateruse, wqp, ) __all__ = [ + # layered configuration (canonical home: ``dataretrieval.configuration``) + "Configuration", + "configure", + "show_configuration", + "ConfigurationError", # service modules "ngwmn", + "nwdc", "nwis", "streamstats", "utils", "waterdata", - "wateruse", "wqp", # error taxonomy (canonical home: ``dataretrieval.exceptions``), re-exported # so callers can ``except dataretrieval.DataRetrievalError`` "exceptions", - "ConfigurationError", "DataCurrencyWarning", "DataRetrievalError", "HTTPError", diff --git a/dataretrieval/_configuration_core.py b/dataretrieval/_configuration_core.py new file mode 100644 index 000000000..d1b1b460a --- /dev/null +++ b/dataretrieval/_configuration_core.py @@ -0,0 +1,1405 @@ +"""Private model, grammar, and file foundation for configuration. + +The public interface and runtime precedence engine live in +``dataretrieval.configuration``. This lower module keeps the mutually +dependent configuration classes, setting grammar, TOML interpretation, +and file caches together so the public facade can stay small without +introducing callback seams or import cycles. +""" + +from __future__ import annotations + +import math +import os +import stat +import sys +import warnings +from collections.abc import Callable, Mapping +from dataclasses import dataclass, field, fields +from functools import partial +from numbers import Integral +from pathlib import Path +from types import MappingProxyType +from typing import Any, ClassVar, TypeVar + +from dataretrieval._ambient import Ambient +from dataretrieval.exceptions import ConfigurationError + +#: Settings only an adapter can carry, because they name one service. No +#: package-wide value could mean anything for them: there is no one base URL. +#: +#: The package-wide roster is :data:`SETTINGS`, declared below the class it is +#: derived from. +ADAPTER_ONLY_SETTINGS: tuple[str, ...] = ("base_url",) + +#: Environment variable backing a setting (precedence step 2). +#: +#: Not every setting has one. ``parallel_chunks`` is deliberately absent: it +#: fans a query into more sub-requests, each of which spends rate-limit quota, +#: and ``dataretrieval.parallel_chunks`` documents why that must stay a +#: deliberate choice rather than a process-wide default. An environment +#: variable is the wrong shape for it -- exported once in a shell profile, +#: inherited by every subprocess, invisible at the call site. A config-file +#: entry is written deliberately and shows up in :func:`show_configuration`, so the +#: file and :func:`configure` block are the only sources for it. +ENV_VARS: dict[str, str] = { + "api_key": "API_USGS_PAT", + "concurrency": "API_USGS_CONCURRENT", + "retries": "API_USGS_RETRIES", + "progress": "API_USGS_PROGRESS", + "stall_timeout": "API_USGS_STALL_TIMEOUT", +} + +#: Variables the environment is *refused* for, by setting. Named rather than +#: simply left out of :data:`ENV_VARS`, because leaving them out only makes the +#: environment silent: a caller who exports ``API_USGS_BASE_URL`` -- the +#: spelling every other setting's variable predicts -- has redirected nothing +#: and would learn that from the traffic rather than from us. The file refuses +#: the same key in the same words (:func:`_accepted_keys`), for the reason ADR +#: 0011 gives: a redirect a shell profile or a config file can set is one no +#: reader of the script can see. +#: +#: Derived from :data:`ADAPTER_ONLY_SETTINGS` rather than written out beside it, +#: because the two would be spelling one fact -- "this setting is code-only" -- +#: in two tables with nothing keeping them in step. A second adapter-only +#: setting added to the roster alone would be refused by the file (which reads +#: that roster) and *silently ignored* from the environment, which is exactly +#: the defect this table exists to prevent. The predicted spelling is the one +#: the comment above names, so deriving it changes nothing today. +_REFUSED_ENV_VARS: dict[str, str] = { + name: f"API_USGS_{name.upper()}" for name in ADAPTER_ONLY_SETTINGS +} + +#: Environment variable holding an explicit path to the configuration file. +CONFIG_PATH_ENV = "DATARETRIEVAL_CONFIG" + +#: Source label for a setting no source supplied. +_BUILT_IN = "built-in default" + +#: The table ADR 0011 retired. Named here only so a file written against the +#: earlier design gets an error that says what to write instead, rather than the +#: generic "unknown table" that would send the reader looking for a typo. +_RETIRED_PROFILES_TABLE = "profiles" + +#: Label for the file's top-level table, where keys are the defaults. +_TOP_LEVEL = "top level" + +#: Settings that warn when written at the top level of the file, and what to +#: say. Declared as data, beside the other per-setting policies -- ``ENV_VARS``, +#: ``_REFUSED_ENV_VARS``, ``_BLANK_MEANS_SET``, ``_VALIDATORS``, ``_DISPLAYS`` -- +#: so "what is special about ``parallel_chunks``?" is answerable from this block +#: rather than from a condition buried in a validation loop, and so a second +#: quota-spending setting is a row here rather than an edit to shared code. +#: +#: Top level only: a value in a ``[.]`` table is opt-in per run, +#: which is the shape a setting that spends quota wants. +_WARN_AT_TOP_LEVEL: dict[str, str] = { + "parallel_chunks": ( + f"'parallel_chunks' at {_TOP_LEVEL} applies to every query in every " + "process and spends rate-limit quota. Prefer a [.] " + "table selected per run, or the dataretrieval.parallel_chunks(n) " + "block for a single call." + ), +} + +# Built-in defaults (precedence step 4). ``concurrency`` and ``retries`` keep the +# values the environment-only implementation used, so behavior is unchanged for +# anyone who configures nothing. +DEFAULT_CONCURRENCY = 32 +DEFAULT_RETRIES = 4 +DEFAULT_PARALLEL_CHUNKS = 1 +DEFAULT_STALL_TIMEOUT = 60.0 +CONCURRENCY_UNBOUNDED = "unbounded" + + +# Values that turn the progress line off. Blank counts: ``API_USGS_PROGRESS=`` +# has always meant "off", not "unset" -- unlike the numeric knobs, where blank +# falls through to the default. +_PROGRESS_FALSEY = frozenset({"", "0", "false", "no", "off"}) + +# Settings for which a *blank* environment variable is a value rather than an +# absence. ``API_USGS_PROGRESS=`` has always meant "off". For every other +# setting a blank variable is what container and CI tooling produces when it +# has nothing to pass (``docker run -e API_USGS_PAT``, a workflow secret that +# is absent on a fork), so treating it as configured would let it shadow the +# config file and silently drop the user's API key. Keeping this a property of +# the setting -- rather than a second, lower visit to the environment -- keeps +# the chain at the three tiers the docstring and ADR 0009 describe. +_BLANK_MEANS_SET = frozenset({"progress"}) + +# Warnings about the config file report the file, not a call site: settings are +# resolved lazily from wherever a getter first needs one, so the user frame is +# a different depth every time and no fixed ``stacklevel`` can name it. Pointing +# at this module consistently at least makes the warnings filterable by module, +# and every message names the offending path and setting. +_WARN_STACKLEVEL = 2 +_PROGRESS_TRUTHY = frozenset({"1", "true", "yes", "on"}) + + +class _Unset: + """Sentinel that distinguishes an omitted override from explicit ``None``.""" + + __slots__ = () + + def __repr__(self) -> str: + return "" + + +# Typed as Any so public annotations describe accepted caller values without +# exposing this private implementation detail in generated signatures. It is +# also every configuration field's default, which is what makes "left unset" +# distinguishable from an explicit ``None`` meaning "suppress lower sources". +_UNSET: Any = _Unset() +_SettingValue = str | None + +# Overrides from the innermost active ``configure`` block, as raw strings so that +# every source shares one parser and one set of error messages. +# A package-wide override is keyed by the setting's name; an adapter-scoped one +# by ``(adapter, name)``. One flat mapping rather than a nested one so that +# nesting, per-key inheritance, and restore-on-exit keep falling out of a +# single merge, whichever scope a block sets. +_ScopeKey = str | tuple[str, str] +# One frame per ``configure`` block, stacked outermost-first. Frames rather than +# a merged mapping are what makes "the innermost block wins" true across *both* +# scopes: an adapter-scoped value outranks a package-wide one only within the +# same frame. Merged, an outer ``configure(WaterdataConfiguration(...))`` would +# beat an inner ``configure(Configuration(concurrency=1))`` -- inverting +# nesting, and silently discarding the per-call ``parallel_chunks(n)`` block. +# +# Each entry pairs the raw value with the label naming where it came from, the +# same shape the file tier returns (:func:`_adapter_file_settings`). The label +# is built while the configuration object is still in hand, because that is the +# only place the *profile* is known: a value from +# ``WaterdataConfiguration.load("bulk")`` and one from +# ``WaterdataConfiguration(...)`` are indistinguishable by the time they reach +# the frame, so a label rebuilt at resolution time could only ever say +# "some block", never which profile. +_Frame = Mapping[_ScopeKey, tuple[_SettingValue, str]] +_scope: Ambient[tuple[_Frame, ...]] = Ambient("dataretrieval_configuration", ()) + +# Resolved config-file path, memoized on the raw ``DATARETRIEVAL_CONFIG`` +# value (see :func:`config_path`). The guard names its own kind so the memo +# never has to infer which branch built it. +_PathGuard = tuple[str, object] +_path_cache: tuple[str | None, _PathGuard | None, Path] | None = None + +# Parsed configuration file, keyed by file identity, change metadata, and raw +# content. POSIX ctime makes metadata hits reliable; Windows ctime is creation +# time, so cache hits there compare content before reusing the parsed result. +_FileStamp = tuple[int, int, int, int, int, int] +_file_cache: tuple[Path, _FileStamp, bytes, _ParsedFile] | None = None + +# Validated ``[]`` tables, keyed by adapter name and memoized on the +# parsed file's identity, because an adapter table is validated only once that +# adapter is actually used. +_adapter_cache: dict[str, tuple[_ParsedFile, Path, Mapping[str, tuple[str, str]]]] = {} + +# Paths already warned about for loose permissions, so the warning fires once. +_permission_warned: set[Path] = set() + + +@dataclass(frozen=True) +class _ParsedFile: + """A parsed configuration file: package-wide keys plus per-adapter tables. + + ``exists`` distinguishes "the file is there and defines nothing" from "there + is no file", which decides which of the two messages a caller selecting a + profile gets (see :func:`_named_profile`). + """ + + base: dict[str, str] = field(default_factory=dict) + #: Raw, *unvalidated* ``[]`` tables, keyed by adapter name. Each + #: holds that adapter's default-profile keys and, as sub-tables, its named + #: profiles. Left unvalidated because a bad value in ``[nldi]`` must not + #: fail a Water Data call that never reads it. + adapters: dict[str, dict[str, Any]] = field(default_factory=dict) + exists: bool = False + + +#: Stand-in for "no configuration file", which is the common case. Shared +#: rather than rebuilt per read so that callers can memoize on the parsed +#: file's identity; nothing mutates a ``_ParsedFile``. +_NO_FILE = _ParsedFile() + + +# --- configuration profiles ---------------------------------------------- +# +# A setting means the same thing wherever it applies, but it does not apply +# everywhere (ADR 0010). Each adapter declares the settings it accepts as the +# fields of a ``BaseConfiguration`` subclass, defined *in the adapter's own +# module* so a setting's definition sits with the code that reads it -- adding +# a Water Data setting no longer edits a service-neutral file (ADR 0011). The +# *which* is the adapter's own knowledge; the setting itself is drawn from the +# shared groups below, so ``retries`` is declared once rather than six times. +# +# Two settings are deliberately absent from every adapter: +# +# ``api_key`` belongs to the gateway fronting a host, not to an adapter. +# Water Data and NGWMN are two adapters on one host sharing +# one key and one quota pool -- measured, see ADR 0010 -- so +# a per-adapter key would model a distinction that does not +# exist. ``credentials`` keeps sole ownership of it. +# ``progress`` describes the caller's terminal, not a service. There is one +# progress line per call, so scoping it per adapter could only +# produce a contradiction. + +#: Bound to the concrete subclass so ``WaterdataConfiguration.load(...)`` is +#: typed as a ``WaterdataConfiguration`` rather than the base. ``typing.Self`` +#: would say this in one word and arrives in 3.11; the floor is 3.10. +_C = TypeVar("_C", bound="BaseConfiguration") + + +#: Memoized :meth:`BaseConfiguration.settings` results, keyed on the class. +#: A hand-rolled dict rather than ``functools.cache`` only because typeshed's +#: wrapper takes ``Hashable`` and mypy does not accept a class for that +#: protocol, and this package carries no ``type: ignore``. +_settings_cache: dict[type[BaseConfiguration], frozenset[str]] = {} + + +def _settings_of(cls: type[BaseConfiguration]) -> frozenset[str]: + """The setting names a configuration class accepts, computed once. + + A class constant in everything but spelling: the fields cannot change after + the class is created, and every adapter-scoped read asks for it -- through + :func:`_accepts`, before the frame walk and before the file, so the cost is + paid even when a ``configure`` block answers. Rebuilding the frozenset per + read measured as a fifth of an adapter-scoped resolution: two generator + passes over :func:`~dataclasses.fields` to rebuild six strings that cannot + have changed. + + Keyed on the *class* rather than on the adapter name because tests replace a + registry entry to stand in for an unimported adapter; a name-keyed memo + would serve them the schema of the class they replaced. + """ + cached = _settings_cache.get(cls) + if cached is None: + cached = _settings_cache[cls] = frozenset(f.name for f in fields(cls)) + return cached + + +@dataclass(frozen=True) +class BaseConfiguration: + """A named set of settings for one adapter -- a *configuration profile*. + + Subclasses declare the settings their adapter reads as fields, and set + :attr:`adapter` to that adapter's module name. Every field is optional, so + an empty configuration is legal and one can be built up conditionally. + + Frozen, because a configuration is a value: two with the same settings are + interchangeable, and one already handed to :func:`configure` must not + change under the block that entered it. + + Values are checked when the configuration is *constructed*, so a typo + raises where it was written rather than at a later ``with`` statement or, + worse, inside a request. + """ + + #: The adapter this configuration targets, by the name of the module a + #: caller imports. ``None`` on the package-wide :class:`Configuration`, + #: which every adapter reads. A ``ClassVar``, not a field: the adapter is a + #: property of the class, which is what stops the caller restating it at + #: every call site and stops the roster being spelled twice. + adapter: ClassVar[str | None] = None + + #: The named profile these settings were read from, or ``None`` for a + #: configuration written in code. Provenance rather than a setting: it + #: records *where the values came from*, which is what lets + #: :func:`show_configuration` name the profile that supplied each value + #: instead of reporting every block alike. + #: + #: A ``ClassVar`` shadowed per instance by :meth:`load`, so it is neither a + #: field nor part of equality -- two configurations carrying the same + #: settings stay interchangeable however each was spelled, which is what + #: "a configuration is a value" means. + profile: ClassVar[str | None] = None + + def __post_init__(self) -> None: + for name, value in self.values().items(): + if value is not None: + # ``None`` is not a value to check: it means "suppress the + # lower sources", which every setting accepts. + _validated_raw(name, value, self._source(name), optional=", or None") + self.validate() + + def validate(self) -> None: + """Check rules that span more than one setting. + + Does nothing by default. Per-setting grammar lives in this module's + parsers and is shared with the file and the environment, so a value + means the same thing whichever source wrote it; override this only for + a rule no single setting can express. + """ + + @classmethod + def settings(cls) -> frozenset[str]: + """The setting names this configuration accepts.""" + return _settings_of(cls) + + def values(self) -> dict[str, Any]: + """The settings actually supplied, omitting those left unset. + + An omitted setting inherits from an outer block or a lower source; an + explicit ``None`` suppresses them. Distinguishing the two is the whole + job of the ``_UNSET`` default, so it is done here rather than by every + reader. + """ + return { + f.name: getattr(self, f.name) + for f in fields(self) + if getattr(self, f.name) is not _UNSET + } + + @classmethod + def load(cls: type[_C], profile: str) -> _C: + """Read a named profile for this adapter from the configuration file. + + ``[.]``. Only the keys that table names are carried, + so the profile still inherits the adapter's default profile and the + package-wide keys per setting from the tiers below. + + Selecting a profile the file does not define raises: a name a caller + just typed is a typo worth reporting, not a silent fall-through to + settings they did not ask for. + + Parameters + ---------- + profile : str + The name after the adapter, so ``[waterdata.bulk]`` is ``"bulk"``. + + Returns + ------- + BaseConfiguration + An instance of the class it was called on, remembering the profile + it was read from so :func:`show_configuration` can name it. + """ + adapter = cls.adapter + if adapter is None: + raise ConfigurationError( + f"{cls.__name__}.load() names a profile for one adapter, and " + "the package-wide configuration has none. Put shared keys at " + "the top level of the file." + ) + loaded = cls(**_named_profile(adapter, profile, cls.settings())) + # The class is frozen, so the provenance goes on the same way the + # dataclass sets its own fields. It is deliberately not one of them: + # the profile name is where these values came from, not one of the + # values, and :meth:`settings` is built from the fields. + object.__setattr__(loaded, "profile", profile) + return loaded + + def _source(self, name: str) -> str: + """How one of this configuration's settings is named in an error.""" + return f"{name}= in {type(self).__name__}()" + + def _provenance(self) -> str: + """How :func:`show_configuration` reports a value this supplied. + + The profile is named in the file's own spelling -- ``[waterdata.bulk]`` + -- so the report answers "which profile set this?" rather than only + "a block did", and the answer is greppable in the file that holds it. + A configuration written in code has no profile, so it names its adapter + alone; the package-wide one narrows to nothing and names neither. + """ + if self.adapter is None: + return "configure() block" + scope = self.adapter + if self.profile is not None: + scope = f"{scope}.{self.profile}" + return f"configure() block [{scope}]" + + +# --- shared setting groups ----------------------------------------------- +# +# Which settings an adapter accepts is the adapter's own knowledge, and it says +# so by naming the groups below. What a setting *is* -- its type, its default, +# the fact that ``None`` suppresses the tiers under it -- is not: that is this +# module's, and it already was, since :func:`_coerce_typed` keys the type check +# by setting *name* and :data:`_VALIDATORS` holds the grammar. Spelling +# ``retries: int | None = _UNSET`` in six adapter modules therefore bought +# nothing and cost a guarantee: the annotations are decorative, so an adapter +# that drifted to ``retries: str | None`` would type-check clean under +# ``mypy --strict`` and fail only when a value reached the chain. +# +# So each group declares one shared setting once, and an adapter composes the +# groups it reads:: +# +# class NgwmnConfiguration( +# _Chunked, _Concurrent, _Redirectable, _Retrying, BaseConfiguration +# ): +# adapter: ClassVar[str] = "ngwmn" +# +# Widening a shared setting's accepted type, or adding one, is now one edit +# rather than six. Each adapter still documents the settings it takes in its own +# ``Parameters`` section, because that is the signature a caller writes and +# ``base_url`` means something different for every service. +# +# Plain mixins rather than ``BaseConfiguration`` subclasses: a group is not a +# configuration -- it has no adapter and cannot be passed to :func:`configure` +# -- and keeping them off that branch of the tree leaves one linear base for the +# behavior. Frozen because a dataclass may not mix frozen and non-frozen bases. +# Fields are collected in reverse MRO order, so an adapter composing all four +# reads ``retries, stall_timeout, base_url, concurrency, parallel_chunks``. + + +@dataclass(frozen=True) +class _Retrying: + """Every adapter's retry dials: transient retries and the stall bound.""" + + retries: int | None = _UNSET + stall_timeout: float | int | None = _UNSET + + +@dataclass(frozen=True) +class _Redirectable: + """An adapter whose requests can be pointed at another base URL.""" + + base_url: str | None = _UNSET + + +@dataclass(frozen=True) +class _Concurrent: + """An adapter that issues more than one request per call.""" + + concurrency: int | str | None = _UNSET + + +@dataclass(frozen=True) +class _Chunked: + """An adapter whose queries divide into sub-requests the caller can fan.""" + + parallel_chunks: int | None = _UNSET + + +@dataclass(frozen=True) +class Configuration(BaseConfiguration): + """Settings that apply to every adapter. + + The package-wide profile: ``adapter`` stays ``None``, so nothing narrows + and every adapter reads what this sets unless its own configuration, or a + block nested inside, overrides that setting. + + Parameters + ---------- + api_key : str, optional + Water Data API key, sent as ``X-Api-Key`` and only ever to + ``api.waterdata.usgs.gov``. Prefer reading it from a secret store, the + environment, or the configuration file over writing a literal into a + script. Pass ``None`` to make a call without an ambient key. + concurrency : int or str, optional + Cap on simultaneous sub-requests: a positive integer, or + ``"unbounded"`` to disable the cap. + retries : int, optional + Retries attempted after a transient failure; ``0`` disables retrying. + progress : bool or str, optional + Whether to draw the progress line. ``None`` leaves the automatic + behavior (on for a TTY or Jupyter kernel, off otherwise). + parallel_chunks : int, optional + Default optional fan-out for multi-value queries. It limits extra + refinement, but URL-byte safety may already require more sub-requests. + Sets the baseline that :func:`dataretrieval.parallel_chunks` overrides + per call. Each sub-request spends rate-limit quota, so raise it only + for pulls you know are large. + stall_timeout : float, optional + Seconds a call may go without receiving *any* data before retrying + stops and the failure surfaces. Bounds the wall-clock cost of a dead + connection, which ``retries`` does not -- it counts attempts, not + seconds. Progress resets the clock; ``0`` disables the bound. + + Examples + -------- + .. code-block:: python + + with dataretrieval.configure(Configuration(api_key=vault.read("usgs"))): + df, md = waterdata.get_daily(monitoring_location_id="USGS-05114000") + """ + + # Spelled out rather than composed from the groups above, because this + # order is also the order :func:`show_configuration` reports the settings + # in -- :data:`SETTINGS` is derived from it just below -- and composing + # would hand that reader-facing sequence to MRO linearization. The two + # adapter-only fields the groups carry are absent by construction here: + # there is no package-wide base URL. + api_key: str | None = _UNSET + concurrency: int | str | None = _UNSET + retries: int | None = _UNSET + progress: bool | str | None = _UNSET + parallel_chunks: int | None = _UNSET + stall_timeout: float | int | None = _UNSET + + +#: The package-wide settings, in the order :func:`show_configuration` reports +#: them -- the fields of :class:`Configuration`, derived rather than restated. +#: An adapter may accept a subset of them plus :data:`ADAPTER_ONLY_SETTINGS`. +#: +#: Derived because the two copies had nothing holding them together, in the one +#: module whose job is to stop rosters being duplicated: a field added to the +#: class and forgotten here would work from :func:`configure` and be silently +#: dropped from the file -- :func:`_accepted_keys` would call it an unknown +#: setting -- and never appear in the report. Which is the "a schema no call +#: site can reach" failure ADR 0011 makes impossible by construction. This is +#: what the adapter side already does (:func:`settings_for`); only the +#: package-wide side was hand-maintained. +#: +#: Declared here, below the class, for the obvious reason: it cannot be derived +#: before the class exists. Every reader is a call-time lookup or a ``def`` +#: default evaluated further down the module. +SETTINGS: tuple[str, ...] = tuple(f.name for f in fields(Configuration)) + +#: Every setting name this module knows a grammar for. +_ALL_SETTINGS: tuple[str, ...] = SETTINGS + ADAPTER_ONLY_SETTINGS + + +#: The adapters that may be configured, by the name of the module a caller +#: imports. Names only, because this module is a standard-library-only leaf +#: every adapter may import and so cannot import them back. +#: +#: Holding the names here rather than deriving them from the registry below is +#: what lets a ``[nldi]`` table stay valid in a file: NLDI is imported on demand +#: for the geopandas extra, so a roster built from imports would reject a +#: perfectly good table until something happened to import that module, and the +#: verdict would vary by what a caller had touched. +ADAPTERS: tuple[str, ...] = ( + "waterdata", + "ngwmn", + "nwdc", + "wqp", + "nldi", + "streamstats", +) + +#: Configuration classes that have registered themselves, keyed by adapter. +#: Populated at adapter import, and consulted only to validate a table's +#: *keys* -- which happens the first time that adapter resolves a setting, by +#: which point it is necessarily imported. +_REGISTRY: dict[str, type[BaseConfiguration]] = {} + + +def _register(cls: type[BaseConfiguration]) -> None: + """Record an adapter's configuration class. Called at adapter import. + + The roster in :data:`ADAPTERS` and the class are the two halves of one + declaration, and this is where they are checked to agree: a class naming an + adapter the roster does not list would be a configuration no file table and + no report could ever reach. + """ + adapter = cls.adapter + if adapter is None or adapter not in ADAPTERS: + raise ConfigurationError( + f"{cls.__name__}.adapter is {adapter!r}, which is not one of " + f"{', '.join(ADAPTERS)}." + ) + _REGISTRY[adapter] = cls + + +def settings_for(adapter: str) -> frozenset[str] | None: + """The settings *adapter* accepts, or ``None`` if it has not been imported. + + ``None`` is not an error and callers must not treat it as one: a file may + name an adapter this process has never loaded, and rejecting that would + make a configuration file conditionally valid depending on which optional + extras happened to be installed. It means "cannot validate these keys yet", + and the adapter cannot be misreading a setting it has not loaded. + """ + cls = _REGISTRY.get(adapter) + return None if cls is None else cls.settings() + + +def _env_source_label(env_var: str) -> str: + """How a value read from ``env_var`` is reported as a source.""" + return f"${env_var}" + + +def _toml_parser() -> Any: + """The TOML parser, imported on first use. + + ``import dataretrieval`` imports this module, but the parser is reachable + only once a configuration file actually exists -- the minority case. + Importing it eagerly costs every caller ~4 ms of ``tomllib`` regex + compilation for a file most of them do not have. + """ + if sys.version_info >= (3, 11): + import tomllib + else: # pragma: no cover - exercised only on Python 3.10 + import tomli as tomllib + return tomllib + + +def config_path() -> Path: + """Path to the configuration file, honoring ``DATARETRIEVAL_CONFIG``. + + Memoized on the raw ``DATARETRIEVAL_CONFIG`` value, because this sits on + the per-request path via :func:`api_key` and building the default costs + more than the ``stat`` it leads to (``Path.home()`` alone dominates the + whole resolution). Returning a stable object also lets :func:`_load_file` + check its cache by identity instead of re-normalizing a fresh ``Path``. + + Returns + ------- + pathlib.Path + The explicit path from ``DATARETRIEVAL_CONFIG`` if set, otherwise + ``~/.dataretrieval/config.toml``. The file need not exist. + """ + global _path_cache + override = os.environ.get(CONFIG_PATH_ENV) + + # Probe the memo before doing any work: this runs once per request via + # ``api_key()``, so the hit path should be a dict lookup and a compare. + cached = _path_cache + if cached is not None and cached[0] == override: + cached_guard, path = cached[1], cached[2] + # The memo is only valid while whatever the path was *derived from* is + # unchanged, so each branch records its own guard. A relative override + # is anchored to the working directory (a later ``os.chdir`` in a + # per-job notebook or scheduler must not keep reading the previous + # job's file); the default branch is anchored to ``$HOME``. An absolute + # override depends on neither and guards with ``None``. ``stat(".")`` + # identifies the directory ~17x cheaper than ``getcwd()``, which + # reifies the whole path string. + if cached_guard is None or cached_guard == _path_guard(cached_guard[0]): + return path + + expanded = ( + Path(override.strip()).expanduser() if override and override.strip() else None + ) + guard: _PathGuard | None + if expanded is None: + path = _default_home_path() + guard = ("home", _home_id()) + elif expanded.is_absolute(): + path = expanded + guard = None + else: + guard = ("cwd", _cwd_id()) + path = _resolve_against_cwd(expanded) + _path_cache = (override, guard, path) + return path + + +def _default_home_path() -> Path: + """The default ``~/.dataretrieval/config.toml``, or an unusable path. + + ``Path.home()`` raises ``RuntimeError`` where no home can be resolved at all + -- a rootless container running as an arbitrary UID with no passwd entry and + no ``HOME``. That is not a misconfiguration to report: such a deployment + simply has no config file, and before settings were layered it worked fine + on the environment alone. So the unexpanded ``~/...`` form is returned + instead: it does not exist, which keeps the whole file layer inert rather + than failing every request from inside the header builder, and it still + reads correctly in :func:`show_configuration` output. + """ + try: + home = Path.home() + except (RuntimeError, OSError): + return Path("~") / ".dataretrieval" / "config.toml" + return home / ".dataretrieval" / "config.toml" + + +def _resolve_against_cwd(relative: Path) -> Path: + """Resolve a relative override, or report a working directory that is gone. + + A scratch-dir job that removes its own cwd cannot resolve a relative + ``DATARETRIEVAL_CONFIG`` at all. That surfaces as a :class:`ConfigurationError` + rather than a bare ``OSError`` escaping onto the request path -- the + taxonomy contract the rest of this module keeps. + """ + try: + return Path.cwd() / relative + except OSError as exc: + raise ConfigurationError( + f"cannot resolve the relative {CONFIG_PATH_ENV} path {str(relative)!r}: " + f"the working directory is unavailable ({exc})." + ) from exc + + +def _path_guard(kind: str) -> _PathGuard: + """Re-read the guard of the given kind: ``"cwd"`` or ``"home"``.""" + return (kind, _cwd_id() if kind == "cwd" else _home_id()) + + +def _cwd_id() -> tuple[int, int]: + """Identify the working directory without building its path string. + + Only identifies the directory; :func:`_resolve_against_cwd` is what turns a + missing cwd into a :class:`ConfigurationError`. Both are needed, because ``stat`` + on a *deleted* working directory still succeeds -- the process holds the + open handle -- while resolving its path does not. + """ + try: + st = os.stat(".") + except OSError as exc: + raise ConfigurationError( + f"cannot resolve the relative {CONFIG_PATH_ENV} path: the working " + f"directory is unavailable ({exc})." + ) from exc + return (st.st_dev, st.st_ino) + + +def _home_id() -> str: + """The home directory as the environment reports it. + + A plain environment read, not ``Path.home()``: this is on the per-request + path and only needs to detect a *change* (a test or notebook that + reassigns the home variable after the first resolution), not to resolve + the path. + + Which variable that is differs by platform, and the memo has to agree with + the resolver or it watches the wrong thing. ``posixpath.expanduser`` reads + ``HOME``; ``ntpath.expanduser`` reads ``USERPROFILE`` (then + ``HOMEDRIVE``/``HOMEPATH``) and ignores ``HOME`` outright. Preferring + ``HOME`` everywhere means that on Windows -- where Git Bash and MSYS do set + it -- the memo invalidates on a variable that cannot move the path, and + misses the ``USERPROFILE`` change that can. + """ + if os.name == "nt": + return ( + os.environ.get("USERPROFILE") + or os.environ.get("HOMEDRIVE", "") + os.environ.get("HOMEPATH", "") + or "" + ) + return os.environ.get("HOME") or "" + + +# --- value grammar ------------------------------------------------------- +# +# One parser drives each setting's grammar, so a value means the same thing and +# reports the same way whichever source wrote it. Source-level adapters retain +# TOML types and reject Python API type errors before producing raw strings. + + +def _type_error(source: str, expected: str, value: object) -> ConfigurationError: + """Build a type error without rendering a possibly secret value.""" + return ConfigurationError( + f"{source} must be {expected} (got {type(value).__name__})." + ) + + +def _coerce_string(value: object, source: str, optional: str) -> str: + if not isinstance(value, str): + raise _type_error(source, "a string" + optional, value) + return value + + +def _coerce_progress(value: object, source: str, optional: str) -> str: + if isinstance(value, bool): + return str(value) + if isinstance(value, str): + return value + raise _type_error(source, "a bool or recognized string" + optional, value) + + +def _coerce_concurrency(value: object, source: str, optional: str) -> str: + if isinstance(value, bool) or not isinstance(value, (Integral, str)): + raise _type_error(source, "an integer or 'unbounded'" + optional, value) + if isinstance(value, str) and value.strip().lower() != CONCURRENCY_UNBOUNDED: + raise ConfigurationError(f"{source} must be an integer or 'unbounded'.") + return str(value) + + +def _coerce_seconds(value: object, source: str, optional: str) -> str: + # Seconds, so a fractional value is meaningful -- unlike the counts, which + # are whole by nature. + if isinstance(value, bool) or not isinstance(value, (Integral, float)): + raise _type_error(source, "a number of seconds" + optional, value) + return str(value) + + +def _coerce_count(value: object, source: str, optional: str) -> str: + if isinstance(value, bool) or not isinstance(value, Integral): + raise _type_error(source, "an integer" + optional, value) + return str(value) + + +#: Each setting's source-level type policy -- one row per setting, like +#: :data:`_VALIDATORS` holds its grammar. A roster with the completeness guard +#: below rather than an if/elif chain with an implicit integer fallback, so a +#: new setting must declare its type here or fail at import -- not silently +#: parse as an integer from the typed surfaces while the untyped environment +#: accepts it. (Integers are matched as :class:`numbers.Integral` -- a numpy +#: or pandas integer is a legitimate count from Python, and ``tomllib`` only +#: ever yields ``int``, so the wider check cannot change a TOML outcome.) +_TYPES: dict[str, Callable[[object, str, str], str]] = { + "api_key": _coerce_string, + "base_url": _coerce_string, + "progress": _coerce_progress, + "concurrency": _coerce_concurrency, + "retries": _coerce_count, + "parallel_chunks": _coerce_count, + "stall_timeout": _coerce_seconds, +} + +if set(_TYPES) != set(_ALL_SETTINGS): # pragma: no cover - guards a coding error + # Not an ``assert``: ``python -O`` strips those, and an unlisted setting + # would otherwise change meaning by omission. + raise RuntimeError( + "every setting needs a type policy in _TYPES; " + f"missing={sorted(set(_ALL_SETTINGS) - set(_TYPES))} " + f"extra={sorted(set(_TYPES) - set(_ALL_SETTINGS))}" + ) + + +def _coerce_typed(name: str, value: object, source: str, *, optional: str = "") -> str: + """Type-check one source-level value and render it as a raw string. + + Shared by the two *typed* surfaces -- a configuration's fields and TOML + scalars -- so a value accepted from one is accepted from the other and a + tightened rule cannot land on only half of them. (The environment is not + typed: it delivers strings, which go straight to :func:`_validate_raw`.) + + ``optional`` is the only thing that differs between them: the Python + surface accepts ``None`` and says so in its messages. + """ + return _TYPES[name](value, source, optional) + + +def _validated_raw(name: str, value: object, source: str, *, optional: str = "") -> str: + """Type-check, render and grammar-check one typed value.""" + raw = _coerce_typed(name, value, source, optional=optional) + _validate_raw(name, raw, source) + return raw + + +def _parse_int( + raw: str, + source: str, + *, + default: int, + minimum: int, + examples: str | None = None, +) -> int: + """Parse a bounded integer setting; blank falls through to *default*. + + Parameters + ---------- + raw : str + The value as written, from whichever source supplied it. + source : str + Human-readable origin, used as the subject of any error message. + default : int + Returned for a blank value, matching the environment-variable + behavior this replaced. + minimum : int + Smallest accepted value. + examples : str, optional + Illustrative values appended to the message (e.g. ``"2, 8, 32"``). + """ + value = raw.strip() + if value == "": + return default + expected = f"an integer >= {minimum}" + (f", e.g. {examples}" if examples else "") + try: + parsed = int(value) + except ValueError as exc: + raise ConfigurationError(f"{source} must be {expected} (got {raw!r}).") from exc + if parsed < minimum: + raise ConfigurationError(f"{source} must be {expected} (got {parsed}).") + return parsed + + +def _parse_seconds(raw: str, source: str) -> float: + """Parse a non-negative duration in seconds; blank falls through. + + Seconds rather than a count, so fractional values are accepted. ``0`` + disables the bound it guards, which is why the floor is zero rather than + one. + """ + value = raw.strip() + if value == "": + return DEFAULT_STALL_TIMEOUT + expected = "a finite, non-negative number of seconds" + try: + parsed = float(value) + except ValueError as exc: + raise ConfigurationError(f"{source} must be {expected} (got {raw!r}).") from exc + # ``inf`` and ``nan`` both parse as floats and both defeat the bound they + # are meant to set: ``inf`` makes every wait allowed, and ``nan`` compares + # false against every threshold. TOML has literal ``inf``/``nan``, so this + # is reachable from the file as well as from Python. + if not math.isfinite(parsed) or parsed < 0: + raise ConfigurationError(f"{source} must be {expected} (got {parsed}).") + return parsed + + +def _parse_concurrency(raw: str, source: str) -> int | None: + """Parse a concurrency cap: a positive int, or ``unbounded`` -> ``None``.""" + if raw.strip().lower() == CONCURRENCY_UNBOUNDED: + return None + try: + return _parse_int(raw, source, default=DEFAULT_CONCURRENCY, minimum=1) + except ConfigurationError as exc: + raise ConfigurationError( + f"{exc} Use '{CONCURRENCY_UNBOUNDED}' to disable the cap." + ) from exc + + +def _parse_base_url(raw: str, source: str) -> str: + """Parse a service base URL: an absolute ``http``/``https`` origin. + + Only the scheme is checked, and deliberately so. This module cannot know + what a given service's paths look like, but it can refuse the shapes that + are never a base URL and would fail far from here -- a bare hostname that + ``httpx`` would reject, or a ``file://`` that is not a service at all. + """ + value = raw.strip() + if not value.startswith(("http://", "https://")): + raise ConfigurationError( + f"{source} must be an absolute http:// or https:// URL (got {raw!r})." + ) + return value + + +def _parse_progress(raw: str, source: str, *, strict: bool) -> bool: + """Parse a progress toggle, optionally preserving legacy env truthiness.""" + value = raw.strip().lower() + if strict and not value: + raise ConfigurationError(f"{source} must not be blank.") + if value in _PROGRESS_FALSEY: + return False + if value in _PROGRESS_TRUTHY: + return True + if not strict: + return True + expected = ", ".join(sorted(_PROGRESS_TRUTHY | _PROGRESS_FALSEY)) + raise ConfigurationError(f"{source} must be one of {expected} (got {raw!r}).") + + +# Each integer setting's grammar, named once. The accessor and the eager +# block/TOML validator below both spell the parser this way, so a change to a +# bound (say ``minimum``) cannot leave a ``configure()`` block validating +# against different rules than the value it later resolves. +_parse_retries = partial(_parse_int, default=DEFAULT_RETRIES, minimum=0) +_parse_parallel_chunks = partial( + _parse_int, default=DEFAULT_PARALLEL_CHUNKS, minimum=1, examples="2, 8, 32" +) + +#: Per-setting validators used for eager configuration and TOML validation. +_VALIDATORS: dict[str, Callable[[str, str], object]] = { + "concurrency": _parse_concurrency, + "retries": _parse_retries, + "progress": partial(_parse_progress, strict=True), + "parallel_chunks": _parse_parallel_chunks, + "stall_timeout": _parse_seconds, + "base_url": _parse_base_url, +} + + +def _validate_raw(name: str, raw: str, source: str) -> None: + """Run a setting's grammar validator when it has one.""" + validate = _VALIDATORS.get(name) + if validate is not None: + validate(raw, source) + + +def _named_profiles(parsed: _ParsedFile, adapter: str) -> dict[str, dict[str, Any]]: + """The named profiles the file defines for *adapter*, by name. + + A sub-table of an adapter's table is a named profile: ``[waterdata.bulk]`` + parses as a sub-table of ``[waterdata]``, and everything else in that table + is a setting of the adapter's default profile. The two readers of that rule + -- selecting a profile and reporting which ones exist -- share this one + definition so they cannot come to disagree about what a profile is. + + Tables are returned raw, since an adapter this process has not imported has + no vocabulary to check them against. That is enough to *name* a profile, + which is all the report needs; reading one still goes through + :func:`_named_profile`. + """ + return { + name: table + for name, table in parsed.adapters.get(adapter, {}).items() + if isinstance(table, dict) + } + + +def _named_profile( + adapter: str, profile: str, allowed: frozenset[str] +) -> dict[str, Any]: + """The ``[.]`` table, checked against *allowed*. + + Returns the TOML scalars as written rather than raw strings, because the + caller is :meth:`BaseConfiguration.load`, which feeds them straight back + into the configuration's own typed fields. Values are still checked here, + with a source that names the file and the table: a grammar error found on + the way *out* of the file should say which line to fix, not merely which + field of which class ended up holding it. + """ + path, parsed = _current_file() + named = _named_profiles(parsed, adapter) + if profile not in named: + if not parsed.exists: + raise ConfigurationError( + f"profile {profile!r} cannot be selected for {adapter}: there " + f"is no configuration file at {path}." + ) + defined = ", ".join(sorted(named)) or "none" + raise ConfigurationError( + f"{path}: no [{adapter}.{profile}] table. Profiles defined for " + f"{adapter}: {defined}." + ) + + where = f"[{adapter}.{profile}]" + table = named[profile] + + # A profile is one flat set of settings for one adapter, so a table inside + # one is a shape the grammar has no reading for -- most likely a file + # migrated from the retired ``[profiles.bulk.ngwmn]``, where a profile did + # carry per-service detail. Dropping it silently would leave the author + # believing they had tuned something. Checked here rather than at parse + # time for the same reason keys are: a malformed profile for one adapter + # must not fail another adapter's call. + nested = sorted(key for key, value in table.items() if isinstance(value, dict)) + if nested: + raise ConfigurationError( + f"{path}: {where} contains a table, [{adapter}.{profile}.{nested[0]}]. " + "A profile names settings for one adapter and nothing else; to " + "configure two adapters for one run, give each its own profile and " + "select both in the same configure() block." + ) + + return { + name: value + for name, (value, _raw) in _checked_table(table, path, where, allowed).items() + } + + +def _current_file() -> tuple[Path, _ParsedFile]: + """The config file as currently loaded: its path and its parsed form. + + One helper so the two always travel together. They are a single fact, and + handing the top-level tier a different ``_ParsedFile`` than the adapter + tier saw in the same resolution is exactly the drift that made an + adapter-scoped read load the file twice. + """ + path = config_path() + return path, _load_file(path) + + +def _adapter_file_settings( + adapter: str, path: Path, parsed: _ParsedFile +) -> Mapping[str, tuple[str, str]]: + """The ``[]`` table's own keys -- its default profile. + + Layers *above* the file's top-level keys rather than being merged into + them: within the file tier an adapter's own value outranks the package-wide + one. The table's sub-tables are its named profiles, which are inert until a + caller selects one, so they are skipped here (see :func:`_accepted_keys`). + + Validated on first use, not at parse time, so a bad value in ``[nldi]`` + cannot fail a Water Data call -- the blast-radius rule ADR 0010 set. + """ + table = parsed.adapters.get(adapter) + if not table: + return {} + + global _adapter_cache + cached = _adapter_cache.get(adapter) + if cached is not None and cached[0] is parsed and cached[1] == path: + return cached[2] + + where = f"[{adapter}]" + # An adapter this process has not imported declares no vocabulary, so its + # table is checked against the package-wide settings alone: refusing a key + # for want of a schema would make the file's validity depend on which + # optional extras happened to be installed. + accepted = settings_for(adapter) + validated = _scalars(table, path, where, SETTINGS if accepted is None else accepted) + label = f"{path} {where}" + result: Mapping[str, tuple[str, str]] = MappingProxyType( + {name: (value, label) for name, value in validated.items()} + ) + _adapter_cache[adapter] = (parsed, path, result) + return result + + +def _load_file(path: Path) -> _ParsedFile: + """Parse the configuration file at *path*, caching until it changes on disk.""" + global _file_cache + try: + st = path.stat() + except FileNotFoundError: + # No file is the normal case: continue to the built-in default. One + # shared empty instance rather than a fresh one per read -- nothing + # mutates a ``_ParsedFile``, and returning the same object each time is + # what lets callers memoize on its identity. + return _NO_FILE + except OSError as exc: + raise ConfigurationError(f"could not access {path}: {exc}") from exc + + if stat.S_ISDIR(st.st_mode): + raise ConfigurationError( + f"configuration path {path} is a directory, not a file." + ) + + # Only a regular file is parsed. Anything else readable -- a character + # device, a FIFO -- is treated as *empty* configuration without being + # opened, which is what ``DATARETRIEVAL_CONFIG=/dev/null`` asks for and the + # only coherent answer for a stream: settings are re-resolved on every + # request, so a FIFO would hand its contents to the first getter and + # nothing to the rest, making the API key vanish mid-run. (It would also + # block on open until a writer appeared.) + if not stat.S_ISREG(st.st_mode): + return _ParsedFile(exists=True) + + # POSIX ``st_ctime_ns`` advances on any inode change, so the metadata stamp + # catches even a rewrite that restores the original mtime (``cp -p``, rsync + # ``--times``, an editor that preserves timestamps). Windows ctime is + # *creation* time, so there the stamp cannot see that class of edit and the + # content compare below is the only correct check -- worth the re-read, + # since serving a stale API key is the alternative. + # + # Dropping this gate (or dropping ctime from the stamp so Windows can use + # it) has been proposed repeatedly on the grounds that the re-read is + # wasteful. It is, but it is also the only thing standing between a + # timestamp-preserving write and a stale credential; a ctime-less stamp is + # identical across exactly that edit. ``test_file_edit_is_picked_up`` + # pins the behavior. Please do not "optimize" it without a Windows-safe + # change detector. + # + # Measured, so the next reviewer does not have to re-derive it: forcing the + # Windows branch costs 27 us per settings read against 5 us with the stamp + # (5 syscalls instead of 1; a 64-byte file and a 6.5 kB one measure the + # same, since the content compare still spares the TOML parse). At the 8 + # reads a one-chunk query performs that is ~175 us against a 100-500 ms + # round trip -- 0.04%, and the alternative is serving a stale key. + cached = _file_cache + if ( + os.name != "nt" + and cached is not None + and cached[0] is path + and cached[1] == _file_stamp(st) + ): + return cached[3] + + try: + with path.open("rb") as handle: + content = handle.read() + opened_st = os.fstat(handle.fileno()) + except OSError as exc: + raise ConfigurationError(f"could not read {path}: {exc}") from exc + + if cached is not None and cached[0] is path and cached[2] == content: + parsed = cached[3] + else: + tomllib = _toml_parser() + try: + data = tomllib.loads(content.decode("utf-8")) + except UnicodeDecodeError as exc: + raise ConfigurationError(f"{path} is not valid UTF-8: {exc}") from exc + except tomllib.TOMLDecodeError as exc: + raise ConfigurationError(f"{path} is not valid TOML: {exc}") from exc + parsed = _interpret(data, path) + _warn_on_loose_permissions(path, opened_st, parsed) + _file_cache = (path, _file_stamp(opened_st), content, parsed) + return parsed + + +def _file_stamp(st: os.stat_result) -> _FileStamp: + """Metadata that changes with file replacement, content, or permissions.""" + return ( + st.st_dev, + st.st_ino, + st.st_mode, + st.st_size, + st.st_mtime_ns, + st.st_ctime_ns, + ) + + +def _interpret(data: dict[str, Any], path: Path) -> _ParsedFile: + """Validate a parsed TOML document into package-wide keys plus adapter tables. + + Only the top-level table is validated here, because it always applies. An + adapter's table is kept raw and validated when that adapter first resolves a + setting: a bad value in ``[nldi]`` must not fail a Water Data call, the same + blast-radius rule :func:`~dataretrieval.utils._default_headers` follows for + the key itself. It is also what lets an adapter's vocabulary live in the + adapter, which this module cannot import. + """ + top: dict[str, Any] = {} + adapters: dict[str, dict[str, Any]] = {} + + for key, value in data.items(): + if key in ADAPTERS: + if not isinstance(value, dict): + raise ConfigurationError( + f"{path}: [{key}] must be a table of settings for the " + f"{key} adapter." + ) + adapters[key] = value + continue + if key == _RETIRED_PROFILES_TABLE: + # A file written against the earlier design, where one profile + # switched every service at once. The generic message below would + # send its author hunting for a typo in a table that is spelled + # exactly as the old docs said, so name the replacement instead. + raise ConfigurationError( + f"{path}: [{_RETIRED_PROFILES_TABLE}] is no longer read. A " + "profile now belongs to one adapter: write [.] " + 'and select it with Configuration.load("").' + ) + if isinstance(value, dict): + raise ConfigurationError( + f"{path}: unknown table [{key}]. Per-adapter tables are " + f"{', '.join(f'[{name}]' for name in ADAPTERS)}; a named profile " + f"goes under one of them, as [.{key}]; top-level keys " + "are the package-wide defaults." + ) + top[key] = value + + return _ParsedFile(_scalars(top, path, _TOP_LEVEL), adapters, exists=True) + + +def _accepted_keys( + table: dict[str, Any], + path: Path, + where: str, + allowed: frozenset[str] | tuple[str, ...], +) -> dict[str, Any]: + """Filter one table down to the settings it is allowed to name. + + The key policy for every table in the file, in one place, so the default + profile and a named profile cannot come to disagree about what is a typo. + An unrecognized name warns rather than raising, so a file written for a + newer release still works; a name this release *does* know but that table + cannot use raises, because that one can never become meaningful. + """ + out: dict[str, Any] = {} + for key, value in table.items(): + if isinstance(value, dict): + # A named profile -- ``[waterdata.bulk]`` parses as a sub-table of + # ``[waterdata]``. Inert until a caller selects it, so it is + # neither a setting here nor an error. Only an adapter's table can + # reach this: the top level rejects unknown tables when it parses, + # and :func:`_named_profile` refuses a table inside a profile, so a + # sub-table here is always a profile rather than deeper nesting. + continue + if key in ADAPTER_ONLY_SETTINGS: + # Rejected from the file wherever it appears. A file that silently + # redirects a data-retrieval library to another host is a + # supply-chain-shaped hazard; an in-code block keeps the redirect + # where a reader of the script sees it (ADR 0011). + raise ConfigurationError( + f"{path}: {key!r} at {where} may only be set in code, in a " + "configure() block, never from a file." + ) + if key not in allowed: + if key in SETTINGS: + # A real setting, in a table that does not read it. Unlike an + # unrecognized name -- which may simply belong to a newer + # release -- this cannot become meaningful later, and silently + # ignoring it would leave a caller believing they had tuned + # something. See ADR 0010. + raise ConfigurationError( + f"{path}: {key!r} at {where} is not a setting that table " + f"accepts. It accepts: {', '.join(sorted(allowed))}." + ) + warnings.warn( + f"{path}: unknown setting {key!r} at {where} (ignored). " + f"Known settings: {', '.join(SETTINGS)}.", + UserWarning, + stacklevel=_WARN_STACKLEVEL, + ) + continue + out[key] = value + return out + + +def _checked_table( + table: dict[str, Any], + path: Path, + where: str, + allowed: frozenset[str] | tuple[str, ...], +) -> dict[str, tuple[Any, str]]: + """Check one table of the file, in both the forms its two readers need. + + Every table in the file comes through here: the top-level keys, an + adapter's default profile, and a named profile. They differ only in what + they do with the result -- the chain wants raw strings, a profile being + loaded wants the TOML scalars to hand back to a configuration's own typed + fields -- so both are returned and each reader takes its half. Written once + because the checks are the interesting part and they must not diverge: a + per-table policy added for one kind of table would otherwise skip the + other, silently. + + ``tomllib`` returns typed scalars (``concurrency = 32`` is an ``int``, + ``concurrency = "unbounded"`` a ``str``), so types are checked here before + values pass through the same grammar used by the other sources. + + Returns + ------- + dict[str, tuple[Any, str]] + Each accepted setting's value as written, and as a raw string. + """ + checked: dict[str, tuple[Any, str]] = {} + for key, value in _accepted_keys(table, path, where, allowed).items(): + if where == _TOP_LEVEL and key in _WARN_AT_TOP_LEVEL: + warnings.warn( + f"{path}: {_WARN_AT_TOP_LEVEL[key]}", + UserWarning, + stacklevel=_WARN_STACKLEVEL, + ) + source = f"{path}: {key!r} at {where}" + raw = _coerce_typed(key, value, source) + _validate_raw(key, raw, source) + checked[key] = (value, raw) + return checked + + +def _scalars( + table: dict[str, Any], + path: Path, + where: str, + allowed: frozenset[str] | tuple[str, ...] = SETTINGS, +) -> dict[str, str]: + """One table's recognized settings, as the raw strings the chain resolves.""" + return { + key: raw + for key, (_value, raw) in _checked_table(table, path, where, allowed).items() + } + + +def _holds_api_key(parsed: _ParsedFile) -> bool: + """Whether the file names an API key anywhere, including inert tables. + + Inert tables count because the question is what the *file* contains, not + what this run resolves: a key sitting in a profile nobody selected is just + as readable to another user on the machine. + """ + if "api_key" in parsed.base: + return True + return any( + "api_key" in table + or any("api_key" in p for p in table.values() if isinstance(p, dict)) + for table in parsed.adapters.values() + ) + + +def _warn_on_loose_permissions( + path: Path, st: os.stat_result, parsed: _ParsedFile +) -> None: + """Warn once if a file holding an API key is readable by other users. + + Follows the ``~/.ssh`` and ``.netrc`` convention, but warns rather than + refusing -- shared filesystems on HPC clusters have their own conventions, + and refusing to read would strand those users. + """ + if os.name != "posix" or path in _permission_warned: + return + if not _holds_api_key(parsed): + return + if stat.S_IMODE(st.st_mode) & 0o077: + _permission_warned.add(path) + warnings.warn( + f"{path} contains an API key and is readable by other users. " + f"Restrict it with: chmod 600 {path}", + UserWarning, + stacklevel=_WARN_STACKLEVEL, + ) + + +def _reset_file_cache() -> None: + """Drop the parsed-file cache. For tests that rewrite the file in place.""" + global _file_cache, _path_cache + _file_cache = None + _path_cache = None + _adapter_cache.clear() + _permission_warned.clear() diff --git a/dataretrieval/_deprecation.py b/dataretrieval/_deprecation.py index fb29c39c0..d8a2917cc 100644 --- a/dataretrieval/_deprecation.py +++ b/dataretrieval/_deprecation.py @@ -26,6 +26,7 @@ REMOVALS: dict[str, str] = { "nwis": "2027-05-06", "waterdata.get_cql(service=)": "2027-08-09", + "wateruse": "2027-08-11", } diff --git a/dataretrieval/_querying.py b/dataretrieval/_querying.py index 9fcde81aa..6a353c77e 100644 --- a/dataretrieval/_querying.py +++ b/dataretrieval/_querying.py @@ -2,7 +2,7 @@ "Compose a USGS query URL, send it, map the status, retry a transient" -- the half of the old ``utils`` module that talks to the network, as used by ``nwis``, -``wqp``, ``nldi``, ``streamstats`` and ``wateruse``. Its other half (pandas +``wqp``, ``nldi``, ``streamstats`` and ``nwdc``. Its other half (pandas column munging) shared nothing with this but a filename: no caller wanted both, and the two have disjoint dependencies -- this one needs ``exceptions`` and ``transport``, that one needs ``codes`` and pandas. @@ -102,7 +102,7 @@ def _raise_for_status( """Raise the typed :class:`DataRetrievalError` for an HTTP error response. A success status returns ``None``. Shared by the legacy :func:`query` path - (and ``streamstats`` / ``wateruse``). Delegates the status-to-type mapping to + (and ``streamstats`` / ``nwdc``). Delegates the status-to-type mapping to :func:`dataretrieval.exceptions.error_for_status`, except a too-long-URL status (413 / 414): that gets the same actionable "split your query" remediation as the client-side over-long-URL case below, rather than a bare @@ -131,14 +131,20 @@ def _raise_for_status( ) -def _single_request_policy() -> RetryPolicy: +def _single_request_policy(adapter: str | None = None) -> RetryPolicy: """Retry policy for the one-shot adapters (WQP, NLDI, StreamStats). These services answer a rejected query with a 500, so only the gateway statuses are worth re-sending; the Water Data chunker keeps the broader default, where a 5xx is an upstream hiccup worth riding out. + + ``adapter`` names which settings table supplies ``retries`` and + ``stall_timeout`` -- these three services share a retry *shape* but not + a settings scope. """ - return RetryPolicy.from_env(retryable_statuses=_GATEWAY_STATUSES) + return RetryPolicy.from_configuration( + retryable_statuses=_GATEWAY_STATUSES, adapter=adapter + ) def _get_with_retry( @@ -146,6 +152,7 @@ def _get_with_retry( *, detail_from: Callable[[httpx.Response], str | None] | None = None, retry_policy: RetryPolicy | None = None, + adapter: str | None = None, **kwargs: Any, ) -> httpx.Response: """GET with status mapping and bounded retry on typed transients.""" @@ -158,7 +165,7 @@ def attempt() -> httpx.Response: try: return retry_sync( attempt, - _single_request_policy() if retry_policy is None else retry_policy, + _single_request_policy(adapter) if retry_policy is None else retry_policy, ) except httpx.InvalidURL as exc: raise _url_too_long_error(f"httpx rejected the URL client-side: {exc}") from exc @@ -171,6 +178,7 @@ def _query_with_retry( ssl_check: bool = True, *, retry_policy: RetryPolicy | None = None, + adapter: str | None = None, ) -> httpx.Response: """Send an active-service query with bounded transient retry by default.""" @@ -188,6 +196,7 @@ def _query_with_retry( headers=user_agent, verify=ssl_check, retry_policy=retry_policy, + adapter=adapter, **HTTPX_DEFAULTS, ) diff --git a/dataretrieval/configuration.py b/dataretrieval/configuration.py new file mode 100644 index 000000000..93a184551 --- /dev/null +++ b/dataretrieval/configuration.py @@ -0,0 +1,767 @@ +"""Layered configuration resolution for ``dataretrieval``. + +Every tunable setting -- the Water Data API key, the fan-out concurrency cap, +the retry count, and the progress line -- resolves through one ordered chain so +a caller never has to mutate ``os.environ`` to configure a single call. + +Sources, highest precedence first: + +1. A configuration passed to :func:`configure` -- delivered through a + :class:`~contextvars.ContextVar`, so a setting applies to the current thread + or asyncio task and cannot leak into another one. +2. The environment variable for that setting (``API_USGS_PAT``, + ``API_USGS_CONCURRENT``, ``API_USGS_RETRIES``, ``API_USGS_PROGRESS``). +3. The configuration file (TOML): ``~/.dataretrieval/config.toml``, or the path + in ``DATARETRIEVAL_CONFIG``. Top-level keys are the package-wide defaults; a + ``[]`` table is that adapter's *default profile*, always in effect; + a ``[.]`` table is a *named profile*, inert until a caller + selects it with ``Configuration.load("")``. +4. The built-in default. + +Those are the four *sources*, which is the decomposition this module is built +around -- one branch each in :func:`_resolve`. ADR 0011 states the same order +as seven rungs by splitting three of them into the scopes inside: source 1 into +a configuration instance and a selected profile, which cannot disagree because +both name one adapter and two configurations for one adapter raise; source 3 +into the ``[]`` table above the top-level keys; and source 4 into an +adapter's own built-in preference above the package default. That last scope is +invisible here because this module never supplies it -- it arrives as the +``default`` a read site like :func:`concurrency` passes for its own service. + +Precedence applies **per setting**, not per source: an environment that sets only +``API_USGS_PAT`` leaves a file-provided ``concurrency`` fully in effect. Putting +the environment above the file follows common deployment conventions and keeps +the original environment-variable interface authoritative (see ADR 0009) -- with +one exception ADR 0011 carves out: a profile named *in code* is a more +deliberate act than a variable inherited from a shell, and a profile reaches the +chain by being passed to :func:`configure`, which is above the environment. + +A caller configures by passing configuration objects, at most one per adapter:: + + with dataretrieval.configure( + Configuration(api_key=vault.read("usgs/pat")), + WaterdataConfiguration.load("bulk"), + NgwmnConfiguration(concurrency=4), + ): + ... + +Settings are scoped **per adapter** (ADR 0010): a ``[ngwmn]`` table in the file, +or an ``NgwmnConfiguration``, applies to NGWMN calls and no others, so one block +can be gentle with one service while leaving the rest alone. Precedence stays +*source-major*: the chain still walks block, then environment, then file, and an +adapter-scoped value outranks a package-wide one only *within* the same source. +So a variable exported for one run still beats a stale adapter table. Within the +block source that tie-break applies per block: an adapter configuration outranks +a package-wide value set by the same ``configure`` call, while a value set by a +block nested inside it wins over both, so the innermost block still decides. + +Which settings an adapter accepts is its own vocabulary -- ``concurrency`` means +nothing to an adapter that issues one request -- so each adapter declares them +on its own :class:`BaseConfiguration` subclass, defined in the module that +*reads* them. The API key is not among them: it belongs to the gateway fronting +a host, which Water Data and NGWMN share. + +This module is a leaf: it imports only the standard library plus the Python 3.10 +``tomli`` backport, so any module can depend on it without an import cycle or +pulling in httpx or pandas. That is also why it holds the adapter *names* but +never imports an adapter -- see :data:`ADAPTERS`. It centralizes each setting's +parser while retaining legacy environment behavior and stricter validation for +the new Python/TOML surfaces. +""" + +from __future__ import annotations + +import os +import sys +from collections.abc import Callable, Iterator, Mapping +from contextlib import contextmanager +from functools import partial +from typing import TextIO, overload + +# Explicit same-name aliases preserve the facade's public and private compatibility +# symbols for runtime users and static analyzers. Ruff otherwise expands these +# aliases into one import statement each, obscuring the boundary inventory. +# isort: off +from dataretrieval._configuration_core import ( + ADAPTERS as ADAPTERS, + CONFIG_PATH_ENV as CONFIG_PATH_ENV, + CONCURRENCY_UNBOUNDED as CONCURRENCY_UNBOUNDED, + DEFAULT_CONCURRENCY as DEFAULT_CONCURRENCY, + DEFAULT_PARALLEL_CHUNKS as DEFAULT_PARALLEL_CHUNKS, + DEFAULT_RETRIES as DEFAULT_RETRIES, + DEFAULT_STALL_TIMEOUT as DEFAULT_STALL_TIMEOUT, + ENV_VARS as ENV_VARS, + SETTINGS as SETTINGS, + BaseConfiguration as BaseConfiguration, + Configuration as Configuration, + _adapter_file_settings as _adapter_file_settings, + _ALL_SETTINGS as _ALL_SETTINGS, + _BLANK_MEANS_SET as _BLANK_MEANS_SET, + _BUILT_IN as _BUILT_IN, + _Chunked as _Chunked, + _coerce_typed as _coerce_typed, + _Concurrent as _Concurrent, + _current_file as _current_file, + _env_source_label as _env_source_label, + _Frame as _Frame, + _named_profiles as _named_profiles, + _NO_FILE as _NO_FILE, + _parse_base_url as _parse_base_url, + _parse_concurrency as _parse_concurrency, + _parse_parallel_chunks as _parse_parallel_chunks, + _parse_progress as _parse_progress, + _parse_retries as _parse_retries, + _parse_seconds as _parse_seconds, + _ParsedFile as _ParsedFile, + _REFUSED_ENV_VARS as _REFUSED_ENV_VARS, + _Redirectable as _Redirectable, + _register as _register, + _REGISTRY as _REGISTRY, + _reset_file_cache as _reset_file_cache, + _Retrying as _Retrying, + _scope as _scope, + _ScopeKey as _ScopeKey, + _SettingValue as _SettingValue, + _UNSET as _UNSET, + _validated_raw as _validated_raw, + config_path as config_path, + settings_for as settings_for, +) + +# isort: on +from dataretrieval.exceptions import ConfigurationError as ConfigurationError + +__all__ = [ + "ADAPTERS", + # The package-wide configuration, and the base every adapter subclasses. + # Public because a caller writes ``Configuration(...)`` at every call site + # that configures anything, and an adapter module names the base in its own + # subclass. + "BaseConfiguration", + "Configuration", + "config_path", + "configure", + "settings_for", + "show_configuration", +] + +# --- public API ---------------------------------------------------------- + + +@contextmanager +def configure(*configurations: BaseConfiguration) -> Iterator[None]: + """Apply configuration profiles for the duration of a ``with`` block. + + The highest-precedence source. Takes configuration objects positionally, at + most one per adapter, and nothing else:: + + with dataretrieval.configure( + Configuration(api_key=secrets["usgs"]), + WaterdataConfiguration.load("bulk"), + NgwmnConfiguration(concurrency=4), + ): + df, md = waterdata.get_daily(monitoring_location_id=sites) + + The adapter a configuration targets is a property of its class, so the + caller never restates it -- which is what keeps the adapter roster from + being spelled once per call site. Naming two configurations for one adapter + raises: they are the one pairing with no defined order between them. + + Because the block is delivered through a :class:`~contextvars.ContextVar`, + a value set here applies to the current thread and to asyncio tasks started + inside the block, and cannot leak into another thread, task, or unrelated + call the way ``os.environ`` does -- which is what makes it safe for a server + or notebook handling several users' credentials at once. + + Blocks nest and merge per setting: an inner block that sets only + ``concurrency`` keeps the outer block's ``api_key``, and an adapter + configuration in an outer block loses to a package-wide value set by a + block nested inside it, so the innermost block always decides. + + Parameters + ---------- + *configurations : BaseConfiguration + A package-wide :class:`Configuration` and/or one configuration per + adapter, in any order. Each adapter's class lives in that adapter's + module -- ``WaterdataConfiguration`` in :mod:`dataretrieval.waterdata`, + ``NgwmnConfiguration`` in :mod:`dataretrieval.ngwmn`, and so on. + + Yields + ------ + None + + Raises + ------ + ConfigurationError + If an argument is not a configuration, or two of them target the same + adapter. Raised on entry, before any request. A bad *value* raises + earlier still, where the configuration was constructed. + + Examples + -------- + .. code-block:: python + + # credentials from a secret store, no environment mutation + with dataretrieval.configure( + Configuration(api_key=vault.read("usgs/pat")) + ): + df, md = waterdata.get_daily(monitoring_location_id="USGS-05114000") + + # a big overnight pull, from a [waterdata.bulk] table in the file + with dataretrieval.configure(WaterdataConfiguration.load("bulk")): + df, md = waterdata.get_daily(monitoring_location_id=many_sites) + + See Also + -------- + show_configuration : Report the effective configuration and where it came from. + """ + with _scope((*_scope.get(), _frame(configurations))): + yield + + +def _frame(configurations: tuple[BaseConfiguration, ...]) -> _Frame: + """Flatten one ``configure`` call's configurations into a scope frame. + + One frame per block, holding both scopes: a package-wide setting keyed by + its name, an adapter-scoped one by ``(adapter, name)``. Values are rendered + back to raw strings here so that every source shares one parser and one set + of error messages; they were already checked when each configuration was + constructed, so nothing new can fail at this point except the two + call-shaped mistakes below. Rendering is therefore all this asks for -- + :func:`_coerce_typed` rather than :func:`_validated_raw`, so a value is not + put through its grammar a second time on every block entry. Construction + stays the single validation point, which is where a typo should raise + anyway: at the line that wrote it, not at a later ``with`` statement. + + Each value is stored with the label naming the configuration it came from, + because this is the last point where that is known -- see :data:`_Frame`. + """ + overrides: dict[_ScopeKey, tuple[_SettingValue, str]] = {} + seen: set[str | None] = set() + for configuration in configurations: + if not isinstance(configuration, BaseConfiguration): + raise ConfigurationError( + "configure() takes configuration objects, not " + f"{type(configuration).__name__}. Package-wide settings go on " + "Configuration(...); a setting for one service goes on that " + "adapter's configuration, e.g. WaterdataConfiguration(...)." + ) + adapter = configuration.adapter + if adapter in seen: + where = f"the {adapter} adapter" if adapter else "the package-wide settings" + raise ConfigurationError( + f"configure() got two configurations for {where}. Precedence " + "between them would be undefined, so combine them into one." + ) + seen.add(adapter) + label = configuration._provenance() + for name, value in configuration.values().items(): + key: _ScopeKey = name if adapter is None else (adapter, name) + raw = ( + None + if value is None + else _coerce_typed(name, value, configuration._source(name)) + ) + overrides[key] = (raw, label) + return overrides + + +def show_configuration(*, stream: TextIO | None = None) -> None: + """Print the effective configuration and the source of each setting. + + A debugging aid for "why is this using my old key?". Every value is + reported with the source that supplied it, named exactly: which variable, + which table of the file, and -- when a caller selected one -- which + profile. The API key is never printed, only whether one is set. + + Parameters + ---------- + stream : file-like, optional + Where to write. Defaults to ``sys.stdout``. + + Examples + -------- + The sample below is generated by running this function, not written by + hand; ``test_show_configuration_sample_output_is_current`` re-runs it and + fails if the two drift apart. + + .. code-block:: text + + >>> with dataretrieval.configure(WaterdataConfiguration.load("bulk")): + ... dataretrieval.show_configuration() + config file /home/u/.dataretrieval/config.toml (found) + api_key /home/u/.dataretrieval/config.toml + concurrency 16 /home/u/.dataretrieval/config.toml + retries 8 $API_USGS_RETRIES + progress auto built-in default + parallel_chunks 1 built-in default + stall_timeout 60s built-in default + + A built-in default is package-wide. An adapter may prefer its own for + its own calls; a value from any source above overrides both. + + adapter overrides + waterdata parallel_chunks 8 configure() block [waterdata.bulk] + ngwmn concurrency 4 /home/u/.dataretrieval/config.toml [ngwmn] + + profiles in the file: [waterdata.bulk] + A profile applies only where a row above names it; select one in + code with Configuration.load(""). + + not reported: nldi (not imported, so the settings each accepts are unknown here) + """ + out = sys.stdout if stream is None else stream + try: + path = config_path() + except ConfigurationError as exc: + # Resolution itself can fail (a relative override with the working + # directory removed). That is precisely a configuration a caller would + # run this to understand, so report it as the file row rather than + # raising out of the explainer. + print(f"config file ", file=out) + return + + # Nothing here raises. This function exists to explain a configuration, and + # the configurations most in need of explaining are the broken ones -- an + # unparseable file, a value that fails its grammar, a profile that no + # longer exists. Each distinct failure is printed once, in the first place + # it shows up; a repeat is collapsed, so one bad file does not bury the + # rows that did resolve under ten copies of the same message. + reported: str | None = None + + def cell(render: Callable[[], object]) -> str: + nonlocal reported + try: + value = render() + except ConfigurationError as exc: + if str(exc) == reported: + return "" + reported = str(exc) + return f"" + return "" if value is None else str(value) + + # Probing the file once here means a whole-file problem -- unparseable + # TOML, a bad value at the top level -- is reported on the file row rather + # than repeated in every setting's row below. The parsed form is kept for + # the profile section, which asks what the file *defines* rather than what + # resolved; an unparseable file defines nothing, and has already said so + # here. + parsed = _NO_FILE + try: + _, parsed = _current_file() + status = "found" if path.exists() else "not found" + except ConfigurationError as exc: + reported = str(exc) + status = f"ERROR: {exc}" + print(f"config file {path} ({status})", file=out) + + rows = [ + (name, cell(partial(_DISPLAYS[name], None)), cell(partial(_source_label, name))) + for name in SETTINGS + ] + name_width = max(len(name) for name, _value, _source in rows) + value_width = max(len(value) for _name, value, _source in rows) + for name, value, source in rows: + print(f"{name:<{name_width}} {value:<{value_width}} {source}", file=out) + + # A built-in default is package-wide, and a service may prefer its own for + # its own calls -- so a row reading "built-in default" is not a promise + # about every service. Saying so is the honest scope of this report: this + # module is a leaf and cannot enumerate the services, and a value from any + # source outranks both kinds of default anyway. + if any(source == _BUILT_IN for _name, _value, source in rows): + print( + "\nA built-in default is package-wide. An adapter may prefer its own " + "for\nits own calls; a value from any source above overrides both.", + file=out, + ) + + _show_adapter_overrides(out, cell, {name: source for name, _value, source in rows}) + _show_profiles(out, parsed) + _show_unimported_adapters(out) + + +def _show_adapter_overrides( + out: TextIO, + cell: Callable[[Callable[[], object]], str], + package_wide: Mapping[str, str], +) -> None: + """Print the adapter-scoped settings that differ from the rows above. + + Only settings actually overridden, and only adapters that override one: a + full adapter-by-setting grid would be mostly inherited values, burying the + answer to "what will this call use" under the rows that change nothing. + + Each row names its source exactly, which for a selected profile is the + profile: ``configure() block [waterdata.bulk]`` rather than a bare block, + so the report answers *which* profile put that value there. + + An adapter this process has not imported has no vocabulary to resolve + against, so it is skipped here and named by + :func:`_show_unimported_adapters` instead. + """ + overrides: list[tuple[str, str, str, str]] = [] + for adapter in ADAPTERS: + accepted = settings_for(adapter) + if accepted is None: + continue + for name in _ALL_SETTINGS: + if name not in accepted: + continue + scoped = cell(partial(_source_label, name, adapter)) + # ``package_wide`` is what the rows above already resolved. Asking + # again would repeat the work once per adapter *and* consume the + # shared error-dedupe state, so a broken config's message could be + # collapsed here before the row that needs it prints. An + # adapter-only setting has no row above, and no package-wide value + # it could inherit, so its baseline is the built-in default. + if scoped == package_wide.get(name, _BUILT_IN): + continue # inherited from the package-wide tier + value = cell(partial(_DISPLAYS[name], adapter)) + overrides.append((adapter, name, value, scoped)) + + if overrides: + print("\nadapter overrides", file=out) + a_width = max(len(a) for a, _n, _v, _s in overrides) + n_width = max(len(n) for _a, n, _v, _s in overrides) + v_width = max(len(v) for _a, _n, v, _s in overrides) + for adapter, name, value, source in overrides: + print( + f" {adapter:<{a_width}} {name:<{n_width}} " + f"{value:<{v_width}} {source}", + file=out, + ) + + +def _show_profiles(out: TextIO, parsed: _ParsedFile) -> None: + """Print the named profiles the file defines, selected or not. + + A named profile does nothing until a caller selects it, and that is the + thing readers of a configuration file get wrong: adding + ``[waterdata.bulk]`` changes no run on its own. A report that mentioned a + profile only when one had been selected would leave that silence with + nothing to explain it -- the file would look ignored. + + Names come from the parsed file, so an unimported adapter's profiles are + listed too. What such a profile *means* is what needs the import; what it + is called is a fact about the file, and withholding it here would make the + section's answer depend on which optional extras happened to be installed. + """ + defined = [ + f"[{adapter}.{name}]" + for adapter in ADAPTERS + for name in sorted(_named_profiles(parsed, adapter)) + ] + if not defined: + return + print(f"\nprofiles in the file: {', '.join(defined)}", file=out) + print( + " A profile applies only where a row above names it; select one in\n" + ' code with Configuration.load("").', + file=out, + ) + + +def _show_unimported_adapters(out: TextIO) -> None: + """Name the adapters this process cannot report on, and say why. + + An adapter is only known to accept a setting once the module declaring that + vocabulary has been imported, and NLDI is deliberately imported on demand + for the geopandas extra. So the rows above cannot cover it. Omitting it + silently would read as "nothing is configured for nldi", which is a + different claim and the wrong one -- this is the honest cost of validating + an adapter's keys lazily (ADR 0011). + """ + unimported = [a for a in ADAPTERS if settings_for(a) is None] + if unimported: + print( + f"\nnot reported: {', '.join(unimported)} " + "(not imported, so the settings each accepts are unknown here)", + file=out, + ) + + +def _source_label(name: str, adapter: str | None = None) -> str: + """The provenance label for one setting, for :func:`show_configuration`.""" + return _resolve(name, adapter)[1] + + +# --- resolved settings --------------------------------------------------- + + +def api_key() -> str | None: + """The Water Data API key, or ``None`` if none is configured. + + Surrounding whitespace is stripped, so a key read from a file with a + trailing newline works; a blank value resolves to ``None``. + """ + raw, _source, _tier = _resolve("api_key") + return raw.strip() or None if raw is not None else None + + +def concurrency( + default: int | None = DEFAULT_CONCURRENCY, *, adapter: str | None = None +) -> int | None: + """Cap on simultaneous chunks; ``None`` means unbounded. + + ``default`` is the caller's own preference for when nothing is configured -- + Water Use ships a lower figure than the OGC getters, because the NWDC is + only stress-tested to that level. A value resolved from the chain always + wins over it: a service able to override an explicit setting would make + ``concurrency=1`` a lie. + """ + raw, source, _tier = _resolve("concurrency", adapter) + if raw is None: + return default + return _parse_concurrency(raw, source) + + +def retries(*, adapter: str | None = None) -> int: + """Retries attempted after the first try; ``0`` disables retrying.""" + raw, source, _tier = _resolve("retries", adapter) + if raw is None: + return DEFAULT_RETRIES + return _parse_retries(raw, source) + + +def progress() -> bool | None: + """Explicit progress-line setting, or ``None`` to auto-detect. + + ``None`` means nothing configured it, so the caller applies its own + default (a TTY or Jupyter kernel gets the line, redirected output + doesn't). + """ + raw, source, tier = _resolve("progress") + if raw is None: + return None + # Preserve the legacy environment behavior (any value outside the false + # set enables progress), while new block/file values are validated strictly. + return _parse_progress(raw, source, strict=tier != _ENV) + + +def parallel_chunks(*, adapter: str | None = None) -> int: + """Configured default fan-out for multi-value queries. + + ``1`` (the default) means "chunk only as much as the URL byte limit + forces". This is the *baseline*; + :func:`dataretrieval.parallel_chunks` overrides it for one call. Shares + the name of that context manager because it is the same setting -- this + is the resolved value, not the scoping block. + """ + raw, source, _tier = _resolve("parallel_chunks", adapter) + if raw is None: + return DEFAULT_PARALLEL_CHUNKS + return _parse_parallel_chunks(raw, source) + + +def stall_timeout(*, adapter: str | None = None) -> float: + """Longest a call may go without receiving data before retrying stops. + + Seconds; ``0`` disables the bound. Bounds the wall-clock cost of a dead + connection, which the retry *count* does not: it counts attempts, not + seconds. See :attr:`dataretrieval.transport.retry.RetryPolicy.stall_timeout`. + """ + raw, source, _tier = _resolve("stall_timeout", adapter) + if raw is None: + return DEFAULT_STALL_TIMEOUT + return _parse_seconds(raw, source) + + +@overload +def base_url(*, adapter: str | None = ...) -> str | None: ... + + +@overload +def base_url(*, adapter: str | None = ..., default: str) -> str: ... + + +def base_url(*, adapter: str | None = None, default: str | None = None) -> str | None: + """An adapter's configured base URL, falling back to *default*. + + Settable from code only: an adapter configuration may carry it, and both + the file and the environment refuse it -- the file at :func:`_accepted_keys` + and the environment at :data:`_REFUSED_ENV_VARS`, each with an error naming + the block to write instead. A file that silently redirects a data-retrieval + library to another host is a supply-chain-shaped hazard, while a + ``configure`` block keeps the redirect where a reader of the script sees it + (ADR 0011). + + There is no package-wide default, because there is no one base URL: what an + adapter's requests are built on is the adapter's own fact, so the service + passes its own -- ``base_url(adapter="nldi", default=NLDI_API_BASE_URL)`` + -- and the URL stays declared beside the service that owns it. What lives + here is the *rule* for choosing between them, which was being spelled at + every read site as ``... or SERVICE_DEFAULT``; a change to it (normalizing + a trailing slash, say) is one edit rather than five. + + Parameters + ---------- + adapter : str, optional + Whose base URL to resolve. + default : str, optional + The service's own base, returned when nothing configured one. Omitted, + the answer is ``None`` -- which is what :func:`show_configuration` asks + for, having no service default to name. + """ + raw, source, _tier = _resolve("base_url", adapter) + if raw is None: + return default + return _parse_base_url(raw, source) + + +# --- resolution ---------------------------------------------------------- + +#: Which tier of the chain answered a resolution. Machine-readable so a +#: per-tier rule reads the tier, never the display label -- :func:`progress` +#: keys its legacy-lenient parsing on ``_ENV``, and the label stays purely +#: presentational. +_BLOCK, _ENV, _FILE, _DEFAULT = "block", "environment", "file", "built-in" + + +def _resolve(name: str, adapter: str | None = None) -> tuple[str | None, str, str]: + """Return the raw value for *name*, a source label, and the tier. + + Precedence is *source-major*: the chain walks block, then environment, then + file, exactly as ADR 0009 defines it -- and *within* each source an + adapter-scoped value outranks a package-wide one. So a variable exported + for one run still beats a stale ``[wqp]`` table in the config file, which + scope-major ordering would have quietly inverted (ADR 0010). + + ``adapter`` names the adapter on whose behalf the setting is being read. + ``None`` resolves the package-wide value, which is also what an adapter + that declares no interest in this setting gets. + + Returns + ------- + tuple[str or None, str, str] + The raw string as written (parsing happens per setting, so each keeps + its own blank-value rule), the human-readable source label, and which + tier answered (one of the constants above) -- ``None`` with + ``_BUILT_IN`` / ``_DEFAULT`` when nothing configured it. + """ + # An adapter name nobody recognizes is a typo in *our* source, and its + # failure mode is silence: ``_accepts`` would wave every setting through, + # the file would hold no table under that name, and the read would fall + # through to the package-wide value -- so a ``[waterdata]`` table, or a + # ``WaterdataConfiguration``, would be ignored with nothing raised + # anywhere. Checked here rather than left to the fitness test that greps + # for ``adapter=""``, which can only see that the string occurs. + if adapter is not None and adapter not in ADAPTERS: + raise ConfigurationError( + f"{adapter!r} is not a configurable adapter. The adapters are " + f"{', '.join(ADAPTERS)}." + ) + + # Refused before anything is consulted, not at the environment's turn in + # the chain. The file refuses ``base_url`` whether or not a block also set + # one -- it raises while the file is read -- and the two surfaces are one + # rule, so a variable that cannot work must not be silently outranked by a + # block that happens to work. Unsetting it is the only fix, and the message + # says so. + refused = _REFUSED_ENV_VARS.get(name) + if refused is not None and refused in os.environ: + raise ConfigurationError( + f"{_env_source_label(refused)} is set, but {name!r} may only be set " + "in code, in a configure() block, never from the environment. Unset " + f"it and pass the value on the adapter's configuration, e.g. " + f"WaterdataConfiguration({name}=...)." + ) + + # ``None`` unless this adapter actually reads this setting, so a setting + # outside its vocabulary resolves package-wide rather than looking for a + # scope it could never have been written into. + scoped: str | None = ( + adapter if adapter is not None and _accepts(adapter, name) else None + ) + + # Innermost block first: a value set by a nested block wins over both + # scopes of an enclosing one. Within one block the adapter-scoped value is + # the more specific of the two, so it is asked first. Each entry already + # carries its own label, which is what keeps the profile a value came from + # reportable (:data:`_Frame`). + for frame in reversed(_scope.get()): + if scoped is not None and (scoped, name) in frame: + return (*frame[(scoped, name)], _BLOCK) + if name in frame: + return (*frame[name], _BLOCK) + + # No per-adapter environment variables: seven adapters times four settings + # is a namespace nobody can hold in mind, and an exported variable is + # invisible at the call site. See ADR 0010. + env = ENV_VARS.get(name) + if env is not None: + raw = os.environ.get(env) + if raw is not None and (raw.strip() or name in _BLANK_MEANS_SET): + return raw, _env_source_label(env), _ENV + + # One load serves both file tiers. Reading the file twice -- once for the + # adapter table, once for the top level -- cost a second stat on every + # adapter-scoped resolution, and the common case (no table for this + # adapter) is the one that paid it. + path, parsed = _current_file() + + if scoped is not None: + from_adapter = _adapter_file_settings(scoped, path, parsed) + if name in from_adapter: + return (*from_adapter[name], _FILE) + + if name in parsed.base: + return parsed.base[name], str(path), _FILE + + return None, _BUILT_IN, _DEFAULT + + +def _accepts(adapter: str, name: str) -> bool: + """Whether *adapter* reads the setting *name*. + + An adapter this process has not imported has no vocabulary to consult, so + every setting is assumed to be in scope for it: the file stays valid either + way, and an adapter cannot be misreading a setting it has not loaded. See + :func:`settings_for`. + """ + accepted = settings_for(adapter) + return name in _ALL_SETTINGS if accepted is None else name in accepted + + +def _display_api_key(adapter: str | None = None) -> str: + """Render the key's presence, never its value.""" + return "" if api_key() else "" + + +def _display_concurrency(adapter: str | None = None) -> str: + value = concurrency(adapter=adapter) + return CONCURRENCY_UNBOUNDED if value is None else str(value) + + +def _display_progress(adapter: str | None = None) -> str: + setting = progress() + return "auto" if setting is None else ("on" if setting else "off") + + +#: How each setting renders in :func:`show_configuration`. Keyed by the same +#: names as :data:`_ALL_SETTINGS`, and asserted to cover them, so a setting +#: added to one without the other fails loudly instead of silently printing a +#: neighbour's value in the one report whose whole job is to be trustworthy. +#: +#: Every renderer takes the adapter to resolve for, so the adapter-override +#: rows use this same table rather than a parallel one that the guard below +#: would not cover. ``api_key`` and ``progress`` ignore it -- neither is +#: adapter-scoped, and :func:`_show_adapter_overrides` never asks them. +_DISPLAYS: dict[str, Callable[[str | None], str]] = { + "api_key": _display_api_key, + "concurrency": _display_concurrency, + "retries": lambda adapter: str(retries(adapter=adapter)), + "progress": _display_progress, + "parallel_chunks": lambda adapter: str(parallel_chunks(adapter=adapter)), + "stall_timeout": lambda adapter: f"{stall_timeout(adapter=adapter):g}s", + "base_url": lambda adapter: base_url(adapter=adapter) or "", +} + +if set(_DISPLAYS) != set(_ALL_SETTINGS): # pragma: no cover - guards a coding error + # Not an ``assert``: ``python -O`` strips those, and this guards the one + # report whose whole job is to be trustworthy about provenance. + raise RuntimeError( + "every setting needs a show_configuration renderer; " + f"missing={sorted(set(_ALL_SETTINGS) - set(_DISPLAYS))} " + f"extra={sorted(set(_DISPLAYS) - set(_ALL_SETTINGS))}" + ) diff --git a/dataretrieval/credentials.py b/dataretrieval/credentials.py index ffb3df02f..a6fb2fed1 100644 --- a/dataretrieval/credentials.py +++ b/dataretrieval/credentials.py @@ -1,25 +1,35 @@ """Which host honors the USGS API key, and how it is attached and withheld. One leaf owns every answer about the ``API_USGS_PAT`` credential: the host that -accepts it, whether a given destination qualifies, and how it is stripped back -off a request bound somewhere else. Splitting those answers across the layers -that happen to need them is how a credential reaches a host nobody authorized: -the code that attaches a key and the code that removes it have to agree, and the -only way to guarantee they agree is to have them read the same predicate. - -This is deliberately a leaf. It sits below HTTP mechanics (which attaches the -header) and below progress reporting (which tells an unauthenticated caller where -to register), so neither has to depend on the other to learn the same fact. +accepts it, whether a given destination qualifies, how it is stripped back off a +request bound somewhere else, and which keyword names are a caller *asking* to +send it. Splitting those answers across the layers that happen to need them is +how a credential reaches a host nobody authorized: the code that attaches a key +and the code that removes it have to agree, and the only way to guarantee they +agree is to have them read the same predicate. + +This sits below HTTP mechanics (which attaches the header) and below progress +reporting (which tells an unauthenticated caller where to register), so neither +has to depend on the other to learn the same fact. Its only first-party +dependency is :mod:`dataretrieval.configuration`, which is itself a +standard-library-only leaf and sits directly beneath this module in the layers +contract -- it supplies the key's *value*, while the questions this module +owns are which host may receive it and how it is withheld from every other. """ from __future__ import annotations -import os +from collections.abc import Iterable import httpx +from dataretrieval import configuration as _configuration + #: Environment variable holding the USGS Water Data personal access token. -API_KEY_ENV = "API_USGS_PAT" +#: Taken from the chain that reads it rather than spelled again here -- the +#: same rule ``test_credential_policy_has_one_definition`` enforces for the +#: authorized host, and for the same reason: two copies stop agreeing silently. +API_KEY_ENV = _configuration.ENV_VARS["api_key"] #: Where to register for a key. Surfaced once, by the progress reporter, when a #: query against the authorized host runs without one -- unauthenticated callers @@ -77,13 +87,77 @@ def without_embedded_credentials(url: httpx.URL) -> httpx.URL: return url.copy_with(userinfo=b"") if url.userinfo else url +# Credential-shaped keyword names must never reach a getter's generic query +# passthrough: URLs are retained by clients, proxies, logs, and response +# metadata. Kept here rather than in the adapter that first needed it, because +# the fact that motivates the check is package-wide -- ``configure()`` now takes +# ``Configuration(api_key=...)``, so a caller who has not read that far reaches +# for ``api_key=`` on whichever getter they are already calling, and every +# adapter with a ``**kwargs`` passthrough is that getter. +# +# Matched as *substrings* of the separator-stripped name, not as exact names: +# an exact-match list missed the spelling the library's own docs make most +# tempting -- ``x_api_key``, after the ``X-Api-Key`` header. +_CREDENTIAL_MARKERS = ( + "apikey", + "authorization", + "credential", + "password", + "passwd", + "secret", + "token", +) + +# Whole names that are credentials on their own but too short to match as +# substrings without catching legitimate query parameters. +# +# ``session`` is deliberately absent from both lists: it carries no secret, so +# rejecting it with a credentials message told users the wrong thing, and as a +# substring it claimed part of a namespace the *server* owns -- any future +# query parameter containing it would have been unreachable behind that message. +_CREDENTIAL_NAMES = frozenset({"auth", "key", "pat", "pw"}) + + +def refuse_credential_keywords(names: Iterable[str]) -> None: + """Raise ``TypeError`` if any of *names* reads as a request for the key. + + For the ``**kwargs`` passthroughs -- Water Data's ``**queryables`` and + WQP's search filters -- where a name the caller invents is forwarded to the + server as a query parameter. Both call this rather than each keeping its + own list, so a spelling learned from one adapter's mistake is refused by + the other on the same day. + + This catches the plausible mistake; it is not a security control. Nothing + inspects *values*, so a secret pasted into ``state_name=`` travels just the + same, and the name space belongs to the server (``get_queryables``) rather + than to us. The point is to answer the caller who reasonably guesses that a + credential goes here, with a ``TypeError`` naming + ``configure(Configuration(api_key=...))`` instead of a token in a URL. It + errs toward rejecting for that reason. + """ + forbidden = set() + for name in names: + flat = name.replace("_", "").replace("-", "").casefold() + if flat in _CREDENTIAL_NAMES or any(m in flat for m in _CREDENTIAL_MARKERS): + forbidden.add(name) + if forbidden: + spellings = ", ".join(f"{name}=" for name in sorted(forbidden)) + raise TypeError( + f"Credentials cannot be passed as query parameters ({spellings}); " + "use dataretrieval.configure(Configuration(api_key=...)) instead." + ) + + def api_key() -> str | None: """The configured token, or ``None``. - Read through a function rather than captured at import so a caller that sets - the variable after import -- or a test that patches it -- is still honored. + Lives here, next to the host check and + :func:`strip_api_key_from_untrusted_host`, so reading the key and the rules + governing where it may travel stay in one module. The value itself resolves + through :func:`dataretrieval.configuration.api_key`, so host scoping applies + identically no matter which source supplied the key. """ - return os.getenv(API_KEY_ENV) + return _configuration.api_key() def strip_api_key_from_untrusted_host(request: httpx.Request) -> None: diff --git a/dataretrieval/exceptions.py b/dataretrieval/exceptions.py index f71be7df1..809d7c246 100644 --- a/dataretrieval/exceptions.py +++ b/dataretrieval/exceptions.py @@ -13,11 +13,14 @@ aren't a plain status: :class:`RequestTooLarge` (with :class:`URLTooLong` / :class:`Unchunkable`), :class:`NetworkError` (a failed connection, per above), :class:`NoSitesError`, and :class:`ConfigurationError` for an unusable setting. -:func:`error_for_status` maps a status to its type. The *warning* side of the -taxonomy lives here too: :class:`SkippedItemWarning` (specialized by -:class:`SkippedRatingWarning`) for a per-item skip inside a batched -retrieval, and :class:`DataCurrencyWarning` for an upstream dataset that has -stopped being updated. +:func:`error_for_status` maps a status to its type. ``ConfigurationError`` is +the one member that is not a request failure at all: it reports an unusable +setting or config file, raised from wherever a setting is first resolved -- +which, because resolution is lazy, is inside whichever getter runs first. The +*warning* side of the taxonomy lives here too: :class:`SkippedItemWarning` +(specialized by :class:`SkippedRatingWarning`) for a per-item skip inside a +batched retrieval, and :class:`DataCurrencyWarning` for an upstream dataset +that has stopped being updated. This module has no third-party runtime dependencies -- ``httpx`` is imported only for type checking. Any module can therefore import it without pulling in pandas @@ -55,7 +58,15 @@ class DataRetrievalError(Exception): - """Base class for every failed-request error in ``dataretrieval``. + """Base class for every ``dataretrieval`` error. + + Almost every member is a failed request, and the read-anywhere fields below + describe one. The exception is :class:`ConfigurationError`, which reports a + configuration the library cannot use; it appears here because configuration + is resolved lazily on the request path, so it surfaces from inside a getter + and one ``except DataRetrievalError`` should cover it too. It carries no + status and is not retryable, so the branching idiom below routes it to the + final ``raise``. Catch it to handle any USGS or EPA service failure uniformly, and branch on the read-anywhere fields below without needing the concrete subclass:: @@ -263,15 +274,18 @@ class NetworkError(DataRetrievalError): class ConfigurationError(DataRetrievalError, ValueError): - """A ``dataretrieval`` setting holds a value that can't be used. - - The setting may be an environment variable or a policy field; either way, - no request was issued. + """A ``dataretrieval`` setting holds a value that can't be used, so no + request was issued -- an environment variable, a policy field, a malformed + ``config.toml``, or a profile the file does not define. It is a :class:`DataRetrievalError` so ``except`` around a retrieval catches - it rather than letting a bare ``ValueError`` escape a request path, and a - :class:`ValueError` so code that already treats a bad setting as one keeps - working. + it rather than letting a bare ``ValueError`` escape a request path. That + matters because settings resolve lazily, on the request path: a broken + config file surfaces from inside whichever getter runs first, and belongs in + the same handler as any other failure of that call. It is *also* a + :class:`ValueError`, so code that already treats a bad setting as one keeps + working whether the value came from the environment, a file, or a + :func:`dataretrieval.configure` block. """ diff --git a/dataretrieval/ngwmn.py b/dataretrieval/ngwmn.py index cc0cf9bfc..93d506963 100644 --- a/dataretrieval/ngwmn.py +++ b/dataretrieval/ngwmn.py @@ -18,11 +18,21 @@ from __future__ import annotations from collections.abc import Iterable -from typing import TYPE_CHECKING, Any +from dataclasses import dataclass +from typing import TYPE_CHECKING, Any, ClassVar import pandas as pd +from dataretrieval import configuration as _configuration from dataretrieval.codes.states import apply_state +from dataretrieval.configuration import ( + BaseConfiguration, + _Chunked, + _Concurrent, + _Redirectable, + _register, + _Retrying, +) from dataretrieval.credentials import WATERDATA_BASE_URL from dataretrieval.ogc import OgcDialect, get_ogc_data, prepare_request_args @@ -30,6 +40,7 @@ from dataretrieval._response_metadata import BaseMetadata __all__ = [ + "NgwmnConfiguration", "get_sites", "get_water_level", "get_lithology", @@ -104,9 +115,14 @@ def _get(service: str, local_vars: dict[str, Any]) -> tuple[pd.DataFrame, BaseMe args, service, output_id=_NGWMN_OUTPUT_ID, - base_url=NGWMN_OGC_API_URL, + # A ``NgwmnConfiguration(base_url=...)`` from an enclosing block, or + # this service's own base. Resolved per call because the block is + # scoped to a ``with`` statement, and read here because this is the one + # place the NGWMN base is named. + base_url=_configuration.base_url(adapter="ngwmn", default=NGWMN_OGC_API_URL), spatial=service == "sites", dialect=NGWMN_DIALECT, + adapter="ngwmn", ) @@ -429,3 +445,48 @@ def get_providers( ... ) """ return _get("providers", locals()) + + +@dataclass(frozen=True) +class NgwmnConfiguration( + _Chunked, _Concurrent, _Redirectable, _Retrying, BaseConfiguration +): + """Settings for NGWMN calls alone. + + NGWMN is a second OGC API on the Water Data host, so its queries + divide along the same URL byte budget and take the same two fan-out + dials. The API key is not among them: one gateway fronts both + adapters, so one key and one quota pool serve them (ADR 0010). + + Lives here rather than in :mod:`dataretrieval.configuration` because + *which* settings a service reads is the service's own knowledge (ADR + 0011); what each of them means is shared, so the fields come from the + setting groups declared beside their grammar. + + Parameters + ---------- + retries : int, optional + Retries attempted after a transient failure; ``0`` disables retrying. + stall_timeout : float, optional + Seconds a call may go without receiving any data before retrying + stops. + base_url : str, optional + OGC API base to send NGWMN requests to, instead of the service's + own (``NGWMN_OGC_API_URL``). Code only: the file and the + environment refuse it. The API key is scoped to the host that + honors it, so a redirected call carries no key. + concurrency : int or str, optional + Cap on simultaneous sub-requests, or ``"unbounded"``. + parallel_chunks : int, optional + Baseline fan-out for multi-value queries. Each sub-request spends + rate-limit quota, so raise it only for pulls you know are large. + """ + + # NGWMN rides the same OGC engine as Water Data, so it reads the same + # groups: retry dials, a redirectable base, and both fan-out dials. The + # settings themselves are declared once in + # :mod:`dataretrieval.configuration`, beside the grammar that parses them. + adapter: ClassVar[str] = "ngwmn" + + +_register(NgwmnConfiguration) diff --git a/dataretrieval/nldi.py b/dataretrieval/nldi.py index cb216d487..bbae1c116 100644 --- a/dataretrieval/nldi.py +++ b/dataretrieval/nldi.py @@ -10,13 +10,22 @@ from __future__ import annotations +from dataclasses import dataclass from json import JSONDecodeError -from typing import Any, Literal, cast +from typing import Any, ClassVar, Literal, cast +from dataretrieval import configuration as _configuration from dataretrieval._querying import _query_with_retry from dataretrieval._validation import require_one_of +from dataretrieval.configuration import ( + BaseConfiguration, + _Redirectable, + _register, + _Retrying, +) __all__ = [ + "NldiConfiguration", "get_flowlines", "get_basin", "get_features", @@ -36,6 +45,23 @@ _VALID_NAVIGATION_MODES = ("UM", "DM", "UT", "DD") +def _api_base() -> str: + """The NLDI base this call targets: a block's redirect, or the service's. + + Every URL below is built from this rather than from + :data:`NLDI_API_BASE_URL` directly, so a ``NldiConfiguration(base_url=...)`` + reaches every navigation, basin, and catalog request alike -- a redirect + that covered only some of them would leave the library asking the real + service about the mirror's data. Resolved per call, because a ``configure`` + block is scoped to a ``with`` statement rather than to the process. + + Six call sites, which is what this seam is for; choosing between the + redirect and the service's own base is the accessor's job, not each + service's. + """ + return _configuration.base_url(adapter="nldi", default=NLDI_API_BASE_URL) + + def _query_nldi( url: str, query_params: dict[str, str], @@ -43,7 +69,7 @@ def _query_nldi( # A helper function to query the NLDI API. ``query()`` already raises a # typed ``DataRetrievalError`` for any HTTP error response, so a returned # response is a success that we only need to parse. - response = _query_with_retry(url, payload=query_params) + response = _query_with_retry(url, payload=query_params, adapter="nldi") response_data: dict[str, Any] | list[Any] = {} try: response_data = response.json() @@ -186,7 +212,7 @@ def get_basin( if not feature_id: raise ValueError("feature_id is required") - url = f"{NLDI_API_BASE_URL}/{feature_source}/{feature_id}/basin" + url = f"{_api_base()}/{feature_source}/{feature_id}/basin" simplified_str = str(simplified).lower() split_catchment_str = str(split_catchment).lower() query_params = { @@ -296,7 +322,7 @@ def _navigation_request( string keeps its documented parameter order. """ origin = f"{feature_source}/{feature_id}" if feature_source else f"comid/{comid}" - url = f"{NLDI_API_BASE_URL}/{origin}/navigation/{navigation_mode}/{tail}" + url = f"{_api_base()}/{origin}/navigation/{navigation_mode}/{tail}" return url, {"distance": str(distance)} @@ -327,7 +353,7 @@ def _get_features_request( "Provide only one origin type - feature_source and feature_id cannot" " be provided with lat or long" ) - return f"{NLDI_API_BASE_URL}/comid/position", {"coords": f"POINT({long} {lat})"} + return f"{_api_base()}/comid/position", {"coords": f"POINT({long} {lat})"} if (comid is not None or data_source is not None) and navigation_mode is None: raise ValueError( @@ -341,7 +367,7 @@ def _get_features_request( _validate_data_source(feature_source) if not navigation_mode: - return f"{NLDI_API_BASE_URL}/{feature_source}/{feature_id}", {} + return f"{_api_base()}/{feature_source}/{feature_id}", {} navigation_mode = _validate_navigation_mode(navigation_mode) url, query_params = _navigation_request( @@ -386,7 +412,7 @@ def get_features_by_data_source(data_source: str) -> gpd.GeoDataFrame: """ # validate the data source _validate_data_source(data_source) - url = f"{NLDI_API_BASE_URL}/{data_source}" + url = f"{_api_base()}/{data_source}" feature_collection = cast("dict[str, Any]", _query_nldi(url, {})) gdf = _features_to_gdf(feature_collection) return gdf @@ -530,7 +556,7 @@ def _validate_data_source(data_source: str) -> None: # get the available data/feature sources - if not already cached if _AVAILABLE_DATA_SOURCES is None: - url = f"{NLDI_API_BASE_URL}/" + url = f"{_api_base()}/" available_data_sources = _query_nldi(url, {}) if not isinstance(available_data_sources, list) or not all( isinstance(ds, dict) and "source" in ds for ds in available_data_sources @@ -576,3 +602,42 @@ def _validate_feature_source_comid( raise ValueError( "Specify one origin type - comid or feature_source is required" ) + + +@dataclass(frozen=True) +class NldiConfiguration(_Redirectable, _Retrying, BaseConfiguration): + """Settings for NLDI calls alone. + + No fan-out dials: an NLDI query is answered by a single request. + + This adapter is imported on demand for the geopandas extra, so this + class registers itself later than the rest -- which is exactly why + the adapter roster lives in :data:`~dataretrieval.configuration.ADAPTERS` + rather than being derived from what has been imported. + + Lives here rather than in :mod:`dataretrieval.configuration` because + *which* settings a service reads is the service's own knowledge (ADR + 0011); what each of them means is shared, so the fields come from the + setting groups declared beside their grammar. + + Parameters + ---------- + retries : int, optional + Retries attempted after a transient failure; ``0`` disables retrying. + stall_timeout : float, optional + Seconds a call may go without receiving any data before retrying + stops. + base_url : str, optional + Linked-data base to send NLDI requests to, instead of the + service's own (``NLDI_API_BASE_URL``). Every navigation, basin + and catalog request is built on it. Code only: the file and the + environment refuse it. + """ + + # One request per call, so this service reads the retry dials and a + # redirectable base and no fan-out dial. Each setting is declared once, + # in :mod:`dataretrieval.configuration`, beside its grammar. + adapter: ClassVar[str] = "nldi" + + +_register(NldiConfiguration) diff --git a/dataretrieval/nwdc.py b/dataretrieval/nwdc.py new file mode 100644 index 000000000..667053f1b --- /dev/null +++ b/dataretrieval/nwdc.py @@ -0,0 +1,505 @@ +"""Retrieve USGS water-use data from the NWDC web service. + +The National Water Availability Assessment Data Companion (NWDC) web services +provide national-scale, USGS-modeled water-use data that underlie the `USGS +National Water Availability Assessment `_. +Estimates are served on a HUC12 (12-digit hydrologic unit) spatial grid and can +be queried for any county, state, or hydrologic unit. This is the modern +replacement for the defunct legacy NWIS water-use service +(``nwis.get_water_use``). + +Unlike the main Water Data getters (:mod:`dataretrieval.waterdata`) and NGWMN +(:mod:`dataretrieval.ngwmn`), the NWDC is a plain CSV REST service rather than +an OGC API Features collection. This module supplies the NWDC-specific bits — +request building, CSV parsing, the ``Link``-header cursor, and the ``{detail}`` +error envelope. The service-neutral transport layer supplies cursor pagination, +response aggregation, client lifecycle, and sync-from-async dispatch. The module +follows the same conventions: host-scoped request headers, the typed +:class:`~dataretrieval.exceptions.DataRetrievalError` taxonomy, and a +``(DataFrame, BaseMetadata)`` return. + +See https://api.water.usgs.gov/docs/nwaa-data/ for the API reference and +https://water.usgs.gov/nwaa-data/ for the catalog of available models and +variables. + +Examples +-------- +.. code-block:: python + + from dataretrieval import nwdc + + # Monthly public-supply withdrawals for Rhode Island, 2020 onward. + df, md = nwdc.get_wateruse( + model="wu-public-supply-wd", + variable=["pswdtot", "pswdgw", "pswdsw"], + state="RI", + start_date="2020-01", + time_resolution="monthly", + ) + +""" + +from __future__ import annotations + +import io +from collections.abc import Callable, Iterable +from dataclasses import dataclass +from typing import Any, ClassVar + +import httpx +import pandas as pd + +from dataretrieval import configuration as _configuration +from dataretrieval._querying import _raise_for_status, to_str +from dataretrieval._response_metadata import BaseMetadata +from dataretrieval.codes.states import to_state +from dataretrieval.configuration import ( + BaseConfiguration, + _Concurrent, + _Redirectable, + _register, + _Retrying, +) +from dataretrieval.exceptions import DataRetrievalError +from dataretrieval.transport.http import default_headers +from dataretrieval.transport.links import resolve_next_url +from dataretrieval.transport.pagination import run_paginated + +__all__ = [ + "NwdcConfiguration", + "get_wateruse", + "WATERUSE_URL", + "MODELS", + "TIME_RESOLUTIONS", + "DEFAULT_CONCURRENT_REQUESTS", +] + +WATERUSE_URL = "https://api.water.usgs.gov/nwaa-data/data" +_WATERUSE_HOST = httpx.URL(WATERUSE_URL).host +# Hosts a ``rel="next"`` cursor may name for this same service; each is +# rewritten to :data:`_WATERUSE_HOST` rather than followed as given. +_WATERUSE_HOST_ALIASES = frozenset({_WATERUSE_HOST, "water.usgs.gov"}) + +#: Water-use models (categories) served by the NWDC. The catalog at +#: https://water.usgs.gov/nwaa-data/ lists the variables available within each. +MODELS = ( + "wu-public-supply-wd", # public-supply withdrawals + "wu-public-supply-cu", # public-supply consumptive use + "wu-thermoelectric", # thermoelectric-power water use + "wu-irrigation-wd", # irrigation withdrawals + "wu-irrigation-cu", # irrigation consumptive use +) + +#: Temporal resolutions: monthly, annual calendar year, annual water year. +TIME_RESOLUTIONS = ("monthly", "annualcy", "annualwy") + +#: This service's preferred in-flight cap when nothing is configured. Lower +#: than the package default of 32 because every location retries +#: independently, so a rate-limit episode bursts this number times the retry +#: count; the NWDC tolerates this level without rate-limit errors (verified by +#: stress test) and higher has not been tested. Any configured concurrency +#: overrides it -- see :func:`dataretrieval.configuration.concurrency` for why the +#: general setting outranks a module's default rather than the reverse. +DEFAULT_CONCURRENT_REQUESTS = 4 + +# Page responses carry the HUC12 identifier in this column; it must stay a +# string so leading zeros (e.g. "010900020502") survive the round trip. +_HUC12_COLUMN = "huc12_id" + + +def get_wateruse( + model: str, + variable: str | Iterable[str] | None = None, + state: str | int | Iterable[str | int] | None = None, + county: str | Iterable[str] | None = None, + huc: str | Iterable[str] | None = None, + time_resolution: str | None = None, + start_date: str | None = None, + end_date: str | None = None, + intersection: str = "overlap", + limit: int = 600, + ssl_check: bool = True, +) -> tuple[pd.DataFrame, BaseMetadata]: + """Get USGS water-use data from the NWDC web service. + + Retrieves modeled water-use estimates from the USGS National Water + Availability Assessment Data Companion. The area is given as exactly one of + ``state``, ``county``, or ``huc``; results are always returned on a HUC12 + grid, in a long (tidy) frame with one row per HUC12 and time step. Large + areas (e.g. a whole region or a populous state) are served across multiple + pages; this function follows those pages transparently and concatenates + them into one frame. + + Each selector also accepts a list of values. The NWDC queries one area per + request, so a list is fanned out into one request per value — up to the + effective ``concurrency`` setting in parallel, defaulting to + :data:`DEFAULT_CONCURRENT_REQUESTS` for this service — and the results are + concatenated in the order given. That cap resolves through the + configuration chain, so it can be raised or lowered for this service alone + (``configure(NwdcConfiguration(concurrency=2))``, or an ``[nwdc]`` table in + the config file) as well as package-wide via ``API_USGS_CONCURRENT``; see + :doc:`the configuration guide `. A fan-out + interrupted by a rate limit or an upstream fault raises a resumable + :class:`~dataretrieval.interruptions.FanOutInterrupted`, whose + ``.call.resume()`` re-issues only the locations that did not complete. + + Parameters + ---------- + model : string + Water-use category to query. See :data:`MODELS` for the available + options (e.g. ``"wu-public-supply-wd"``). The full catalog of models + and their variables is at https://water.usgs.gov/nwaa-data/. + variable : string or iterable of strings, optional + One or more variable IDs within ``model`` (e.g. ``"pswdtot"`` for total + public-supply withdrawals, or ``["pswdgw", "pswdsw"]`` for the + groundwater and surface-water components). Multiple variables are + comma-joined into a single request. The service requires at least one + variable; omitting it returns a 400 listing the model's valid variable + IDs (surfaced as a :class:`~dataretrieval.exceptions.DataRetrievalError`). + state : string, int, or iterable, optional + One or more US states/territories to query. Each accepts a full name + (``"Wisconsin"``), a two-letter postal code (``"WI"``), or a two-digit + ANSI/FIPS code (``"55"`` or ``55``), mirroring + :func:`dataretrieval.ngwmn.get_sites`. + county : string or iterable, optional + One or more five-digit county FIPS codes — state FIPS + county FIPS, + e.g. ``"55025"`` for Dane County, Wisconsin. + huc : string or iterable, optional + One or more hydrologic unit codes. Each code's level is taken from its + length: a 2-digit code queries a HUC2 region, 8-digit a HUC8 subbasin, + 12-digit a single HUC12, and so on (even lengths 2-12, e.g. ``"04"``, + ``"07070005"``, ``"010900020502"``). + + Provide exactly one of ``state``, ``county``, or ``huc`` (each may be a + single value or a list). + time_resolution : string, optional + Temporal resolution: ``"monthly"``, ``"annualcy"`` (annual, calendar + year), or ``"annualwy"`` (annual, water year). See + :data:`TIME_RESOLUTIONS`. + start_date : string, optional + Start of the query window, formatted ``"YYYY"`` for annual data or + ``"YYYY-MM"`` for monthly data. + end_date : string, optional + End of the query window, in the same format as ``start_date``. + intersection : string, optional + How to select HUC12s that straddle the queried-area boundary: + ``"overlap"`` (any overlap, the default) or ``"envelop"`` (fully + enclosed). + limit : int, optional + Maximum number of HUC12s returned per page. Queries spanning more than + ``limit`` HUC12s are split across pages and reassembled. Default 600. + ssl_check : bool, optional + If True (default), verify SSL certificates; set False to skip + verification (e.g. behind a TLS-intercepting proxy). + + Returns + ------- + df : ``pandas.DataFrame`` + Water-use estimates in long form: a ``huc12_id`` column (string, + leading zeros preserved), a time column (``year_month`` for monthly + data or ``year`` for annual data), and one value column per requested + variable (suffixed with its unit, e.g. ``pswdtot_mgd`` for million + gallons per day). + md : :class:`dataretrieval.utils.BaseMetadata` + Metadata describing the request (URL, query time, response headers). + + Raises + ------ + ValueError + If not exactly one of ``state``, ``county``, or ``huc`` is given, or a + given selector is malformed (an unrecognized state, a county code that + is not five digits, or a HUC of invalid length). + DataRetrievalError + On an HTTP error response, the typed subclass for the status (see + :func:`dataretrieval.exceptions.error_for_status`). A transient 429, + 5xx, or recoverable connection failure that exhausts inline retries is + raised as a resumable + :class:`~dataretrieval.interruptions.FanOutInterrupted`; a deterministic + connection failure (for example, a permanently unresolvable host) + remains a :class:`~dataretrieval.exceptions.NetworkError`. + + Examples + -------- + .. doctest:: + :skipif: True # network + + >>> from dataretrieval import nwdc + >>> df, md = nwdc.get_wateruse( + ... model="wu-public-supply-wd", + ... variable=["pswdtot", "pswdgw", "pswdsw"], + ... state="RI", + ... start_date="2020-01", + ... time_resolution="monthly", + ... ) + + """ + # The public parameters are idiomatic snake_case (consistent with + # ``waterdata.get_samples``); the NWDC service expects compact lowercase + # query names, so map to those here as the request is built. + base_params: dict[str, Any] = { + "format": "csv", + "model": model, + "variable": to_str(variable), + "timeres": time_resolution, + "startdate": start_date, + "enddate": end_date, + "intersection": intersection, + "limit": limit, + } + # Drop params the caller left unset; the service rejects empty values. + base_params = {k: v for k, v in base_params.items() if v is not None} + + # An ``NwdcConfiguration(base_url=...)`` from an enclosing block, or this + # service's own endpoint. Resolved once per call -- the block is scoped to + # a ``with`` statement -- and threaded through every request and the page + # walk, so a redirected call cannot half-follow the redirect. + service_url = _configuration.base_url(adapter="nwdc", default=WATERUSE_URL) + + # The NWDC queries one location per request, so fan a multi-value selector + # out into one request per location, each handled by shared transport + # pagination, and concatenate the results. + headers = default_headers(service_url) + requests = [ + httpx.Request( + "GET", + service_url, + params={**base_params, "location": location}, + headers=headers, + ) + for location in _resolve_locations(state, county, huc) + ] + return _fan_out(requests, headers, ssl_check, host=httpx.URL(service_url).host) + + +# Valid HUC code lengths (digits) → the hydrologic-unit level they query. +_HUC_LENGTHS = (2, 4, 6, 8, 10, 12) + +# Maps each selector to the NWDC ``location=:`` value(s) it produces. +# A value may be a single code or a list; ``_as_list`` normalizes both (``state`` +# additionally normalizes to the two-letter postal code, and ``to_state`` may +# itself return a scalar or list, which ``_as_list`` flattens the same way). +# Since NWDC takes one location per request, a list value fans out — one request +# per location (see :func:`_fan_out`). +_LOCATION_BUILDERS: dict[str, Callable[[Any], list[str]]] = { + "state": lambda v: [f"stateCd:{c}" for c in _as_list(to_state(v, to="postal"))], + "county": lambda v: [f"countyCd:{_validate_county(c)}" for c in _as_list(v)], + "huc": lambda v: [f"huc{len(c)}:{c}" for c in map(_validate_huc, _as_list(v))], +} + + +def _resolve_locations( + state: str | int | Iterable[str | int] | None, + county: str | Iterable[str] | None, + huc: str | Iterable[str] | None, +) -> list[str]: + """Build the NWDC ``location=:`` value(s) from the selectors. + + Exactly one of ``state`` / ``county`` / ``huc`` must be given; each may be a + single value or a list. ``state`` is normalized to the two-letter postal + code ``stateCd`` requires; ``county`` is a five-digit FIPS code; and a + ``huc`` code's length selects its level (``huc2`` … ``huc12``). Returns one + location string per value — the caller issues one request per location. + """ + selected = { + name: value + for name, value in (("state", state), ("county", county), ("huc", huc)) + if value is not None + } + if len(selected) != 1: + raise ValueError( + "Specify exactly one of state, county, or huc " + f"(got: {', '.join(selected) or 'none'})." + ) + [(name, value)] = selected.items() + locations = _LOCATION_BUILDERS[name](value) + if not locations: + raise ValueError( + "The chosen location selector is empty; pass at least one value." + ) + return locations + + +def _as_list(value: object) -> list[Any]: + """Normalize a value to a list. + + A scalar becomes a one-element list; any non-string iterable (list, tuple, + Series, ndarray, generator) is materialized to a list. A string is treated + as a scalar so it isn't exploded into characters. + """ + if isinstance(value, Iterable) and not isinstance(value, str): + return list(value) + return [value] + + +def _validate_county(value: object) -> str: + """Validate and normalize a five-digit state+county FIPS code.""" + code = str(value).strip() + if not (code.isdigit() and len(code) == 5): + raise ValueError( + "county must be a five-digit state+county FIPS code " + f"(e.g. '55025'), got {value!r}." + ) + return code + + +def _validate_huc(value: object) -> str: + """Validate a HUC code (even length 2-12 digits; level set by length).""" + code = str(value).strip() + if not (code.isdigit() and len(code) in _HUC_LENGTHS): + raise ValueError( + "huc must be a hydrologic unit code of even length 2-12 digits " + f"(e.g. '04', '07070005', '010900020502'), got {value!r}." + ) + return code + + +def _fan_out( + requests: list[httpx.Request], + headers: dict[str, str], + ssl_check: bool, + *, + host: str = _WATERUSE_HOST, +) -> tuple[pd.DataFrame, BaseMetadata]: + """Fetch every request (each paginated) over the shared fan-out executor. + + This function is only the NWDC-specific half: parse a CSV page and read + its ``Link`` header cursor, follow that cursor, raise the typed error + carrying the NWDC ``detail``, and shape the result. + :func:`~dataretrieval.transport.pagination.run_paginated` owns the rest. + + The plan is the request list itself. The executor asks a plan only to be + sized and iterable, and the NWDC accepts one ``location=`` per request, so + the caller's locations arrive already separate -- there is nothing to + divide and so nothing for a plan class to hold. + + The broad retry status set is on purpose: NWDC reports a bad query as a 400 + with a ``{"detail": ...}`` envelope, so unlike WQP and StreamStats its 5xx + really is an upstream fault worth re-sending. + """ + + def parse(response: httpx.Response) -> tuple[pd.DataFrame, str | None]: + return _read_csv_page(response), _next_page_url(response, host=host) + + async def follow(cursor: str, sess: httpx.AsyncClient) -> httpx.Response: + return await sess.get(cursor, headers=headers) + + def raise_for_status(response: httpx.Response) -> None: + _raise_for_status(response, detail_from=_nwdc_error_detail) + + def finalize( + frame: pd.DataFrame, response: httpx.Response + ) -> tuple[pd.DataFrame, BaseMetadata]: + return frame, BaseMetadata(response) + + return run_paginated( + requests, + parse_response=parse, + follow_up=follow, + raise_for_status=raise_for_status, + finalize=finalize, + client_options={"verify": ssl_check}, + default_concurrent=DEFAULT_CONCURRENT_REQUESTS, + service="nwdc", + adapter="nwdc", + ) + + +def _read_csv_page(response: httpx.Response) -> pd.DataFrame: + """Parse one CSV page; ``huc12_id`` stays a string to keep leading zeros.""" + try: + return pd.read_csv(io.BytesIO(response.content), dtype={_HUC12_COLUMN: str}) + except pd.errors.EmptyDataError as exc: + # NWDC normally signals "no data" with a 400 (handled above) or rows of + # zeros, never an empty body — but keep the typed-error contract if it + # ever returns one rather than leaking a bare pandas exception. + raise DataRetrievalError( + f"NWDC returned an empty response body (URL: {response.url})." + ) from exc + + +def _next_page_url( + response: httpx.Response, *, host: str = _WATERUSE_HOST +) -> str | None: + """Return the absolute URL of the next page, or None if this is the last. + + Reads the standard ``Link: <...>; rel="next"`` header (parsed by httpx into + ``response.links``). The cursor is normalized before it is trusted, because + the service spells it inconsistently. A relative reference is resolved + against the page it came from, and the bare ``water.usgs.gov`` host is + rewritten to the public ``api.water.usgs.gov`` gateway (over https, whatever + scheme the link used) so the follow-up request reaches the API. Only a + cursor that still points somewhere else after that is refused -- following + it would send Water Use requests, and any credentials on them, to a host the + caller never asked for. + + ``host`` is the host this call is actually talking to, which is not the + NWDC's when a ``configure`` block redirected the adapter. The alias list and + the rewrite are facts about *this* service -- nothing else answers for + ``water.usgs.gov`` -- so a redirected call gets the general rule instead: + follow a link only back to the host that served the page. Applying the + NWDC's rewrite there would send page two of a mirrored query to the USGS. + """ + url = response.links.get("next", {}).get("url") + if not url: + return None + if host != _WATERUSE_HOST: + return resolve_next_url(url, response, service="Water Use") + return resolve_next_url( + url, + response, + service="Water Use", + allowed_hosts=_WATERUSE_HOST_ALIASES, + rewrite_host=_WATERUSE_HOST, + ) + + +def _nwdc_error_detail(response: httpx.Response) -> str | None: + """Pull the ``detail`` message out of an NWDC JSON error envelope, if any. + + The NWDC reports errors as ``{"detail": "Invalid model name: ..."}``. Passed + to :func:`~dataretrieval.utils._raise_for_status` as ``detail_from`` so the + service's wording surfaces in the typed error message. + """ + try: + body = response.json() + except ValueError: + return None + return body.get("detail") if isinstance(body, dict) else None + + +@dataclass(frozen=True) +class NwdcConfiguration(_Concurrent, _Redirectable, _Retrying, BaseConfiguration): + """Settings for NWDC calls alone. + + No ``parallel_chunks``: the NWDC is a plain CSV service, so a query + fans out per location rather than being divided along a URL byte + budget. There is nothing for the planner to divide more finely. + + Lives here rather than in :mod:`dataretrieval.configuration` because + *which* settings a service reads is the service's own knowledge (ADR + 0011); what each of them means is shared, so the fields come from the + setting groups declared beside their grammar. + + Parameters + ---------- + retries : int, optional + Retries attempted after a transient failure; ``0`` disables retrying. + stall_timeout : float, optional + Seconds a call may go without receiving any data before retrying + stops. + base_url : str, optional + Endpoint to send NWDC requests to, instead of the service's own + (``WATERUSE_URL``). A ``rel="next"`` cursor is then followed only + back to that host, since the service's own host aliases mean + nothing there. Code only: the file and the environment refuse it. + concurrency : int or str, optional + Cap on simultaneous sub-requests, or ``"unbounded"``. + """ + + # One request per location, fanned out but never chunked, so this service + # reads the retry dials, a redirectable base and ``concurrency`` -- but not + # ``parallel_chunks``, which divides a query it never divides. + adapter: ClassVar[str] = "nwdc" + + +_register(NwdcConfiguration) diff --git a/dataretrieval/nwis.py b/dataretrieval/nwis.py index 2e1f4b132..8703e6794 100644 --- a/dataretrieval/nwis.py +++ b/dataretrieval/nwis.py @@ -736,16 +736,16 @@ def get_pmcodes(**kwargs: Any) -> NoReturn: def get_water_use(**kwargs: Any) -> NoReturn: - """Defunct: use ``dataretrieval.wateruse.get_wateruse`` instead. + """Defunct: use ``dataretrieval.nwdc.get_wateruse`` instead. The legacy NWIS water-use service has been retired. Modeled water-use estimates are now served by the National Water Availability Assessment Data Companion (NWDC); retrieve them with - :func:`dataretrieval.wateruse.get_wateruse`. + :func:`dataretrieval.nwdc.get_wateruse`. """ raise NameError( "`nwis.get_water_use` is defunct; use " - "`dataretrieval.wateruse.get_wateruse` instead." + "`dataretrieval.nwdc.get_wateruse` instead." ) diff --git a/dataretrieval/ogc/chunking.py b/dataretrieval/ogc/chunking.py index c958b49d9..8ed4783d7 100644 --- a/dataretrieval/ogc/chunking.py +++ b/dataretrieval/ogc/chunking.py @@ -24,8 +24,9 @@ :meth:`ChunkPlan._refine`; see ``parallel_chunks`` for the why and the when. Concurrency, retries, and interruption semantics are documented on -:mod:`dataretrieval.transport.fanout`; ``API_USGS_CONCURRENT`` and -``API_USGS_RETRIES`` are read there. +:mod:`dataretrieval.transport.fanout`; the ``concurrency`` and ``retries`` +settings are resolved there, through the chain in +:mod:`dataretrieval.configuration`. Dedup: list-axis chunks don't overlap; filter-axis chunks can, so ``_combine_chunk_frames`` dedupes by feature ``id``. ``properties``, @@ -44,7 +45,7 @@ import httpx import pandas as pd -from dataretrieval._ambient import Ambient +from dataretrieval import configuration as _configuration from dataretrieval.transport.fanout import ( FanOut, _active_client, @@ -56,7 +57,6 @@ from dataretrieval.transport.retry import RetryPolicy from .planning import ChunkPlan -from .policy import _require_positive_int # Compatibility aliases. ``ChunkedCall`` was this module's executor before it # moved down to transport as the API-neutral ``FanOut``; ``get_active_client`` @@ -78,15 +78,6 @@ _OGC_URL_BYTE_LIMIT = 8000 -# Parallel-chunks dial: opt-in to fan a query out *more finely* than the byte -# limit alone requires. Scoped to a ``with parallel_chunks(...):`` block (a -# ContextVar), deliberately NOT an env var (see :func:`parallel_chunks` for -# why). The ambient holds ``n`` — the requested cap on the plan's total -# chunk count; ``1`` (the default, outside any block) means "off — chunk -# only as much as the byte limit needs, no extra fan-out". -_parallel_chunks: Ambient[int] = Ambient("ogc_parallel_chunks", 1) - - @contextmanager def parallel_chunks(n: int) -> Iterator[None]: """ @@ -133,9 +124,9 @@ def parallel_chunks(n: int) -> Iterator[None]: Each chunk fetches at least one page, so it costs at least one request against your hourly rate limit — a larger ``n`` spends more quota. How many chunks run *at once* is capped separately by - ``API_USGS_CONCURRENT`` (default 32), so an ``n`` beyond that adds - quota without adding parallelism; the useful range is roughly ``2`` - up to ``API_USGS_CONCURRENT``. + the ``concurrency`` setting (default 32), so an ``n`` beyond that + adds quota without adding parallelism; the useful range is roughly + ``2`` up to the effective ``concurrency``. Yields ------ @@ -183,10 +174,23 @@ def parallel_chunks(n: int) -> Iterator[None]: -------- ChunkPlan._refine : the planning-side effect of ``n``. """ - # Fail loudly on a bad ``n`` at ``with`` entry, before any request. Shared - # rules with ``max_rows`` via the helper (accepts numpy ints, rejects bool). - _require_positive_int(n, "parallel_chunks(n)", examples="2, 8, 32") - with _parallel_chunks(n): + # Fail loudly on a bad ``n`` at ``with`` entry, before any request -- and + # fail by the *setting's* grammar, not a second one written here. ``n`` is + # ``parallel_chunks``: the same bool/Integral rejection and the same lower + # bound, from the table that owns them, so raising the floor there cannot + # leave this block accepting a value the chain would then refuse. Spelled + # with the source label this block is written as, so the message names + # ``parallel_chunks(n)`` rather than the ``Configuration`` built below. + # ``ConfigurationError`` is a ``ValueError``, so callers catching that + # still catch this. + _configuration._validated_raw("parallel_chunks", n, "parallel_chunks(n)") + # Sugar for a package-wide ``Configuration`` rather than a second scope of + # its own: two competing ContextVars would let ``show_configuration()`` report a + # value the chunker does not use. Sharing one means the innermost block + # wins, whichever spelling opened it -- and package-wide rather than scoped + # to one adapter, because this block is a per-call request that must reach + # whichever adapter the call goes to. + with _configuration.configure(_configuration.Configuration(parallel_chunks=n)): yield @@ -194,6 +198,7 @@ def multi_value_chunked( *, build_request: Callable[..., httpx.Request], url_limit: int | None = None, + adapter: str | None = None, ) -> Callable[[_Fetch[dict[str, Any]]], Callable[..., tuple[pd.DataFrame, Any]]]: """ Decorate an async fetcher to transparently chunk over-budget requests. @@ -251,17 +256,21 @@ def wrapper( finalize: _Finalize = _passthrough_result, ) -> tuple[pd.DataFrame, Any]: limit = _OGC_URL_BYTE_LIMIT if url_limit is None else url_limit - # Read the parallel_chunks dial ``n`` from the ambient set by - # ``parallel_chunks`` (1 = off outside any such block; otherwise the - # requested total chunk cap). It only affects *planning*, done - # here up front, so a later resume — which re-issues the - # already-planned chunks — needs no snapshot. + # Resolve the parallel_chunks dial ``n`` through the configuration + # chain (1 = off unless a ``parallel_chunks``/``configure`` block or + # the config file raised it; otherwise the requested total chunk + # cap). It only affects *planning*, done here up front, so a later + # resume — which re-issues the already-planned chunks — reuses this + # plan rather than resolving again. plan = ChunkPlan( - args, build_request, limit, max_chunks=_parallel_chunks.get() + args, + build_request, + limit, + max_chunks=_configuration.parallel_chunks(adapter=adapter), ) - retry_policy = RetryPolicy.from_env() - # The concurrency cap is resolved inside ``resume()`` from - # ``API_USGS_CONCURRENT``; ``1`` is a sequential gather, + retry_policy = RetryPolicy.from_configuration(adapter=adapter) + # The concurrency cap is resolved inside ``resume()`` through the + # configuration chain; ``1`` is a sequential gather, # ``total <= 1`` a one-element gather — no special branch. return ChunkedCall( plan, @@ -272,6 +281,7 @@ def wrapper( # The collection name, for the progress line the executor # opens. ``get_ogc_data`` puts it in ``args``. service=args.get("collection"), + adapter=adapter, ).resume() return wrapper diff --git a/dataretrieval/ogc/engine.py b/dataretrieval/ogc/engine.py index 064cc6451..b50216957 100644 --- a/dataretrieval/ogc/engine.py +++ b/dataretrieval/ogc/engine.py @@ -225,6 +225,7 @@ def get_ogc_data( extra_id_cols: frozenset[str] | set[str] = frozenset(), dialect: OgcDialect | None = None, cql_body: str | None = None, + adapter: str | None = None, ) -> tuple[pd.DataFrame, BaseMetadata]: """ Retrieves OGC (Open Geospatial Consortium) data as a DataFrame with metadata. @@ -364,10 +365,11 @@ def get_ogc_data( include_geometry=include_geometry, row_cap=max_rows, ), - RetryPolicy.from_env(), + RetryPolicy.from_configuration(adapter=adapter), finalize, canonical_url=str(req.url), service=collection, + adapter=adapter, ).resume() # Bind the API target and quirks into the request builder and fetcher the @@ -386,7 +388,9 @@ def get_ogc_data( include_geometry=include_geometry, row_cap=max_rows, ) - run = chunking.multi_value_chunked(build_request=build_request)(fetch) + run = chunking.multi_value_chunked(build_request=build_request, adapter=adapter)( + fetch + ) # No progress block here: the executor that emits the events owns the line # (see :meth:`~dataretrieval.transport.fanout.FanOut.resume`). return run(args, finalize=finalize) @@ -410,9 +414,9 @@ async def _fetch_once( URL fits, and iterates the cartesian product. With no chunkable inputs the decorator passes args through unchanged. The decorator gathers every chunk over one shared :class:`httpx.AsyncClient` (concurrency - bounded by a semaphore, sized from ``API_USGS_CONCURRENT``) and - returns a *synchronous* wrapper, so ``get_ogc_data`` drives it - synchronously. The return shape is ``(frame, response)``. + bounded by a semaphore, sized from the effective ``concurrency`` + setting) and returns a *synchronous* wrapper, so ``get_ogc_data`` drives + it synchronously. The return shape is ``(frame, response)``. """ req = build_request(**args) return await _walk_pages( diff --git a/dataretrieval/progress.py b/dataretrieval/progress.py index d59bff48f..ea7cc2d37 100644 --- a/dataretrieval/progress.py +++ b/dataretrieval/progress.py @@ -26,12 +26,12 @@ from __future__ import annotations -import os import sys from collections.abc import Iterator from contextlib import contextmanager from typing import TYPE_CHECKING, TextIO +from dataretrieval import configuration as _configuration from dataretrieval._ambient import Ambient from dataretrieval.credentials import SIGNUP_URL, accepts_api_key, api_key @@ -83,9 +83,12 @@ def _enabled_default(stream: TextIO) -> bool: a TTY or a Jupyter/IPython kernel — and stay quiet for redirected output, logs, and CI. """ - override = os.getenv("API_USGS_PROGRESS") + # config owns the grammar, so this is already a bool: the same value means + # the same thing whether it came from a configure() block, the environment, + # or the file. Re-parsing here is what let those three disagree. + override = _configuration.progress() if override is not None: - return override.strip().lower() not in {"", "0", "false", "no", "off"} + return override if _in_jupyter_kernel(): return True return hasattr(stream, "isatty") and stream.isatty() diff --git a/dataretrieval/streamstats.py b/dataretrieval/streamstats.py index 22b3bf451..f3b67074b 100644 --- a/dataretrieval/streamstats.py +++ b/dataretrieval/streamstats.py @@ -7,18 +7,43 @@ from __future__ import annotations import json -from typing import Any, cast +from dataclasses import dataclass +from typing import Any, ClassVar, cast import httpx +from dataretrieval import configuration as _configuration from dataretrieval._querying import _get_with_retry +from dataretrieval.configuration import ( + BaseConfiguration, + _Redirectable, + _register, + _Retrying, +) from dataretrieval.transport.http import HTTPX_DEFAULTS -__all__ = ["download_workspace", "get_sample_watershed", "get_watershed", "Watershed"] +__all__ = [ + "StreamstatsConfiguration", + "Watershed", + "download_workspace", + "get_sample_watershed", + "get_watershed", +] STREAMSTATS_URL = "https://streamstats.usgs.gov/streamstatsservices" +def _service_base() -> str: + """The StreamStats base this call targets: a block's redirect, or its own. + + Both endpoints below hang off this, so a + ``StreamstatsConfiguration(base_url=...)`` moves the whole service rather + than the one endpoint a caller happened to reach first. Resolved per call, + because a ``configure`` block is scoped to a ``with`` statement. + """ + return _configuration.base_url(adapter="streamstats", default=STREAMSTATS_URL) + + def download_workspace(workspaceID: str, format: str = "") -> httpx.Response: """Download a StreamStats workspace. @@ -39,9 +64,9 @@ def download_workspace(workspaceID: str, format: str = "") -> httpx.Response: """ payload = {"workspaceID": workspaceID, "format": format} - url = f"{STREAMSTATS_URL}/download" + url = f"{_service_base()}/download" - r = _get_with_retry(url, params=payload, **HTTPX_DEFAULTS) + r = _get_with_retry(url, params=payload, adapter="streamstats", **HTTPX_DEFAULTS) return r # data = r.raw.read() @@ -142,9 +167,9 @@ def get_watershed( "includefeatures": includefeatures, "simplify": simplify, } - url = f"{STREAMSTATS_URL}/watershed.geojson" + url = f"{_service_base()}/watershed.geojson" - r = _get_with_retry(url, params=payload, **HTTPX_DEFAULTS) + r = _get_with_retry(url, params=payload, adapter="streamstats", **HTTPX_DEFAULTS) if format == "geojson": return r @@ -215,3 +240,37 @@ def _populate(self, streamstats_json: dict[str, Any]) -> None: self.watershed_polygon = streamstats_json["featurecollection"][1]["feature"] self.parameters = streamstats_json["parameters"] self._workspaceID = streamstats_json["workspaceID"] + + +@dataclass(frozen=True) +class StreamstatsConfiguration(_Redirectable, _Retrying, BaseConfiguration): + """Settings for StreamStats calls alone. + + No fan-out dials: a StreamStats query is answered by a single + request. + + Lives here rather than in :mod:`dataretrieval.configuration` because + *which* settings a service reads is the service's own knowledge (ADR + 0011); what each of them means is shared, so the fields come from the + setting groups declared beside their grammar. + + Parameters + ---------- + retries : int, optional + Retries attempted after a transient failure; ``0`` disables retrying. + stall_timeout : float, optional + Seconds a call may go without receiving any data before retrying + stops. + base_url : str, optional + Services base to send StreamStats requests to, instead of its own + (``STREAMSTATS_URL``). Both endpoints hang off it. Code only: + the file and the environment refuse it. + """ + + # One request per call, so this service reads the retry dials and a + # redirectable base and no fan-out dial. Each setting is declared once, + # in :mod:`dataretrieval.configuration`, beside its grammar. + adapter: ClassVar[str] = "streamstats" + + +_register(StreamstatsConfiguration) diff --git a/dataretrieval/transport/env.py b/dataretrieval/transport/env.py deleted file mode 100644 index 2bfaf472b..000000000 --- a/dataretrieval/transport/env.py +++ /dev/null @@ -1,57 +0,0 @@ -"""Environment parsing for the ``API_USGS_*`` numeric knobs. - -A dependency-free leaf: every transport setting read from the environment -shares one grammar and one error voice, and no policy module has to be -imported to get at the parser. -""" - -from __future__ import annotations - -import math -import os -from collections.abc import Callable -from typing import TypeVar - -from dataretrieval.exceptions import ConfigurationError - -_Number = TypeVar("_Number", int, float) - - -def _read_env_number( - name: str, - default: _Number, - cast: Callable[[str], _Number], - expected: str, - *, - minimum: float = 0, - hint: str = "", -) -> _Number: - """Read a bounded number from the environment, or ``default`` if unset. - - The single parser behind every ``API_USGS_*`` numeric knob, so they share - one grammar and one error voice rather than each adapter hand-rolling the - read-cast-validate sequence its own way. - - Raises :class:`~dataretrieval.exceptions.ConfigurationError` -- a - ``DataRetrievalError`` *and* a ``ValueError`` -- for an unusable value, so a - typo in the environment doesn't escape a request path as a bare - ``ValueError`` that ``except DataRetrievalError`` misses. ``hint`` appends - a sentence pointing at the fix when a setting has one (e.g. the keyword - that disables a cap). - """ - raw = os.environ.get(name, "").strip() - if not raw: - return default - try: - value = cast(raw) - except ValueError as exc: - raise ConfigurationError( - f"{name} must be {expected} (got {raw!r}).{hint}" - ) from exc - # ``nan`` passes every ordering test, so a bare ``< minimum`` guard lets it - # through and then silently makes each budget comparison false. - if not math.isfinite(value): - raise ConfigurationError(f"{name} must be {expected} (got {raw!r}).{hint}") - if value < minimum: - raise ConfigurationError(f"{name} must be >= {minimum:g} (got {value}).{hint}") - return value diff --git a/dataretrieval/transport/fanout.py b/dataretrieval/transport/fanout.py index 53b46c8a5..5f9d76468 100644 --- a/dataretrieval/transport/fanout.py +++ b/dataretrieval/transport/fanout.py @@ -27,9 +27,11 @@ ``asyncio.Semaphore`` -- not the client's connection pool, which is merely sized to match -- caps the chunks in flight at ``N``; see :meth:`FanOut._run` for why the gate must be the semaphore rather than the pool. -``API_USGS_CONCURRENT`` resolves ``N``: an integer N > 1 allows N chunks -in flight; ``1`` forces sequential dispatch; the literal ``unbounded`` lifts the -cap. ``N`` bounds only how many of a query's chunks are in flight at once +The ``concurrency`` setting resolves ``N`` -- a ``configure()`` block, then +``API_USGS_CONCURRENT``, then the config file, and per adapter as well as +package-wide: an integer N > 1 allows N chunks in flight; ``1`` forces +sequential dispatch; the literal ``unbounded`` lifts the cap. ``N`` bounds only +how many of a query's chunks are in flight at once -- a client-side trade-off between open connections and fan-out latency. It does not affect the API rate limit: a fanned-out call issues the same number of chunks regardless of ``N``, so ``N`` changes their timing, not the total @@ -41,9 +43,10 @@ Retries: each chunk is retried on a transient failure (429, 5xx, connect/read timeout) with exponential backoff + full jitter, honoring a server -``Retry-After`` when present. ``API_USGS_RETRIES`` sets the cap (default 4; -``0`` disables). A ``Retry-After`` longer than the per-call ceiling escalates to -a resumable interruption. +``Retry-After`` when present. The ``retries`` setting caps them (default 4; +``0`` disables), resolved through the same chain and scopable per adapter. A +``Retry-After`` longer than the per-call ceiling escalates to a resumable +interruption. Interruption: any mid-stream transient failure surfaces as a :class:`~dataretrieval.interruptions.FanOutInterrupted` subclass carrying @@ -56,7 +59,6 @@ import asyncio import functools -import os from collections.abc import Awaitable, Callable, Iterator from typing import Any, Generic, Protocol, TypeVar, cast @@ -64,6 +66,7 @@ import pandas as pd from anyio.from_thread import start_blocking_portal +from dataretrieval import configuration as _configuration from dataretrieval import progress as _progress from dataretrieval._ambient import Ambient from dataretrieval.combining import ( @@ -75,7 +78,6 @@ _classify_chunk_error, _walk_causes, ) -from dataretrieval.transport.env import _read_env_number from dataretrieval.transport.http import network_error, open_async_client from dataretrieval.transport.retry import _NO_RETRY, RetryPolicy from dataretrieval.transport.retry import retry_async as _retry @@ -88,56 +90,12 @@ #: supertype, the way ``Iterable`` is covariant for the same reason. _ChunkCo = TypeVar("_ChunkCo", covariant=True) -# Fan-out concurrency cap, read at call time (not import) so test -# ``monkeypatch.setenv`` applies. Value grammar in :func:`_read_concurrency_env`; -# the concurrency model is in the module docstring. -_CONCURRENCY_ENV = "API_USGS_CONCURRENT" -_CONCURRENCY_DEFAULT = 32 -_CONCURRENCY_UNBOUNDED = "unbounded" - - -def _resolve_concurrency(default: int = _CONCURRENCY_DEFAULT) -> int | None: - """ - Resolve the parallelism cap: the general setting, or a module's default. - - ``API_USGS_CONCURRENT`` is the general knob and applies to every fanned-out - call in the package. A module may pass a different ``default`` when its - service warrants one — Water Use ships a lower figure than the OGC getters, - because the NWDC is only stress-tested to that level. - - The ordering is deliberate: an explicitly set environment variable wins over - a module's default, never the reverse. A module that could override the - general setting would make ``API_USGS_CONCURRENT=1`` a lie — the user - dialing concurrency down to be polite to the service would find one adapter - quietly ignoring them, which is precisely the defect this consolidates away. - Module defaults express "absent instruction, this service prefers N"; they - do not express "this service knows better than you". - - Parameters - ---------- - default : int - Cap to use when ``API_USGS_CONCURRENT`` is unset or empty. - - Returns - ------- - int or None - ``1`` for sequential dispatch (one chunk at a time); an - integer >1 for bounded concurrency; ``None`` to disable the - per-call cap entirely (the ``unbounded`` keyword). - """ - # Only the ``unbounded`` keyword is specific to this knob; the rest is the - # same read-cast-validate every ``API_USGS_*`` number gets, so it delegates - # rather than growing a third copy with its own error wording. - if os.environ.get(_CONCURRENCY_ENV, "").strip().lower() == _CONCURRENCY_UNBOUNDED: - return None - return _read_env_number( - _CONCURRENCY_ENV, - default, - int, - f"a positive integer or '{_CONCURRENCY_UNBOUNDED}'", - minimum=1, - hint=f" Use '{_CONCURRENCY_UNBOUNDED}' to disable the cap.", - ) +# The fan-out concurrency cap resolves through +# :func:`dataretrieval.configuration.concurrency`, which owns the setting's name, its +# grammar (``1`` sequential, >1 bounded, ``unbounded`` uncapped) and its +# built-in default. Naming any of those here too would let this module and the +# chain disagree about what a value means. The concurrency model -- why the cap +# is a semaphore rather than the connection pool -- is in the module docstring. # --------------------------------------------------------------------------- @@ -281,8 +239,8 @@ class FanOut(Generic[_Chunk]): Extra ``httpx.AsyncClient`` options for the shared client this run opens (e.g. ``{"verify": False}``). default_concurrent : int, optional - This service's preferred in-flight cap when ``API_USGS_CONCURRENT`` - is unset. Defaults to 32. + This adapter's preferred in-flight cap for when nothing is + configured. Any resolved ``concurrency`` outranks it. Defaults to 32. canonical_url : str or None, optional URL identifying the query as a whole, restored onto the combined response so the caller sees the request they made rather than @@ -290,7 +248,7 @@ class FanOut(Generic[_Chunk]): :meth:`resume` labels its progress line with. service : str or None, optional Human-facing name of what is being retrieved (e.g. ``"daily"``, - ``"wateruse"``), used to label the progress line :meth:`resume` + ``"nwdc"``), used to label the progress line :meth:`resume` opens. ``None`` leaves the line unlabelled. Attributes @@ -319,10 +277,11 @@ def __init__( retry_policy: RetryPolicy = _NO_RETRY, finalize: _Finalize = _passthrough_result, client_options: dict[str, Any] | None = None, - default_concurrent: int = _CONCURRENCY_DEFAULT, + default_concurrent: int = _configuration.DEFAULT_CONCURRENCY, *, canonical_url: str | None = None, service: str | None = None, + adapter: str | None = None, ) -> None: self.plan = plan self.fetch = fetch @@ -333,10 +292,17 @@ def __init__( # to ``canonical_url``, because this class is what emits the progress # events — see :meth:`resume`. self.service = service - # This service's preferred cap when the user has not set - # ``API_USGS_CONCURRENT``. Resolved at resume time, not here, so a - # test's ``monkeypatch.setenv`` still applies. See - # :func:`_resolve_concurrency` for why the env var outranks it. + # Which adapter's settings this drive resolves, so a ``[ngwmn]`` table + # or an ``NgwmnConfiguration`` reaches only NGWMN calls. Distinct from + # ``service`` above, which is a *display label* for the progress line + # and is variously a collection name or prose. ``None`` resolves + # package-wide. See ADR 0010. + self.adapter = adapter + # This service's preferred cap for when nothing is configured. Resolved + # at resume time, not here, so a setting that arrives after this call + # was built still applies. Anything the chain resolves outranks it -- + # see :func:`dataretrieval.configuration.concurrency` for why a service + # preference must not override an explicit setting. self.default_concurrent = default_concurrent # Extra ``httpx.AsyncClient`` options merged into the shared client this # run opens (``verify`` for the Water Use ``ssl_check`` flag, say). The @@ -543,7 +509,14 @@ def resume(self) -> tuple[pd.DataFrame, Any]: with _progress.progress_context( service=self.service, target_url=self.canonical_url ): - concurrency = _resolve_concurrency(self.default_concurrent) + # Resolve concurrency here, per drive, rather than at construction. + # It is the one dial a caller adjusts precisely *while* retrying -- + # the documented recovery from QuotaExhausted is to wait and + # re-issue more gently -- so a ``configure()`` block entered + # between the interruption and the resume has to win. + concurrency = _configuration.concurrency( + self.default_concurrent, adapter=self.adapter + ) with start_blocking_portal() as portal: # ``portal.call`` returns ``Any`` because ``functools.partial`` # erases ``_run``'s return type; restore the declared tuple. diff --git a/dataretrieval/transport/http.py b/dataretrieval/transport/http.py index 1b2e29b3e..b14d5bb61 100644 --- a/dataretrieval/transport/http.py +++ b/dataretrieval/transport/http.py @@ -48,16 +48,26 @@ def default_headers(target_url: str | httpx.URL | None = None) -> dict[str, str]: - """Build standard headers, scoping the API key to its authorized host.""" + """Build standard headers, scoping the API key to its authorized host. + + The host is checked *before* the key is resolved, and the key is resolved + only for the authorized host. Order matters now that settings come from a + layered chain: resolution reads the config file and can raise + :class:`~dataretrieval.exceptions.ConfigurationError` for a malformed file or + a profile it no longer defines. Resolving first would let a Water Data + configuration problem break a legacy NWIS, WQP, or NGWMN call that would + never have received the key. + """ headers = { "Accept-Encoding": "compress, gzip", "Accept": "application/json", "User-Agent": USER_AGENT, "lang": "en-US", } - token = api_key() - if token and accepts_api_key(target_url): - headers["X-Api-Key"] = token + if accepts_api_key(target_url): + token = api_key() + if token: + headers["X-Api-Key"] = token return headers diff --git a/dataretrieval/transport/pagination.py b/dataretrieval/transport/pagination.py index 052ad940e..1cc2e27cb 100644 --- a/dataretrieval/transport/pagination.py +++ b/dataretrieval/transport/pagination.py @@ -14,6 +14,7 @@ import httpx import pandas as pd +from dataretrieval import configuration as _configuration from dataretrieval import progress as _progress from dataretrieval.combining import ( _QUOTA_HEADER, @@ -24,7 +25,6 @@ # One-way: ``fanout`` does not import this module, so this edge cannot cycle. from dataretrieval.transport.fanout import ( - _CONCURRENCY_DEFAULT, FanOut, _Finalize, _passthrough_result, @@ -161,8 +161,9 @@ def run_paginated( finalize: _Finalize = _passthrough_result, client: httpx.AsyncClient | None = None, client_options: dict[str, Any] | None = None, - default_concurrent: int = _CONCURRENCY_DEFAULT, + default_concurrent: int = _configuration.DEFAULT_CONCURRENCY, canonical_url: str | None = None, + adapter: str | None = None, ) -> tuple[pd.DataFrame, Any]: """Drive one full page walk per request through the shared executor. @@ -175,7 +176,9 @@ def run_paginated( Raw transport errors need no mapping in the strategies: the executor retries them and normalizes a deterministic one into the typed - :class:`~dataretrieval.exceptions.NetworkError`. + :class:`~dataretrieval.exceptions.NetworkError`. ``adapter`` names the + adapter for the settings chain, so a ``[waterdata] retries = 2`` table + scopes to that adapter alone. """ async def fetch(request: httpx.Request) -> tuple[pd.DataFrame, httpx.Response]: @@ -192,10 +195,11 @@ async def fetch(request: httpx.Request) -> tuple[pd.DataFrame, httpx.Response]: return FanOut( requests, fetch, - RetryPolicy.from_env(), + RetryPolicy.from_configuration(adapter=adapter), finalize=finalize, client_options=client_options, default_concurrent=default_concurrent, canonical_url=canonical_url, service=service, + adapter=adapter, ).resume() diff --git a/dataretrieval/transport/retry.py b/dataretrieval/transport/retry.py index 95dd1b882..d38d61454 100644 --- a/dataretrieval/transport/retry.py +++ b/dataretrieval/transport/retry.py @@ -11,6 +11,7 @@ import httpx +from dataretrieval import configuration as _configuration from dataretrieval import progress as _progress from dataretrieval.exceptions import ( ConfigurationError, @@ -18,7 +19,6 @@ TransientError, ) from dataretrieval.interruptions import _deterministic_failure -from dataretrieval.transport.env import _read_env_number from dataretrieval.transport.liveness import ( credit_wait, elapsed_since_progress, @@ -38,8 +38,6 @@ # on a request that can never succeed and delays the caller's error. _RETRYABLE_STATUSES = frozenset({429, *range(500, 600)}) _GATEWAY_STATUSES = frozenset({429, 502, 503, 504}) -_RETRIES_ENV = "API_USGS_RETRIES" -_RETRIES_DEFAULT = 4 _RETRY_BASE_BACKOFF = 0.5 _RETRY_MAX_BACKOFF = 30.0 _RETRY_AFTER_CAP = 60.0 @@ -49,8 +47,6 @@ _RETRY_AFTER_JITTER = 1.0 # Attempts the no-progress budget never withholds; see RetryPolicy.allows_wait. _STALL_EXEMPT_ATTEMPTS = 1 -_STALL_TIMEOUT_ENV = "API_USGS_STALL_TIMEOUT" -_STALL_TIMEOUT_DEFAULT = 60.0 _T = TypeVar("_T") @@ -64,8 +60,10 @@ class RetryPolicy: call may go on receiving nothing. """ - #: Attempts after the first. ``0`` disables retry entirely. - max_retries: int = _RETRIES_DEFAULT + #: Attempts after the first. ``0`` disables retry entirely. The default is + #: ``config``'s, not a second copy of it: a directly-constructed policy and + #: one built by :meth:`from_configuration` must agree on the retry budget. + max_retries: int = _configuration.DEFAULT_RETRIES #: First backoff ceiling; doubles per attempt up to :attr:`max_backoff`. base_backoff: float = _RETRY_BASE_BACKOFF #: Ceiling for our own exponential backoff. @@ -91,7 +89,7 @@ class RetryPolicy: #: productive download is never cut short, and an attempt already in flight #: is never interrupted. ``0`` disables the bound. See :meth:`allows_wait` #: for how it is applied. - stall_timeout: float = _STALL_TIMEOUT_DEFAULT + stall_timeout: float = _configuration.DEFAULT_STALL_TIMEOUT def __post_init__(self) -> None: if self.max_retries < 0: @@ -107,25 +105,32 @@ def __post_init__(self) -> None: raise ConfigurationError("retry backoff settings must be non-negative.") @classmethod - def from_env(cls, retryable_statuses: frozenset[int] | None = None) -> RetryPolicy: - """Build a policy from current environment and module defaults.""" + def from_configuration( + cls, + retryable_statuses: frozenset[int] | None = None, + *, + adapter: str | None = None, + ) -> RetryPolicy: + """Build a policy from the effective configuration and module defaults. + + ``max_retries`` and ``stall_timeout`` both resolve through + :mod:`dataretrieval.configuration` -- a ``configure()`` block, then the + environment variable, then the config file. ``adapter`` names the + adapter this policy is for, so a ``[wqp] retries = 2`` table applies to + WQP calls and nothing else; ``None`` resolves package-wide. The pure + timing knobs stay module constants read at call time so a test's + ``monkeypatch.setattr`` still applies. + """ statuses = ( _RETRYABLE_STATUSES if retryable_statuses is None else retryable_statuses ) return cls( retryable_statuses=statuses, - max_retries=_read_env_number( - _RETRIES_ENV, _RETRIES_DEFAULT, int, "a non-negative integer" - ), + max_retries=_configuration.retries(adapter=adapter), base_backoff=_RETRY_BASE_BACKOFF, max_backoff=_RETRY_MAX_BACKOFF, retry_after_cap=_RETRY_AFTER_CAP, - stall_timeout=_read_env_number( - _STALL_TIMEOUT_ENV, - _STALL_TIMEOUT_DEFAULT, - float, - "a non-negative number of seconds", - ), + stall_timeout=_configuration.stall_timeout(adapter=adapter), ) def should_retry(self, attempt: int, retry_after: float | None) -> bool: @@ -272,7 +277,7 @@ async def retry_async( silence. A caller that gated its own body would have to rediscover both, and nothing would catch it getting them wrong. """ - policy = RetryPolicy.from_env() if policy is None else policy + policy = RetryPolicy.from_configuration() if policy is None else policy attempt = 0 note_progress() @@ -303,7 +308,7 @@ def retry_sync(fn: Callable[[], _T], policy: RetryPolicy | None = None) -> _T: not caught because the loop handles ``Exception`` rather than ``BaseException``. """ - policy = RetryPolicy.from_env() if policy is None else policy + policy = RetryPolicy.from_configuration() if policy is None else policy attempt = 0 note_progress() while True: diff --git a/dataretrieval/waterdata/__init__.py b/dataretrieval/waterdata/__init__.py index 48c4d9fb9..988d3e620 100644 --- a/dataretrieval/waterdata/__init__.py +++ b/dataretrieval/waterdata/__init__.py @@ -33,6 +33,7 @@ get_stats_por, get_time_series_metadata, ) +from .configuration import WaterdataConfiguration from .nearest import get_nearest_continuous from .ratings import get_ratings from .types import ( @@ -46,6 +47,7 @@ __all__ = [ "CODE_SERVICES", "FILTER_LANG", + "WaterdataConfiguration", "PROFILES", "PROFILE_LOOKUP", "SERVICES", diff --git a/dataretrieval/waterdata/configuration.py b/dataretrieval/waterdata/configuration.py new file mode 100644 index 000000000..8813b3c4e --- /dev/null +++ b/dataretrieval/waterdata/configuration.py @@ -0,0 +1,69 @@ +"""The settings the Water Data adapter reads -- its configuration profile. + +A file of its own because :mod:`dataretrieval.waterdata` is a package rather +than a single module; every other adapter declares its class in the module a +caller imports. Either way the point is the same: a setting's definition sits +with the code that reads it, so adding one no longer edits a service-neutral +file (ADR 0011). +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import ClassVar + +from dataretrieval.configuration import ( + BaseConfiguration, + _Chunked, + _Concurrent, + _Redirectable, + _register, + _Retrying, +) + +__all__ = ["WaterdataConfiguration"] + + +@dataclass(frozen=True) +class WaterdataConfiguration( + _Chunked, _Concurrent, _Redirectable, _Retrying, BaseConfiguration +): + """Settings for Water Data calls alone. + + Pass one to :func:`dataretrieval.configure` to narrow a setting to this + service, leaving every other adapter on whatever the tiers below it + resolve:: + + with dataretrieval.configure(WaterdataConfiguration(concurrency=8)): + df, md = waterdata.get_daily(monitoring_location_id=sites) + + Parameters + ---------- + retries : int, optional + Retries attempted after a transient failure; ``0`` disables retrying. + stall_timeout : float, optional + Seconds a call may go without receiving any data before retrying stops. + base_url : str, optional + Root to send Water Data requests to, instead of the service's own. The + package appends its own paths, so one value moves all four families + together -- ``/ogcapi/v0``, ``/samples-data``, ``/statistics/v0`` and + ``/stac/v0``. Code only: the file and the environment refuse it. The + API key is scoped to the host that honors it, so a redirected call + carries no key. + concurrency : int or str, optional + Cap on simultaneous sub-requests, or ``"unbounded"``. + parallel_chunks : int, optional + Baseline fan-out for multi-value queries. Each sub-request spends + rate-limit quota, so raise it only for pulls you know are large. + """ + + # The settings this service reads, named by the groups they come from: + # every adapter's retry dials, a redirectable base, and -- because Water + # Data queries divide along a URL byte budget and are executed concurrently + # -- both fan-out dials. Each group declares the setting itself once, in + # :mod:`dataretrieval.configuration`, which is also where its grammar and + # its coercion live. + adapter: ClassVar[str] = "waterdata" + + +_register(WaterdataConfiguration) diff --git a/dataretrieval/waterdata/endpoints.py b/dataretrieval/waterdata/endpoints.py index 7611d1204..4a8cc6e63 100644 --- a/dataretrieval/waterdata/endpoints.py +++ b/dataretrieval/waterdata/endpoints.py @@ -7,35 +7,61 @@ key -- while the paths below stay here rather than importing OGC policy internals. -This module imports nothing but that leaf, so a family module can name its -endpoint without also taking on an OGC or transport edge. +This module imports only leaves -- the credentials host and the configuration +chain -- so a family module can name its endpoint, and honor a caller's +redirect, without also taking on an OGC or transport edge. """ from __future__ import annotations +from dataretrieval import configuration as _configuration from dataretrieval.credentials import WATERDATA_BASE_URL -#: Root of the modernized Water Data APIs. -BASE_URL = WATERDATA_BASE_URL +#: Canonical paths below the Water Data root. They are not endpoints on their +#: own: callers obtain complete destinations through the functions below, which +#: makes scoped redirection part of endpoint acquisition rather than a wrapper +#: every use site must remember. +_OGC_API_PATH = "/ogcapi/v0" +_SAMPLES_PATH = "/samples-data" +_STATISTICS_API_PATH = "/statistics/v0" +_RATINGS_CATALOG_PATH = "/stac/v0" -#: OGC API - Features service backing the typed collection getters. -OGC_API_URL = f"{BASE_URL}/ogcapi/v0" +# Default-value compatibility for the documented ``waterdata.utils`` constants. +# Production collection-family modules do not import these raw values. +_DEFAULT_BASE_URL = WATERDATA_BASE_URL +_DEFAULT_OGC_API_URL = f"{_DEFAULT_BASE_URL}{_OGC_API_PATH}" +_DEFAULT_SAMPLES_URL = f"{_DEFAULT_BASE_URL}{_SAMPLES_PATH}" -#: Samples database (discrete water-quality results, WQX3 CSV). -SAMPLES_URL = f"{BASE_URL}/samples-data" -#: Daily-statistics service (period-of-record and date-range normals). -STATISTICS_API_VERSION = "v0" -STATISTICS_API_URL = f"{BASE_URL}/statistics/{STATISTICS_API_VERSION}" +def _endpoint(path: str) -> str: + """Return *path* beneath the effective Water Data root for this call.""" + root = _configuration.base_url(adapter="waterdata", default=WATERDATA_BASE_URL) + return f"{root}{path}" + + +def ogc_api_url() -> str: + """Return the OGC collections endpoint for the effective configuration.""" + return _endpoint(_OGC_API_PATH) + + +def samples_url() -> str: + """Return the Samples endpoint for the effective configuration.""" + return _endpoint(_SAMPLES_PATH) + + +def statistics_api_url() -> str: + """Return the Statistics endpoint for the effective configuration.""" + return _endpoint(_STATISTICS_API_PATH) + + +def ratings_catalog_url() -> str: + """Return the Ratings catalog endpoint for the effective configuration.""" + return _endpoint(_RATINGS_CATALOG_PATH) -#: STAC catalog serving NWIS rating-curve assets. -STAC_URL = f"{BASE_URL}/stac/v0" __all__ = [ - "BASE_URL", - "OGC_API_URL", - "SAMPLES_URL", - "STAC_URL", - "STATISTICS_API_URL", - "STATISTICS_API_VERSION", + "ogc_api_url", + "ratings_catalog_url", + "samples_url", + "statistics_api_url", ] diff --git a/dataretrieval/waterdata/ratings.py b/dataretrieval/waterdata/ratings.py index e00c949d1..d5aecd4a3 100644 --- a/dataretrieval/waterdata/ratings.py +++ b/dataretrieval/waterdata/ratings.py @@ -30,7 +30,7 @@ from dataretrieval.transport.links import resolve_next_url from dataretrieval.transport.pagination import run_paginated from dataretrieval.transport.retry import RetryPolicy -from dataretrieval.waterdata.endpoints import STAC_URL +from dataretrieval.waterdata.endpoints import ratings_catalog_url __all__ = ["get_ratings"] @@ -256,7 +256,7 @@ def _search( if bbox is not None: query_params["bbox"] = ",".join(map(str, bbox)) - url = f"{STAC_URL}/search" + url = f"{ratings_catalog_url()}/search" req = httpx.Request("GET", url, params=query_params, headers=_default_headers(url)) def parse_response(resp: httpx.Response) -> tuple[pd.DataFrame, str | None]: @@ -288,6 +288,7 @@ async def follow_up(cursor: str, sess: httpx.AsyncClient) -> httpx.Response: raise_for_status=_raise_for_non_200, client_options={"verify": ssl_check}, service="ratings", + adapter="waterdata", ) # Every page frame is built with a ``feature`` column, and the combine # helpers preserve it, so the empty case needs no special branch. @@ -392,7 +393,7 @@ async def fetch(feature: dict[str, Any]) -> tuple[pd.DataFrame, httpx.Response]: ) # 204: completed, no content. return pd.DataFrame(), _inert_response( - 204, _asset_href(feature) or f"{STAC_URL}/search" + 204, _asset_href(feature) or f"{ratings_catalog_url()}/search" ) out[fid] = df return df, _inert_response( @@ -402,11 +403,12 @@ async def fetch(feature: dict[str, Any]) -> tuple[pd.DataFrame, httpx.Response]: FanOut( features, fetch, - RetryPolicy.from_env(), + RetryPolicy.from_configuration(adapter="waterdata"), client_options={"verify": ssl_check}, # No single URL expresses "all of these assets" -- the aggregate # reports the first, matching what a single-feature call would show. canonical_url=_asset_href(features[0]), service="ratings", + adapter="waterdata", ).resume() return out diff --git a/dataretrieval/waterdata/reference.py b/dataretrieval/waterdata/reference.py index 506506a5e..38ab0c95d 100644 --- a/dataretrieval/waterdata/reference.py +++ b/dataretrieval/waterdata/reference.py @@ -14,13 +14,11 @@ from dataretrieval._validation import require_one_of from dataretrieval.ogc.schema import queryables_frame +from dataretrieval.waterdata.endpoints import ogc_api_url from dataretrieval.waterdata.types import ( METADATA_COLLECTIONS, ) -from dataretrieval.waterdata.utils import ( - OGC_API_URL, - get_ogc_data, -) +from dataretrieval.waterdata.utils import get_ogc_data if TYPE_CHECKING: from dataretrieval._response_metadata import BaseMetadata @@ -160,8 +158,10 @@ def get_queryables(collection: str) -> tuple[pd.DataFrame, BaseMetadata]: 'string' """ # Reading the queryables document is OGC protocol work; this getter only - # names the API to ask. - return queryables_frame(collection, base_url=OGC_API_URL) + # names the API to ask -- which is the redirected one when a ``configure`` + # block set a base URL, so the queryables describe the API the getters are + # actually querying. + return queryables_frame(collection, base_url=ogc_api_url()) __all__ = ["get_reference_table", "get_queryables"] diff --git a/dataretrieval/waterdata/samples.py b/dataretrieval/waterdata/samples.py index 2f21d26cb..b73ef3e1c 100644 --- a/dataretrieval/waterdata/samples.py +++ b/dataretrieval/waterdata/samples.py @@ -32,6 +32,7 @@ from dataretrieval.transport.http import ( get as _get, ) +from dataretrieval.waterdata.endpoints import samples_url from dataretrieval.waterdata.types import ( CODE_SERVICES, PROFILES, @@ -39,7 +40,6 @@ _check_profiles, ) from dataretrieval.waterdata.utils import ( - SAMPLES_URL, _accept_legacy_kwargs, _get_args, ) @@ -66,7 +66,7 @@ def get_codes(code_service: CODE_SERVICES) -> tuple[pd.DataFrame, BaseMetadata]: """ require_one_of(code_service, get_args(CODE_SERVICES), name="code_service") - url = f"{SAMPLES_URL}/codeservice/{code_service}?mimeType=application%2Fjson" + url = f"{samples_url()}/codeservice/{code_service}?mimeType=application%2Fjson" response = _get(url, headers=_default_headers(url), **HTTPX_DEFAULTS) @@ -362,7 +362,7 @@ def get_samples( if "boundingBox" in params: params["boundingBox"] = to_str(params["boundingBox"]) - url = f"{SAMPLES_URL}/{service}/{profile}" + url = f"{samples_url()}/{service}/{profile}" df, response = _get_samples_csv(url, params, ssl_check) df = _attach_datetime_columns(df) @@ -424,7 +424,7 @@ def get_samples_summary( f"request, got {type(monitoring_location_id).__name__}." ) - url = f"{SAMPLES_URL}/summary/{quote(monitoring_location_id, safe='')}" + url = f"{samples_url()}/summary/{quote(monitoring_location_id, safe='')}" params = {"mimeType": "text/csv"} df, response = _get_samples_csv(url, params, ssl_check) diff --git a/dataretrieval/waterdata/stats.py b/dataretrieval/waterdata/stats.py index 9239f324f..09ee8468e 100644 --- a/dataretrieval/waterdata/stats.py +++ b/dataretrieval/waterdata/stats.py @@ -28,7 +28,7 @@ ) from dataretrieval.transport.http import default_headers from dataretrieval.transport.pagination import run_paginated -from dataretrieval.waterdata.endpoints import STATISTICS_API_URL +from dataretrieval.waterdata.endpoints import statistics_api_url __all__ = ["get_data"] @@ -247,7 +247,7 @@ def get_data( :doc:`/userguide/errors`). """ - url = f"{STATISTICS_API_URL}/{service}" + url = f"{statistics_api_url()}/{service}" req = httpx.Request( method="GET", url=url, @@ -278,6 +278,7 @@ async def follow_up(cursor: str, sess: httpx.AsyncClient) -> httpx.Response: raise_for_status=_raise_for_non_200, client=client, service=service, + adapter="waterdata", ) if expand_percentiles: diff --git a/dataretrieval/waterdata/utils.py b/dataretrieval/waterdata/utils.py index f5df3f4e3..5b9a269a4 100644 --- a/dataretrieval/waterdata/utils.py +++ b/dataretrieval/waterdata/utils.py @@ -24,12 +24,22 @@ from dataretrieval._deprecation import warn_deprecated from dataretrieval.codes.states import apply_state +from dataretrieval.credentials import refuse_credential_keywords from dataretrieval.ogc import OgcDialect, prepare_request_args from dataretrieval.ogc import get_ogc_data as _facade_get_ogc_data -# Endpoint constants live in one place for the whole collection; they are re-bound -# here because ``waterdata.utils.OGC_API_URL`` is a documented path. -from dataretrieval.waterdata.endpoints import BASE_URL, OGC_API_URL, SAMPLES_URL +# Default endpoint constants remain at their documented compatibility paths. +# Production retrieval resolves scoped destinations through ``ogc_api_url``. +from dataretrieval.waterdata.endpoints import ( + _DEFAULT_BASE_URL, + _DEFAULT_OGC_API_URL, + _DEFAULT_SAMPLES_URL, + ogc_api_url, +) + +BASE_URL = _DEFAULT_BASE_URL +OGC_API_URL = _DEFAULT_OGC_API_URL +SAMPLES_URL = _DEFAULT_SAMPLES_URL if TYPE_CHECKING: from dataretrieval._response_metadata import BaseMetadata @@ -130,7 +140,14 @@ def _flatten_queryables(local_vars: dict[str, Any]) -> dict[str, Any]: popped, so this is a no-op on getters without the passthrough and idempotent if called twice. """ - local_vars.update(local_vars.pop("queryables", {})) + queryables = local_vars.pop("queryables", {}) + # A credential-shaped name would go out in the query string, which is the + # one thing this passthrough must not forward. The predicate lives in the + # credentials leaf rather than here: what motivates it -- ``api_key=`` being + # a plausible guess now that ``configure()`` takes it -- is package-wide, + # and WQP's ``**kwargs`` search filters read the same list. + refuse_credential_keywords(queryables) + local_vars.update(queryables) return local_vars @@ -225,11 +242,19 @@ def get_ogc_data( collection, output_id, max_rows=max_rows, - base_url=OGC_API_URL, + # Endpoint acquisition resolves the active ContextVar at request time; + # the documented ``OGC_API_URL`` constant remains the default-value + # compatibility path rather than a production request destination. + base_url=ogc_api_url(), spatial=spatial, extra_id_cols=_EXTRA_ID_COLS, dialect=WATERDATA_DIALECT, cql_body=cql_body, + # Which settings table these calls read. Declared here, in the one + # wrapper every Water Data getter goes through, rather than derived + # from ``base_url``: NGWMN is served from the same host, so a URL + # cannot tell the two adapters apart (ADR 0010). + adapter="waterdata", ) diff --git a/dataretrieval/wateruse.py b/dataretrieval/wateruse.py index 39fc23f01..35346a61b 100644 --- a/dataretrieval/wateruse.py +++ b/dataretrieval/wateruse.py @@ -1,432 +1,53 @@ -"""Retrieve USGS water-use data from the NWDC web service. - -The National Water Availability Assessment Data Companion (NWDC) web services -provide national-scale, USGS-modeled water-use data that underlie the `USGS -National Water Availability Assessment `_. -Estimates are served on a HUC12 (12-digit hydrologic unit) spatial grid and can -be queried for any county, state, or hydrologic unit. This is the modern -replacement for the defunct legacy NWIS water-use service -(``nwis.get_water_use``). - -Unlike the main Water Data getters (:mod:`dataretrieval.waterdata`) and NGWMN -(:mod:`dataretrieval.ngwmn`), the NWDC is a plain CSV REST service rather than -an OGC API Features collection. This module supplies the NWDC-specific bits — -request building, CSV parsing, the ``Link``-header cursor, and the ``{detail}`` -error envelope. The service-neutral transport layer supplies cursor pagination, -response aggregation, client lifecycle, and sync-from-async dispatch. The module -follows the same conventions: host-scoped request headers, the typed -:class:`~dataretrieval.exceptions.DataRetrievalError` taxonomy, and a -``(DataFrame, BaseMetadata)`` return. - -See https://api.water.usgs.gov/docs/nwaa-data/ for the API reference and -https://water.usgs.gov/nwaa-data/ for the catalog of available models and -variables. - -Examples --------- -.. code-block:: python - - from dataretrieval import wateruse - - # Monthly public-supply withdrawals for Rhode Island, 2020 onward. - df, md = wateruse.get_wateruse( - model="wu-public-supply-wd", - variable=["pswdtot", "pswdgw", "pswdsw"], - state="RI", - start_date="2020-01", - time_resolution="monthly", - ) - +"""Deprecated alias for :mod:`dataretrieval.nwdc`. + +The module was named for one subset of what the service offers. The National +Water Availability Assessment Data Companion serves ten modeled datasets, of +which the water-use models are five; the rest are hydrologic, +atmospheric-forcing, and assessment outputs. Every other adapter in this +package is named for its service -- ``ngwmn``, ``nldi``, ``wqp``, +``streamstats``, ``nwis`` -- so this one is now ``nwdc``. + +Importing this module emits a :class:`DeprecationWarning` and re-exports +:mod:`dataretrieval.nwdc`'s public surface. The re-exported objects are the +*same objects*, not copies -- ``wateruse.get_wateruse is nwdc.get_wateruse`` +-- so calls and identity comparisons behave identically through either +spelling. + +It is an alias for reading, not a second name for the module. This is a +distinct module object holding its own references to the five public names, +so it does not forward *assignment* or private names: rebinding +``wateruse.get_wateruse`` leaves ``nwdc``'s global untouched (and so has no +effect on anything ``nwdc`` does internally), and ``wateruse._WATERUSE_HOST`` +does not exist. Code that monkeypatches, or that reaches for a private, must +name :mod:`dataretrieval.nwdc` directly -- which is the point of the +deprecation. + +``dataretrieval.__init__`` deliberately imports :mod:`dataretrieval.nwdc` +rather than this module, so ``import dataretrieval`` stays silent. The warning +fires only for code that names ``wateruse`` itself. """ from __future__ import annotations -import io -from collections.abc import Callable, Iterable -from typing import Any - -import httpx -import pandas as pd - -from dataretrieval._querying import _raise_for_status, to_str -from dataretrieval._response_metadata import BaseMetadata -from dataretrieval.codes.states import to_state -from dataretrieval.exceptions import DataRetrievalError -from dataretrieval.transport.http import default_headers -from dataretrieval.transport.links import resolve_next_url -from dataretrieval.transport.pagination import run_paginated - -__all__ = [ - "get_wateruse", - "WATERUSE_URL", - "MODELS", - "TIME_RESOLUTIONS", - "DEFAULT_CONCURRENT_REQUESTS", -] - -WATERUSE_URL = "https://api.water.usgs.gov/nwaa-data/data" -_WATERUSE_HOST = httpx.URL(WATERUSE_URL).host -# Hosts a ``rel="next"`` cursor may name for this same service; each is -# rewritten to :data:`_WATERUSE_HOST` rather than followed as given. -_WATERUSE_HOST_ALIASES = frozenset({_WATERUSE_HOST, "water.usgs.gov"}) - -#: Water-use models (categories) served by the NWDC. The catalog at -#: https://water.usgs.gov/nwaa-data/ lists the variables available within each. -MODELS = ( - "wu-public-supply-wd", # public-supply withdrawals - "wu-public-supply-cu", # public-supply consumptive use - "wu-thermoelectric", # thermoelectric-power water use - "wu-irrigation-wd", # irrigation withdrawals - "wu-irrigation-cu", # irrigation consumptive use +from dataretrieval import nwdc as _nwdc +from dataretrieval._deprecation import REMOVALS, warn_deprecated +from dataretrieval.nwdc import * # noqa: F403 (re-export the public surface) + +#: When the alias may be deleted. Read from the shared horizon table rather +#: than spelled here, so it is audited and bumped with every other published +#: removal; matches the dated-removal convention :mod:`dataretrieval.nwis` +#: uses. +NWDC_RENAME_REMOVAL_DATE = REMOVALS["wateruse"] + +__all__ = list(_nwdc.__all__) + +warn_deprecated( + "`dataretrieval.wateruse`", + replacement="`dataretrieval.nwdc`", + removal=NWDC_RENAME_REMOVAL_DATE, + detail="The service is the National Water Availability Assessment Data " + "Companion, and water use is one of the ten datasets it serves.", + # 1 lands the warning on the line that imported this module -- an import + # has no deeper user frame to point at. + stacklevel=1, ) - -#: Temporal resolutions: monthly, annual calendar year, annual water year. -TIME_RESOLUTIONS = ("monthly", "annualcy", "annualwy") - -#: This service's preferred in-flight cap when ``API_USGS_CONCURRENT`` is -#: unset. Lower than the package default of 32 because every location retries -#: independently, so a rate-limit episode bursts this number times the retry -#: count; the NWDC tolerates this level without rate-limit errors (verified by -#: stress test) and higher has not been tested. Setting ``API_USGS_CONCURRENT`` -#: overrides it -- see :func:`dataretrieval.transport.fanout._resolve_concurrency` -#: for why the general setting outranks a module's default rather than the -#: reverse. -DEFAULT_CONCURRENT_REQUESTS = 4 - -# Page responses carry the HUC12 identifier in this column; it must stay a -# string so leading zeros (e.g. "010900020502") survive the round trip. -_HUC12_COLUMN = "huc12_id" - - -def get_wateruse( - model: str, - variable: str | Iterable[str] | None = None, - state: str | int | Iterable[str | int] | None = None, - county: str | Iterable[str] | None = None, - huc: str | Iterable[str] | None = None, - time_resolution: str | None = None, - start_date: str | None = None, - end_date: str | None = None, - intersection: str = "overlap", - limit: int = 600, - ssl_check: bool = True, -) -> tuple[pd.DataFrame, BaseMetadata]: - """Get USGS water-use data from the NWDC web service. - - Retrieves modeled water-use estimates from the USGS National Water - Availability Assessment Data Companion. The area is given as exactly one of - ``state``, ``county``, or ``huc``; results are always returned on a HUC12 - grid, in a long (tidy) frame with one row per HUC12 and time step. Large - areas (e.g. a whole region or a populous state) are served across multiple - pages; this function follows those pages transparently and concatenates - them into one frame. - - Each selector also accepts a list of values. The NWDC queries one area per - request, so a list is fanned out into one request per value — up to - ``API_USGS_CONCURRENT`` in parallel, defaulting to - :data:`DEFAULT_CONCURRENT_REQUESTS` for this service — and the results are - concatenated in the order given. A fan-out interrupted by a rate limit or an - upstream fault raises a resumable - :class:`~dataretrieval.interruptions.FanOutInterrupted`, whose - ``.call.resume()`` re-issues only the locations that did not complete. - - Parameters - ---------- - model : string - Water-use category to query. See :data:`MODELS` for the available - options (e.g. ``"wu-public-supply-wd"``). The full catalog of models - and their variables is at https://water.usgs.gov/nwaa-data/. - variable : string or iterable of strings, optional - One or more variable IDs within ``model`` (e.g. ``"pswdtot"`` for total - public-supply withdrawals, or ``["pswdgw", "pswdsw"]`` for the - groundwater and surface-water components). Multiple variables are - comma-joined into a single request. The service requires at least one - variable; omitting it returns a 400 listing the model's valid variable - IDs (surfaced as a :class:`~dataretrieval.exceptions.DataRetrievalError`). - state : string, int, or iterable, optional - One or more US states/territories to query. Each accepts a full name - (``"Wisconsin"``), a two-letter postal code (``"WI"``), or a two-digit - ANSI/FIPS code (``"55"`` or ``55``), mirroring - :func:`dataretrieval.ngwmn.get_sites`. - county : string or iterable, optional - One or more five-digit county FIPS codes — state FIPS + county FIPS, - e.g. ``"55025"`` for Dane County, Wisconsin. - huc : string or iterable, optional - One or more hydrologic unit codes. Each code's level is taken from its - length: a 2-digit code queries a HUC2 region, 8-digit a HUC8 subbasin, - 12-digit a single HUC12, and so on (even lengths 2-12, e.g. ``"04"``, - ``"07070005"``, ``"010900020502"``). - - Provide exactly one of ``state``, ``county``, or ``huc`` (each may be a - single value or a list). - time_resolution : string, optional - Temporal resolution: ``"monthly"``, ``"annualcy"`` (annual, calendar - year), or ``"annualwy"`` (annual, water year). See - :data:`TIME_RESOLUTIONS`. - start_date : string, optional - Start of the query window, formatted ``"YYYY"`` for annual data or - ``"YYYY-MM"`` for monthly data. - end_date : string, optional - End of the query window, in the same format as ``start_date``. - intersection : string, optional - How to select HUC12s that straddle the queried-area boundary: - ``"overlap"`` (any overlap, the default) or ``"envelop"`` (fully - enclosed). - limit : int, optional - Maximum number of HUC12s returned per page. Queries spanning more than - ``limit`` HUC12s are split across pages and reassembled. Default 600. - ssl_check : bool, optional - If True (default), verify SSL certificates; set False to skip - verification (e.g. behind a TLS-intercepting proxy). - - Returns - ------- - df : ``pandas.DataFrame`` - Water-use estimates in long form: a ``huc12_id`` column (string, - leading zeros preserved), a time column (``year_month`` for monthly - data or ``year`` for annual data), and one value column per requested - variable (suffixed with its unit, e.g. ``pswdtot_mgd`` for million - gallons per day). - md : :class:`dataretrieval.utils.BaseMetadata` - Metadata describing the request (URL, query time, response headers). - - Raises - ------ - ValueError - If not exactly one of ``state``, ``county``, or ``huc`` is given, or a - given selector is malformed (an unrecognized state, a county code that - is not five digits, or a HUC of invalid length). - DataRetrievalError - On an HTTP error response, the typed subclass for the status (see - :func:`dataretrieval.exceptions.error_for_status`). A transient 429, - 5xx, or recoverable connection failure that exhausts inline retries is - raised as a resumable - :class:`~dataretrieval.interruptions.FanOutInterrupted`; a deterministic - connection failure (for example, a permanently unresolvable host) - remains a :class:`~dataretrieval.exceptions.NetworkError`. - - Examples - -------- - .. doctest:: - :skipif: True # network - - >>> from dataretrieval import wateruse - >>> df, md = wateruse.get_wateruse( - ... model="wu-public-supply-wd", - ... variable=["pswdtot", "pswdgw", "pswdsw"], - ... state="RI", - ... start_date="2020-01", - ... time_resolution="monthly", - ... ) - - """ - # The public parameters are idiomatic snake_case (consistent with - # ``waterdata.get_samples``); the NWDC service expects compact lowercase - # query names, so map to those here as the request is built. - base_params: dict[str, Any] = { - "format": "csv", - "model": model, - "variable": to_str(variable), - "timeres": time_resolution, - "startdate": start_date, - "enddate": end_date, - "intersection": intersection, - "limit": limit, - } - # Drop params the caller left unset; the service rejects empty values. - base_params = {k: v for k, v in base_params.items() if v is not None} - - # The NWDC queries one location per request, so fan a multi-value selector - # out into one request per location, each handled by shared transport - # pagination, and concatenate the results. - headers = default_headers(WATERUSE_URL) - requests = [ - httpx.Request( - "GET", - WATERUSE_URL, - params={**base_params, "location": location}, - headers=headers, - ) - for location in _resolve_locations(state, county, huc) - ] - return _fan_out(requests, headers, ssl_check) - - -# Valid HUC code lengths (digits) → the hydrologic-unit level they query. -_HUC_LENGTHS = (2, 4, 6, 8, 10, 12) - -# Maps each selector to the NWDC ``location=:`` value(s) it produces. -# A value may be a single code or a list; ``_as_list`` normalizes both (``state`` -# additionally normalizes to the two-letter postal code, and ``to_state`` may -# itself return a scalar or list, which ``_as_list`` flattens the same way). -# Since NWDC takes one location per request, a list value fans out — one request -# per location (see :func:`_fan_out`). -_LOCATION_BUILDERS: dict[str, Callable[[Any], list[str]]] = { - "state": lambda v: [f"stateCd:{c}" for c in _as_list(to_state(v, to="postal"))], - "county": lambda v: [f"countyCd:{_validate_county(c)}" for c in _as_list(v)], - "huc": lambda v: [f"huc{len(c)}:{c}" for c in map(_validate_huc, _as_list(v))], -} - - -def _resolve_locations( - state: str | int | Iterable[str | int] | None, - county: str | Iterable[str] | None, - huc: str | Iterable[str] | None, -) -> list[str]: - """Build the NWDC ``location=:`` value(s) from the selectors. - - Exactly one of ``state`` / ``county`` / ``huc`` must be given; each may be a - single value or a list. ``state`` is normalized to the two-letter postal - code ``stateCd`` requires; ``county`` is a five-digit FIPS code; and a - ``huc`` code's length selects its level (``huc2`` … ``huc12``). Returns one - location string per value — the caller issues one request per location. - """ - selected = { - name: value - for name, value in (("state", state), ("county", county), ("huc", huc)) - if value is not None - } - if len(selected) != 1: - raise ValueError( - "Specify exactly one of state, county, or huc " - f"(got: {', '.join(selected) or 'none'})." - ) - [(name, value)] = selected.items() - locations = _LOCATION_BUILDERS[name](value) - if not locations: - raise ValueError( - "The chosen location selector is empty; pass at least one value." - ) - return locations - - -def _as_list(value: object) -> list[Any]: - """Normalize a value to a list. - - A scalar becomes a one-element list; any non-string iterable (list, tuple, - Series, ndarray, generator) is materialized to a list. A string is treated - as a scalar so it isn't exploded into characters. - """ - if isinstance(value, Iterable) and not isinstance(value, str): - return list(value) - return [value] - - -def _validate_county(value: object) -> str: - """Validate and normalize a five-digit state+county FIPS code.""" - code = str(value).strip() - if not (code.isdigit() and len(code) == 5): - raise ValueError( - "county must be a five-digit state+county FIPS code " - f"(e.g. '55025'), got {value!r}." - ) - return code - - -def _validate_huc(value: object) -> str: - """Validate a HUC code (even length 2-12 digits; level set by length).""" - code = str(value).strip() - if not (code.isdigit() and len(code) in _HUC_LENGTHS): - raise ValueError( - "huc must be a hydrologic unit code of even length 2-12 digits " - f"(e.g. '04', '07070005', '010900020502'), got {value!r}." - ) - return code - - -def _fan_out( - requests: list[httpx.Request], headers: dict[str, str], ssl_check: bool -) -> tuple[pd.DataFrame, BaseMetadata]: - """Fetch every request (each paginated) over the shared fan-out executor. - - This function is only the NWDC-specific half: parse a CSV page and read - its ``Link`` header cursor, follow that cursor, raise the typed error - carrying the NWDC ``detail``, and shape the result. - :func:`~dataretrieval.transport.pagination.run_paginated` owns the rest. - - The plan is the request list itself. The executor asks a plan only to be - sized and iterable, and the NWDC accepts one ``location=`` per request, so - the caller's locations arrive already separate -- there is nothing to - divide and so nothing for a plan class to hold. - - The broad retry status set is on purpose: NWDC reports a bad query as a 400 - with a ``{"detail": ...}`` envelope, so unlike WQP and StreamStats its 5xx - really is an upstream fault worth re-sending. - """ - - def parse(response: httpx.Response) -> tuple[pd.DataFrame, str | None]: - return _read_csv_page(response), _next_page_url(response) - - async def follow(cursor: str, sess: httpx.AsyncClient) -> httpx.Response: - return await sess.get(cursor, headers=headers) - - def raise_for_status(response: httpx.Response) -> None: - _raise_for_status(response, detail_from=_nwdc_error_detail) - - def finalize( - frame: pd.DataFrame, response: httpx.Response - ) -> tuple[pd.DataFrame, BaseMetadata]: - return frame, BaseMetadata(response) - - return run_paginated( - requests, - parse_response=parse, - follow_up=follow, - raise_for_status=raise_for_status, - finalize=finalize, - client_options={"verify": ssl_check}, - default_concurrent=DEFAULT_CONCURRENT_REQUESTS, - service="wateruse", - ) - - -def _read_csv_page(response: httpx.Response) -> pd.DataFrame: - """Parse one CSV page; ``huc12_id`` stays a string to keep leading zeros.""" - try: - return pd.read_csv(io.BytesIO(response.content), dtype={_HUC12_COLUMN: str}) - except pd.errors.EmptyDataError as exc: - # NWDC normally signals "no data" with a 400 (handled above) or rows of - # zeros, never an empty body — but keep the typed-error contract if it - # ever returns one rather than leaking a bare pandas exception. - raise DataRetrievalError( - f"NWDC returned an empty response body (URL: {response.url})." - ) from exc - - -def _next_page_url(response: httpx.Response) -> str | None: - """Return the absolute URL of the next page, or None if this is the last. - - Reads the standard ``Link: <...>; rel="next"`` header (parsed by httpx into - ``response.links``). The cursor is normalized before it is trusted, because - the service spells it inconsistently. A relative reference is resolved - against the page it came from, and the bare ``water.usgs.gov`` host is - rewritten to the public ``api.water.usgs.gov`` gateway (over https, whatever - scheme the link used) so the follow-up request reaches the API. Only a - cursor that still points somewhere else after that is refused -- following - it would send Water Use requests, and any credentials on them, to a host the - caller never asked for. - """ - url = response.links.get("next", {}).get("url") - if not url: - return None - return resolve_next_url( - url, - response, - service="Water Use", - allowed_hosts=_WATERUSE_HOST_ALIASES, - rewrite_host=_WATERUSE_HOST, - ) - - -def _nwdc_error_detail(response: httpx.Response) -> str | None: - """Pull the ``detail`` message out of an NWDC JSON error envelope, if any. - - The NWDC reports errors as ``{"detail": "Invalid model name: ..."}``. Passed - to :func:`~dataretrieval.utils._raise_for_status` as ``detail_from`` so the - service's wording surfaces in the typed error message. - """ - try: - body = response.json() - except ValueError: - return None - return body.get("detail") if isinstance(body, dict) else None diff --git a/dataretrieval/wqp.py b/dataretrieval/wqp.py index 86124ffa5..605fe44fd 100644 --- a/dataretrieval/wqp.py +++ b/dataretrieval/wqp.py @@ -11,19 +11,29 @@ from __future__ import annotations import warnings +from dataclasses import dataclass from io import StringIO -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, ClassVar import pandas as pd +from dataretrieval import configuration as _configuration from dataretrieval._response_metadata import BaseMetadata from dataretrieval._validation import require_one_of +from dataretrieval.configuration import ( + BaseConfiguration, + _Redirectable, + _register, + _Retrying, +) +from dataretrieval.credentials import refuse_credential_keywords from dataretrieval.exceptions import DataCurrencyWarning from ._querying import _query_with_retry from ._wqx import _attach_datetime_columns __all__ = [ + "WqpConfiguration", "get_results", "what_sites", "what_organizations", @@ -44,6 +54,11 @@ from pandas import DataFrame +#: Root the Water Quality Portal serves both its interfaces from. Private +#: because the two builders below are the documented way to name a WQP URL; +#: this is only the piece they share, and the piece a redirect replaces. +_WQP_BASE_URL = "https://www.waterqualitydata.us" + result_profiles_wqx3 = ["basicPhysChem", "fullPhysChem", "narrow"] result_profiles_legacy = ["biological", "narrowResult", "resultPhysChem"] activity_profiles_legacy = ["activityAll"] @@ -200,7 +215,9 @@ def get_results( if legacy is not True and profile is None: kwargs["dataProfile"] = "fullPhysChem" - response = _query_with_retry(url, kwargs, delimiter=";", ssl_check=ssl_check) + response = _query_with_retry( + url, kwargs, delimiter=";", ssl_check=ssl_check, adapter="wqp" + ) df = _read_wqp_csv(response.text) df = _attach_datetime_columns(df) @@ -230,7 +247,7 @@ def _what( url = _legacy_only_url(service, legacy=legacy) response = _query_with_retry( - url, payload=kwargs, delimiter=";", ssl_check=ssl_check + url, payload=kwargs, delimiter=";", ssl_check=ssl_check, adapter="wqp" ) df = _read_wqp_csv(response.text) return df, WQP_Metadata(response, **kwargs) @@ -628,22 +645,33 @@ def _validate_service(service: str, valid_services: list[str], profile: str) -> require_one_of(service, valid_services, name="service", context=profile) +def _service_base() -> str: + """The WQP root this call targets: a block's redirect, or the portal's own. + + The portal serves the legacy and WQX3 interfaces from one root under + different paths, so a ``WqpConfiguration(base_url=...)`` names that root and + both follow it. Redirecting only the interface a caller happened to use + first would leave the other pointed at the service they were trying not to + talk to. Resolved per call, because a ``configure`` block is scoped to a + ``with`` statement. + """ + return _configuration.base_url(adapter="wqp", default=_WQP_BASE_URL) + + def wqp_url(service: str) -> str: """Construct the WQP URL for a given service.""" - base_url = "https://www.waterqualitydata.us/data/" _warn_legacy_use() _validate_service(service, services_legacy, "Legacy") - return f"{base_url}{service}/Search?" + return f"{_service_base()}/data/{service}/Search?" def wqx3_url(service: str) -> str: """Construct the WQP URL for a given WQX 3.0 service.""" - base_url = "https://www.waterqualitydata.us/wqx3/" _warn_wqx3_use() _validate_service(service, services_wqx3, "WQX3.0") - return f"{base_url}{service}/search?" + return f"{_service_base()}/wqx3/{service}/search?" class WQP_Metadata(BaseMetadata): @@ -702,7 +730,18 @@ def site_info(self) -> tuple[DataFrame, WQP_Metadata] | None: def _check_kwargs(kwargs: dict[str, Any]) -> dict[str, Any]: - """Check kwargs for unsupported parameters.""" + """Check kwargs for unsupported parameters. + + Every WQP getter's ``**kwargs`` funnels through here on its way to the + query payload, so this is the choke point where a credential-shaped name is + refused. The predicate is the credentials leaf's, shared with Water Data's + ``**queryables`` passthrough: ``api_key=`` is a plausible guess on any + getter now that ``configure(Configuration(api_key=...))`` is the spelling, + and this is the adapter with the widest passthrough -- ten getters, whose + filter names the portal rather than this package defines. + """ + refuse_credential_keywords(kwargs) + mimetype = kwargs.get("mimeType") if mimetype == "geojson": raise NotImplementedError("GeoJSON not yet supported. Set 'mimeType=csv'.") @@ -758,3 +797,38 @@ def _legacy_only_url(service: str, legacy: bool) -> str: _warn_wqx3_unavailable() warnings.simplefilter("ignore", DataCurrencyWarning) return wqp_url(service) + + +@dataclass(frozen=True) +class WqpConfiguration(_Redirectable, _Retrying, BaseConfiguration): + """Settings for Water Quality Portal calls alone. + + No fan-out dials: a WQP query is answered by a single request, so a + concurrency cap could only report a number nothing honours. + + Lives here rather than in :mod:`dataretrieval.configuration` because + *which* settings a service reads is the service's own knowledge (ADR + 0011); what each of them means is shared, so the fields come from the + setting groups declared beside their grammar. + + Parameters + ---------- + retries : int, optional + Retries attempted after a transient failure; ``0`` disables retrying. + stall_timeout : float, optional + Seconds a call may go without receiving any data before retrying + stops. + base_url : str, optional + Root to send WQP requests to, instead of the portal's own. Both + interfaces hang off it, so one value moves the legacy ``/data/`` + and the WQX3 ``/wqx3/`` paths together. Code only: the file and + the environment refuse it. + """ + + # One request per call, so this service reads the retry dials and a + # redirectable base and no fan-out dial. Each setting is declared once, + # in :mod:`dataretrieval.configuration`, beside its grammar. + adapter: ClassVar[str] = "wqp" + + +_register(WqpConfiguration) diff --git a/demos/USGS_WaterUse_Examples.ipynb b/demos/USGS_WaterUse_Examples.ipynb index 0017f0482..418457d62 100644 --- a/demos/USGS_WaterUse_Examples.ipynb +++ b/demos/USGS_WaterUse_Examples.ipynb @@ -14,7 +14,7 @@ "the modern replacement for the retired legacy NWIS water-use service.\n", "\n", "`dataretrieval` exposes the service through a single function,\n", - "`wateruse.get_wateruse`, which returns a tidy `pandas.DataFrame` plus a\n", + "`nwdc.get_wateruse`, which returns a tidy `pandas.DataFrame` plus a\n", "metadata object. Available **models** (categories) include:\n", "\n", "| model | description |\n", @@ -50,7 +50,7 @@ "import matplotlib.pyplot as plt\n", "import pandas as pd\n", "\n", - "from dataretrieval import wateruse" + "from dataretrieval import nwdc" ] }, { @@ -74,7 +74,7 @@ "metadata": {}, "outputs": [], "source": [ - "df, md = wateruse.get_wateruse(\n", + "df, md = nwdc.get_wateruse(\n", " model=\"wu-public-supply-wd\",\n", " variable=[\"pswdtot\", \"pswdgw\", \"pswdsw\"],\n", " state=\"WI\",\n", diff --git a/docs/source/architecture/decisions/0008-fan-out-execution.rst b/docs/source/architecture/decisions/0008-fan-out-execution.rst index d5431f1ef..44b324fc4 100644 --- a/docs/source/architecture/decisions/0008-fan-out-execution.rst +++ b/docs/source/architecture/decisions/0008-fan-out-execution.rst @@ -130,8 +130,8 @@ Consequences Compliance ---------- -``tests/architecture_test.py`` asserts three things. That ``wateruse`` -contains no ``asyncio.gather``, ``Semaphore``, or ``TaskGroup``, so the +``tests/architecture_test.py`` asserts three things. That ``nwdc`` +(named ``wateruse`` when this decision was taken) contains no ``asyncio.gather``, ``Semaphore``, or ``TaskGroup``, so the duplication cannot return. That both plan types are sized and *repeatably* iterable -- resume keys completed work by position, so a generator mistaken for a collection would re-issue the wrong chunks. And that an interruption diff --git a/docs/source/architecture/decisions/0009-layered-configuration.rst b/docs/source/architecture/decisions/0009-layered-configuration.rst new file mode 100644 index 000000000..43ba1e8d7 --- /dev/null +++ b/docs/source/architecture/decisions/0009-layered-configuration.rst @@ -0,0 +1,194 @@ +ADR 0009: Layered configuration resolution +========================================== + +Status +------ + +Accepted, with clauses superseded twice. + +:doc:`0010-adapter-scoped-settings` supersedes "One flat set of setting names" +and "Per-service overrides are deferred" below, having found the premise of the +first -- that every service accepts the same settings -- to be false. + +:doc:`0011-configuration-profiles` supersedes three more: + +- **The** ``[profiles.]`` **table** in step 3 of the chain, and the + recommendation in "``parallel_chunks`` at the top level of the file warns" to + put the setting in one. A profile is now named under the adapter it + configures (``[.]``); the global table and + ``DATARETRIEVAL_PROFILE`` are retired, since a table that switched every + service at once could not carry per-service detail. +- **"The environment ranks above the file"**, inverted for -- and only for -- a + profile selected in code. Everything the caller did not name in code still + follows the rule as written here. +- **The refusal of a configuration object**, stated in the leaf clause ("a + scoped action, not a ``Configuration`` dataclass") and in "A configuration + object would have no way to reach the call". ``configure()`` now takes + exactly such objects. The grounds were that an instance had no way to reach a + free function; the ``ContextVar`` this ADR established is that way, and ADR + 0010 had already narrowed the objection to a payload-shape preference. + +The chain itself, the ``ContextVar`` delivery, host-scoped credentials, and the +leaf constraint stand. + +Context +------- + +Settings reached the library through one mechanism: process-global environment +variables (``API_USGS_PAT``, ``API_USGS_CONCURRENT``, ``API_USGS_RETRIES``, +``API_USGS_PROGRESS``), each with its own hand-rolled parser at its point of +use. Nothing could report the effective configuration, and the grammars were +free to drift apart. + +That mechanism cannot express a per-call credential. An application holding +keys in a secret store, a notebook pulling for two accounts, or a server +handling concurrent users must assign to ``os.environ`` — which is +process-global, so it races across threads and tasks (issue #352). + +The obvious fix, an ``api_key=`` parameter on the public getters, is unsafe +here. Every Water Data getter ends in ``_get_args(locals())`` with a +``**queryables`` catch-all that forwards unrecognized keywords to the API as +query parameters. A credential parameter missed in one of ~20 signatures would +be serialized into a URL. The maintainers also object to an ``api_key=`` +parameter on the separate ground that it invites keys pasted into shared +scripts. + +Decision +-------- + +Every setting resolves through one ordered chain, owned by a new +``dataretrieval.configuration`` module: + +1. An active ``dataretrieval.configure(...)`` block (a ``ContextVar``). +2. The setting's environment variable. +3. The configuration file: ``~/.dataretrieval/config.toml``, or the path in + ``DATARETRIEVAL_CONFIG``. Top-level keys are the defaults; a + ``[profiles.]`` table layers over them per setting when selected. +4. The built-in default. + +Supporting decisions: + +- **Precedence is per setting, not per source.** An environment that sets only + ``API_USGS_PAT`` leaves a file-provided ``concurrency`` in effect. A + *blank* environment variable does not count as set, so it cannot shadow the + file: container and CI tooling routinely materializes one. The exception is + ``progress``, where a blank ``API_USGS_PROGRESS`` has always meant "off" -- + so "does blank count as a value?" is a property of the setting + (``configuration._BLANK_MEANS_SET``) rather than an extra tier in the chain. +- **The environment ranks above the file.** This follows the established + precedence used by `pip + `_ + and `AWS + `_, + supports deployment-time overrides without editing mounted files, and keeps + the pre-existing ``API_USGS_*`` interface authoritative. +- **Omitted and explicitly cleared values differ.** An omitted + ``configure()`` argument inherits from lower sources. Explicit ``None`` is a + scoped reset to built-in behavior, so a server can guarantee an anonymous + call rather than accidentally falling through to its process credential. +- **No public getter grows a credential parameter.** ``configure`` is the only + programmatic path, and a fitness function asserts no getter accepts + ``api_key`` / ``session`` / ``token``. The generic ``**queryables`` path also + rejects those names before request construction so they cannot enter a URL. +- **The module owns each setting's parser.** ``unbounded``, bounds, and + rejection messages live in one place. ``tomllib`` returns typed scalars, so + the file and Python API validate source-level types before normalized values + pass through the shared parsers. Legacy environment-only forms, including a + blank numeric value and arbitrary non-empty progress value, remain compatible + without making the new surfaces equally permissive. +- **TOML, read with** ``tomllib``. Stdlib from Python 3.11; the ``tomli`` + backport is a marker-scoped dependency that disappears when + ``requires-python`` moves to ``>=3.11``. YAML was rejected because PyYAML is + a dependency at every Python version and the settings are flat. +- **Not every setting gets an environment variable.** ``parallel_chunks`` + spends rate-limit quota, and ADR-adjacent documentation on + ``dataretrieval.parallel_chunks`` argues it must stay a deliberate choice. + It does not add a new exported process-global knob; the file and ``configure`` + block are its only sources, with a scoped block as the recommended use. +- **Names distinguish execution capacity from planning granularity.** + ``concurrency`` is the noun for the maximum in-flight subrequests and maps to + the established ``API_USGS_CONCURRENT`` variable. ``parallel_chunks`` asks + the planner for optional extra chunks; it does not promise that many requests + execute simultaneously. The name is retained because the context manager is + already public. ``parallelism`` and ``chunk_parallelism`` were rejected + because they would conflate this planning hint with ``concurrency``. +- **Configuration errors are in the error taxonomy.** ``ConfigurationError`` is a + ``DataRetrievalError`` *and* a ``ValueError``. Configuration resolves lazily + on the request path, so a broken file surfaces from inside whichever getter + runs first; ``except DataRetrievalError`` around a call has to catch it like + any other failure of that call, while the ``ValueError`` base keeps the + handlers that predate the file layer working. +- **``parallel_chunks`` at the top level of the file warns.** It is the one + setting that spends rate-limit quota, so a value left there applies to every + splittable query in every process that reads the file. A + ``[profiles.]`` table is opt-in per run, which is the shape this + setting wants; the top-level form still works but says so. +- **``dataretrieval.configuration`` is a lightweight leaf.** It uses only the standard + library, the ``tomli`` backport on Python 3.10, and + ``dataretrieval.exceptions`` -- itself a dependency-free leaf, so this adds + no weight and cannot cycle. It is read by ``utils`` + (headers), ``ogc.chunking``, ``ogc.retry``, and ``ogc.progress``, so under ADR + 0003 it must import none of them. The public callable is named ``configure`` + rather than ``config`` so it does not shadow the module. It is a scoped + action, not a ``Configuration`` dataclass: a value object would imply + snapshot, equality, serialization, and representation contracts while + risking disclosure of the API key through generated helpers. + +- **One flat set of setting names, shared by every service.** ``concurrency`` + means the same thing to every adapter, so the chain resolves one name rather + than one per service. Services differ in the *value* they want, not the + vocabulary, and that difference is expressed as a caller-supplied default: + ``wateruse`` passes its ``DEFAULT_CONCURRENT_REQUESTS`` of 4 to + ``configuration.concurrency()`` where the OGC getters take the package default of + 32, and the single-shot adapters pass ``_GATEWAY_STATUSES`` to + ``RetryPolicy.from_configuration()`` because WQP and StreamStats report a rejected + query as a 500. A value resolved from the chain always outranks a caller + default -- a service able to override an explicit setting would make + ``concurrency=1`` a lie. + +- **Per-service overrides are deferred, not refused.** One ``configure()`` + block cannot currently ask for a gentler Water Use than Water Data. Every + known service difference is a default, which the caller already supplies, so + nothing needs it yet. If something does, the shape is a namespace inside this + chain -- a ``[wateruse]`` table beside the top-level keys, read as + ``configuration.concurrency(default, service=...)``. It costs a second dimension in + resolution, which ``show_configuration()`` must then render as a matrix rather than + a list, and that cost should buy a real requirement before it is paid. + +- **A configuration object would have no way to reach the call.** The public + surface is free functions -- ``waterdata.get_daily(...)``, not a client with + methods. An instance would therefore arrive either as a parameter on every + getter, which is the threading the ``ContextVar`` exists to remove and which + the ``**queryables`` catch-all makes unsafe, or through a module-level + global, which restores the cross-thread and cross-task leakage this ADR + exists to end. A library entered through a constructed client can hold + settings on that client; one entered through free functions cannot, and the + scoped block follows from that. + +Consequences +------------ + +- A credential can be supplied per thread or per task without touching + ``os.environ``, which is what issue #352 asked for. +- Host scoping is unchanged and unconditional: a key from any source is sent + only to ``api.waterdata.usgs.gov`` and is stripped on cross-host redirects. +- ``show_configuration()`` reports the effective value and provenance of each setting + without ever printing the key. +- Behavior is unchanged when no file exists and no block is active, so + existing environment-variable users are unaffected. +- A configuration file becomes a supported artifact whose format is now a + compatibility surface. +- The Python floor and the file format are coupled: raising + ``requires-python`` to ``>=3.11`` drops the ``tomli`` dependency with no + other change. + +Compliance +---------- + +``tests/architecture_test.py::test_config_is_a_standard_library_only_leaf`` +asserts the module imports nothing from ``dataretrieval`` other than the +``exceptions`` taxonomy leaf, and no third-party package other than the +``tomli`` backport. +``tests/configuration_test.py`` covers the precedence chain, per-setting merging, +thread and asyncio isolation, host scoping for file-sourced keys, redaction in +``show_configuration``, and rejection of credential parameters on public getters. diff --git a/docs/source/architecture/decisions/0010-adapter-scoped-settings.rst b/docs/source/architecture/decisions/0010-adapter-scoped-settings.rst new file mode 100644 index 000000000..8f1df66b6 --- /dev/null +++ b/docs/source/architecture/decisions/0010-adapter-scoped-settings.rst @@ -0,0 +1,286 @@ +ADR 0010: Adapter-scoped settings +================================= + +Status +------ + +Accepted, except for two clauses. Supersedes the "One flat set of setting +names" and "Per-service overrides are deferred" clauses of +:doc:`0009-layered-configuration`; the rest of ADR 0009 stands, subject to what +ADR 0011 supersedes there. + +:doc:`0011-configuration-profiles` supersedes decisions 5 and 8 below -- +adapter schemas held centrally as ``TypedDict``, and each adapter a named +keyword on ``configure()``. Each adapter now declares a ``BaseConfiguration`` +subclass in the module that *reads* those settings, and ``configure()`` takes +instances of them positionally, which is what removes the adapter roster from +the call site. The spelling shown in decision 1 goes with decision 8. Decisions +2, 3, 4, 6 and 7 stand: the tiers, source-major precedence, package-wide +environment variables, the host-scoped key, and the adapter names. + +Context +------- + +ADR 0009 resolved every setting through one flat namespace, on the premise that +"a setting means the same thing to every service; services differ in the value +they want, not the vocabulary." Surveying the seven APIs this package retrieves +from shows the premise is false. The settings themselves differ: + +.. list-table:: + :header-rows: 1 + + * - Adapter + - Host / path + - Fan-out + - Retryable statuses + - ``ssl_check`` + * - ``waterdata`` + - ``api.waterdata.usgs.gov/ogcapi`` + - yes (OGC chunking) + - all 5xx + 429 + - yes, on 3 of its getters + * - ``ngwmn`` + - ``api.waterdata.usgs.gov/ngwmn/ogcapi`` + - yes (OGC chunking) + - all 5xx + 429 + - -- + * - ``nwdc`` + - ``api.water.usgs.gov/nwaa-data`` + - yes (fan-out) + - all 5xx + 429 + - yes + * - ``nldi`` + - ``api.water.usgs.gov/nldi/linked-data`` + - no + - gateway only + - no + * - ``wqp`` + - ``www.waterqualitydata.us`` + - no + - gateway only + - yes + * - ``streamstats`` + - ``streamstats.usgs.gov`` + - no + - gateway only + - no + * - ``nwis`` (deprecated, not an adapter key) + - ``waterservices.usgs.gov`` + - no + - fixed, no retry + - yes + +``concurrency`` and ``parallel_chunks`` are meaningless for the four +single-shot adapters -- there is nothing to fan out. ``ssl_check`` applies to +four adapters (``waterdata``, ``nwdc``, ``nwis``, ``wqp``) and is currently a +per-call keyword outside the chain entirely; it reaches ``httpx``'s ``verify``, +verified by spying on the client. A flat namespace accepts +``configure(streamstats={"parallel_chunks": 8})`` without complaint, which is +the typo class ADR 0009 exists to catch. + +The credential is a separate axis, and measurement settled it. Probing the live +APIs with and without a key: + +* NGWMN and Water Data are served from the *same host* + (``ngwmn.py`` derives its base URL from ``credentials.WATERDATA_BASE_URL``). +* Both return ``200`` with no key, and both return ``x-ratelimit-limit: 1000`` + with one. +* Alternating authenticated calls decrement a *single* counter + (997, 996, 996, 994, 993, 992), so the two adapters share one quota pool. +* Water Data's OpenAPI declares ``ApiKeyHeader``/``ApiKeyQuery``; NGWMN's + declares no security scheme at all across 34 paths -- yet the gateway meters + it regardless. Every response carries ``via: ... api-umbrella``. + +The key is therefore a credential of the **gateway fronting the host**, not of +either adapter. It cannot meaningfully vary per adapter: two keys against one +quota pool is not a state the gateway can be in. + +Decision +-------- + +Settings are scoped to the **adapter**, not the service, and not the host. + +1. **The configuration file gains one table per adapter**, beside the existing + top-level keys:: + + concurrency = 16 # every adapter + + [ngwmn] + concurrency = 4 # this adapter only + + ``configure()`` takes the same shape, so one block configures several + adapters at once:: + + with dataretrieval.configure(ngwmn={"concurrency": 4}, + wqp={"retries": 2}): + ... + + .. note:: + + The file table stands; the ``configure()`` spelling above is superseded + by :doc:`0011-configuration-profiles` along with decision 8. One block + still configures several adapters at once, now as + ``configure(NgwmnConfiguration(concurrency=4), + WqpConfiguration(retries=2))``. + +2. **The top-level tier survives.** An adapter table *overrides* it per key; it + does not replace it. Every setting still has a package-wide spelling, and + the shipped ``API_USGS_*`` variables are package-wide by construction. + ``retries`` and ``stall_timeout`` are additionally adapter-scopable, because + a service that answers slowly or refuses often warrants its own budget + without changing anyone else's. ``progress`` is not: it describes the + caller's terminal, and there is one progress line per call, so scoping it + per adapter could only produce a contradiction. + +3. **Precedence stays source-major.** Resolution walks block, then environment, + then file, as ADR 0009 defines; *within* each source an adapter-scoped value + outranks a top-level one. The environment therefore still outranks the file, + so a stale adapter table cannot quietly beat a variable exported for one run. + +4. **Adapter-scoped settings get no environment variables.** Every entry in + ``ENV_VARS`` stays package-wide, for the reason ``parallel_chunks`` already + has none: an exported variable is inherited by every subprocess and + invisible at the call site. Six adapters times four settings would be a + namespace nobody could hold in mind. + +5. **Each adapter's schema is a** ``TypedDict``. Its ``__annotations__`` *are* + the schema -- there is no second table to maintain, ``mypy --strict`` checks + literal dicts at call sites, and the file path validates against the same + annotations. A key an adapter does not accept raises ``ConfigurationError`` + at block entry, the way an unknown profile already does. + + *Superseded by* :doc:`0011-configuration-profiles`. The schema is now a + frozen dataclass owned by the adapter, for the same "the annotations are the + schema" reason -- what changed is where it lives. A ``TypedDict`` had to be + declared centrally to annotate a central keyword, which put a Water Data + setting's definition in a module that knows nothing about Water Data. + +6. **The API key stays host-scoped and is not an adapter setting.** + ``credentials`` keeps sole ownership of which host honors the key. There is + no ``[ngwmn] api_key``. + +7. **Adapters are keyed by their service's name**, matching the module: + ``waterdata``, ``ngwmn``, ``nwdc``, ``wqp``, ``nldi``, ``streamstats``. + The deprecated ``nwis`` is deliberately absent: its calls pin + ``max_retries=0``, so a ``[nwis]`` table could only be reported as live and + then ignored -- the failure this decision exists to prevent. + +8. **Each adapter is a named, typed parameter on** ``configure()``, annotated + with its own ``TypedDict``, so a type checker rejects a setting the adapter + does not read before the code runs. A ``**unknown`` catch-all remains, and + exists to turn a misspelled *setting* into a message naming the settings -- + ``configure(concurrancy=8)`` would otherwise be a bare ``TypeError``. + + *Superseded by* :doc:`0011-configuration-profiles`. ``configure()`` takes + configuration objects positionally instead, so the adapter is named by the + class rather than by a keyword. The type checking survives -- a setting an + adapter does not read is not a field of its class -- and the catch-all is + no longer needed for a misspelling, because + ``WaterdataConfiguration(concurrancy=8)`` is already a ``TypeError`` naming + the keyword that does not exist. What the change buys is that ``configure()`` + no longer enumerates the adapters at all: that enumeration was the roster + this ADR left spelled in four places. + +Consequences +------------ + +- **A caller can be gentle with one adapter without throttling the rest** -- + the requirement ADR 0009 deferred. Because NGWMN and Water Data share a quota + pool, throttling NGWMN now measurably preserves quota for Water Data. + +- **The schema stops being a separate mechanism.** Choosing ``TypedDict`` over + a hand-maintained table removes the failure mode where a new adapter setting + is added and the validation table is not, and over a dataclass per adapter it + keeps the payload a plain mapping, so the file and block paths share one + validator and ``configuration`` grows no runtime classes. *Superseded with + decision 5*: the classes exist, and live with their adapters rather than in + the leaf. + +- **A configuration object is still refused, but on narrower grounds than ADR + 0009 stated.** That ADR rejected an object because it had no way to *reach* + the call. A per-adapter payload type does not have that problem -- the + ``ContextVar`` remains the delivery mechanism and the type is only the + payload's shape. ``TypedDict`` is chosen over a dataclass for the reason + above, not because an object could not be delivered. + + *Withdrawn by* :doc:`0011-configuration-profiles`, which took the remaining + step. Narrowing the objection to a payload-shape preference is what left it + open, and a dataclass turned out to buy the thing a mapping could not: an + instance knows which adapter it targets, so the caller stops naming one and + the roster stops being duplicated. + +- **``show_configuration()`` grows a second section, not a matrix.** It prints + the top-level tier as today, then only those adapter overrides actually set. + A seven-by-eight grid of mostly-inherited values would bury the answer to + "what will this call use". + +- **The shared quota pool is not modelled.** ``[waterdata]`` and ``[ngwmn]`` + read as independent dials but draw on one 1000/hour allowance. A host or + gateway tier would express it; that is deferred until someone is confused by + it, since the pool is a property of the credential, which is already + host-scoped. + +- **``stall_timeout`` joins the chain.** ``API_USGS_STALL_TIMEOUT`` was read + directly from ``os.environ``, so it could not be set by a block or the file + and never appeared in ``show_configuration()`` -- a gap in ADR 0009's own + claim that every setting resolves through one chain. It is package-wide by + default and adapter-scopable. ``dataretrieval/transport/env.py`` existed only + to parse it and is deleted, so ``configuration`` is now the only module in + the package that reads ``os.environ`` for a setting. + +- **``ssl_check`` stays a per-call argument and does not become a setting.** + It is a defaulted keyword on 23 shipped getters across four adapters -- + ``wqp`` (9), ``nwis`` (10), ``waterdata`` (3) and ``nwdc`` (1) -- and it does + reach ``httpx``'s ``verify``. It was added in 2023 to what were then the only + modules; the OGC getters arrived later and never adopted it, so its + distribution records the package's history rather than a boundary. + + Three reasons not to promote it. It disables certificate verification, so as + a per-call keyword it is a visible, scoped decision, while a config-file key + or environment variable would make a security downgrade process-wide and + invisible at the call site -- the opposite of the direction this chain + narrows everything else. It does not respect adapter boundaries: within + ``waterdata`` it applies only to the getters that bypass the OGC engine, so + ``[waterdata] ssl_check`` would be honored by three getters and silently + ignored by the rest, exactly the shape this ADR refuses elsewhere. And the + need it serves is already met better: the legitimate case is a + TLS-intercepting corporate proxy, and ``httpx`` natively honors + ``SSL_CERT_FILE`` and ``SSL_CERT_DIR`` on both its sync and async clients -- + so that mechanism already covers *every* getter, including the OGC ones that + have no ``ssl_check``, and it trusts the corporate CA rather than trusting + nothing. The ``bool`` type cannot even carry a CA bundle path, which is the + value a caller actually wants. + + The configuration guide documents ``SSL_CERT_FILE`` for that case. Whether + ``ssl_check`` should be deprecated outright is a public-API question left to + its own change. + +- ``tests/configuration_test.py`` covers adapter-table resolution, top-level + inheritance per setting, source-major precedence (the environment still + outranks an adapter table), an adapter block outranking a package-wide one, + and rejection of a setting an adapter does not read -- from both the file and + ``configure()``. +- ``test_api_key_is_never_adapter_scoped`` asserts no adapter configuration + accepts ``api_key``. +- ``test_adapter_roster_names_real_modules_that_register_themselves`` imports + every name in the roster, so a renamed adapter cannot leave a configuration + pointing at nothing. +- ``lint-imports`` continues to place ``configuration`` between ``credentials`` + and ``exceptions``. + +The two entries covering decision 8's ``**adapters`` catch-all +(``test_a_misspelled_setting_is_not_taken_for_an_adapter``) and the central +``TypedDict`` registry (``test_adapter_schema_names_a_real_module``) went with +the clauses ADR 0011 superseded; the checks they stood for are named above in +their current form. + +Notes +----- + +- Supersedes two clauses of :doc:`0009-layered-configuration`; the chain, the + ``ContextVar`` delivery, and the leaf constraint are unchanged. +- Live-API measurements behind the credential decision were taken 2026-08-11 + against ``api.waterdata.usgs.gov`` and ``api.water.usgs.gov``. +- The ``wateruse`` module is renamed ``nwdc`` under separate cover; the service + names itself "National Water Availability Assessment Data Companion" and + serves ten models, only five of which are water use. diff --git a/docs/source/architecture/decisions/0011-configuration-profiles.rst b/docs/source/architecture/decisions/0011-configuration-profiles.rst new file mode 100644 index 000000000..95a5efa5b --- /dev/null +++ b/docs/source/architecture/decisions/0011-configuration-profiles.rst @@ -0,0 +1,241 @@ +ADR 0011: Configuration profiles, scoped to one adapter +======================================================== + +Status +------ + +Accepted. Supersedes two clauses of :doc:`0010-adapter-scoped-settings` -- +decision 5 (adapter schemas held centrally as ``TypedDict``) and decision 8 +(each adapter a named keyword on ``configure``) -- and three of +:doc:`0009-layered-configuration`: the global ``[profiles.]`` table; the +environment-above-file rule, inverted for a profile selected in code; and the +refusal of a configuration object, which ADR 0010 had already narrowed to a +preference about the payload's shape. The chain, the ``ContextVar`` delivery, +host-scoped credentials and the leaf constraint stand. + +Context +------- + +ADR 0010 gave each adapter its own slice of the chain, so ``[ngwmn]`` narrows a +setting to NGWMN. That covers "tune one service" but not the case a +multi-service caller actually has: + +- **Several named configurations per adapter.** A caller with an overnight + bulk shape and a polite daytime shape for Water Data cannot store both. The + only named construct is ``[profiles.]``, which switches *every* + service at once. +- **Composing them.** The two mechanisms do not compose: + ``[profiles.bulk.ngwmn]`` raises, so a profile cannot carry per-service + detail. That refusal was recorded in ADR 0010 on the grounds that layering + them needed a fourth precedence rule nobody had asked for. Someone has now + asked for it, and it is the primary use case. + +Two further problems ADR 0010 left open feed into the same decision. The +adapter roster is spelled in four places, only one of which is derived -- +adding an adapter needs coordinated edits, and forgetting one leaves a schema +no call site can reach, which happened to three adapters and shipped +undetected until a fitness test was written. And a setting's definition lives +in ``config`` rather than in the module that reads it, so adding a Water Data +setting edits a file that knows nothing about Water Data. + +Decision +-------- + +**A configuration profile is a named set of settings for one adapter.** The +file gains named profiles beside each adapter's default profile:: + + concurrency = 16 # package-wide defaults + + [waterdata] # waterdata's DEFAULT profile: always active + concurrency = 32 + + [waterdata.bulk] # a NAMED profile: only when selected + parallel_chunks = 8 + + [ngwmn.gentle] + concurrency = 4 + +A named profile never enters the chain unless a caller selects it. The global +``[profiles.]`` table and ``DATARETRIEVAL_PROFILE`` are retired; nothing +has shipped, so nothing is deprecated. + +**``configure()`` takes configuration objects.** Positionally, one per +adapter, and nothing else:: + + with dataretrieval.configure( + Configuration(api_key=vault.read("usgs/pat")), + WaterdataConfiguration.load("bulk"), + NgwmnConfiguration(concurrency=4), + ): + ... + +The adapter an instance targets is a property of its class, so the caller +never restates it -- which is what removes the roster duplication. Naming two +configurations for one adapter raises: they would be the one pairing with no +defined order. + +Keyword settings are removed, so ``configure(api_key=...)`` no longer works. +This is the most-typed line the feature exists to enable, and making it wordier +is a real cost, accepted deliberately for one shape everywhere. + +**Schemas live with their adapter; names live centrally.** ``configuration`` +is a standard-library-only leaf every adapter may import, so it cannot import +adapters. It holds the tuple of adapter *names*, which is what parsing a file +needs (is ``[ngwmn]`` a table or a typo?). Each adapter package owns its +subclass, which is what a setting's definition needs to be local to the +service that reads it. + +Registration at import alone would not do: ``dataretrieval`` imports six of +seven adapters eagerly, but NLDI is deliberately on demand for the geopandas +extra, so a registry built from imports would reject a valid ``[nldi]`` table +until something imported it, and the report would vary by what a caller had +touched. + +**Precedence**, highest first: + +1. A configuration instance passed to ``configure()`` +2. A profile selected in code, ``WaterdataConfiguration.load("bulk")`` +3. The setting's environment variable (package-wide settings only) +4. The adapter's default profile in the file +5. Package-wide defaults in the file +6. The adapter's built-in preference in code +7. The package built-in default + +Each level overrides the one below **per key**, so a named profile still +inherits its adapter's default profile and the package-wide keys. Positions 1 +and 2 are both code and both target one adapter, so the same-adapter rule +means they cannot tie. + +Position 2 above 3 inverts ADR 0009's environment-above-file rule for this one +case. A profile named in code is a more deliberate act than a variable +inherited from a shell, and losing to that variable is the behaviour a caller +would file a bug about. Everything the caller did *not* name in code still +follows the original rule. + +**Validation is lazy.** A file's structure is checked when it is parsed; a +table's keys are checked when that adapter first resolves a setting. This +keeps the blast-radius rule ADR 0010 established -- a malformed ``[nldi]`` +table must not fail a Water Data call -- and it is what allows the schema to +live in a module the parser cannot import. + +**Base URLs may be configured, from code only.** An adapter's configuration +may carry its base URL, settable in a ``configure()`` block and rejected from +the file and the environment. A file that silently redirects a data-retrieval +library to another host is a supply-chain-shaped hazard; an in-code block +keeps the redirect where a reader sees it. + +**The module is renamed** ``dataretrieval.config`` to +``dataretrieval.configuration``, +and ADR 0009's rule reserving ``config`` as an abbreviation for the module and +the file is withdrawn. The path has never been released, so no alias is +needed. + +**Credentials are unchanged, and measurement settled why.** The API key stays +one package-wide setting scoped to the single host that honours it. Probing +the live services: + +.. list-table:: + :header-rows: 1 + + * - Host + - No key + - With key + - Bad key + * - ``api.waterdata.usgs.gov`` (waterdata, ngwmn) + - no limit header + - ``x-ratelimit-limit: 4000`` + - 403 + * - ``api.water.usgs.gov`` (nwdc) + - ``1000`` + - ``1000`` + - 403 + * - ``api.water.usgs.gov`` (nldi) + - ``3600`` + - ``3600`` + - 403 + +NWDC and NLDI meter by address and report the *same* limit with or without a +key; the gateway validates one only if present. Sending the key there would +gain nothing and would turn a stale key into 403s on calls that work +anonymously today. The three hosts also keep independent counters, so ADR +0010's "one key, one quota pool" is true of waterdata and ngwmn only. + +Consequences +------------ + +- **The multi-service case gets a spelling**, which is the point. One block, + several adapters, at most one configuration each, any of them from the file + or from code. +- **The roster stops being duplicated.** An adapter declares itself once. The + failure mode where a schema exists that nothing passes becomes impossible by + construction rather than caught by a fitness test. +- **A setting's definition moves next to the code that reads it.** Adding a + Water Data setting no longer edits a service-neutral module. +- **``configure(api_key=...)`` breaks.** The README, the configuration guide, + the PR description and ADR 0009's examples all use it and all must change in + the same commit. +- **``show_configuration()`` can only resolve the settings an adapter accepts + once that adapter has been imported.** It names the adapters it could not + check rather than omitting them silently, which is the honest cost of lazy + validation. The *profile list* is not import-limited: what a profile is + called is a fact about the file, so every ``[.]`` table it + defines is listed, imported or not -- withholding one would make the + section's answer depend on which optional extras happened to be installed. +- **Two names differ only by case** -- the ``configuration`` module and the + ``Configuration`` class. The module stays out of the package's public + exports so the confusing import line cannot arise. +- **Separate quota pools are still not modelled.** Three exist. Nothing in the + library needs to know yet. +- **``ssl_check`` is unaffected** and remains a per-call argument, for the + reasons in ADR 0010. + +Compliance +---------- + +Satisfied. In ``tests/configuration_test.py``: + +- ``test_several_named_profiles_are_selected_independently`` -- one block, + a different profile per adapter. +- ``test_a_named_profile_layers_per_key_over_the_tiers_below`` -- a profile + inherits its adapter's default profile and the package-wide keys per key. +- ``test_adding_a_named_profile_changes_nothing_until_it_is_selected`` -- a + named profile is inert until something selects it. +- ``test_two_configurations_for_one_adapter_raise``. +- ``test_a_code_selected_profile_outranks_the_environment``, plus a case per + rung of the seven-rung ladder above, each written against one file that + populates every rung with a distinct value. +- ``test_inner_block_can_lower_a_setting_an_outer_block_scoped`` -- the + innermost block wins, including over an adapter-scoped outer one. +- ``test_a_table_for_an_unimported_adapter_stays_valid`` and + ``test_a_malformed_table_does_not_fail_another_adapters_call`` -- the + blast-radius rule under lazy validation. +- ``test_base_url_applies_from_code_and_is_refused_from_the_file``, with + ``test_base_url_is_refused_from_the_environment`` for the other source, and + ``test_a_code_base_url_redirects_every_water_data_endpoint_family`` -- one + public-getter contract covering OGC, Samples, Statistics, and Ratings. The + endpoint module exposes request-time acquisition functions rather than raw + usable endpoints, so a family module gets the active scoped root without a + wrapper obligation at every use site. +- ``test_adapter_roster_names_real_modules_that_register_themselves`` and + ``test_every_adapter_is_actually_wired_to_a_read_site`` -- the roster + resolves, and no configuration exists that nothing reads. An adapter name + the code does not recognize now raises out of ``_resolve`` rather than + falling through to the package-wide value, so the grep is a backstop rather + than the only guard. + +``tests/architecture_test.py::test_config_is_a_standard_library_only_leaf`` +asserts the module imports no adapter -- ``dataretrieval.exceptions`` is its +only first-party import -- and ``lint-imports`` keeps ``configuration`` below +``credentials``. + +Notes +----- + +- Live measurements taken 2026-08-11 against ``api.waterdata.usgs.gov`` and + ``api.water.usgs.gov``. +- Open, not decided here: whether ``parallel_chunks`` is renamed. ``fan_out`` + was suggested and conflicts with the glossary, where fan-out is *executing* + chunks concurrently -- which ``concurrency`` already governs -- while + ``parallel_chunks`` asks the planner to *divide* more finely. ADR 0009 + rejected ``parallelism`` and ``chunk_parallelism`` for the same conflation. + ``chunk_count`` or ``target_chunks`` would stay on the correct side of it. diff --git a/docs/source/architecture/decisions/index.rst b/docs/source/architecture/decisions/index.rst index fb1315283..ad468728a 100644 --- a/docs/source/architecture/decisions/index.rst +++ b/docs/source/architecture/decisions/index.rst @@ -25,4 +25,7 @@ records sequentially. 0006-service-neutral-transport 0007-adapter-facades 0008-fan-out-execution + 0009-layered-configuration + 0010-adapter-scoped-settings + 0011-configuration-profiles template diff --git a/docs/source/architecture/index.rst b/docs/source/architecture/index.rst index 7aad9e663..c292b63a7 100644 --- a/docs/source/architecture/index.rst +++ b/docs/source/architecture/index.rst @@ -80,7 +80,7 @@ Public service facades configures with an NGWMN-specific base URL, output identifiers, state translation, and :class:`OgcDialect`. -``dataretrieval.wateruse`` +``dataretrieval.nwdc`` NWDC Water Use facade. Builds CSV requests, follows ``Link`` headers, and uses service-neutral transport for bounded fan-out, retry, pagination, response aggregation, and synchronous dispatch. It does not depend on OGC modules. @@ -96,6 +96,14 @@ Public service facades Shared components ^^^^^^^^^^^^^^^^^ +``dataretrieval.configuration`` + Lightweight configuration leaf: standard library plus the ``tomli`` + backport on Python 3.10. It resolves scoped overrides, environment + variables, a TOML file with optional profiles, and built-in defaults in + that order. Service and protocol modules may depend on it; it must not + depend back on them. Scoped overrides use ``ContextVar`` so concurrent + threads and asyncio tasks can carry distinct credentials. + ``dataretrieval.ogc`` Protocol subsystem for Water Data and NGWMN. A small facade (``__init__.py``) exposes the service-adapter seam: ``OgcDialect``, @@ -162,7 +170,7 @@ Shared components ``dataretrieval._querying`` The one-shot HTTP query path the single-request adapters (``nwis``, - ``wqp``, ``nldi``, ``streamstats``, ``wateruse``) use: compose the URL, send + ``wqp``, ``nldi``, ``streamstats``, ``nwdc``) use: compose the URL, send it, map the status, retry a transient. It left ``utils`` because the two halves shared only a filename -- this one depends on ``exceptions`` and ``transport``, the shaping half on ``codes`` and pandas, and no caller @@ -269,17 +277,19 @@ used by Water Use because the NWDC accepts only one location per request. Resource and configuration view ------------------------------- -``API_USGS_PAT`` - Optional USGS API token. It is attached only to requests for - ``api.waterdata.usgs.gov``. Shared synchronous and asynchronous clients - re-check every redirected request and strip the token before following a - link to any other host, including external rating assets. +Every setting resolves per key through an active ``configure()`` block, its +environment variable when one exists, the adapter's own table and the top-level +values in ``~/.dataretrieval/config.toml``, then its built-in default. +``configure()`` takes configuration objects -- a package-wide ``Configuration`` +and at most one per adapter -- and each adapter's class is defined in the module +that reads those settings, so ``configuration`` stays a leaf holding only the +adapter roster. ``show_configuration()`` reports the effective source while +redacting credentials. -``API_USGS_CONCURRENT`` - Fan-out concurrency cap; defaults to 32 for OGC and 4 for Water Use when - unset. An explicit value applies to every service, ``1`` is sequential, and - ``unbounded`` removes the explicit cap. A semaphore, not pool waiting, is - the execution throttle. +The settings themselves -- names, defaults, environment variables, and the +config-file format -- are catalogued once in the +:doc:`configuration guide `. What matters +architecturally is the behavior around them: ``API_USGS_RETRIES`` Number of retries after the first attempt on supported active request paths; @@ -306,6 +316,18 @@ Resource and configuration view ``API_USGS_PROGRESS`` Controls best-effort progress display. Reporting failures must never change retrieval results. +* The API token is attached only to requests for ``api.waterdata.usgs.gov``. + Shared synchronous and asynchronous clients re-check every redirected request + and strip the token before following a link to any other host, including + external rating assets. +* A semaphore, not connection-pool waiting, is the execution throttle for + sub-request concurrency. +* Retry backoff is exponential with full jitter and honors bounded + ``Retry-After`` values. +* Progress reporting is best-effort: a reporting failure must never change + retrieval results. +* ``dataretrieval.configuration`` is a stdlib-only leaf, so any module may depend on + it without an import cycle. ``dataretrieval.transport`` centralizes HTTP timeout, redirect, and authentication policy. OGC chunk fan-out and Water Use location diff --git a/docs/source/reference/config.rst b/docs/source/reference/config.rst new file mode 100644 index 000000000..174c81b46 --- /dev/null +++ b/docs/source/reference/config.rst @@ -0,0 +1,18 @@ +.. _config: + +dataretrieval.configuration +--------------------------- + +Layered configuration: a ``dataretrieval.configure(...)`` block holding one +configuration per adapter, then the ``API_USGS_*`` environment variables, then +the adapter's table and the package-wide keys in +``~/.dataretrieval/config.toml``, then built-in defaults. A +``[.]`` table is a named profile, selected in code with +``Configuration.load("")``. See the +:doc:`configuration guide ` for the settings and +worked examples. + +.. automodule:: dataretrieval.configuration + :members: configure, Configuration, BaseConfiguration, show_configuration, + config_path, settings_for, ConfigurationError + :show-inheritance: diff --git a/docs/source/reference/index.rst b/docs/source/reference/index.rst index 959d26750..5cbebce15 100644 --- a/docs/source/reference/index.rst +++ b/docs/source/reference/index.rst @@ -7,6 +7,7 @@ API reference .. toctree:: :maxdepth: 1 + config exceptions ngwmn nldi @@ -14,5 +15,5 @@ API reference streamstats utils waterdata - wateruse + nwdc wqp diff --git a/docs/source/reference/nwdc.rst b/docs/source/reference/nwdc.rst new file mode 100644 index 000000000..52087b4fd --- /dev/null +++ b/docs/source/reference/nwdc.rst @@ -0,0 +1,12 @@ +.. _nwdc: +.. _wateruse: + +dataretrieval.nwdc +------------------ + +The National Water Availability Assessment Data Companion. Water use is one of +the datasets it serves; the module was named ``wateruse`` until that became +misleading, and the old name remains as a deprecated alias. + +.. automodule:: dataretrieval.nwdc + :members: diff --git a/docs/source/reference/wateruse.rst b/docs/source/reference/wateruse.rst deleted file mode 100644 index db4e49620..000000000 --- a/docs/source/reference/wateruse.rst +++ /dev/null @@ -1,7 +0,0 @@ -.. _wateruse: - -dataretrieval.wateruse ----------------------- - -.. automodule:: dataretrieval.wateruse - :members: diff --git a/docs/source/userguide/configuration.rst b/docs/source/userguide/configuration.rst new file mode 100644 index 000000000..99db38095 --- /dev/null +++ b/docs/source/userguide/configuration.rst @@ -0,0 +1,598 @@ +.. _configuration: + +============= +Configuration +============= + +``dataretrieval`` retrieves from several services, and most of what you would +want to adjust — a concurrency cap, a retry budget, where requests go — belongs +to *one* of them. So a **configuration profile** is a named set of settings for +one adapter, written in code or stored in your configuration file, and a +``configure`` block puts one profile per adapter into effect for the calls +inside it. The Water Data API key is the exception that proves the rule: it +authenticates to a gateway rather than to an adapter, so it stays package-wide. + +.. contents:: + :local: + :depth: 1 + + +.. _configuration-one-block: + +One block, several services +--------------------------- + +This is the case the mechanism exists for. Say the file holds what you would +write once and keep — the key, a retry budget, and Water Data's everyday +concurrency — plus two named profiles for the shapes you only sometimes want: + +.. code-block:: toml + + api_key = "your_api_key_here" # package-wide: every adapter that reads it + retries = 6 + + [waterdata] + concurrency = 16 # waterdata's default profile: always active + + [waterdata.overnight] # a named profile: only when selected + concurrency = "unbounded" + parallel_chunks = 8 + + [ngwmn.gentle] + concurrency = 2 + +Then one block configures three services, taking two of them from the file by +name and building the third on the spot: + +.. code-block:: python + + import dataretrieval + from dataretrieval import ngwmn, waterdata, wqp + from dataretrieval.ngwmn import NgwmnConfiguration + from dataretrieval.waterdata import WaterdataConfiguration + from dataretrieval.wqp import WqpConfiguration + + with dataretrieval.configure( + WaterdataConfiguration.load("overnight"), # from the file, by name + NgwmnConfiguration.load("gentle"), # from the file, by name + WqpConfiguration(retries=2), # built here + ): + flow, _ = waterdata.get_daily(monitoring_location_id=sites, time="P30D") + levels, _ = ngwmn.get_water_level(monitoring_location_id=wells) + samples, _ = wqp.get_results(siteid=sites) + +Inside the block Water Data runs unbounded and asks the planner for eight +chunks, NGWMN runs two requests at a time, and WQP retries twice. Everything a +configuration does *not* name still comes from below it, per setting: Water +Data and NGWMN both retry six times and both send the ``api_key``, written once +at the top of the file, because a configuration contributes what it names and +inherits the rest. Only WQP named ``retries``, so only WQP departs from the +file's six. + +Outside the block nothing has changed, and putting those two profiles in the +file changed nothing on its own — a named profile is inert until a caller +selects it, which is what makes one safe to add to a file other people's jobs +also read. + +Two rules keep a block like that unambiguous. A configuration knows which +adapter it targets — that is a property of its class — so you never restate it, +and ``Configuration`` targets none of them, which is what makes it +package-wide. And there is at most one configuration per adapter: naming two +raises rather than picking one, because there would be no defined order between +them — combine them into one instead. + + +Settings +-------- + +.. list-table:: + :header-rows: 1 + :widths: 18 12 26 44 + + * - Setting + - Default + - Environment variable + - What it does + * - ``api_key`` + - none + - ``API_USGS_PAT`` + - Water Data API key. Raises your hourly request quota substantially; + `register for one `_. + * - ``concurrency`` + - ``32`` + - ``API_USGS_CONCURRENT`` + - Cap on sub-requests in flight at once for a chunked query. A positive + integer, ``1`` to run them one at a time, or ``"unbounded"`` to remove + the cap. Does not change how many requests are made, only how many run + simultaneously. + * - ``retries`` + - ``4`` + - ``API_USGS_RETRIES`` + - Retries after a transient failure (429, 5xx, timeout). ``0`` disables. + * - ``progress`` + - auto + - ``API_USGS_PROGRESS`` + - Whether to draw the status line. Auto means on for a terminal or + Jupyter kernel, off for redirected output and CI. + * - ``parallel_chunks`` + - ``1`` + - *(none — see below)* + - Default fan-out for multi-value queries. ``1`` means split only as far + as the URL byte limit forces. + * - ``stall_timeout`` + - ``60`` + - ``API_USGS_STALL_TIMEOUT`` + - Seconds a call may go without receiving *any* data before retrying + stops and the failure surfaces. Bounds the wall-clock cost of a dead + connection, which ``retries`` alone does not — it counts attempts, not + seconds. Progress resets the clock; ``0`` disables the bound. + * - ``base_url`` + - the service's own + - *(none — code only)* + - Where to send one service's requests. Per adapter, and settable only in + a ``configure`` block: a file that silently redirected the library to + another host would be a supply-chain hazard. See + :ref:`configuration-redirect`. + + +Where settings come from +------------------------ + +Highest precedence first: + +1. A configuration passed to an active ``dataretrieval.configure(...)`` block. +2. A named profile you selected in that block — + ``WaterdataConfiguration.load("bulk")``. +3. The environment variable for that setting. +4. The adapter's default profile in the configuration file: the + ``[]`` table. +5. The package-wide keys at the top of the configuration file — + ``~/.dataretrieval/config.toml``, or the path in ``DATARETRIEVAL_CONFIG``. +6. The adapter's own built-in preference, where it has one — NWDC asks for a + ``concurrency`` of 4, because that is as far as the service is + stress-tested. It is a default, not a cap: anything you set above outranks + it. +7. The package built-in default, which for ``concurrency`` is 32. + +The top two rungs both name a single adapter, and naming two configurations +for one adapter raises, so they cannot disagree inside one block. Between +nested blocks the innermost decides, as it does for everything else. + +Precedence applies **per setting**. An environment that sets only +``API_USGS_PAT`` leaves a file-provided ``concurrency`` fully in effect — +sources are merged, not replaced. + +A variable that is *set but empty* (``export API_USGS_PAT=``, or a CI secret +that resolves to nothing) does not count as configured, so an empty variable +your tooling happened to create cannot silently discard the key in your config +file. The one exception is ``API_USGS_PROGRESS``, where blank has always meant +"off" and so is treated as a real value. + +.. note:: + + The environment ranks above the file, matching common deployment tools and + preserving the existing ``API_USGS_*`` variables as authoritative runtime + overrides. The reasoning is in :doc:`ADR 0009 + `. + + The one exception is rung 2 above rung 3 — a profile you name in code. That + is a more deliberate act than a variable inherited from whatever started + your process, and having it lose to that variable is the kind of thing you + would file a bug about. The inversion covers what the profile names and + nothing else: every setting you did *not* name still follows the + environment-above-file rule, in the same block. See :doc:`ADR 0011 + `. + + +An environment variable +----------------------- + +Still fully supported, and the simplest option for a single key on one +machine: + +.. code-block:: bash + + export API_USGS_PAT="your_api_key_here" + +This is also the mechanism the `R dataRetrieval package +`_ uses, under the same variable +name, so one export serves both. + + +A configuration file +-------------------- + +Better when you would rather not have a credential in your shell environment, +where it is inherited by every process you start. Create +``~/.dataretrieval/config.toml``: + +.. code-block:: toml + + api_key = "your_api_key_here" + +Restrict it so other users on the machine cannot read it — ``dataretrieval`` +warns once if a file containing a key is group- or world-readable: + +.. code-block:: bash + + chmod 600 ~/.dataretrieval/config.toml + +Any setting can go in the file: + +.. code-block:: toml + + api_key = "your_api_key_here" + concurrency = 16 + retries = 8 + +Point ``DATARETRIEVAL_CONFIG`` at a different path to override the location — +useful for a container or a job scheduler that mounts secrets elsewhere. + + +Per-adapter settings +~~~~~~~~~~~~~~~~~~~~ + +To tune one service and leave the rest alone, name the adapter — the same name +you import: + +.. code-block:: toml + + concurrency = 16 # every adapter + + [ngwmn] + concurrency = 4 # NGWMN only + + [wqp] + retries = 2 + +.. code-block:: python + + from dataretrieval.ngwmn import NgwmnConfiguration + from dataretrieval.wqp import WqpConfiguration + + with dataretrieval.configure( + NgwmnConfiguration(concurrency=4), WqpConfiguration(retries=2) + ): + ... + +An adapter table *overrides* the top-level one per setting, so ``[ngwmn]`` +above still inherits ``retries`` and the ``api_key``. Precedence is unchanged +otherwise: an adapter-scoped value outranks a package-wide one only within the +same source, so ``API_USGS_CONCURRENT`` exported for one run still beats a +``[ngwmn] concurrency`` in the file. + +Between ``configure`` blocks that tie-break applies per block: an adapter +configuration beats a package-wide value set by the *same* block, while +anything set by a block nested inside it wins over both. So a +``configure(Configuration(concurrency=1))`` can still throttle a call an +enclosing block had scoped to one adapter, and the innermost block decides. + +Each adapter accepts only the settings it reads, and they are the fields of its +configuration class — ``concurrency`` and ``parallel_chunks`` are meaningless to +an adapter that issues a single request, so ``StreamstatsConfiguration`` has no +such field and ``[streamstats] parallel_chunks = 8`` is an error rather than a +line that quietly does nothing: + +==================================== ====================================== ======================================== +Adapter Configuration Accepts +==================================== ====================================== ======================================== +``waterdata`` ``waterdata.WaterdataConfiguration`` ``concurrency``, ``parallel_chunks``, + ``retries``, ``stall_timeout``, + ``base_url`` +``ngwmn`` ``ngwmn.NgwmnConfiguration`` the same five +``nwdc`` ``nwdc.NwdcConfiguration`` ``concurrency``, ``retries``, + ``stall_timeout``, ``base_url`` +``wqp``, ``nldi``, ``streamstats`` ``wqp.WqpConfiguration`` and so on ``retries``, ``stall_timeout``, + ``base_url`` +==================================== ====================================== ======================================== + +Each class lives in the module whose code reads those settings, so a setting's +definition sits next to its use rather than in a service-neutral file. + +``api_key`` is deliberately not per-adapter. It authenticates to the *gateway* +in front of a host, and Water Data and NGWMN are served from the same host — +one key, one hourly quota shared between them — so a per-adapter key would +describe a distinction the service does not have. ``progress`` is likewise +package-wide: there is one progress line per call. + + +Named profiles +~~~~~~~~~~~~~~ + +An adapter can hold more than one shape at a time. The ``[]`` table is +that adapter's **default profile** — always in effect, as above — while a +``[.]`` table is a **named profile**, inert until you select it: + +.. code-block:: toml + + [waterdata] + concurrency = 16 # the default profile: always in effect + + [waterdata.bulk-pull] + concurrency = "unbounded" # only when selected + parallel_chunks = 8 + +So one file can hold an overnight bulk shape beside a polite daytime one, and +name as many of each as an adapter has uses for. + +A named profile states only what differs: everything it does not name still +comes from the adapter's default profile, the package-wide keys, and the tiers +below — per setting. + +``load`` reads the table and hands you a configuration object, so a name the +file does not define raises there and then, listing the names it does define — +a profile you just typed is more likely a typo than a request to fall through +to settings you did not ask for. What comes back is inert until you pass it to +``configure``; that is what puts a selected profile above the environment, +since selecting one is something your code did. + +A profile holds settings and nothing else: ``[waterdata.bulk-pull.ngwmn]`` is +not a Water Data profile carrying NGWMN detail, and selecting it says so rather +than quietly ignoring the nested table. Two adapters means two profiles, +selected in the same block, as in :ref:`the example above +`. + + +A ``configure`` block +--------------------- + +The highest-precedence source, and the one to use when a setting must apply to +*this* call and no other: + +.. code-block:: python + + import dataretrieval + from dataretrieval import Configuration, waterdata + + with dataretrieval.configure(Configuration(api_key=secrets["usgs"])): + df, md = waterdata.get_daily( + monitoring_location_id="USGS-05114000", + parameter_code="00060", + time="P7D", + ) + +``configure`` takes configuration objects positionally, and nothing else. The +adapter a configuration targets is a property of its class, so you never +restate it — and ``Configuration`` targets none of them in particular, which is +what makes it package-wide. + +.. note:: + + Settings are not keywords on ``configure``. ``configure(api_key=...)`` and + the per-adapter mappings ``configure(ngwmn={"concurrency": 4})`` were an + earlier spelling and are gone; write ``Configuration(api_key=...)`` and + ``NgwmnConfiguration(concurrency=4)`` instead. Passing anything that is not + a configuration raises and names the replacement, so an old script says what + to write rather than failing obscurely. + +Because it is backed by a :class:`~contextvars.ContextVar`, the value applies +to the current thread and to asyncio tasks started inside the block, and +cannot leak into another thread or task. That is what makes it usable from a +web service or a notebook working with more than one account: + +.. code-block:: python + + # each thread keeps its own key; no os.environ mutation, no race + def fetch_for(user): + with dataretrieval.configure(Configuration(api_key=vault.read(user.key_path))): + return waterdata.get_daily(monitoring_location_id=user.sites) + +Blocks nest and merge per setting, so an inner block that tunes one thing +keeps the rest: + +.. code-block:: python + + with dataretrieval.configure(Configuration(api_key=key, concurrency=8)): + ... + # api_key still applies + with dataretrieval.configure(Configuration(concurrency=1)): + ... + +Values are validated when the configuration is *constructed*, so a typo raises +on the line you wrote it on rather than deep inside a later request. + +Omitted settings inherit from an outer block or a lower-precedence source. +Passing ``None`` explicitly suppresses those sources and restores built-in +behavior for that block. ``Configuration(api_key=None)``, for example, makes an +anonymous call even if ``API_USGS_PAT`` is set. + +.. tip:: + + Prefer reading the key from a secret store, environment, or config file + over writing a literal into a script — a literal is what ends up committed + or pasted into a shared notebook. + + +Checking what is in effect +-------------------------- + +``show_configuration()`` reports each setting's effective value and where it came +from. It never prints the key itself. The report below is what a file holding a +key, a package-wide ``concurrency``, an ``[ngwmn]`` table and a +``[waterdata.bulk]`` profile produces, with ``API_USGS_RETRIES`` exported and +the ``bulk`` profile selected for the block: + +.. code-block:: python + + >>> with dataretrieval.configure(WaterdataConfiguration.load("bulk")): + ... dataretrieval.show_configuration() + config file /home/u/.dataretrieval/config.toml (found) + api_key /home/u/.dataretrieval/config.toml + concurrency 16 /home/u/.dataretrieval/config.toml + retries 8 $API_USGS_RETRIES + progress auto built-in default + parallel_chunks 1 built-in default + stall_timeout 60s built-in default + + A built-in default is package-wide. An adapter may prefer its own for + its own calls; a value from any source above overrides both. + + adapter overrides + waterdata parallel_chunks 8 configure() block [waterdata.bulk] + ngwmn concurrency 4 /home/u/.dataretrieval/config.toml [ngwmn] + + profiles in the file: [waterdata.bulk] + A profile applies only where a row above names it; select one in + code with Configuration.load(""). + + not reported: nldi (not imported, so the settings each accepts are unknown here) + +Each line names the exact source, including which table inside the file, which +is usually enough to answer "why is it still using my old key?". A value that +came from a profile names the profile — ``configure() block +[waterdata.bulk]``, not merely "a block" — so a report taken from inside a +``with`` block says which selection produced it. Only settings actually +overridden for an adapter get a row in the second section; everything else is +inherited from the rows above it. + +The profile section lists what the *file* defines, whether or not this run +selected any of it. A named profile does nothing until a caller selects it, so +seeing ``[waterdata.bulk]`` there while no row above mentions it is the answer +to "I added a profile and nothing changed". + +The last line is the honest cost of validating an adapter's settings lazily: +``dataretrieval`` cannot say what ``nldi`` accepts until something imports it, +so it says that rather than quietly omitting the service. It is named rather +than left out, because an omitted service would read as "nothing is configured +for it", which is a different claim. + +It never raises. A malformed file or a value that fails its grammar is reported +in place — on the ``config file`` line for a whole-file problem, or in that +setting's own row — because a broken configuration is exactly when you reach +for this. + + +Why ``parallel_chunks`` has no environment variable +--------------------------------------------------- + +Every other setting can be set from the environment. ``parallel_chunks`` +cannot, on purpose. + +Raising it splits a query into more sub-requests, and *each sub-request spends +rate-limit quota*. Whether that is a good trade depends on the size of the +query — which the library cannot know in advance. The setting therefore does +not add another process-global environment knob that could be exported once +and inherited by every subprocess. + +Set it per call, which is almost always what you want: + +.. code-block:: python + + with waterdata.parallel_chunks(8): + df, md = waterdata.get_daily(monitoring_location_id=many_sites) + +or as a baseline in the config file — deliberately written, and visible in +``show_configuration()``. Put it in a ``[.]`` table rather than +at the top level: a named profile applies only to runs that select it, while a +top-level value applies to every query in every process that reads the file, +which is how a setting added for one bulk pull quietly exhausts an hourly quota +months later. ``dataretrieval`` warns if it finds one at the top level. + +The value limits optional refinement only. URL-byte safety can require more +sub-requests than the configured value, and an input with nothing to split +stays a single request. + +``parallel_chunks(n)`` is sugar for +``configure(Configuration(parallel_chunks=n))``: one scoping mechanism, so the +innermost block wins whichever spelling set it, and ``show_configuration()`` +always reports the value the chunker will actually use. + + +.. _configuration-redirect: + +Pointing an adapter at another host +----------------------------------- + +``base_url`` sends one adapter's requests somewhere else — a staging instance, +a mirror, or a recording proxy — for the duration of a block: + +.. code-block:: python + + import dataretrieval + from dataretrieval import waterdata + from dataretrieval.waterdata import WaterdataConfiguration + + with dataretrieval.configure( + WaterdataConfiguration(base_url="https://staging.example/waterdata") + ): + df, md = waterdata.get_daily(monitoring_location_id="USGS-05114000") + +It names one adapter, so nothing else moves: NGWMN is served from the same host +as Water Data, and a ``WaterdataConfiguration`` still leaves it alone. What the +value replaces is that adapter's own base, and the package appends its usual +paths to it — for Water Data that is the root all four of its APIs hang off, so +one value moves the OGC collections, the Samples database, the statistics +service and the STAC catalog together. + +**Code only.** The configuration file and the environment both refuse it. A +``base_url`` key anywhere in the file, and an exported ``API_USGS_BASE_URL``, +each raise a ``ConfigurationError`` saying the setting *may only be set in +code, in a configure() block* and naming the configuration to pass it on +instead. + +A file or a shell export that silently redirected a data-retrieval library to +another host would be a supply-chain hazard: nothing at the call site would +show it, and a script that reads correctly would be talking to someone else's +service. A ``with`` block keeps the redirect where a reader of the script sees +it. The refusal is loud rather than silent for the same reason — a variable +that was quietly ignored would leave you believing you had redirected +something. + +**The API key does not follow.** It is scoped to the one host that honors it +(:ref:`below `), so a redirected call goes out +without it. That is deliberate: the host you redirected to is not the host you +gave a credential to. If the mirror needs its own credential, it needs its own +mechanism. + + +.. _configuration-secret-store: + +Keeping a key out of your environment entirely +---------------------------------------------- + +If your credentials live in a secret manager, nothing needs to touch +``os.environ``: + +.. code-block:: python + + import dataretrieval + import boto3 + from dataretrieval import Configuration, waterdata + + secrets = boto3.client("secretsmanager") + key = secrets.get_secret_value(SecretId="usgs-pat")["SecretString"] + + with dataretrieval.configure(Configuration(api_key=key)): + df, md = waterdata.get_continuous(monitoring_location_id="USGS-05114000") + +Wherever the key comes from, it is sent only to ``api.waterdata.usgs.gov`` and +is stripped from any cross-host redirect, so it cannot leak to another host. + + +Behind a TLS-intercepting proxy +------------------------------- + +On a corporate network that re-signs HTTPS traffic, requests fail with a +certificate-verification error. Point the standard OpenSSL variables at your +organization's CA bundle: + +.. code-block:: bash + + export SSL_CERT_FILE=/path/to/corporate-ca.pem + # or, for a directory of hashed certificates: + export SSL_CERT_DIR=/etc/ssl/certs + +``httpx`` honors these natively, so they apply to **every** getter in the +package — including the OGC collection getters (``get_daily``, +``get_continuous``, and the rest), which take no SSL parameter of their own. + +Prefer this to ``ssl_check=False``. That argument exists on some of the older +getters and switches certificate verification *off* rather than trusting your +CA, so it accepts any certificate a network path offers — and it is not +available on the OGC getters at all. A CA bundle keeps verification on and +works everywhere. + +.. note:: + + ``SSL_CERT_FILE`` is read by OpenSSL, not by ``dataretrieval``, so it does + not appear in :func:`~dataretrieval.show_configuration`. diff --git a/docs/source/userguide/errors.rst b/docs/source/userguide/errors.rst index 046f7a5d1..0708c1bec 100644 --- a/docs/source/userguide/errors.rst +++ b/docs/source/userguide/errors.rst @@ -102,7 +102,7 @@ mid-stream, the work already completed is preserved: catch except FanOutInterrupted as again: exc = again -The same loop works for ``wateruse.get_wateruse`` with a list of states, +The same loop works for ``nwdc.get_wateruse`` with a list of states, counties, or HUCs. Chunk a large request more finely diff --git a/docs/source/userguide/index.rst b/docs/source/userguide/index.rst index 96ca88fcc..7acd4cbb6 100644 --- a/docs/source/userguide/index.rst +++ b/docs/source/userguide/index.rst @@ -13,6 +13,7 @@ Contents .. toctree:: :maxdepth: 1 + configuration errors timeconventions dataportals diff --git a/pyproject.toml b/pyproject.toml index 5a184141f..1147e6889 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,6 +33,10 @@ dependencies = [ # Directly imported by ``waterdata`` (``anyio.from_thread.start_blocking_portal``), # so declared here rather than relied on transitively via httpx. "anyio>=4.0", + # ``dataretrieval.config`` reads a TOML config file. ``tomllib`` is stdlib + # from 3.11, so this marker installs the backport only on 3.10 and drops + # itself when ``requires-python`` moves to >=3.11. + "tomli>=1.1.0; python_version < '3.11'", ] dynamic = ["version"] diff --git a/tests/architecture_test.py b/tests/architecture_test.py index 88606b76b..4af377b77 100644 --- a/tests/architecture_test.py +++ b/tests/architecture_test.py @@ -178,6 +178,47 @@ def test_runtime_import_graph_is_acyclic() -> None: raise AssertionError(f"Runtime import cycle: {cycle}") from exc +def test_config_is_a_standard_library_only_leaf() -> None: + """The two-module configuration subsystem must stay cheap and cycle-safe. + + ``dataretrieval.configuration`` remains the public runtime interface and may + depend on its private foundation, ``dataretrieval._configuration_core``. + The core may depend only on the package's dependency-free leaves, + ``dataretrieval._ambient`` and ``dataretrieval.exceptions``. Neither module + may reach adapters or runtime third-party packages. ``tomli`` remains the + one third-party exception: it is the ``tomllib`` backport used on Python + 3.10. + """ + subsystem = { + PACKAGE_ROOT / "configuration.py": { + "dataretrieval._configuration_core", + "dataretrieval.exceptions", + }, + PACKAGE_ROOT / "_configuration_core.py": { + "dataretrieval._ambient", + "dataretrieval.exceptions", + }, + } + for path, allowed_first_party in subsystem.items(): + imports = _runtime_imports(path) + first_party = {name for name in imports if name.startswith("dataretrieval")} + unexpected_first_party = first_party - allowed_first_party + assert not unexpected_first_party, ( + f"{_module_name(path)} crossed the configuration subsystem boundary: " + f"{sorted(unexpected_first_party)}" + ) + + roots = {module.partition(".")[0] for module in imports} + # Static analysis sees both sides of the version guard. Python 3.10's + # stdlib inventory does not yet include the unreachable ``tomllib`` branch. + allowed_roots = {"dataretrieval", "tomli", "tomllib"} + third_party = roots - sys.stdlib_module_names - allowed_roots + assert not third_party, ( + f"{_module_name(path)} gained third-party dependencies: " + f"{sorted(third_party)}" + ) + + def test_engine_request_import_surface_does_not_grow() -> None: """Engine imports only request names it uses and may not grow a new hub. @@ -366,11 +407,11 @@ def test_credential_policy_has_one_definition() -> None: def _waterdata_family_paths() -> tuple[str, ...]: """Read the collection-family inventory from its dependency contract.""" - config = configparser.ConfigParser() - config.read(PACKAGE_ROOT.parent / ".importlinter") + parser = configparser.ConfigParser() + parser.read(PACKAGE_ROOT.parent / ".importlinter") return tuple( module.removeprefix("dataretrieval.").replace(".", "/") + ".py" - for module in config["importlinter:contract:waterdata-families"][ + for module in parser["importlinter:contract:waterdata-families"][ "modules" ].split() ) @@ -386,7 +427,7 @@ def _waterdata_family_paths() -> tuple[str, ...]: "ngwmn.py", "nldi.py", "streamstats.py", - "wateruse.py", + "nwdc.py", "wqp.py", "waterdata/nearest.py", "waterdata/ratings.py", @@ -515,7 +556,7 @@ def test_empty_result_shaping_consults_the_schema_endpoint() -> None: ) -def test_wateruse_does_not_reimplement_fan_out_orchestration() -> None: +def test_nwdc_does_not_reimplement_fan_out_orchestration() -> None: """Water Use must drive its locations through the shared fan-out executor. It previously ran its own ``asyncio.gather`` with a private semaphore and a @@ -524,7 +565,7 @@ def test_wateruse_does_not_reimplement_fan_out_orchestration() -> None: resume, progress, and the shared concurrency setting. Assert the duplication cannot quietly return. """ - source = (PACKAGE_ROOT / "wateruse.py").read_text(encoding="utf-8") + source = (PACKAGE_ROOT / "nwdc.py").read_text(encoding="utf-8") tree = ast.parse(source) offenders = { f"{node.value.id}.{node.attr}" diff --git a/tests/configuration_test.py b/tests/configuration_test.py new file mode 100644 index 000000000..609d16456 --- /dev/null +++ b/tests/configuration_test.py @@ -0,0 +1,2026 @@ +"""Tests for layered configuration resolution (``dataretrieval.configuration``).""" + +from __future__ import annotations + +import asyncio +import inspect +import io +import json +import os +import pathlib +import re +import textwrap +import threading +from dataclasses import dataclass +from typing import ClassVar + +import pytest + +import dataretrieval +from dataretrieval import configuration, streamstats, waterdata +from dataretrieval.configuration import Configuration +from dataretrieval.ngwmn import NgwmnConfiguration +from dataretrieval.nwdc import DEFAULT_CONCURRENT_REQUESTS, NwdcConfiguration +from dataretrieval.streamstats import StreamstatsConfiguration +from dataretrieval.utils import _default_headers +from dataretrieval.waterdata import WaterdataConfiguration +from dataretrieval.wqp import WqpConfiguration + +WATERDATA_URL = "https://api.waterdata.usgs.gov/ogcapi/v0/collections/daily/items" + +# Where the base-URL tests redirect to. A host the suite can never reach, so a +# redirect that failed to apply shows up as an unmocked request rather than as +# a real one. +_MIRROR = "https://mirror.example/waterdata" +_MIRROR_RE = re.compile(r"^https://mirror\.example/") +_WATERDATA_RE = re.compile(r"^https://api\.waterdata\.usgs\.gov/") + +# One committed page of the ``daily`` collection, shared with the Water Data +# suite. Real response shape rather than a hand-made stub, so a redirect is +# exercised through the same shaping the getters normally do; it carries no +# ``links``, so nothing paginates. +_DAILY_PAGE = json.loads( + (pathlib.Path(__file__).parent / "data" / "waterdata_ogc_fixtures.json").read_text() +)["daily"] + + +@pytest.fixture +def config_file(tmp_path, monkeypatch): + """Write a config file and point ``DATARETRIEVAL_CONFIG`` at it.""" + + def write(text: str): + path = tmp_path / "config.toml" + path.write_text(text) + path.chmod(0o600) # keep the loose-permission warning out of the way + for env in configuration.ENV_VARS.values(): + monkeypatch.delenv(env, raising=False) + monkeypatch.setenv(configuration.CONFIG_PATH_ENV, str(path)) + configuration._reset_file_cache() + return path + + return write + + +# --- precedence ---------------------------------------------------------- + + +def test_default_when_nothing_is_configured(monkeypatch): + for env in configuration.ENV_VARS.values(): + monkeypatch.delenv(env, raising=False) + assert configuration.api_key() is None + assert configuration.concurrency() == configuration.DEFAULT_CONCURRENCY + assert configuration.retries() == configuration.DEFAULT_RETRIES + assert configuration.parallel_chunks() == configuration.DEFAULT_PARALLEL_CHUNKS + assert configuration.progress() is None + + +def test_env_is_used_when_no_file_or_block(monkeypatch): + monkeypatch.setenv("API_USGS_PAT", "env-key") + monkeypatch.setenv("API_USGS_CONCURRENT", "4") + assert configuration.api_key() == "env-key" + assert configuration.concurrency() == 4 + + +def test_env_outranks_file(config_file, monkeypatch): + config_file('api_key = "file-key"\n') + monkeypatch.setenv("API_USGS_PAT", "env-key") + assert configuration.api_key() == "env-key" + + +def test_block_outranks_file_and_env(config_file, monkeypatch): + config_file('api_key = "file-key"\n') + monkeypatch.setenv("API_USGS_PAT", "env-key") + with dataretrieval.configure(Configuration(api_key="block-key")): + assert configuration.api_key() == "block-key" + assert configuration.api_key() == "env-key" + + +def test_precedence_is_per_setting_not_per_source(config_file, monkeypatch): + """An environment key must not blank out file-provided settings.""" + config_file("concurrency = 16\n") + monkeypatch.setenv("API_USGS_PAT", "env-key") + monkeypatch.setenv("API_USGS_RETRIES", "9") + assert configuration.concurrency() == 16 # from the file + assert configuration.api_key() == "env-key" # still from the env + assert configuration.retries() == 9 # still from the env + + +# --- the configure() block ----------------------------------------------- + + +def test_blocks_nest_and_merge_per_setting(): + with dataretrieval.configure(Configuration(api_key="outer", concurrency=4)): + with dataretrieval.configure(Configuration(concurrency=8)): + assert configuration.concurrency() == 8 + assert configuration.api_key() == "outer" # inherited from the outer block + assert configuration.concurrency() == 4 # inner block restored on exit + + +def test_omitted_setting_inherits_lower_source(monkeypatch): + monkeypatch.setenv("API_USGS_PAT", "env-key") + with dataretrieval.configure(Configuration(concurrency=2)): + assert configuration.api_key() == "env-key" + + +def test_explicit_none_suppresses_lower_sources(monkeypatch): + monkeypatch.setenv("API_USGS_PAT", "env-key") + monkeypatch.setenv("API_USGS_CONCURRENT", "4") + monkeypatch.setenv("API_USGS_PROGRESS", "true") + with dataretrieval.configure( + Configuration(api_key=None, concurrency=None, progress=None) + ): + assert configuration.api_key() is None + assert configuration.concurrency() == configuration.DEFAULT_CONCURRENCY + assert configuration.progress() is None + assert configuration.api_key() == "env-key" + assert configuration.concurrency() == 4 + assert configuration.progress() is True + + +@pytest.mark.parametrize( + "settings", + [ + {"concurrency": 0}, + {"retries": -1}, + {"parallel_chunks": 0}, + {"progress": "flase"}, + ], +) +def test_a_configuration_validates_its_own_settings(settings): + """A bad value raises where it was written, not inside a later request. + + Construction is earlier than the ``with``, which is earlier than the + request the value would otherwise have broken. + """ + with pytest.raises(configuration.ConfigurationError): + Configuration(**settings) + + +@pytest.mark.parametrize( + ("settings", "expected"), + [ + ({"api_key": 123}, "string"), + ({"concurrency": 1.5}, "integer"), + ({"concurrency": "8"}, "integer"), + ({"retries": "2"}, "integer"), + ({"progress": []}, "bool"), + ({"parallel_chunks": True}, "integer"), + ], +) +def test_configuration_rejects_values_outside_annotated_types(settings, expected): + with pytest.raises(configuration.ConfigurationError, match=expected): + Configuration(**settings) + + +def test_block_accepts_ints_and_strings(): + with dataretrieval.configure(Configuration(concurrency="unbounded")): + assert configuration.concurrency() is None + with dataretrieval.configure(Configuration(concurrency=8)): + assert configuration.concurrency() == 8 + with dataretrieval.configure(Configuration(progress=False)): + assert configuration.progress() is False + with dataretrieval.configure(Configuration(progress=True)): + assert configuration.progress() is True + + +def test_configure_takes_configurations_and_nothing_else(): + """The argument is an object, so a stray mapping or keyword cannot pass. + + ``configure(ngwmn={"concurrency": 2})`` was the earlier spelling, and it is + exactly what a reader of an old script will try. Naming the replacement in + the error is the difference between a two-minute fix and a search. + """ + with pytest.raises(configuration.ConfigurationError, match="configuration objects"): + with dataretrieval.configure({"concurrency": 2}): + pass + with pytest.raises(configuration.ConfigurationError, match="configuration objects"): + with dataretrieval.configure("waterdata"): + pass + # Settings are no longer keywords on ``configure`` at all. + with pytest.raises(TypeError): + with dataretrieval.configure(api_key="k"): + pass + + +def test_two_configurations_for_one_adapter_raise(): + """They are the one pairing with no defined order between them. + + Silently letting the last win would make a block's meaning depend on + argument order, which nothing in the surrounding chain does. + """ + with pytest.raises(configuration.ConfigurationError, match="two configurations"): + with dataretrieval.configure( + WaterdataConfiguration(concurrency=2), + WaterdataConfiguration(retries=1), + ): + pass + + # Same rule for the package-wide configuration, which targets no adapter. + with pytest.raises(configuration.ConfigurationError, match="package-wide"): + with dataretrieval.configure( + Configuration(retries=1), Configuration(retries=2) + ): + pass + + # Two *different* adapters in one block is the whole point of the feature. + with dataretrieval.configure( + WaterdataConfiguration(concurrency=2), NgwmnConfiguration(concurrency=8) + ): + assert configuration.concurrency(adapter="waterdata") == 2 + assert configuration.concurrency(adapter="ngwmn") == 8 + + +def test_a_configuration_resolves_end_to_end(config_file, monkeypatch): + """Every tier below a passed configuration still applies, per setting.""" + config_file('api_key = "file-key"\nstall_timeout = 15\n') + monkeypatch.setenv("API_USGS_RETRIES", "9") + + with dataretrieval.configure(Configuration(concurrency=3)): + assert configuration.concurrency() == 3 # from the configuration + assert configuration.retries() == 9 # still from the environment + assert configuration.api_key() == "file-key" # still from the file + assert configuration.stall_timeout() == 15 # still from the file + assert ( + configuration.parallel_chunks() == configuration.DEFAULT_PARALLEL_CHUNKS + ) # still the built-in default + + assert configuration.concurrency() == configuration.DEFAULT_CONCURRENCY + + +def test_an_adapter_configuration_narrows_to_one_adapter(monkeypatch): + """The adapter is a property of the class, so nothing else moves.""" + monkeypatch.delenv("API_USGS_RETRIES") # pinned by the autouse fixture + + with dataretrieval.configure(NgwmnConfiguration(retries=1)): + assert configuration.retries(adapter="ngwmn") == 1 + # Every other adapter, and the package-wide read, are untouched -- + # including waterdata, which shares NGWMN's host and its API key. + for other in ("waterdata", "nwdc", "wqp", "streamstats"): + assert configuration.retries(adapter=other) == configuration.DEFAULT_RETRIES + assert configuration.retries() == configuration.DEFAULT_RETRIES + + +# --- isolation (the point of issue #352) --------------------------------- + + +def test_threads_do_not_leak_credentials_into_each_other(): + """Two threads in different blocks see different keys. + + This is the concurrency complaint in #352: ``os.environ`` is + process-global, so it cannot express this. + """ + seen: dict[str, str | None] = {} + started = threading.Barrier(2) + + def worker(name: str, key: str) -> None: + with dataretrieval.configure(Configuration(api_key=key)): + started.wait(timeout=5) # force the blocks to overlap in time + seen[name] = configuration.api_key() + + threads = [ + threading.Thread(target=worker, args=("a", "key-a")), + threading.Thread(target=worker, args=("b", "key-b")), + ] + for thread in threads: + thread.start() + for thread in threads: + thread.join(timeout=10) + + assert seen == {"a": "key-a", "b": "key-b"} + + +def test_asyncio_tasks_do_not_leak_credentials_into_each_other(): + """Concurrent asyncio tasks each keep their own key.""" + + async def worker(key: str) -> str | None: + with dataretrieval.configure(Configuration(api_key=key)): + await asyncio.sleep(0) # yield, letting the other task interleave + return configuration.api_key() + + async def main() -> list[str | None]: + return list(await asyncio.gather(worker("key-a"), worker("key-b"))) + + assert asyncio.run(main()) == ["key-a", "key-b"] + + +# --- the file ------------------------------------------------------------ + + +def test_named_profile_is_selected_in_code(config_file): + """``[.]`` reaches the chain only when a caller loads it.""" + config_file( + 'api_key = "shared"\nconcurrency = 4\n\n' + "[waterdata]\nretries = 2\n\n" + '[waterdata.bulk]\nconcurrency = "unbounded"\n' + ) + + # Inert until selected: the file alone changes nothing about concurrency. + assert configuration.concurrency(adapter="waterdata") == 4 + + with dataretrieval.configure(WaterdataConfiguration.load("bulk")): + assert configuration.concurrency(adapter="waterdata") is None # the profile + assert configuration.retries(adapter="waterdata") == 2 # default profile + assert configuration.api_key() == "shared" # package-wide, from the file + # It narrows to one adapter, so a sibling on the same host is untouched. + assert configuration.concurrency(adapter="ngwmn") == 4 + + assert configuration.concurrency(adapter="waterdata") == 4 + + +def test_a_code_selected_profile_outranks_the_environment(config_file, monkeypatch): + """ADR 0011 inverts ADR 0009's environment-above-file rule for this case. + + A profile named in code is a more deliberate act than a variable inherited + from a shell, and losing to that variable is what a caller would file a bug + about. + """ + config_file("[waterdata.gentle]\nconcurrency = 2\n") + monkeypatch.setenv("API_USGS_CONCURRENT", "16") + + assert configuration.concurrency(adapter="waterdata") == 16 + with dataretrieval.configure(WaterdataConfiguration.load("gentle")): + assert configuration.concurrency(adapter="waterdata") == 2 + + +def test_several_named_profiles_are_selected_independently(config_file): + """One block, two adapters, a different named profile for each.""" + config_file( + "[waterdata.bulk]\nconcurrency = 32\n\n" + "[waterdata.polite]\nconcurrency = 2\n\n" + "[ngwmn.gentle]\nconcurrency = 4\n" + ) + + with dataretrieval.configure( + WaterdataConfiguration.load("polite"), NgwmnConfiguration.load("gentle") + ): + assert configuration.concurrency(adapter="waterdata") == 2 + assert configuration.concurrency(adapter="ngwmn") == 4 + + +def test_a_named_profile_layers_per_key_over_the_tiers_below(config_file): + """Selecting a profile replaces keys, never whole tiers. + + Every level of the file overrides the one below it *per key* (ADR 0011), so + one adapter-scoped read here draws each of its four settings from a + different table. + """ + config_file( + "concurrency = 16\nretries = 3\nstall_timeout = 30\n\n" + "[waterdata]\nretries = 2\n\n" + '[waterdata.bulk]\nconcurrency = "unbounded"\nparallel_chunks = 8\n' + ) + + with dataretrieval.configure(WaterdataConfiguration.load("bulk")): + # the profile, over a package-wide key it names... + assert configuration.concurrency(adapter="waterdata") is None + # ...the default profile, over a package-wide key the profile is silent + # about... + assert configuration.retries(adapter="waterdata") == 2 + # ...the package-wide key, which neither table touched... + assert configuration.stall_timeout(adapter="waterdata") == 30 + # ...and a setting only the profile names. + assert configuration.parallel_chunks(adapter="waterdata") == 8 + + +def _resolved_settings() -> dict[object, object]: + """Every setting this process can resolve, package-wide and per adapter. + + A snapshot rather than a handful of assertions, because the claim under + test is about what a file does *not* change -- and naming the settings + individually would only prove it for the ones the author thought of. + """ + snapshot: dict[object, object] = { + "api_key": configuration.api_key(), + "progress": configuration.progress(), + } + for adapter in (None, *configuration.ADAPTERS): + snapshot[(adapter, "concurrency")] = configuration.concurrency(adapter=adapter) + snapshot[(adapter, "retries")] = configuration.retries(adapter=adapter) + snapshot[(adapter, "parallel_chunks")] = configuration.parallel_chunks( + adapter=adapter + ) + snapshot[(adapter, "stall_timeout")] = configuration.stall_timeout( + adapter=adapter + ) + snapshot[(adapter, "base_url")] = configuration.base_url(adapter=adapter) + return snapshot + + +def test_adding_a_named_profile_changes_nothing_until_it_is_selected(config_file): + """Inertness is what makes a profile safe to add to a file others share. + + A named profile that could shift a setting on its own would make every + addition to a shared ``config.toml`` a change to every script reading it, + which is the failure the retired global ``[profiles.]`` table had. + """ + shared = 'api_key = "shared"\nconcurrency = 4\n\n[waterdata]\nretries = 2\n' + config_file(shared) + before = _resolved_settings() + + config_file( + shared + '\n[waterdata.bulk]\nconcurrency = "unbounded"\n' + "retries = 9\nparallel_chunks = 8\nstall_timeout = 5\n" + ) + assert _resolved_settings() == before + + # ...and the profile does reach the chain once it is named in code, so the + # comparison above is inertness rather than a profile nothing can select. + with dataretrieval.configure(WaterdataConfiguration.load("bulk")): + assert configuration.parallel_chunks(adapter="waterdata") == 8 + + +def test_a_named_profile_cannot_hold_a_nested_table(config_file): + """``[waterdata.bulk.ngwmn]`` is the retired shape, not a deeper profile. + + A profile carries settings for the one adapter it belongs to, so a table + inside one has no reading. Refused rather than skipped: silently dropping + it would leave the author believing they had tuned NGWMN. + """ + config_file( + "[waterdata.bulk]\nparallel_chunks = 8\n\n" + "[waterdata.bulk.ngwmn]\nconcurrency = 2\n" + ) + + # Still inert, like every other problem inside an unselected profile: an + # unrelated call resolves without ever reading it. + assert configuration.retries(adapter="ngwmn") == configuration.DEFAULT_RETRIES + assert configuration.parallel_chunks(adapter="waterdata") == ( + configuration.DEFAULT_PARALLEL_CHUNKS + ) + + with pytest.raises( + configuration.ConfigurationError, match=r"\[waterdata\.bulk\.ngwmn\]" + ): + WaterdataConfiguration.load("bulk") + + +def test_loading_an_undefined_profile_raises(config_file): + """A name the caller just typed is a typo, not a silent fall-through. + + The message lists what the file *does* define, because a misspelling is + only obvious next to the spelling that was meant -- and only for this + adapter, since selecting a profile is per adapter and another service's + profile names are not candidates for what the caller meant to type. + """ + config_file( + "[waterdata]\nconcurrency = 4\n\n" + "[waterdata.bulk]\nretries = 8\n\n" + "[waterdata.polite]\nretries = 1\n\n" + "[ngwmn.gentle]\nconcurrency = 2\n" + ) + with pytest.raises(configuration.ConfigurationError) as excinfo: + WaterdataConfiguration.load("bluk") + message = str(excinfo.value) + assert "no [waterdata.bluk]" in message + assert "bulk, polite" in message + assert "gentle" not in message + + # An adapter with no profiles at all says so rather than trailing off after + # the colon, which would read as a truncated message. + config_file("[waterdata]\nconcurrency = 4\n") + with pytest.raises(configuration.ConfigurationError, match="waterdata: none"): + WaterdataConfiguration.load("bulk") + + +def test_loading_a_profile_with_no_file_says_so(tmp_path, monkeypatch): + monkeypatch.delenv("API_USGS_CONCURRENT") # pinned by the autouse fixture + monkeypatch.setenv(configuration.CONFIG_PATH_ENV, str(tmp_path / "absent.toml")) + configuration._reset_file_cache() + + with pytest.raises(configuration.ConfigurationError, match="no configuration file"): + WaterdataConfiguration.load("also-gone") + + +def test_the_package_wide_configuration_has_no_profiles(config_file): + """A profile belongs to one adapter, so ``Configuration`` cannot name one.""" + config_file("[waterdata.bulk]\nconcurrency = 8\n") + with pytest.raises(configuration.ConfigurationError, match="package-wide"): + Configuration.load("bulk") + + +def test_missing_file_is_not_an_error(tmp_path, monkeypatch): + monkeypatch.delenv("API_USGS_CONCURRENT") # pinned by the autouse fixture + monkeypatch.setenv(configuration.CONFIG_PATH_ENV, str(tmp_path / "absent.toml")) + configuration._reset_file_cache() + assert configuration.concurrency() == configuration.DEFAULT_CONCURRENCY + + +def test_malformed_file_raises_pointing_at_the_file(config_file): + path = config_file("api_key = \n") + with pytest.raises(configuration.ConfigurationError) as excinfo: + configuration.api_key() + assert "not valid TOML" in str(excinfo.value) + assert str(path) in str(excinfo.value) + + +def test_non_utf8_file_raises_config_error(config_file): + path = config_file("") + path.write_bytes(b'api_key = "\xff"\n') + with pytest.raises(configuration.ConfigurationError, match="not valid UTF-8"): + configuration.api_key() + + +def test_config_path_must_not_be_a_directory(tmp_path, monkeypatch): + monkeypatch.setenv(configuration.CONFIG_PATH_ENV, str(tmp_path)) + monkeypatch.delenv("API_USGS_CONCURRENT", raising=False) + configuration._reset_file_cache() + with pytest.raises(configuration.ConfigurationError, match="directory"): + configuration.concurrency() + + +@pytest.mark.skipif(os.name != "posix", reason="POSIX directory permissions") +def test_inaccessible_config_path_raises(tmp_path, monkeypatch): + parent = tmp_path / "blocked" + parent.mkdir() + path = parent / "config.toml" + path.write_text("concurrency = 4\n") + parent.chmod(0) + monkeypatch.setenv(configuration.CONFIG_PATH_ENV, str(path)) + monkeypatch.delenv("API_USGS_CONCURRENT", raising=False) + configuration._reset_file_cache() + try: + try: + path.stat() + except PermissionError: + pass + else: # pragma: no cover - root or a filesystem that ignores mode bits + pytest.skip("filesystem does not enforce directory mode bits") + with pytest.raises(configuration.ConfigurationError, match="could not access"): + configuration.concurrency() + finally: + parent.chmod(0o700) + + +def test_unknown_setting_warns_but_is_ignored(config_file): + config_file('concurrency = 4\napi_kye = "typo"\n') + with pytest.warns(UserWarning, match="unknown setting"): + assert configuration.concurrency() == 4 + + +def test_unknown_table_raises(config_file): + """A profile written as ``[bulk]`` instead of ``[waterdata.bulk]``.""" + config_file("[bulk]\nconcurrency = 4\n") + with pytest.raises(configuration.ConfigurationError, match="unknown table"): + configuration.concurrency() + + +def test_the_retired_profiles_table_names_its_replacement(config_file): + """Nothing shipped with ``[profiles.]``, but the docs described it. + + The generic "unknown table" message would send its author hunting for a + typo in a table spelled exactly as they had been told to spell it. + """ + config_file("[profiles.bulk]\nconcurrency = 4\n") + with pytest.raises( + configuration.ConfigurationError, match=r"\[\.\]" + ): + configuration.concurrency() + + +def test_the_retired_profile_environment_variable_is_ignored(config_file, monkeypatch): + """``DATARETRIEVAL_PROFILE`` went with the table it selected (ADR 0011). + + A profile is now named in code. A variable exported once in a shell profile + and inherited by every subprocess is the opposite shape: invisible at the + call site, and able to switch every service at once. Honoring it under the + new grammar would restore exactly what the grammar removed. + """ + config_file('concurrency = 4\n\n[waterdata.bulk]\nconcurrency = "unbounded"\n') + monkeypatch.setenv("DATARETRIEVAL_PROFILE", "bulk") + + assert configuration.concurrency(adapter="waterdata") == 4 + assert "DATARETRIEVAL_PROFILE" not in configuration.ENV_VARS.values() + + +def test_typed_toml_values_are_normalized(config_file): + """``tomllib`` returns typed values that normalize into shared parsers.""" + config_file("concurrency = 16\nretries = 0\nprogress = true\n") + assert configuration.concurrency() == 16 + assert configuration.retries() == 0 + assert configuration.progress() is True + + +@pytest.mark.parametrize( + "text", + [ + "api_key = true\n", + 'concurrency = "8"\n', + 'retries = "2"\n', + "progress = 17\n", + "parallel_chunks = true\n", + ], +) +def test_toml_rejects_wrong_scalar_types(config_file, text): + config_file(text) + with pytest.raises(configuration.ConfigurationError): + configuration.parallel_chunks() + + +def test_file_edit_is_picked_up(config_file, monkeypatch): + path = config_file("concurrency = 4\n") + assert configuration.concurrency() == 4 + original = path.stat() + path.write_text("concurrency = 8\n") + os.utime(path, ns=(original.st_atime_ns, original.st_mtime_ns)) + # Windows ctime is creation time, so unchanged metadata must fall back to + # comparing raw content before the parsed cache is reused. + monkeypatch.setattr(configuration.os, "name", "nt") + assert configuration.concurrency() == 8 + + +def test_explicit_config_path_is_expanded(monkeypatch): + monkeypatch.setenv(configuration.CONFIG_PATH_ENV, "~/somewhere/config.toml") + assert str(configuration.config_path()).startswith(os.path.expanduser("~")) + assert "~" not in str(configuration.config_path()) + + +def test_relative_config_path_follows_the_working_directory(tmp_path, monkeypatch): + """A relative ``DATARETRIEVAL_CONFIG`` is resolved against the *current* cwd. + + The path memo keys on the working directory for exactly this reason: a + scheduler or notebook that sets a relative path and chdirs per job would + otherwise keep serving the first job's credentials for the life of the + process, with ``show_configuration()`` reporting the stale path as current. + """ + first = tmp_path / "first" + second = tmp_path / "second" + first.mkdir() + second.mkdir() + (first / "config.toml").write_text("concurrency = 4\n") + (second / "config.toml").write_text("concurrency = 9\n") + monkeypatch.delenv("API_USGS_CONCURRENT", raising=False) + monkeypatch.setenv(configuration.CONFIG_PATH_ENV, "config.toml") + + monkeypatch.chdir(first) + configuration._reset_file_cache() + assert configuration.config_path() == first / "config.toml" + assert configuration.concurrency() == 4 + + monkeypatch.chdir(second) + assert configuration.config_path() == second / "config.toml" + assert configuration.concurrency() == 9 + + +@pytest.mark.skipif(os.name != "posix", reason="POSIX file modes") +def test_world_readable_file_with_a_key_warns(tmp_path, monkeypatch): + path = tmp_path / "config.toml" + path.write_text('api_key = "secret"\n') + path.chmod(0o644) + monkeypatch.setenv(configuration.CONFIG_PATH_ENV, str(path)) + monkeypatch.delenv("API_USGS_PAT", raising=False) + configuration._reset_file_cache() + with pytest.warns(UserWarning, match="readable by other users"): + assert configuration.api_key() == "secret" + + +@pytest.mark.skipif(os.name != "posix", reason="POSIX file modes") +def test_permission_change_is_checked_on_cached_file(config_file): + path = config_file('api_key = "secret"\n') + assert configuration.api_key() == "secret" + path.chmod(0o644) + with pytest.warns(UserWarning, match="readable by other users"): + assert configuration.api_key() == "secret" + + +@pytest.mark.skipif(os.name != "posix", reason="POSIX file modes") +def test_no_permission_warning_without_a_key(tmp_path, monkeypatch, recwarn): + path = tmp_path / "config.toml" + path.write_text("concurrency = 4\n") + path.chmod(0o644) + monkeypatch.setenv(configuration.CONFIG_PATH_ENV, str(path)) + monkeypatch.delenv("API_USGS_CONCURRENT", raising=False) + configuration._reset_file_cache() + assert configuration.concurrency() == 4 + assert not [w for w in recwarn if "readable by other users" in str(w.message)] + + +# --- value grammar ------------------------------------------------------- + + +def test_api_key_is_stripped_and_blank_means_none(monkeypatch): + monkeypatch.setenv("API_USGS_PAT", " key-with-newline\n") + assert configuration.api_key() == "key-with-newline" + monkeypatch.setenv("API_USGS_PAT", " ") + assert configuration.api_key() is None + + +def test_blank_numeric_env_falls_back_to_the_default(monkeypatch): + monkeypatch.setenv("API_USGS_CONCURRENT", "") + monkeypatch.setenv("API_USGS_RETRIES", "") + assert configuration.concurrency() == configuration.DEFAULT_CONCURRENCY + assert configuration.retries() == configuration.DEFAULT_RETRIES + + +def test_blank_progress_env_means_off_not_unset(monkeypatch): + """Preserved from the pre-config behavior: blank disables the line.""" + monkeypatch.setenv("API_USGS_PROGRESS", "") + assert configuration.progress() is False + + +@pytest.mark.parametrize("value", ["0", "false", "no", "off", "FALSE"]) +def test_progress_falsey_values(monkeypatch, value): + monkeypatch.setenv("API_USGS_PROGRESS", value) + assert configuration.progress() is False + + +@pytest.mark.parametrize("value", ["1", "true", "yes", "on"]) +def test_progress_truthy_values(monkeypatch, value): + monkeypatch.setenv("API_USGS_PROGRESS", value) + assert configuration.progress() is True + + +def test_legacy_unknown_progress_env_still_means_on(monkeypatch): + monkeypatch.setenv("API_USGS_PROGRESS", "legacy-nonempty-value") + assert configuration.progress() is True + + +@pytest.mark.parametrize("value", ["nope", "-1", "0"]) +def test_invalid_concurrency_raises(monkeypatch, value): + monkeypatch.setenv("API_USGS_CONCURRENT", value) + with pytest.raises(ValueError): # ConfigurationError is a ValueError + configuration.concurrency() + + +def test_unbounded_concurrency(monkeypatch): + monkeypatch.setenv("API_USGS_CONCURRENT", "unbounded") + assert configuration.concurrency() is None + + +def test_error_message_names_the_source(config_file, monkeypatch): + monkeypatch.setenv("API_USGS_CONCURRENT", "nope") + with pytest.raises( + configuration.ConfigurationError, match=r"\$?API_USGS_CONCURRENT" + ): + configuration.concurrency() + monkeypatch.delenv("API_USGS_CONCURRENT") + path = config_file('concurrency = "nope"\n') + # ``match`` is a regex, and a Windows path is mostly escapes: + # ``C:\\Users\\...`` makes ``\\U`` an invalid escape. + with pytest.raises(configuration.ConfigurationError, match=re.escape(str(path))): + configuration.concurrency() + + +# --- security ------------------------------------------------------------ + + +def test_show_config_never_prints_the_key(monkeypatch): + monkeypatch.setenv("API_USGS_PAT", "super-secret-value") + out = io.StringIO() + dataretrieval.show_configuration(stream=out) + text = out.getvalue() + assert "super-secret-value" not in text + assert "" in text + assert "$API_USGS_PAT" in text # provenance is still reported + + +def test_show_config_reports_absent_key(monkeypatch): + monkeypatch.delenv("API_USGS_PAT", raising=False) + out = io.StringIO() + dataretrieval.show_configuration(stream=out) + assert "" in out.getvalue() + + +def test_file_sourced_key_is_still_host_scoped(config_file): + """A key from a file gets the same host scoping as one from the env.""" + config_file('api_key = "file-key"\n') + assert _default_headers(WATERDATA_URL)["X-Api-Key"] == "file-key" + assert "X-Api-Key" not in _default_headers("https://example.com/data") + assert "X-Api-Key" not in _default_headers( + "https://api.waterdata.usgs.gov.evil.com/x" + ) + + +def test_block_sourced_key_is_still_host_scoped(): + with dataretrieval.configure(Configuration(api_key="block-key")): + assert _default_headers(WATERDATA_URL)["X-Api-Key"] == "block-key" + assert "X-Api-Key" not in _default_headers("https://example.com/data") + + +def test_no_public_getter_accepts_a_credential_parameter(): + """Guards the ``**queryables`` catch-all. + + Every Water Data getter forwards unknown keywords as OGC query + parameters, so a getter that grew an ``api_key`` or ``session`` + parameter could serialize a credential into a URL. Credentials must + arrive through ``dataretrieval.configure`` instead. + """ + import inspect + + from dataretrieval import waterdata + + offenders = [] + for name in waterdata.__all__: + obj = getattr(waterdata, name) + if not callable(obj) or inspect.isclass(obj): + continue + try: + params = inspect.signature(obj).parameters + except (TypeError, ValueError): # pragma: no cover - builtins + continue + for forbidden in ("api_key", "session", "token", "apikey"): + if forbidden in params: + offenders.append(f"{name}({forbidden}=)") + assert not offenders, ( + "public getters must not take credential parameters: " + ", ".join(offenders) + ) + + +@pytest.mark.parametrize("allowed", ["session", "session_id", "sampling_session"]) +def test_session_is_not_treated_as_a_credential(allowed): + """``session`` carries no secret, and the queryable namespace is the + server's — a substring rule would make any future field containing it + unreachable behind a credentials message that misstates the problem.""" + from dataretrieval.waterdata.utils import _flatten_queryables + + assert _flatten_queryables({"queryables": {allowed: 1}}) == {allowed: 1} + + +@pytest.mark.parametrize( + "forbidden", + ["api_key", "apikey", "apiKey", "API_KEY", "api-key", "token"], +) +def test_credential_keyword_cannot_enter_queryables(forbidden): + from dataretrieval import waterdata + + with pytest.raises(TypeError, match=forbidden): + waterdata.get_daily( + monitoring_location_id="USGS-01646500", **{forbidden: "secret"} + ) + + +# --- wiring into the rest of the package --------------------------------- + + +def test_retry_policy_reads_the_block(): + from dataretrieval.transport.retry import RetryPolicy + + with dataretrieval.configure(Configuration(retries=3)): + assert RetryPolicy.from_configuration().max_retries == 3 + + +def test_parallel_chunks_baseline_comes_from_config(config_file): + from dataretrieval.ogc.chunking import parallel_chunks + + assert configuration.parallel_chunks() == 1 + config_file("parallel_chunks = 8\n") + assert configuration.parallel_chunks() == 8 + with parallel_chunks(2): # an explicit block still wins over the file + assert configuration.parallel_chunks() == 2 + assert configuration.parallel_chunks() == 8 + + +def test_parallel_chunks_and_configure_share_one_mechanism(): + """``parallel_chunks(n)`` is sugar for a package-wide ``Configuration``. + + They must not be two competing scopes: whichever block is innermost wins, + so ``show_configuration()`` always reports the value the chunker will use. + """ + from dataretrieval.ogc.chunking import parallel_chunks + + with parallel_chunks(2): + with dataretrieval.configure(Configuration(parallel_chunks=8)): + assert configuration.parallel_chunks() == 8 + assert configuration.parallel_chunks() == 2 + + with dataretrieval.configure(Configuration(parallel_chunks=8)): + with parallel_chunks(2): + assert configuration.parallel_chunks() == 2 + assert configuration.parallel_chunks() == 8 + + +def test_parallel_chunks_has_no_environment_variable(): + """It spends quota, so it is deliberately file/block-only (see ENV_VARS).""" + assert "parallel_chunks" not in configuration.ENV_VARS + assert "parallel_chunks" in configuration.SETTINGS + + +def test_progress_reporter_reads_the_block(): + from dataretrieval.progress import ProgressReporter + + with dataretrieval.configure(Configuration(progress=True)): + assert ProgressReporter(stream=io.StringIO()).enabled + with dataretrieval.configure(Configuration(progress=False)): + assert not ProgressReporter(stream=io.StringIO()).enabled + + +# --- review regressions -------------------------------------------------- + + +def test_blank_env_does_not_mask_the_config_file(config_file, monkeypatch): + """A blank-but-set env var must not shadow a configured file. + + Container and CI tooling routinely materializes one (``docker run -e + API_USGS_PAT`` with nothing to pass, a workflow secret absent on a fork). + Letting that outrank the file silently dropped the API key and sent every + request unauthenticated. + """ + config_file('api_key = "file-key"\nconcurrency = 4\nretries = 7\nprogress = true\n') + for env in configuration.ENV_VARS.values(): + monkeypatch.setenv(env, "") + + assert configuration.api_key() == "file-key" + assert configuration.concurrency() == 4 + assert configuration.retries() == 7 + # ``progress`` is the documented exception: a blank API_USGS_PROGRESS has + # always meant "off", so for that setting blank *is* a value and outranks + # the file. The asymmetry is declared once, in configuration._BLANK_MEANS_SET. + assert configuration.progress() is False + assert set(configuration._BLANK_MEANS_SET) == {"progress"} + + +def test_blank_progress_env_keeps_its_legacy_meaning(monkeypatch): + """With no file, blank keeps the environment-only meaning it always had.""" + monkeypatch.setenv("API_USGS_PROGRESS", "") + monkeypatch.setenv("API_USGS_CONCURRENT", "") + assert configuration.progress() is False # blank has always meant "off" + assert configuration.concurrency() == configuration.DEFAULT_CONCURRENCY + + +def test_config_error_is_in_the_error_taxonomy(): + """A broken config surfaces from inside a getter, so it must be catchable.""" + import dataretrieval.exceptions as exceptions + + assert issubclass(configuration.ConfigurationError, exceptions.DataRetrievalError) + assert issubclass( + configuration.ConfigurationError, ValueError + ) # legacy handlers still work + assert configuration.ConfigurationError is exceptions.ConfigurationError + + +def test_show_config_reports_a_broken_file_instead_of_raising(config_file): + """The tool that explains a configuration must survive a broken one.""" + config_file("this is not = valid toml [[[\n") + out = io.StringIO() + dataretrieval.show_configuration(stream=out) # must not raise + text = out.getvalue() + assert "ERROR:" in text + # Every setting still gets a row rather than the report dying part-way. + for name in configuration.SETTINGS: + assert name in text + + +def test_show_config_reports_a_bad_value_in_its_own_row(monkeypatch): + monkeypatch.setenv("API_USGS_CONCURRENT", "nope") + out = io.StringIO() + dataretrieval.show_configuration(stream=out) + text = out.getvalue() + assert "= {"retries", "stall_timeout", "base_url"} + + +def test_registering_an_adapter_outside_the_roster_raises(): + """The roster is the authority, so a class cannot invent an adapter.""" + + @dataclass(frozen=True) + class BogusConfiguration(configuration.BaseConfiguration): + adapter: ClassVar[str] = "not-an-adapter" + + with pytest.raises(configuration.ConfigurationError, match="not one of"): + configuration._register(BogusConfiguration) + + +def test_settings_for_an_unimported_adapter_is_not_an_error(monkeypatch): + """``None`` means "cannot validate these keys yet", never "invalid". + + NLDI is imported on demand for the geopandas extra, so a roster built from + imports would reject a perfectly good ``[nldi]`` table until something + happened to import that module. + """ + monkeypatch.delitem(configuration._REGISTRY, "nldi", raising=False) + assert configuration.settings_for("nldi") is None + assert "nldi" in configuration.ADAPTERS + + +def test_every_adapter_is_actually_wired_to_a_read_site(): + """A schema nothing passes is worse than no schema. + + ``show_configuration()`` would report a ``[nwis]`` override as live while + every call ignored it -- the report whose whole job is answering "what will + this call use" being confidently wrong. Importability is the weaker half of + the invariant: it passed while ``waterdata.get_cql``, eight of nine WQP + getters, and all of ``nwis`` silently resolved package-wide. + """ + import pathlib + + source = "\n".join( + p.read_text(encoding="utf-8") + for p in pathlib.Path(configuration.__file__).parent.rglob("*.py") + if p.name != "configuration.py" + ) + missing = [a for a in configuration.ADAPTERS if f'adapter="{a}"' not in source] + assert not missing, ( + f"adapters with a schema but no read site: {missing}. Either pass " + 'adapter="" where that adapter builds its policy or fan-out, or ' + "drop it from configuration.ADAPTERS." + ) + + +def test_a_misspelled_adapter_at_a_read_site_raises(): + """The other half of the invariant above, which a grep cannot check. + + ``adapter="waterdatas"`` used to resolve *silently* package-wide: no table + matches the typo, every setting is accepted because nothing knows the + schema, and a ``[waterdata]`` table or a ``WaterdataConfiguration`` is then + ignored with nothing raised anywhere. The grep only sees that the correctly + spelled string occurs somewhere; it cannot see a second, wrong one. + """ + with pytest.raises(configuration.ConfigurationError, match="not a configurable"): + configuration.retries(adapter="waterdatas") + + # Every read site funnels through one resolver, so the check reaches them + # all -- including the accessors that would otherwise return a default. + with pytest.raises(configuration.ConfigurationError, match="not a configurable"): + configuration.base_url(adapter="nwis", default="https://example.invalid") + + +def test_a_non_finite_stall_timeout_is_refused(): + """``inf`` parses as a float and silently disables the bound it sets.""" + for bad in (float("inf"), float("nan")): + with pytest.raises(configuration.ConfigurationError, match="finite"): + Configuration(stall_timeout=bad) + + +def test_stall_timeout_resolves_through_the_chain(config_file, monkeypatch): + """It was read straight from os.environ, so a block and the file were mute.""" + config_file("stall_timeout = 15\n\n[wqp]\nstall_timeout = 300\n") + + assert configuration.stall_timeout() == 15 + assert configuration.stall_timeout(adapter="wqp") == 300 + + monkeypatch.setenv("API_USGS_STALL_TIMEOUT", "42") + assert configuration.stall_timeout() == 42 + + with dataretrieval.configure(Configuration(stall_timeout=2.5)): + assert configuration.stall_timeout() == 2.5 + + +def test_base_url_applies_from_code_and_is_refused_from_the_file(config_file): + """A redirect belongs where a reader of the script sees it (ADR 0011). + + A configuration file that silently sent a data-retrieval library to another + host would be a supply-chain-shaped hazard, so the file refuses the setting + outright rather than accepting it and being trusted. + """ + config_file("") + + with dataretrieval.configure( + WaterdataConfiguration(base_url="https://mirror.example/ogcapi") + ): + assert configuration.base_url(adapter="waterdata") == ( + "https://mirror.example/ogcapi" + ) + # It names one service, so it never reaches another. + assert configuration.base_url(adapter="ngwmn") is None + assert configuration.base_url(adapter="waterdata") is None + + for text in ( + 'base_url = "https://evil.example"\n', + "[waterdata]\nbase_url = 'x'\n", + ): + config_file(text) + with pytest.raises( + configuration.ConfigurationError, match="only be set in code" + ): + configuration.base_url(adapter="waterdata") + + +def test_base_url_must_be_an_absolute_http_url(): + """A bare host would fail far from here, inside the request builder.""" + with pytest.raises(configuration.ConfigurationError, match="absolute"): + WaterdataConfiguration(base_url="mirror.example") + with pytest.raises(configuration.ConfigurationError, match="absolute"): + WaterdataConfiguration(base_url="file:///etc/passwd") + + +def test_base_url_is_refused_from_the_environment(monkeypatch): + """The environment is refused out loud, not merely unread. + + ``API_USGS_BASE_URL`` is the spelling every other setting's variable + predicts, so a caller who exports it believes they have redirected + something. Leaving it out of ``ENV_VARS`` would make that belief wrong and + silent; the error names the block to write instead. + """ + monkeypatch.setenv("API_USGS_BASE_URL", "https://evil.example") + + with pytest.raises(configuration.ConfigurationError, match="only be set in code"): + configuration.base_url(adapter="waterdata") + + # Refused even under a block that sets one, matching the file: the variable + # cannot work, and being quietly outranked is how it survives to a run where + # nothing outranks it. Unsetting it is the only fix. + with dataretrieval.configure(WaterdataConfiguration(base_url=_MIRROR)): + with pytest.raises( + configuration.ConfigurationError, match="only be set in code" + ): + configuration.base_url(adapter="waterdata") + + # A configuration in this state is exactly what show_configuration() exists + # to explain, so it reports the failure rather than raising out of it. + out = io.StringIO() + dataretrieval.show_configuration(stream=out) + assert "only be set in code" in out.getvalue() + + +def test_a_code_base_url_redirects_every_water_data_endpoint_family(httpx_mock): + """One Water Data configuration moves every endpoint family together.""" + httpx_mock.add_response(json=_DAILY_PAGE) + httpx_mock.add_response(json={"data": []}) + httpx_mock.add_response(json={"features": []}) + httpx_mock.add_response(json={"features": []}) + + with dataretrieval.configure(WaterdataConfiguration(base_url=_MIRROR)): + waterdata.get_daily(monitoring_location_id="USGS-05427718") + waterdata.get_codes("states") + waterdata.get_stats_por( + monitoring_location_id="USGS-05427718", + parameter_code="00060", + start_date="01-01", + end_date="01-01", + ) + waterdata.get_ratings( + monitoring_location_id="USGS-05427718", + download_and_parse=False, + ) + + requested = [str(request.url) for request in httpx_mock.get_requests()] + assert requested[0].startswith(f"{_MIRROR}/ogcapi/v0/collections/daily/items") + assert requested[1].startswith(f"{_MIRROR}/samples-data/codeservice/states") + assert requested[2].startswith(f"{_MIRROR}/statistics/v0/observationNormals") + assert requested[3].startswith(f"{_MIRROR}/stac/v0/search") + assert all(_WATERDATA_RE.match(url) is None for url in requested) + + +def test_a_code_base_url_redirects_the_adapters_requests(httpx_mock): + """The setting has to move real traffic, not just resolve to a string. + + Two adapters with unrelated request machinery -- the OGC engine and a plain + one-shot GET -- because "the configuration reaches the request" is a claim + about each adapter's wiring, and one of them passing says nothing about the + other. + """ + httpx_mock.add_response(method=None, url=_MIRROR_RE, json=_DAILY_PAGE) + httpx_mock.add_response(method=None, url=_WATERDATA_RE, json=_DAILY_PAGE) + + with dataretrieval.configure(WaterdataConfiguration(base_url=_MIRROR)): + waterdata.get_daily(monitoring_location_id="USGS-05427718") + redirected_url = str(httpx_mock.get_requests()[-1].url) + + # Nothing configured: back to the service's own base, so the redirect is + # scoped to the block rather than latched somewhere at import. + waterdata.get_daily(monitoring_location_id="USGS-05427718") + direct_url = str(httpx_mock.get_requests()[-1].url) + + assert redirected_url.startswith(f"{_MIRROR}/ogcapi/v0/collections/daily/items") + assert direct_url.startswith(WATERDATA_URL) + + streamstats_mirror = "https://mirror.example/streamstats" + with dataretrieval.configure(StreamstatsConfiguration(base_url=streamstats_mirror)): + streamstats.download_workspace("workspace-id") + assert str(httpx_mock.get_requests()[-1].url).startswith( + f"{streamstats_mirror}/download" + ) + + +def test_a_redirected_adapter_is_not_sent_the_api_key(httpx_mock): + """The key is scoped to the host that honors it, and a mirror is not it. + + ``credentials.accepts_api_key`` is checked where the header is attached, so + a redirect needs no second rule to be safe -- but "needs no rule" is exactly + the kind of claim that stops being true silently, and the cost of it being + wrong is a credential handed to whatever host the block named. + """ + httpx_mock.add_response(method=None, url=_MIRROR_RE, json=_DAILY_PAGE) + httpx_mock.add_response(method=None, url=_WATERDATA_RE, json=_DAILY_PAGE) + + with dataretrieval.configure(Configuration(api_key="secret")): + with dataretrieval.configure(WaterdataConfiguration(base_url=_MIRROR)): + waterdata.get_daily(monitoring_location_id="USGS-05427718") + redirected_request = httpx_mock.get_requests()[-1] + + # The same key, the same call, the service's own host: the control that + # keeps this test from passing because no key was configured at all. + waterdata.get_daily(monitoring_location_id="USGS-05427718") + direct_request = httpx_mock.get_requests()[-1] + + assert "X-Api-Key" not in redirected_request.headers + assert direct_request.headers["X-Api-Key"] == "secret" + + +def test_the_validate_hook_can_refuse_a_combination(): + """Per-setting grammar is shared with the file; this is for the rest.""" + + @dataclass(frozen=True) + class Fussy(configuration.BaseConfiguration): + adapter: ClassVar[str] = "waterdata" + + concurrency: int | str | None = configuration._UNSET + parallel_chunks: int | None = configuration._UNSET + + def validate(self) -> None: + supplied = self.values() + if supplied.get("parallel_chunks", 1) > supplied.get("concurrency", 1): + raise configuration.ConfigurationError( + "parallel_chunks above concurrency only queues sub-requests." + ) + + assert Fussy(concurrency=8, parallel_chunks=4).settings() == { + "concurrency", + "parallel_chunks", + } + with pytest.raises(configuration.ConfigurationError, match="only queues"): + Fussy(concurrency=2, parallel_chunks=8) + + +def test_show_configuration_lists_only_real_overrides(config_file): + """A full adapter-by-setting grid would bury the answer in inherited rows.""" + config_file("concurrency = 16\n\n[ngwmn]\nconcurrency = 4\n") + out = io.StringIO() + + dataretrieval.show_configuration(stream=out) + text = out.getvalue() + + assert "adapter overrides" in text + assert "ngwmn" in text + # waterdata inherits every setting, so it must not appear as an override. + override_section = text.split("adapter overrides", 1)[1] + assert "waterdata" not in override_section + assert "streamstats" not in override_section + + +def test_inner_block_can_lower_a_setting_an_outer_block_scoped(config_file): + """The innermost block wins across *both* scopes, not just within one. + + An adapter-scoped value is the more specific of two written by the same + block. It must not outrank one written by a block nested *inside* it, or + the documented recovery from QuotaExhausted -- wait, then re-issue more + gently -- cannot be expressed once any adapter table is in play. + """ + config_file("") + + with dataretrieval.configure(WaterdataConfiguration(concurrency=32)): + with dataretrieval.configure(Configuration(concurrency=1)): + assert configuration.concurrency(adapter="waterdata") == 1 + assert configuration.concurrency(adapter="waterdata") == 32 + + +def test_adapter_scope_still_wins_within_one_block(config_file): + """Depth breaks ties between blocks, never within one.""" + config_file("") + + with dataretrieval.configure( + Configuration(concurrency=16), WaterdataConfiguration(concurrency=4) + ): + assert configuration.concurrency(adapter="waterdata") == 4 + assert configuration.concurrency(adapter="wqp") == 16 + + +def test_parallel_chunks_block_survives_an_adapter_scoped_outer_block(): + """``parallel_chunks(n)`` is a per-call request and must not be discarded. + + It delegates to a package-wide ``Configuration``, so it writes the + package-wide key -- and before blocks were kept as separate frames, any + enclosing ``WaterdataConfiguration(parallel_chunks=...)`` outranked it. + """ + from dataretrieval.waterdata import parallel_chunks + + with dataretrieval.configure(WaterdataConfiguration(parallel_chunks=2)): + with parallel_chunks(16): + assert configuration.parallel_chunks(adapter="waterdata") == 16 + assert configuration.parallel_chunks(adapter="waterdata") == 2 + + +# --- the precedence ladder (ADR 0011) ------------------------------------ +# +# ADR 0011 states the ladder in seven rungs, highest first: +# +# 1 a configuration instance passed to configure() +# 2 a profile selected in code, Configuration.load("") +# 3 the setting's environment variable +# 4 the adapter's default profile in the file, [] +# 5 the package-wide keys at the top of the file +# 6 the adapter's built-in preference, passed by the adapter's own read site +# 7 the package built-in default +# +# The tests below walk it as a *chain*: each one knocks the rung above out and +# asserts the next takes over. Seven independent single-rung assertions would +# all still pass if two rungs collapsed into one, which is the mistake worth +# catching -- rungs 2 and 3 are the pair a refactor is most likely to fuse, +# since 2 above 3 is the one place ADR 0011 inverts ADR 0009. +# +# ``nwdc`` and ``concurrency`` are the pair that can express all seven. NWDC is +# the adapter that ships a built-in preference of its own -- 4 concurrent +# requests, because the service is only stress-tested that far -- distinct from +# the package default of 32, and that difference is the only way rungs 6 and 7 +# can be told apart at all. + +#: Rungs 5, 4 and 2, with a distinct value per rung so a resolved number +#: identifies the table it came from. Top-level keys are written first because +#: TOML assigns a bare key to whichever table header precedes it: moved below +#: ``[nwdc]``, ``concurrency = 15`` would quietly stop being a rung-5 key and +#: become a second rung-4 one, and the tests would still pass by coincidence. +_LADDER_FILE = ( + "concurrency = 15\n" # rung 5: the package-wide keys + "retries = 7\n" # rung 5 again, for a setting no profile names + "\n[nwdc]\n" + "concurrency = 14\n" # rung 4: the adapter's default profile + "\n[nwdc.tuned]\n" + "concurrency = 12\n" # rung 2: inert until load() selects it +) + +#: Rung 3, which is not in the file. +_LADDER_ENV = 13 + +#: Rung 1, which is not in the file either. +_LADDER_INSTANCE = 11 + + +def _nwdc_concurrency() -> int | None: + """Resolve ``concurrency`` the way NWDC's own fan-out does. + + Through the adapter's read site rather than a bare ``concurrency()``, so + the built-in preference at rung 6 is really in the chain and the ladder is + exercised as the adapter experiences it. + """ + return configuration.concurrency(DEFAULT_CONCURRENT_REQUESTS, adapter="nwdc") + + +def test_a_configuration_instance_tops_the_ladder(config_file, monkeypatch): + """Rung 1 over every other rung, all six of them present at once.""" + config_file(_LADDER_FILE) + monkeypatch.setenv("API_USGS_CONCURRENT", str(_LADDER_ENV)) + + with dataretrieval.configure(NwdcConfiguration(concurrency=_LADDER_INSTANCE)): + assert _nwdc_concurrency() == _LADDER_INSTANCE + + +def test_a_loaded_profile_beats_the_environment(config_file, monkeypatch): + """Rung 2 over rung 3 -- the one inversion ADR 0011 exists to make. + + ADR 0009 put the environment above the file, and a named profile lives in + the file, so the naive reading is that ``API_USGS_CONCURRENT`` in the shell + wins. It does not: what reaches the chain is the caller *naming* the + profile in code, which is a more deliberate act than a variable inherited + from whatever started the process, and losing to that variable is the + behaviour a caller would file a bug about. + + The inversion is also bounded, which the second half asserts: it covers + what the profile names and nothing else, so ``retries`` -- which the file + sets at the top level and no selected profile mentions -- still follows the + original environment-above-file rule inside the very same block. + """ + config_file(_LADDER_FILE) + monkeypatch.setenv("API_USGS_CONCURRENT", str(_LADDER_ENV)) + monkeypatch.setenv("API_USGS_RETRIES", "9") + + assert _nwdc_concurrency() == _LADDER_ENV # rung 3, until a profile is selected + with dataretrieval.configure(NwdcConfiguration.load("tuned")): + assert _nwdc_concurrency() == 12 # rung 2 wins for the key it names... + assert configuration.retries(adapter="nwdc") == 9 # ...and only that key + assert _nwdc_concurrency() == _LADDER_ENV # and the shell has it back on exit + + +def test_the_environment_beats_the_adapters_default_profile(config_file, monkeypatch): + """Rung 3 over rung 4: the file's always-on table is still just the file.""" + config_file(_LADDER_FILE) + monkeypatch.setenv("API_USGS_CONCURRENT", str(_LADDER_ENV)) + + assert _nwdc_concurrency() == _LADDER_ENV + monkeypatch.delenv("API_USGS_CONCURRENT") + assert _nwdc_concurrency() == 14 + + +def test_the_adapters_default_profile_beats_the_package_wide_keys(config_file): + """Rung 4 over rung 5: within the file, the narrower table decides.""" + config_file(_LADDER_FILE) + + assert _nwdc_concurrency() == 14 + config_file("concurrency = 15\n") # the [nwdc] table gone + assert _nwdc_concurrency() == 15 + + +def test_the_package_wide_keys_beat_the_adapters_built_in_preference(config_file): + """Rung 5 over rung 6: a user-written value outranks an adapter's taste. + + The adapter's preference is a default, not a cap. One able to override a + setting the user actually wrote would make that setting a lie -- so a + top-level key the user never scoped to NWDC still reaches NWDC's calls. + """ + config_file("concurrency = 15\n") + + assert _nwdc_concurrency() == 15 + config_file("") + assert _nwdc_concurrency() == DEFAULT_CONCURRENT_REQUESTS + + +def test_the_adapters_built_in_preference_beats_the_package_built_in_default( + config_file, +): + """Rung 6 over rung 7, and only for the adapter that stated a preference.""" + config_file("") + + assert _nwdc_concurrency() == DEFAULT_CONCURRENT_REQUESTS + assert DEFAULT_CONCURRENT_REQUESTS != configuration.DEFAULT_CONCURRENCY + # It is the read site's own figure, not a property of the adapter, so a + # caller that states no preference lands on the package default instead -- + # which is what makes rungs 6 and 7 two rungs rather than one. + assert ( + configuration.concurrency(adapter="nwdc") == configuration.DEFAULT_CONCURRENCY + ) + + +def test_the_package_built_in_default_is_the_floor(config_file): + """Rung 7: with the six rungs above it empty, every setting still resolves. + + The floor is what makes the whole chain optional -- a caller who has + configured nothing at all gets working values rather than an error. + """ + config_file("") + + for adapter in (None, *configuration.ADAPTERS): + assert configuration.concurrency(adapter=adapter) == ( + configuration.DEFAULT_CONCURRENCY + ) + assert configuration.retries(adapter=adapter) == configuration.DEFAULT_RETRIES + assert configuration.parallel_chunks(adapter=adapter) == ( + configuration.DEFAULT_PARALLEL_CHUNKS + ) + assert configuration.stall_timeout(adapter=adapter) == ( + configuration.DEFAULT_STALL_TIMEOUT + ) + + +def test_the_top_two_rungs_cannot_tie(config_file): + """Rungs 1 and 2 both target one adapter, so no block can hold both. + + That is what stops the ladder needing a tie-break nobody could remember: + the same-adapter rule refuses the pairing where the order would matter, + and between *nested* blocks the ordinary rule applies -- the innermost + decides, whichever kind of configuration it holds. + """ + config_file(_LADDER_FILE) + + with pytest.raises(configuration.ConfigurationError, match="two configurations"): + with dataretrieval.configure( + NwdcConfiguration(concurrency=_LADDER_INSTANCE), + NwdcConfiguration.load("tuned"), + ): + pass + + with dataretrieval.configure(NwdcConfiguration(concurrency=_LADDER_INSTANCE)): + with dataretrieval.configure(NwdcConfiguration.load("tuned")): + assert _nwdc_concurrency() == 12 + with dataretrieval.configure(NwdcConfiguration.load("tuned")): + with dataretrieval.configure(NwdcConfiguration(concurrency=_LADDER_INSTANCE)): + assert _nwdc_concurrency() == _LADDER_INSTANCE + + # "Rung 1 above rung 2" is a claim about one adapter, so a *package-wide* + # instance is not the thing it is talking about: it targets no adapter at + # all. Alongside a loaded profile in one block the adapter-scoped value is + # the more specific of the two and wins for that adapter (ADR 0010), while + # the package-wide value still governs every other adapter. + with dataretrieval.configure( + Configuration(concurrency=_LADDER_INSTANCE), NwdcConfiguration.load("tuned") + ): + assert _nwdc_concurrency() == 12 + assert configuration.concurrency(adapter="wqp") == _LADDER_INSTANCE + + +def test_load_returns_an_instance_carrying_only_the_profiles_keys(config_file): + """``load`` is a constructor: it reads one table and returns the class. + + Only what the table names is carried, so every other setting stays unset + and keeps inheriting from the rungs below rather than being pinned to a + default the profile never asked for. That is what makes a profile a + *contribution* to the chain rather than a replacement for it. + """ + config_file( + "concurrency = 16\n\n" + "[waterdata]\nretries = 2\n\n" + '[waterdata.bulk]\nconcurrency = "unbounded"\nparallel_chunks = 8\n' + ) + + loaded = WaterdataConfiguration.load("bulk") + + assert isinstance(loaded, WaterdataConfiguration) + assert loaded.values() == {"concurrency": "unbounded", "parallel_chunks": 8} + assert loaded.retries is configuration._UNSET + + +# --- show_configuration() reports profiles -------------------------------- +# +# The report exists to answer "why is this call using that value?", so every +# row names the source that supplied it. A value from a profile is the case a +# bare "configure() block" answers badly: a configuration written in code and +# one loaded from a table reach the chain by the same route, and only the +# latter has a name in a file the caller can go and read. + +#: The file the documented sample is generated from. Exercises every section: +#: package-wide keys, an adapter's default profile, and a named profile. +_SAMPLE_FILE = ( + 'api_key = "0123456789abcdef"\n' + "concurrency = 16\n" + "\n[ngwmn]\n" + "concurrency = 4\n" + "\n[waterdata.bulk]\n" + "parallel_chunks = 8\n" +) + +#: The illustrative path the samples print, standing in for the temporary file +#: the test actually writes. Substituting it is the *only* edit made to the +#: captured output -- everything else has to match what the function printed. +_SAMPLE_PATH = "/home/u/.dataretrieval/config.toml" + +#: The two lines above the captured output in both samples. +_SAMPLE_PROMPT = ( + '>>> with dataretrieval.configure(WaterdataConfiguration.load("bulk")):\n' + "... dataretrieval.show_configuration()" +) + + +def _documented_sample() -> str: + """The sample output embedded in ``show_configuration``'s docstring.""" + doc = inspect.getdoc(dataretrieval.show_configuration) or "" + _, _, block = doc.partition(".. code-block:: text\n\n") + return textwrap.dedent(block).strip("\n") + + +def test_show_configuration_names_the_profile_a_value_came_from(config_file): + """A value from a profile is reported with that profile, not with "a block". + + ``WaterdataConfiguration.load("bulk")`` and ``WaterdataConfiguration(...)`` + enter the chain by the same route and are indistinguishable once their + values are in the block, so a report that said only ``configure() block`` + left a caller who selected the wrong profile -- or who had forgotten a + profile was selected at all -- with nothing to look at. The label is the + table's own spelling, so it is greppable in the file that defines it. + """ + config_file("[waterdata.bulk]\nconcurrency = 6\n") + out = io.StringIO() + + with dataretrieval.configure(WaterdataConfiguration.load("bulk")): + dataretrieval.show_configuration(stream=out) + + assert "configure() block [waterdata.bulk]" in out.getvalue() + + # A configuration written in code has no profile to name, so it names its + # adapter alone rather than inventing one -- and the package-wide one + # narrows to nothing, so it names neither. + out = io.StringIO() + with dataretrieval.configure( + WaterdataConfiguration(concurrency=6), Configuration(retries=3) + ): + dataretrieval.show_configuration(stream=out) + text = out.getvalue() + + assert "configure() block [waterdata]" in text + # The file still *defines* the profile, so it is still listed as available; + # what must not happen is a value being attributed to it. + assert "configure() block [waterdata.bulk]" not in text + retries_row = next(line for line in text.splitlines() if line.startswith("retries")) + assert retries_row.endswith("configure() block") + + +def test_a_loaded_profile_remembers_its_name_without_becoming_a_setting(config_file): + """The profile name is provenance, so it is not a field and not a value. + + Keeping it off the fields is what stops it reaching :meth:`settings`, the + ``configure()`` frame, and equality: two configurations carrying the same + settings stay interchangeable however each was spelled, which is what + makes a configuration a value rather than a record of how it was built. + """ + config_file("[waterdata.bulk]\nconcurrency = 6\n") + + loaded = WaterdataConfiguration.load("bulk") + written = WaterdataConfiguration(concurrency=6) + + assert loaded.profile == "bulk" + assert written.profile is None + assert "profile" not in loaded.settings() + assert loaded == written + + +def test_show_configuration_lists_the_profiles_the_file_defines( + config_file, monkeypatch +): + """A named profile is inert until selected, so the file's are listed too. + + "I added ``[waterdata.bulk]`` and nothing changed" is the confusion this + section exists for: the profiles are there, and no row above names one + because no caller selected one. A report that mentioned a profile only + once it had been selected would leave that silence unexplained. + + Names are read from the parsed file, so an adapter this process never + imported still has its profiles listed: what a table *means* needs the + import, what it is called does not, and hiding it would make the section + depend on which optional extras happened to be installed. + """ + monkeypatch.delitem(configuration._REGISTRY, "nldi", raising=False) + config_file( + "[ngwmn]\nconcurrency = 4\n\n" + "[ngwmn.gentle]\nconcurrency = 2\n\n" + "[waterdata.bulk]\nparallel_chunks = 8\n\n" + "[nldi.gentle]\nretries = 1\n" + ) + out = io.StringIO() + + dataretrieval.show_configuration(stream=out) + text = out.getvalue() + listed = text.split("profiles in the file: ", 1)[1].splitlines()[0] + + assert listed == "[waterdata.bulk], [ngwmn.gentle], [nldi.gentle]" + # The adapter's *default* profile is not a named one: it is always in + # effect and already shows up as a source, so listing it here is noise. + assert "[ngwmn]" not in listed + # Inert, and the report says so by never naming one as a source. + assert "configure() block" not in text + + +def test_show_configuration_reports_an_unimported_adapter(config_file, monkeypatch): + """An adapter this process cannot report on is named, never omitted. + + NLDI is imported on demand for the geopandas extra, so a process that has + not touched it cannot say which settings it accepts -- the honest cost of + validating an adapter's keys lazily (ADR 0011). Leaving it out of the + report would read as "nothing is configured for nldi", which is a + different claim from "this report could not check", and the caller cannot + tell which one they are looking at. + """ + config_file("") + monkeypatch.delitem(configuration._REGISTRY, "nldi", raising=False) + out = io.StringIO() + + dataretrieval.show_configuration(stream=out) + text = out.getvalue() + + assert "not reported: nldi" in text + assert "not imported" in text + # An adapter that *was* imported is covered by the rows above, so it must + # not be named as uncoverable. + assert "waterdata" not in text.split("not reported:", 1)[1] + + # The line is a statement about this process, not about nldi: once the + # module is imported its configuration registers and the caveat goes away. + @dataclass(frozen=True) + class _AsImported(configuration.BaseConfiguration): + adapter: ClassVar[str] = "nldi" + + retries: int | None = configuration._UNSET + + monkeypatch.setitem(configuration._REGISTRY, "nldi", _AsImported) + out = io.StringIO() + dataretrieval.show_configuration(stream=out) + assert "not reported" not in out.getvalue() + + +def test_show_configuration_sample_output_is_current(config_file, monkeypatch): + """The documented samples are this function's real output, not a drawing. + + Both had drifted from it -- the docstring wrapped a line the function + prints whole, the user guide had lost a paragraph -- because a sample kept + by hand is only ever as fresh as the last person who remembered it. So + the scenario is rebuilt here and the output compared verbatim; the only + edit is swapping the temporary path for the illustrative one. + + Regenerate by running this test and copying the reported ``actual`` into + both places, never by editing them to taste. + """ + path = config_file(_SAMPLE_FILE) + monkeypatch.setenv("API_USGS_RETRIES", "8") + # The sample shows the report a caller with the geopandas extra uninstalled + # sees; in this suite something has usually imported nldi already. + monkeypatch.delitem(configuration._REGISTRY, "nldi", raising=False) + + out = io.StringIO() + with dataretrieval.configure(WaterdataConfiguration.load("bulk")): + dataretrieval.show_configuration(stream=out) + actual = out.getvalue().replace(str(path), _SAMPLE_PATH).strip("\n") + + assert _documented_sample() == f"{_SAMPLE_PROMPT}\n{actual}" + + # The user guide shows the same sample, indented into its code block, and + # goes stale the same way. Checked here rather than in a docs test because + # the thing that makes it stale is a change to this function's output. + guide = ( + pathlib.Path(__file__).resolve().parents[1] + / "docs" + / "source" + / "userguide" + / "configuration.rst" + ) + if not guide.exists(): # pragma: no cover - docs are absent from an sdist + pytest.skip("docs tree not present") + block = textwrap.indent(f"{_SAMPLE_PROMPT}\n{actual}", " ") + assert block in guide.read_text(encoding="utf-8") + + +def test_show_configuration_survives_a_malformed_profile(config_file): + """Explaining a broken configuration is the job, so nothing here validates. + + The section lists what the file *defines*; a profile's keys are checked when a + caller selects it. So a profile holding a value that fails its grammar -- + or the nested table a file migrated from the retired ``[profiles.]`` + layout still carries -- is reported rather than taking the report down + with it, which is the one moment a caller most needs it. + """ + config_file( + '[waterdata.bulk]\nconcurrency = "nope"\n\n' + "[ngwmn.gentle]\n\n[ngwmn.gentle.nested]\nconcurrency = 2\n" + ) + out = io.StringIO() + + dataretrieval.show_configuration(stream=out) # must not raise + + listed = out.getvalue().split("profiles in the file: ", 1)[1].splitlines()[0] + assert listed == "[waterdata.bulk], [ngwmn.gentle]" + # Selecting one is where the grammar is checked, and it still is. + with pytest.raises(configuration.ConfigurationError, match="integer"): + WaterdataConfiguration.load("bulk") + with pytest.raises(configuration.ConfigurationError, match="contains a table"): + NgwmnConfiguration.load("gentle") diff --git a/tests/conftest.py b/tests/conftest.py index de5cfd360..e7a30288d 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -13,6 +13,8 @@ import pytest +from dataretrieval import configuration + def pytest_collection_modifyitems(config, items): """Apply relaxed ``pytest-httpx`` strict-mode settings to every test @@ -34,7 +36,7 @@ def non_mocked_hosts() -> list[str]: @pytest.fixture(autouse=True) -def _pin_chunker_env(monkeypatch): +def _pin_chunker_env(monkeypatch, tmp_path): """Pin every test to one connection, no retries, and no stall budget. Production defaults ``API_USGS_CONCURRENT`` to 32, @@ -56,3 +58,9 @@ def _pin_chunker_env(monkeypatch): monkeypatch.setenv("API_USGS_CONCURRENT", "1") monkeypatch.setenv("API_USGS_RETRIES", "0") monkeypatch.setenv("API_USGS_STALL_TIMEOUT", "0") + # Point DATARETRIEVAL_CONFIG at a path that does not exist, so a developer's + # real ~/.dataretrieval/config.toml -- which may hold an API key or a raised + # concurrency -- can never influence a test run. Config tests opt in by + # pointing the variable at a file they wrote. + monkeypatch.setenv("DATARETRIEVAL_CONFIG", str(tmp_path / "no-such-config.toml")) + configuration._reset_file_cache() diff --git a/tests/contracts/README.md b/tests/contracts/README.md index d05bed14d..72886e9f0 100644 --- a/tests/contracts/README.md +++ b/tests/contracts/README.md @@ -5,7 +5,7 @@ The suite uses four dependency-oriented layers without moving established tests: - **Public contract** (`tests/contracts/`): imports, exports, signatures, return annotations, metadata/error promises, and compatibility paths. These tests use public modules and no live services. -- **Adapter contract** (`waterdata_test.py`, `ngwmn_test.py`, `wateruse_test.py`, +- **Adapter contract** (`waterdata_test.py`, `ngwmn_test.py`, `nwdc_test.py`, `wqp_test.py`, `nldi_test.py`, `streamstats_test.py`): service request wiring, response parsing, and documented protocol behavior. - **Component** (`transport_test.py`, `waterdata_chunking_test.py`, diff --git a/tests/contracts/public_api_test.py b/tests/contracts/public_api_test.py index 53e28aebf..b0ed4be4a 100644 --- a/tests/contracts/public_api_test.py +++ b/tests/contracts/public_api_test.py @@ -20,6 +20,7 @@ _EXPECTED_WATERDATA_ALL = [ "CODE_SERVICES", "FILTER_LANG", + "WaterdataConfiguration", "PROFILES", "PROFILE_LOOKUP", "SERVICES", diff --git a/tests/ngwmn_test.py b/tests/ngwmn_test.py index 78dd0fac1..38e24cc32 100644 --- a/tests/ngwmn_test.py +++ b/tests/ngwmn_test.py @@ -18,7 +18,8 @@ import pytest from pandas import DataFrame -from dataretrieval import ngwmn +import dataretrieval +from dataretrieval import configuration, ngwmn from dataretrieval.utils import BaseMetadata # Agency-qualified ids in the multi-agency form NGWMN uses (not all ``USGS-``). @@ -565,6 +566,37 @@ def test_empty_result_returns_typed_empty_frame(httpx_mock): assert "geometry" not in df.columns +def test_a_configured_base_url_redirects_ngwmn_alone(httpx_mock): + """Two adapters share this host, and a redirect must still name only one. + + NGWMN and Water Data are served from ``api.waterdata.usgs.gov``, so a URL + cannot tell them apart -- which is why the settings table an OGC call reads + is declared by the adapter rather than derived from its base. Redirecting + NGWMN therefore has to leave Water Data where it was, and the Water Data + mock here is never requested: the assertion is on the whole request list. + """ + mirror = "https://mirror.example/ngwmn" + httpx_mock.add_response( + method="GET", + url=re.compile(rf"^{re.escape(mirror)}/collections/sites/items"), + json=_SITES, + ) + _mock(httpx_mock, "sites", _SITES) + + with dataretrieval.configure(ngwmn.NgwmnConfiguration(base_url=mirror)): + df, md = ngwmn.get_sites(state="Wisconsin", limit=10) + + assert len(df) == 2 + assert str(md.url).startswith(f"{mirror}/collections/sites/items") + assert [urlsplit(str(r.url)).netloc for r in httpx_mock.get_requests()] == [ + "mirror.example" + ] + + # And Water Data, the other adapter on the real host, was never named by it. + with dataretrieval.configure(ngwmn.NgwmnConfiguration(base_url=mirror)): + assert configuration.base_url(adapter="waterdata") is None + + # --- live upstream monitor --------------------------------------------------- diff --git a/tests/nldi_test.py b/tests/nldi_test.py index 6356f8ded..8b8f4af04 100644 --- a/tests/nldi_test.py +++ b/tests/nldi_test.py @@ -3,6 +3,7 @@ import pytest from geopandas import GeoDataFrame +import dataretrieval import dataretrieval.nldi as nldi from dataretrieval.nldi import ( NLDI_API_BASE_URL, @@ -56,7 +57,9 @@ def test_query_nldi_opts_into_retry(monkeypatch): monkeypatch.setattr(nldi, "_query_with_retry", query) assert nldi._query_nldi("https://example.test", {}) == {} - query.assert_called_once_with("https://example.test", payload={}) + # ``adapter`` names whose settings the retry resolves, so a ``[nldi]`` + # table reaches these calls and no others. + query.assert_called_once_with("https://example.test", payload={}, adapter="nldi") def mock_request(httpx_mock, request_url, file_path): @@ -440,3 +443,33 @@ def test_query_504_raises_service_unavailable(httpx_mock): # legacy query path renders verbatim as "HTTP 504 (URL: ...)". with pytest.raises(ServiceUnavailable, match="504"): query(url, {"a": "1"}) + + +def test_a_configured_base_url_redirects_every_nldi_request(httpx_mock): + """The block moves the catalog probe and the query alike. + + NLDI validates a feature source against a catalog it fetches itself, so a + redirect that reached only the getter's own URL would leave the library + asking the real service whether the mirror's sources exist -- and the mirror + exists precisely because the caller cannot or should not reach the service. + Both mocks are on the mirror, so either one straying fails this. + """ + mirror = "https://mirror.example/nldi" + httpx_mock.add_response( + method="GET", url=f"{mirror}/", json=[{"source": "WQP"}, {"source": "comid"}] + ) + with open("tests/data/nldi_get_basin.json") as body: + httpx_mock.add_response( + method="GET", + url=( + f"{mirror}/WQP/USGS-054279485/basin" + "?simplified=true&splitCatchment=false" + ), + text=body.read(), + ) + + with dataretrieval.configure(nldi.NldiConfiguration(base_url=mirror)): + gdf = get_basin(feature_source="WQP", feature_id="USGS-054279485") + + assert isinstance(gdf, GeoDataFrame) + assert {str(r.url).startswith(mirror) for r in httpx_mock.get_requests()} == {True} diff --git a/tests/wateruse_test.py b/tests/nwdc_test.py similarity index 83% rename from tests/wateruse_test.py rename to tests/nwdc_test.py index 9e003e65b..c85544c47 100644 --- a/tests/wateruse_test.py +++ b/tests/nwdc_test.py @@ -1,10 +1,11 @@ -"""Offline tests for :mod:`dataretrieval.wateruse`. +"""Offline tests for :mod:`dataretrieval.nwdc`. All HTTP is mocked with ``pytest-httpx``; no live calls (per AGENTS.md). """ import re import socket +import warnings from urllib.parse import parse_qs, urlsplit import httpx @@ -12,11 +13,12 @@ import pytest import dataretrieval +from dataretrieval import configuration, nwdc from dataretrieval import progress as _progress -from dataretrieval import wateruse +from dataretrieval.exceptions import DataRetrievalError +from dataretrieval.nwdc import _next_page_url, _resolve_locations, get_wateruse from dataretrieval.transport import fanout as _fanout from dataretrieval.utils import BaseMetadata -from dataretrieval.wateruse import _next_page_url, _resolve_locations, get_wateruse # Match the NWDC endpoint regardless of query string, so assertions can drill # into the captured params without coupling registration to param order. @@ -514,9 +516,70 @@ def capture(request: httpx.Request) -> httpx.Response: assert sent["auth"] is None +def test_a_configured_base_url_redirects_the_request(httpx_mock): + """The whole call moves, page walk included, or the redirect is a half-truth. + + The page-two mock is served from the mirror and its cursor names the mirror: + if either the request or the ``rel="next"`` walk had stayed on the NWDC's + host, one of them would go unmocked and this would fail rather than quietly + talk to the service the block redirected away from. + """ + mirror = re.compile(r"^https://mirror\.example/data") + httpx_mock.add_response( + method="GET", + url=mirror, + text=_CSV_P1, + headers={"link": '; rel="next"'}, + ) + httpx_mock.add_response(method="GET", url=mirror, text=_CSV_P2) + + with dataretrieval.configure( + nwdc.NwdcConfiguration(base_url="https://mirror.example/data") + ): + df, _ = get_wateruse(model="wu-public-supply-wd", state="RI") + + assert len(df) == 3 + assert [urlsplit(str(r.url)).netloc for r in httpx_mock.get_requests()] == [ + "mirror.example", + "mirror.example", + ] + + +def test_next_page_url_drops_the_service_rewrite_when_redirected(): + """The alias list and the rewrite are facts about the NWDC, not about URLs. + + Nothing but the NWDC answers for ``water.usgs.gov``, so a call an + ``NwdcConfiguration(base_url=...)`` pointed elsewhere gets the general rule + instead: follow a link only back to the host that served the page. Keeping + the rewrite would send page two of a mirrored query to the USGS -- and + refusing the mirror's own cursor would throw away page one. + """ + mirrored = httpx.Response( + 200, + text="", + headers={"link": '; rel="next"'}, + request=httpx.Request("GET", "https://mirror.example/data"), + ) + + assert _next_page_url(mirrored, host="mirror.example") == ( + "https://mirror.example/data?skip=600" + ) + + # A cursor back to the real service is now the cross-host case, refused for + # the same reason a foreign link is refused on an ordinary call. + strayed = httpx.Response( + 200, + text="", + headers={"link": '; rel="next"'}, + request=httpx.Request("GET", "https://mirror.example/data"), + ) + with pytest.raises(DataRetrievalError, match="cross-host"): + _next_page_url(strayed, host="mirror.example") + + def test_module_exposes_catalog_constants(): - assert "wu-public-supply-wd" in wateruse.MODELS - assert set(wateruse.TIME_RESOLUTIONS) == {"monthly", "annualcy", "annualwy"} + assert "wu-public-supply-wd" in nwdc.MODELS + assert set(nwdc.TIME_RESOLUTIONS) == {"monthly", "annualcy", "annualwy"} def test_initial_transient_is_retried(httpx_mock, monkeypatch): @@ -584,12 +647,12 @@ async def open_mock_client(**overrides): monkeypatch.setattr(_fanout, "open_async_client", open_mock_client) requests = [ - httpx.Request("GET", wateruse.WATERUSE_URL, params={"location": location}) + httpx.Request("GET", nwdc.WATERUSE_URL, params={"location": location}) for location in ("stateCd:AA", "stateCd:BB") ] with pytest.raises(dataretrieval.DataRetrievalError, match="Invalid model"): - wateruse._fan_out(requests, {}, True) + nwdc._fan_out(requests, {}, True) assert pages["n"] == 2, "the sibling finished its walk rather than being abandoned" @@ -655,15 +718,15 @@ def test_fan_out_honors_the_general_concurrency_setting(monkeypatch): quietly ignoring them -- the defect that motivated consolidating the knob. """ monkeypatch.setenv("API_USGS_CONCURRENT", "7") - assert _fanout._resolve_concurrency(wateruse.DEFAULT_CONCURRENT_REQUESTS) == 7 + assert configuration.concurrency(nwdc.DEFAULT_CONCURRENT_REQUESTS) == 7 monkeypatch.delenv("API_USGS_CONCURRENT", raising=False) assert ( - _fanout._resolve_concurrency(wateruse.DEFAULT_CONCURRENT_REQUESTS) - == wateruse.DEFAULT_CONCURRENT_REQUESTS + configuration.concurrency(nwdc.DEFAULT_CONCURRENT_REQUESTS) + == nwdc.DEFAULT_CONCURRENT_REQUESTS ) # The service default is deliberately below the package-wide 32. - assert wateruse.DEFAULT_CONCURRENT_REQUESTS < _fanout._CONCURRENCY_DEFAULT + assert nwdc.DEFAULT_CONCURRENT_REQUESTS < configuration.DEFAULT_CONCURRENCY def test_fan_out_reports_progress(httpx_mock, monkeypatch): @@ -831,3 +894,78 @@ def test_mid_page_walk_transient_is_still_resumable(httpx_mock): assert excinfo.value.call is not None assert excinfo.value.completed_chunks == 1 assert excinfo.value.total_chunks == 2 + + +# --------------------------------------------------------------------------- +# Deprecated ``wateruse`` alias +# --------------------------------------------------------------------------- + + +def _reimport_wateruse(): + """Import the alias fresh, so its module-level warning fires again.""" + import importlib + import sys + + sys.modules.pop("dataretrieval.wateruse", None) + return importlib.import_module("dataretrieval.wateruse") + + +def test_wateruse_alias_warns_and_names_the_replacement(): + """Importing the old name is deprecated, dated, and points at ``nwdc``.""" + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + _reimport_wateruse() + + deprecations = [w for w in caught if issubclass(w.category, DeprecationWarning)] + assert len(deprecations) == 1, [str(w.message) for w in caught] + message = str(deprecations[0].message) + assert "`dataretrieval.wateruse` is deprecated" in message + assert "`dataretrieval.nwdc`" in message + # Dated removal, per the convention nwis follows. + from dataretrieval.wateruse import NWDC_RENAME_REMOVAL_DATE + + assert NWDC_RENAME_REMOVAL_DATE in message + + +def test_wateruse_alias_re_exports_the_same_objects(): + """The alias forwards, it does not copy: identity must survive it. + + A caller monkeypatching through one spelling and asserting through the + other would otherwise see two different objects. + """ + alias = _reimport_wateruse() + + assert alias.get_wateruse is nwdc.get_wateruse + assert alias.MODELS is nwdc.MODELS + assert alias.TIME_RESOLUTIONS is nwdc.TIME_RESOLUTIONS + assert alias.DEFAULT_CONCURRENT_REQUESTS == nwdc.DEFAULT_CONCURRENT_REQUESTS + assert alias.__all__ == nwdc.__all__ + + +def test_importing_dataretrieval_does_not_warn(): + """``import dataretrieval`` must stay silent. + + The package imports ``nwdc`` directly; only code naming ``wateruse`` + itself should see the warning. If ``__init__`` ever imports the alias, + every user of the library gets a DeprecationWarning they cannot act on. + + Runs in a subprocess: a fresh interpreter is the only honest way to test + an import side effect, and clearing ``sys.modules`` in-process would hand + every later test a second copy of the package. + """ + import subprocess + import sys + + result = subprocess.run( + [ + sys.executable, + "-W", + "error::DeprecationWarning", + "-c", + "import dataretrieval", + ], + capture_output=True, + text=True, + ) + + assert result.returncode == 0, result.stderr diff --git a/tests/transport_test.py b/tests/transport_test.py index dcd4b5c01..0957da4e8 100644 --- a/tests/transport_test.py +++ b/tests/transport_test.py @@ -441,15 +441,15 @@ def test_bad_retry_environment_raises_a_catchable_error(monkeypatch) -> None: """ monkeypatch.setenv("API_USGS_RETRIES", "off") with pytest.raises(DataRetrievalError): - retry.RetryPolicy.from_env() + retry.RetryPolicy.from_configuration() monkeypatch.setenv("API_USGS_RETRIES", "2") monkeypatch.setenv("API_USGS_STALL_TIMEOUT", "none") with pytest.raises(ConfigurationError): - retry.RetryPolicy.from_env() + retry.RetryPolicy.from_configuration() monkeypatch.setenv("API_USGS_STALL_TIMEOUT", "10") - assert retry.RetryPolicy.from_env().stall_timeout == 10.0 + assert retry.RetryPolicy.from_configuration().stall_timeout == 10.0 # Still a ValueError, so existing handling of a bad setting keeps working. assert issubclass(ConfigurationError, ValueError) diff --git a/tests/waterdata_chunking_test.py b/tests/waterdata_chunking_test.py index 3454dc317..beb7a908c 100644 --- a/tests/waterdata_chunking_test.py +++ b/tests/waterdata_chunking_test.py @@ -32,11 +32,14 @@ import pandas as pd import pytest +import dataretrieval +from dataretrieval import configuration as _configuration from dataretrieval.combining import ( _QUOTA_HEADER, _combine_chunk_frames, _combine_chunk_responses, ) +from dataretrieval.configuration import Configuration from dataretrieval.exceptions import ( DataRetrievalError, NetworkError, @@ -50,7 +53,6 @@ from dataretrieval.ogc.chunking import ( ChunkedCall, _chunked_client, - _parallel_chunks, get_active_client, multi_value_chunked, parallel_chunks, @@ -75,7 +77,6 @@ ) from dataretrieval.transport import retry as _retry_mod from dataretrieval.transport.retry import ( - _RETRIES_DEFAULT, RetryPolicy, _retryable, ) @@ -731,6 +732,45 @@ async def fetch(args, *, base): assert sorted(df["id"].tolist()) == sorted(sites) +def test_resume_reads_concurrency_from_the_caller_not_the_snapshot(monkeypatch): + """A ``configure()`` block around a ``resume()`` must actually take effect. + + The concurrency cap is the one dial a caller adjusts precisely *when* + retrying -- the documented recovery from ``QuotaExhausted`` is to wait and + re-issue more gently -- so ``resume()`` resolves it per drive rather than + carrying a value fixed when the call was constructed. A ``configure()`` + block entered between the interruption and the resume therefore wins. + """ + state = {"calls": 0} + + async def fetch(args): + state["calls"] += 1 + if state["calls"] == 3: + raise RateLimited("429: Too many requests made.") + sites = list(args["sites"]) + return (pd.DataFrame({"id": sites}), _quota_response(500)) + + sites = ["S" * 10 + str(i) for i in range(16)] + decorated = multi_value_chunked(build_request=_fake_build, url_limit=240)(fetch) + with pytest.raises(QuotaExhausted) as excinfo: + decorated({"sites": sites}) + + # Spy on the real, decorator-built ChunkedCall rather than a hand-made one. + seen: list[int | None] = [] + original_run = _chunking.ChunkedCall._run + + async def spy_run(self, max_concurrent): + seen.append(max_concurrent) + return await original_run(self, max_concurrent) + + monkeypatch.setattr(_chunking.ChunkedCall, "_run", spy_run) + + with dataretrieval.configure(Configuration(concurrency=2)): + excinfo.value.call.resume() + + assert seen == [2], seen + + def test_chunker_passes_through_non_429_runtime_error(): """A non-429 ``RuntimeError`` (e.g. a 500) is not a quota signal; it must propagate unchanged so callers see the real cause.""" @@ -1421,8 +1461,8 @@ def test_iter_chunk_args_passthrough_yields_a_copy(): # --- async fan-out path ---------------------------------------------------- # # Every chunk is gathered over one ``httpx.AsyncClient`` and -# concurrency is bounded by an ``asyncio.Semaphore`` sized from -# ``API_USGS_CONCURRENT`` (the client's connection pool is sized to +# concurrency is bounded by an ``asyncio.Semaphore`` sized from the effective +# configuration (the client's connection pool is sized to # match, but the semaphore is the throttle — see ``ChunkedCall._run``). # The conftest's ``_pin_chunker_env`` autouse pins # ``API_USGS_CONCURRENT=1`` (sequential dispatch) for the whole suite; @@ -1650,6 +1690,21 @@ def test_fan_out_in_flight_high_water_mark_is_the_cap( assert in_flight["max"] == expected_high_water +def test_configure_concurrency_controls_dispatch(monkeypatch): + """The highest-precedence block setting reaches the execution semaphore.""" + monkeypatch.setenv("API_USGS_CONCURRENT", "1") + in_flight = {"now": 0, "max": 0} + fetch = multi_value_chunked(build_request=_fake_build, url_limit=240)( + _concurrency_probe(in_flight) + ) + + with dataretrieval.configure(Configuration(concurrency=2)): + df, _ = fetch({"sites": list(_EIGHT_SINGLETON_SITES)}) + + assert len(df) == len(_EIGHT_SINGLETON_SITES) + assert in_flight["max"] == 2 + + def test_fan_out_outlives_pool_timeout_on_real_transport(monkeypatch): """End-to-end regression for the pool-timeout starvation bug: the fan-out must survive every pooled connection staying busy past the @@ -1847,19 +1902,21 @@ def test_retry_policy_long_retry_after_escalates(): assert not policy.should_retry(attempt=1, retry_after=120.0) # escalates -def test_retry_policy_from_env(monkeypatch): +def test_retry_policy_from_config(monkeypatch): monkeypatch.setenv("API_USGS_RETRIES", "2") - assert RetryPolicy.from_env().max_retries == 2 + assert RetryPolicy.from_configuration().max_retries == 2 monkeypatch.setenv("API_USGS_RETRIES", "0") - assert RetryPolicy.from_env().max_retries == 0 + assert RetryPolicy.from_configuration().max_retries == 0 monkeypatch.delenv("API_USGS_RETRIES", raising=False) - assert RetryPolicy.from_env().max_retries == _RETRIES_DEFAULT + assert ( + RetryPolicy.from_configuration().max_retries == _configuration.DEFAULT_RETRIES + ) monkeypatch.setenv("API_USGS_RETRIES", "-1") with pytest.raises(ValueError): - RetryPolicy.from_env() + RetryPolicy.from_configuration() monkeypatch.setenv("API_USGS_RETRIES", "lots") with pytest.raises(ValueError): - RetryPolicy.from_env() + RetryPolicy.from_configuration() def test_retry_policy_rejects_invalid_settings(): @@ -1871,12 +1928,12 @@ def test_retry_policy_rejects_invalid_settings(): RetryPolicy(max_backoff=-1.0) -def test_retry_policy_from_env_honors_monkeypatched_constants(monkeypatch): +def test_retry_policy_from_config_honors_monkeypatched_constants(monkeypatch): # The timing knobs are read from the module constants at call time, so # monkeypatching them (as the module comment promises) takes effect. monkeypatch.setattr(_retry_mod, "_RETRY_MAX_BACKOFF", 0.0) monkeypatch.setattr(_retry_mod, "_RETRY_BASE_BACKOFF", 0.0) - policy = RetryPolicy.from_env() + policy = RetryPolicy.from_configuration() assert policy.max_backoff == 0.0 and policy.base_backoff == 0.0 @@ -2384,16 +2441,26 @@ def test_cap_does_not_mask_unchunkable(): ChunkPlan(args, _fake_build, url_limit=10, max_chunks=32) -def test_parallel_chunks_publishes_n_on_the_ambient(): - """The context manager publishes ``n`` on the ambient for the block and - restores the previous value on exit — including proper nesting.""" - assert _parallel_chunks.get() == 1 # default (off, = no extra fan-out) +def test_parallel_chunks_publishes_n_as_the_effective_setting(): + """The context manager sets ``n`` for the block and restores the previous + value on exit — including proper nesting. + + ``parallel_chunks(n)`` is sugar for ``configure(parallel_chunks=n)``, so + both forms share one scoping mechanism and the innermost block wins. + Outside any block the configured baseline applies, which is ``1`` — off — + unless a config file raised it.""" + assert _configuration.parallel_chunks() == 1 # default (off, = no extra fan-out) with parallel_chunks(32): - assert _parallel_chunks.get() == 32 + assert _configuration.parallel_chunks() == 32 with parallel_chunks(2): - assert _parallel_chunks.get() == 2 - assert _parallel_chunks.get() == 32 # outer restored - assert _parallel_chunks.get() == 1 # default (off) outside any block + assert _configuration.parallel_chunks() == 2 + assert _configuration.parallel_chunks() == 32 # outer restored + with dataretrieval.configure( + Configuration(parallel_chunks=4) + ): # the other spelling + assert _configuration.parallel_chunks() == 4 + assert _configuration.parallel_chunks() == 32 + assert _configuration.parallel_chunks() == 1 # default (off) outside any block @pytest.mark.parametrize( @@ -2413,11 +2480,16 @@ def test_parallel_chunks_rejects_non_positive_int(bad): """``n`` must be a positive integer; every other shape — zero, negative, a float, a string (including a numeric one and the old level names), ``None``, a ``bool``, a list — raises ``ValueError`` at ``with`` entry, before any - request, and leaves the ambient untouched.""" - with pytest.raises(ValueError, match="must be a positive integer"): + request, and leaves the ambient untouched. + + The message comes from the ``parallel_chunks`` grammar in the configuration + chain, which is the one that owns this setting's bound; a + ``ConfigurationError`` is a ``ValueError``, which is the contract callers + were given here.""" + with pytest.raises(ValueError, match="must be an integer"): with parallel_chunks(bad): pass - assert _parallel_chunks.get() == 1 # default (off) — unchanged by a rejected call + assert _configuration.parallel_chunks() == 1 # unchanged by a rejected call def test_parallel_chunks_drives_end_to_end_fan_out(): diff --git a/tests/waterdata_utils_test.py b/tests/waterdata_utils_test.py index 804d4cb2c..27e4b9b79 100644 --- a/tests/waterdata_utils_test.py +++ b/tests/waterdata_utils_test.py @@ -44,6 +44,7 @@ _EXTRA_ID_COLS, OGC_API_URL, WATERDATA_DIALECT, + _flatten_queryables, _get_args, ) @@ -1323,3 +1324,24 @@ def fake_engine_get_ogc_data(args, collection, output_id, **k): ): _utils_module.get_ogc_data({"state": "WI"}, "monitoring-locations") assert captured["args"] == {"state": "WI"} + + +@pytest.mark.parametrize( + "name", ["x_api_key", "x-api-key", "api_token", "access_token", "pat", "auth"] +) +def test_credential_shaped_queryables_are_rejected(name): + """The denylist matches spellings, not just a few exact names. + + ``x_api_key`` is the tempting one -- it mirrors the ``X-Api-Key`` header + the README documents -- and an exact-match list let it through into the + query string. + """ + with pytest.raises(TypeError, match="Credentials cannot be passed"): + _flatten_queryables({"queryables": {name: "SECRET"}}) + + +@pytest.mark.parametrize( + "name", ["state_name", "site_type_code", "monitoring_location_id", "qualifier"] +) +def test_real_queryables_still_pass_through(name): + assert _flatten_queryables({"queryables": {name: "v"}}) == {name: "v"} diff --git a/tests/wqp_test.py b/tests/wqp_test.py index e125e3fd2..ecc9828f4 100644 --- a/tests/wqp_test.py +++ b/tests/wqp_test.py @@ -4,6 +4,7 @@ import pytest from pandas import DataFrame +import dataretrieval import dataretrieval.wqp as wqp from dataretrieval.exceptions import DataCurrencyWarning from dataretrieval.wqp import ( @@ -143,6 +144,30 @@ def test_wqp_url_profiles(builder, service, expected, warning): assert builder(service) == expected +def test_a_configured_base_url_moves_both_interfaces(): + """One root, both paths: the portal serves legacy and WQX3 from one host. + + Redirecting only the interface a caller happened to use first would leave + the other pointed at the service they were redirecting away from, which is + the failure a redirect exists to prevent. + """ + mirror = "https://mirror.example/wqp" + + with dataretrieval.configure(wqp.WqpConfiguration(base_url=mirror)): + with pytest.warns(DataCurrencyWarning): + legacy = wqp.wqp_url("Result") + with pytest.warns(UserWarning): + wqx3 = wqp.wqx3_url("Result") + + assert legacy == f"{mirror}/data/Result/Search?" + assert wqx3 == f"{mirror}/wqx3/Result/search?" + + # Outside the block, the portal's own root again -- the redirect is scoped + # to the ``with`` statement, not latched at import. + with pytest.warns(DataCurrencyWarning): + assert wqp.wqp_url("Result").startswith("https://www.waterqualitydata.us/") + + @pytest.mark.parametrize( ("builder", "profile", "valid_services", "warning"), [ @@ -233,6 +258,30 @@ def test_check_kwargs(): kwargs = _check_kwargs(kwargs) +@pytest.mark.parametrize( + "name", ["api_key", "x_api_key", "access_token", "password", "pat", "auth"] +) +def test_credential_shaped_wqp_kwargs_are_rejected(name): + """WQP has the widest ``**kwargs`` passthrough in the package. + + Its ten getters forward whatever the caller names straight into the query + string, so ``api_key=`` -- the plausible guess now that ``configure()`` + takes ``Configuration(api_key=...)`` -- would put a secret in a URL that + clients, proxies and logs retain. Same predicate and same message as Water + Data's ``**queryables`` guard, because it is the same mistake. + """ + with pytest.raises(TypeError, match="Credentials cannot be passed"): + _check_kwargs({name: "SECRET"}) + + +@pytest.mark.parametrize( + "name", ["siteid", "characteristicName", "statecode", "providers", "pCode"] +) +def test_real_wqp_filters_still_pass_through(name): + """The denylist must not claim names the portal owns.""" + assert _check_kwargs({name: "v"})[name] == "v" + + def test_get_results_wqx3_preserves_user_dataProfile(httpx_mock): """A valid user-supplied WQX3.0 profile must not be overwritten.