diff --git a/docs/charityNavigator.md b/docs/charityNavigator.md new file mode 100644 index 0000000..947e711 --- /dev/null +++ b/docs/charityNavigator.md @@ -0,0 +1,197 @@ +# Charity Navigator Integration — Technical Report + +**Source file:** [`../src/charityNavigator.ts`](../src/charityNavigator.ts) +**Generated:** 2026-08-06 +**Purpose:** Fetch nonprofit profile data from the Charity Navigator Premier GraphQL API and import it into the Philanthropy Data Commons (PDC) as changemaker field values. + +--- + +## 1. Overview + +`charityNavigator.ts` defines a [yargs](https://yargs.js.org/) command module (`charityNavigator`) that is registered in [`../src/index.ts`](../src/index.ts) as part of the `data-scripts` CLI. It exposes three subcommands: + +| Subcommand | Purpose | Writes to PDC? | Needs PDC auth? | +| --------------- | ------------------------------------------------------------------------------------------------------------- | -------------- | -------------------------- | +| `lookup` | Fetch Charity Navigator data for an explicit list of EINs and print or save it | No | No | +| `lookupFromPdc` | Pull all changemaker EINs from PDC, look them up in Charity Navigator, print/save the result (dry-run style) | No | No (reads PDC anonymously) | +| `updateAll` | Full sync: read PDC changemakers → look up in Charity Navigator → write results back into PDC as field values | **Yes** | **Yes (OIDC)** | + +Invocation examples: + +```bash +data-scripts charityNavigator lookup --eins 13-1837418 --charity-navigator-api-key +``` + +```bash +data-scripts charityNavigator updateAll --pdc-api-base-url --oidc-base-url --oidc-client-id --oidc-client-secret --charity-navigator-api-key +``` + +The API key may also be supplied via the `DS_CHARITY_NAVIGATOR_API_KEY` environment variable. + +--- + +## 2. External Systems + +### Charity Navigator Premier API (source) + +- **Endpoint:** `https://api.charitynavigator.org/graphql` (constant `API_URL`) +- **Client:** Apollo Client (`@apollo/client`) with an in-memory cache +- **Auth:** Bearer token in the `Authorization` header (`Bearer `), injected by a `SetContextLink` auth link (`apolloInit`) +- **Query:** `NonprofitsPublic` (`QueryNonprofitsPublic`) against the `nonprofitsPublic` field, filtered by a set of EINs + +### PDC API (destination) + +- **Client:** Axios wrapper in [`../src/pdc-api.ts`](../src/pdc-api.ts) / [`../src/client.ts`](../src/client.ts) +- **Auth:** OIDC `client_credentials` grant via [`../src/oidc.ts`](../src/oidc.ts) (`getToken`), producing a Bearer access token. Only `updateAll` authenticates; reads of `/changemakers` are anonymous. +- **Relevant endpoints:** + - `GET /changemakers` — list all changemakers (shallow, anonymous) + - `GET /sources` — find the Charity Navigator source + - `POST /sources` — create the source (admin-only; typically fails for non-admins) + - `POST /changemakerFieldValueBatches` — open a batch + - `POST /changemakerFieldValues` — write one field value + +--- + +## 3. Data Flow (the `updateAll` path) + +``` +PDC /changemakers ──► extract taxId (EIN) list + │ + ▼ +Validate EINs (isValidEin) ──► strip hyphens ──► split valid / invalid + │ │ + │ (invalid logged as warning, skipped) + ▼ +Charity Navigator GraphQL (getCharityNavigatorProfiles) + └─ paginated fetch (fetchAllPages, perPage=100) ──► NonprofitPublic[] edges + │ + ▼ +Authenticate to PDC (OIDC client_credentials) ──► access token + │ + ▼ +Find or create PDC Source (dataProviderShortCode = "charitynav") + │ + ▼ +Open a ChangemakerFieldValueBatch (sourceId + notes with timestamp) + │ + ▼ +For each nonprofit: + match to a PDC changemaker by EIN (getChangemakerByEin) + For each mapped field (baseFieldMap): + if the CN attribute is present ──► POST /changemakerFieldValues + (batchId, changemakerId, baseFieldShortCode, value, goodAsOf) + └─ 403 Forbidden ──► warn + record changemakerId, continue +``` + +### Key processing details + +- **EIN normalization:** PDC stores tax IDs possibly with a hyphen (`NN-NNNNNNN`). `isValidEin` ([`../src/ein.ts`](../src/ein.ts)) accepts `^\d{2}-?\d{7}$`. Before querying Charity Navigator, hyphens are stripped (`e.replace('-', '')`) because the CN API expects unhyphenated EINs. When matching results back to changemakers, `getChangemakerByEin` also strips hyphens from the PDC `taxId` for comparison. +- **EIN → changemaker matching:** `getChangemakerByEin` returns a changemaker only when **exactly one** matches an EIN. Zero matches → logged at info and skipped; more than one match → logged as a warning and skipped (ambiguous, so nothing is written). +- **Pagination:** `fetchAllPages` loops page-by-page (starting at page 1, `perPage = 100`) accumulating `edges` until `currentPage >= totalPages`. It **hard-fails** if `totalPages` is not a positive integer (guards against `undefined` → infinite loop, and `null`/`0` → silent partial import). `extractPageFromResponse` throws a clear error if a page returns `null`/`undefined` data instead of throwing a cryptic `TypeError`. +- **Runtime validation:** GraphQL responses are untyped at runtime. `isNonprofitPublic` verifies that `ein`, `name`, and `updatedAt` are strings before an edge is treated as a valid `NonprofitPublic`. +- **Sequential writes:** Field values are POSTed one at a time in a `for` loop (not `Promise.all`) because the PDC API times out under concurrent POSTs to `/changemakerFieldValues`. +- **403 handling:** `postChangemakerFieldValueWarnOnForbidden` swallows HTTP 403 (no permission) — it logs a warning and records the `changemakerId` in a set for a summary log at the end — while re-throwing any other error. This lets a run continue past changemakers the client lacks permission to write. +- **Source resolution:** `getOrCreateSource` looks for an existing source with `dataProviderShortCode === "charitynav"`. If not found it attempts to create one, but warns that this usually requires a `pdc-admin` and may fail. + +--- + +## 4. Field Mapping: Charity Navigator → PDC + +The mapping is defined by the `baseFieldMap` constant in [`../src/charityNavigator.ts:44`](../src/charityNavigator.ts). Each Charity Navigator `NonprofitPublic` attribute is written to a PDC **base field** identified by its short code. + +### 4.1 Fields actually written to PDC + +| Charity Navigator attribute (`NonprofitPublic`) | PDC base field short code | Notes | +| ----------------------------------------------- | -------------------------------- | --------------------------- | +| `name` | `organization_name` | Organization legal name | +| `website` | `organization_website` | Optional; skipped if absent | +| `phone` | `organization_phone` | Optional; skipped if absent | +| `mission` | `organization_mission_statement` | Optional; skipped if absent | + +Each written value becomes a **`ChangemakerFieldValue`** with this shape: + +| PDC field value property | Value / source | +| ------------------------ | ------------------------------------------------------------ | +| `changemakerId` | Resolved via `getChangemakerByEin` (EIN match) | +| `batchId` | ID of the batch opened for this run | +| `baseFieldShortCode` | From the mapping table above | +| `value` | The CN attribute, coerced with `.toString()` | +| `goodAsOf` | The nonprofit's `updatedAt` timestamp from Charity Navigator | + +A value is only posted when the CN attribute is neither `undefined` nor `null` (an explicit runtime null-check, since the API can return null despite the TypeScript types). + +### 4.2 EIN — used for matching, not stored as a field + +`ein` is the join key between the two systems. It is used to match a Charity Navigator record to a PDC changemaker, but it is **not** itself written as a field value. + +### 4.3 Fields fetched but NOT mapped/imported + +The GraphQL query requests several attributes that are **not** part of `baseFieldMap` and are therefore fetched but never written to PDC. They are available in the raw output of `lookup`/`lookupFromPdc` (when `--output-file` is used) but ignored by `updateAll`: + +| Charity Navigator attribute | Currently imported? | +| --------------------------- | ------------------------------------------------------------------ | +| `updatedAt` | Not a field value, but reused as `goodAsOf` on every written field | +| `encompassRatingId` | No | +| `encompassScore` | No | +| `encompassStarRating` | No | +| `encompassPublicationDate` | No | +| `size` | No | +| `cause` | No | + +> These represent an opportunity for future mapping (e.g. ratings/scores) if corresponding PDC base fields exist. Adding them would be a matter of extending `baseFieldMap` and ensuring the target base field short codes exist in PDC. + +--- + +## 5. Command Reference + +### `lookup` + +- **Args:** `--eins` (array, validated by `isValidEin`), `--charity-navigator-api-key` (or env var), `--output-file`/`--write` (optional). +- **Behavior:** Calls `getCharityNavigatorProfiles` for the given EINs. Writes JSON to the output file if given, otherwise logs the result. No PDC interaction. + +### `lookupFromPdc` + +- **Args:** `--pdc-api-base-url` (required), `--charity-navigator-api-key`, `--output-file`. +- **Behavior:** Reads all PDC changemakers, extracts + validates + de-hyphenates their EINs, looks them up in Charity Navigator. If no output file, logs which changemaker IDs were found in Charity Navigator; otherwise writes the raw CN response to file. **Read-only** with respect to PDC (does not write field values). + +### `updateAll` + +- **Args:** `--pdc-api-base-url` (required), all `oidcOptions` (`--oidc-base-url`, `--oidc-client-id`, `--oidc-client-secret`, all required), `--charity-navigator-api-key`. +- **Behavior:** The full sync described in Section 3. This is the only subcommand that writes to PDC. + +--- + +## 6. Error Handling & Resilience Summary + +| Concern | Handling | +| -------------------------------------- | -------------------------------------------------------------------------------------- | +| Missing API key | Explicit check; throws with guidance to use CLI flag or `DS_CHARITY_NAVIGATOR_API_KEY` | +| Invalid EINs in PDC | Filtered out and logged as a warning; valid EINs still processed | +| Malformed `totalPages` | `fetchAllPages` throws loudly (prevents infinite loop / silent partial import) | +| Null/undefined GraphQL data | `extractPageFromResponse` throws a clear, page-numbered error | +| Untyped edges | `isNonprofitPublic` runtime type guard | +| Ambiguous EIN → changemaker (>1 match) | Skipped with a warning; nothing written | +| Null CN attribute values | Skipped (not posted) | +| HTTP 403 on write | Warned + changemakerId recorded; run continues; summary warning at end | +| PDC concurrency timeouts | Field values POSTed sequentially | +| Missing Charity Navigator source | Attempt to create (usually admin-only); warns it may fail | + +--- + +## 7. Testing + +Unit tests in [`../src/charityNavigator.unit.test.ts`](../src/charityNavigator.unit.test.ts) cover the two exported helpers: + +- **`fetchAllPages`** — accumulates edges across multiple pages, handles a single page, handles an empty first page, and throws on invalid `totalPages` values (`undefined`, `null`, `0`). +- **`extractPageFromResponse`** — returns `nonprofitsPublic` for valid data and throws clear, page-numbered errors when `data` is `null` or `undefined`. + +The network-facing functions (`getCharityNavigatorProfiles`, the command handlers, and all PDC writes) are not directly unit-tested; testability is achieved by extracting the pure pagination/extraction logic into the two exported helpers. + +--- + +## 8. Notable Constants & TODOs in the Code + +- `CN_SHORT_CODE = 'charitynav'` — the PDC data provider short code for Charity Navigator. +- `PER_PAGE = 100` — fixed GraphQL page size. +- `HTTP_STATUS_FORBIDDEN = 403` — with a code comment noting that a shared `@pdc/http-status-codes` package should replace this once available. +- [`../src/pdc-api.ts`](../src/pdc-api.ts) contains a `TODO` to replace locally-copied `ChangemakerFieldValue*` types with the `@pdc/sdk` equivalents. diff --git a/docs/getMetrics.md b/docs/getMetrics.md new file mode 100644 index 0000000..82b0922 --- /dev/null +++ b/docs/getMetrics.md @@ -0,0 +1,329 @@ +# PDC Metrics Report Generator — Technical Report + +**Source file:** [`getMetrics.ts`](getMetrics.ts) +**Generated:** 2026-08-10 +**Purpose:** Read the Philanthropy Data Commons (PDC) API and produce a high‑level metrics report: a count of items in each top‑level collection endpoint. This is a **read‑only** script — it never writes to PDC. + +--- + +## 1. Overview + +`getMetrics.ts` defines a [yargs](https://yargs.js.org/) command module (`getMetrics`) registered in [`index.ts`](index.ts) as part of the `data-scripts` CLI. Unlike the Charity Navigator and GivingTuesday integrations (which pull data from a third party and write it back into PDC), this script does exactly one thing: it queries a fixed list of PDC collection endpoints and reports how many items each one contains. + +The item count for each endpoint comes from the `total` field of the PDC "bundle" response (see [§4](#4-how-counts-are-determined)). + +Invocation examples: + +```bash +# Interactive browser login, print a table (the default) +npm run getMetrics +``` + +```bash +# Use an existing bearer token, write CSV to a file +npm run getMetrics -- --access-token "$MY_TOKEN" --format csv --write metrics.csv +``` + +```bash +# Non-interactive: client-credentials grant (no browser), single command +npm run getMetrics -- --oidc-client-id YOUR_CLIENT_ID --oidc-client-secret YOUR_CLIENT_SECRET +``` + +```bash +# No authentication at all — only public endpoints report a count +npm run getMetrics -- --skip-auth +``` + +> `npm run getMetrics` is a convenience wrapper for `npm start -- getMetrics`; both accept the same options after `--`. + +### Using `--access-token` (no browser / no Keycloak reconfiguration) + +Because the default `pdc-metrics` client does not allow a loopback redirect (see [§3](#3-authentication-the-interactive-browser-flow)), the quickest way to count the authenticated endpoints today is to supply a bearer token you already have. Any of these work: + +```bash +# 1. Token as a CLI flag +npm run getMetrics -- --access-token "eyJhbGciOi...your.jwt...here" +``` + +```bash +# 2. Token from an environment variable (the CLI reads any DS_-prefixed var) +export DS_ACCESS_TOKEN="eyJhbGciOi...your.jwt...here" +npm run getMetrics +``` + +```bash +# 3. Token in a file, expanded inline — CSV to stdout +npm run getMetrics -- --access-token "$(cat token.txt)" --format csv +``` + +```bash +# 4. Token in a file, JSON written to disk +npm run getMetrics -- --access-token "$(cat token.txt)" --format json --write metrics.json +``` + +**Where to get a token:** + +- If you have OIDC **client credentials** (client id + secret), reuse the existing `auth` command in this same CLI to mint one and save it to a file, then feed it straight into `getMetrics`: + + ```bash + npm start -- auth \ + --oidc-base-url https://auth.philanthropydatacommons.org/realms/pdc \ + --oidc-client-id "$MY_CLIENT_ID" \ + --oidc-client-secret "$MY_CLIENT_SECRET" \ + --write token.txt + + npm run getMetrics -- --access-token "$(cat token.txt)" --format csv --write metrics.csv + ``` + +- Or copy the bearer token from an authenticated session in the PDC web app (browser dev tools → a request to `api.philanthropydatacommons.org` → `Authorization: Bearer …` header). Note these tokens are short‑lived, so re‑run promptly. + +> The token counts endpoints exactly as far as its permissions allow: any collection the token cannot read is reported as `unauthorized` (401) or `forbidden` (403) rather than failing the run. + +--- + +## 2. External Systems + +### PDC API (data source — read only) + +- **Base URL:** `https://api.philanthropydatacommons.org/` (default; override with `--pdc-api-base-url`) +- **Client:** the shared Axios `client` ([`client.ts`](client.ts)) — plain REST `GET` per endpoint +- **Auth:** Bearer access token in the `Authorization` header. Most endpoints require it; `baseFields` and `changemakers` are readable anonymously. +- **Query parameters:** a cheap `?_page=1&_count=1` probe is tried first; endpoints whose `total` is ambiguous at that size are re-fetched in full with `?_page=1&_count=1000000` (see [§4](#4-how-counts-are-determined)). + +### PDC Keycloak realm (authentication) + +- **Realm base URL:** `https://auth.philanthropydatacommons.org/realms/pdc` (default; override with `--oidc-base-url`) +- **Discovery:** OIDC metadata is fetched from the realm's `.well-known/openid-configuration` via `openid-client`'s `Issuer.discover`. +- **Flow:** OAuth 2.0 **Authorization Code with PKCE** (`S256`), driven through the caller's browser (see [§3](#3-authentication-the-interactive-browser-flow)). +- **Default client:** `pdc-metrics` — the public client used by the PDC web application (override with `--oidc-client-id`). + +--- + +## 3. Authentication + +`resolveAccessToken` picks a token source in priority order, so the same command works interactively or headlessly: + +1. `--skip-auth` → no token (public endpoints only). +2. `--access-token` / `DS_ACCESS_TOKEN` → use the supplied bearer token as‑is. +3. `--oidc-client-secret` / `DS_OIDC_CLIENT_SECRET` present → non‑interactive **client‑credentials** grant (see [§3.2](#32-non-interactive-client-credentials-grant)). +4. otherwise → the **interactive browser** login (see [§3.1](#31-interactive-browser-flow)). + +### 3.1 Interactive browser flow + +This is the part that differs most from the other data-scripts, which use the OIDC **client‑credentials** grant ([`oidc.ts`](oidc.ts)). Here the caller authenticates as _themselves_ in a browser, so the report reflects exactly what that user is permitted to read. + +The flow (`authenticateInteractively`) is a textbook RFC 8252 native‑app login: + +``` +1. Issuer.discover(oidc-base-url) → realm metadata (auth + token endpoints) +2. Build a public client (token_endpoint_auth_method: 'none') + redirect_uris = [ http://localhost:/callback ] +3. Generate PKCE code_verifier + code_challenge (S256) and a random state +4. Start a throwaway localhost HTTP server on +5. Open the caller's browser to the realm's authorization endpoint + (scope=openid, code_challenge, state) +6. Caller signs in; Keycloak redirects to http://localhost:/callback?code=…&state=… +7. The local server catches the redirect, exchanges the code (+ code_verifier) at + the token endpoint, and resolves with the access_token +8. The browser tab shows a "you may close this tab" page; the server shuts down +``` + +If the caller does not finish within `AUTH_TIMEOUT_MS` (5 minutes), the attempt is aborted with a clear timeout error. If the browser cannot be opened automatically, the authorization URL is also logged so it can be pasted manually. + +### ⚠️ Loopback redirect URI must be registered + +The authorization‑code flow redirects to `http://localhost:/callback`. **That exact redirect URI must be registered on the OIDC client in Keycloak**, or Keycloak refuses the request with _"Invalid parameter: redirect_uri"_ and the browser tab never returns to the CLI (the script then times out). + +As of this writing the default `pdc-metrics` client only permits the web app's own origin (`https://app.philanthropydatacommons.org/`) as a redirect, **not** a loopback URL. So the interactive flow works only after one of these one‑time setups: + +- **Preferred:** a PDC Keycloak admin adds `http://localhost/*` (or a specific `http://localhost:9736/callback`) to the valid redirect URIs of a public client, and you point `--oidc-client-id` at it; **or** +- **No config needed:** skip the browser entirely and pass a token you already have via `--access-token` (or the `DS_ACCESS_TOKEN` environment variable). This is the quickest way to count the auth‑required endpoints today. + +The two public endpoints (`baseFields`, `changemakers`) always report a count regardless, so `--skip-auth` produces a partial report with no setup at all. + +### 3.2 Non‑interactive client‑credentials grant + +If you have OIDC **client credentials** (a client id + secret), pass the secret and the script skips the browser entirely — it reuses `getToken` from [`oidc.ts`](oidc.ts) to fetch a token via the `client_credentials` grant. This needs no loopback redirect and no separate `auth` step, so the whole report is a single command: + +```bash +npm run getMetrics -- --oidc-client-id YOUR_CLIENT_ID --oidc-client-secret YOUR_CLIENT_SECRET +``` + +The secret can also come from the environment (`DS_OIDC_CLIENT_SECRET`), matching the other data‑scripts: + +```bash +export DS_OIDC_CLIENT_ID=YOUR_CLIENT_ID +export DS_OIDC_CLIENT_SECRET=YOUR_CLIENT_SECRET +npm run getMetrics -- --format csv --write metrics.csv +``` + +The report then reflects exactly what that service client is permitted to read; collections it cannot read are reported as `unauthorized`/`forbidden` rather than failing the run. + +--- + +## 4. How counts are determined + +Every top‑level PDC collection endpoint returns a **bundle**: + +```json +{ "entries": [ … ], "total": 282 } +``` + +`getEndpointCount` reads the count in up to two steps: + +1. **Cheap probe** — `GET {baseUrl}/{path}?_page=1&_count=1`. Well-behaved endpoints report the full `total` even on a one-item page (verified: `baseFields` returns `total: 282` at any `_count`). Any **`total > 1`** is therefore an unambiguous grand total and is used as-is — one request, no payload. +2. **Full fetch** — when the probe's `total` is **missing, `0`, or `1`** (indistinguishable from a page-scoped value at `_count=1`), the whole collection is fetched with `?_page=1&_count=1000000`, and the count is: + + ``` + count = max(total ?? 0, number of entries returned) + ``` + + Taking the larger of the two means neither a **missing/short `total`** nor a **truncated page** can undercount. If the entries returned hit the `1000000` ceiling, the count is reported as a floor with a note (bump `_count` for an exact figure). + +3. If a full response carries neither a `total` nor an `entries` array, the endpoint is marked `error`. + +> **Why this matters:** an earlier version trusted `total` from the `_count=1` probe and, when it was absent, counted only the single fetched page — so an endpoint like `applicationForms` (which does not report a usable `total` at `_count=1`) was reported as **1** instead of its true size. The two-step `max(total, entries)` approach fixes that by actually evaluating the whole result. + +--- + +## 5. Endpoints counted + +The list lives in the `PDC_ENDPOINTS` constant and is trivial to edit. It covers the top‑level bundle collections exposed by `@pdc/sdk` that resolve to a real route on the live API: + +| Path | Label | Public? | +| ------------------------------- | ------------------------------- | ------------ | +| `/baseFields` | Base Fields | ✅ anonymous | +| `/changemakers` | Changemakers | ✅ anonymous | +| `/proposals` | Proposals | 🔒 token | +| `/sources` | Sources | 🔒 token | +| `/dataProviders` | Data Providers | 🔒 token | +| `/funders` | Funders | 🔒 token | +| `/opportunities` | Opportunities | 🔒 token | +| `/users` | Users | 🔒 token | +| `/changemakerFieldValues` | Changemaker Field Values | 🔒 token | +| `/changemakerFieldValueBatches` | Changemaker Field Value Batches | 🔒 token | +| `/changemakerProposals` | Changemaker–Proposal Links | 🔒 token | +| `/applicationForms` | Application Forms | 🔒 token | +| `/terminologySets` | Terminology Sets | 🔒 token | +| `/permissionGrants` | Permission Grants | 🔒 token | +| `/files` | Files | 🔒 token | + +The `public` flag is **informational only** — the script always reports the _actual_ outcome of each request, so an endpoint that changes its auth requirements is reflected truthfully rather than assumed. + +> Some PDC resources (e.g. base field localizations, funder‑collaborative members, bulk‑upload tasks) are only reachable as _nested_ routes and return `404` at the top level; they are intentionally omitted. `POST`‑only routes such as `platformProviderResponses` are likewise excluded. + +--- + +## 6. Data flow + +``` +resolveAccessToken(args) + ├─ --skip-auth → no token + ├─ --access-token/env → use supplied token + └─ otherwise → interactive browser login (authenticateInteractively) + │ + ▼ +collectMetrics(baseUrl, PDC_ENDPOINTS, token) + └─ for each endpoint (sequentially): + getEndpointCount → GET /{path}?_page=1&_count=1 [+ Bearer token] + ├─ 2xx with total → { count: total, status: ok } + ├─ 2xx, entries only → { count: entries.length, status: ok, note } + ├─ 401 → unauthorized 403 → forbidden + ├─ 404 → not_found other → error + │ + ▼ +renderReport(metrics, format) → table | csv | json + └─ logged, or written to --output-file + └─ summary line: items counted, endpoints ok/total, endpoints unavailable +``` + +Requests are issued **sequentially** (a `for … of` loop, not `Promise.all`), matching the other data‑scripts' deliberate gentleness toward the PDC API. + +--- + +## 7. Command reference + +`getMetrics` options (all optional; sensible defaults for the production PDC): + +| Option | Default | Purpose | +| --------------------------- | ----------------------------------------------------- | ------------------------------------------------------------------------------------ | +| `--pdc-api-base-url` | `https://api.philanthropydatacommons.org/` | Which PDC instance to query | +| `--oidc-base-url` | `https://auth.philanthropydatacommons.org/realms/pdc` | Keycloak realm for the browser login | +| `--oidc-client-id` | `pdc-metrics` | Client used for the browser login or client-credentials grant | +| `--oidc-client-secret` | — | Secret for a non-interactive client-credentials login (env: `DS_OIDC_CLIENT_SECRET`) | +| `--format` | `table` | Output format: `table`, `csv`, or `json` | +| `--callback-port` | `9736` | Local port for the OAuth loopback listener | +| `--access-token` | — | Bearer token to use instead of the browser login (env: `DS_ACCESS_TOKEN`) | +| `--skip-auth` | `false` | Query only public endpoints; no login | +| `--output-file` / `--write` | — | Write the report to a file instead of logging it | + +As with all unified `data-scripts`, any option can also be supplied via a `DS_`‑prefixed environment variable (e.g. `DS_FORMAT=csv`) or a `--config` JSON file. + +--- + +## 8. Output formats + +Every report opens with the **`pdc-api-base-url`** it was run against, so a saved report is self-identifying about its environment (production vs. a test instance). Its placement is format-appropriate: a plain header line for `table`, a leading `#` comment line for `csv`, and a top-level `pdcApiBaseUrl` field for `json`. + +In every format the rows are sorted **alphabetically by endpoint path** (`sortMetrics`), regardless of the order the endpoints are fetched in. + +- **`table`** (default): a header line (`PDC API base URL: `), a blank line, then a column‑aligned plain‑text table (`ENDPOINT | COUNT | STATUS | NOTE`). Counts are grouped with thousands separators; unavailable endpoints show `—`. +- **`csv`**: a `# PDC API base URL: ` comment line, then RFC‑4180 CSV with header `endpoint,label,count,status,note`. Fields containing commas, quotes, or newlines are quoted and internal quotes doubled (`csvField`). +- **`json`**: `{ pdcApiBaseUrl, generatedAt, summary, metrics }`, where `summary` carries `endpointCount`, `okCount`, `failedCount`, and `itemTotal`. + +Example (`--skip-auth --format table`): + +``` +PDC API base URL: https://api.philanthropydatacommons.org/ + +ENDPOINT COUNT STATUS NOTE +/applicationForms — unauthorized Authentication required (no valid token supplied) +/baseFields 282 ok +/changemakers 17 ok +… +``` + +--- + +## 9. Error handling & resilience + +| Concern | Handling | +| ---------------------------------------- | --------------------------------------------------------------------------------------------------------------------------- | +| Endpoint requires auth, no/invalid token | `401` → `unauthorized` (count `—`); the run continues | +| Token lacks permission | `403` → `forbidden`; the run continues | +| Endpoint route missing/moved | `404` → `not_found`; the run continues | +| Unexpected response shape | No `total` and no `entries` → `error` with a descriptive note | +| Transport/other error | Caught per endpoint; message recorded as `error`; other endpoints still processed | +| Browser login not completed | Aborts after `AUTH_TIMEOUT_MS` (5 min) with a clear timeout error | +| Browser cannot be opened | Authorization URL is logged for manual paste | +| Redirect URI not registered | Documented prominently ([§3](#3-authentication-the-interactive-browser-flow)); `--access-token` is the no‑config workaround | + +`getEndpointCount` **never throws** — every failure mode is folded into the endpoint's `status`/`note`, so a single unreadable collection never aborts the whole report. + +--- + +## 10. Testing + +Unit tests in [`getMetrics.unit.test.ts`](getMetrics.unit.test.ts) cover the pure, side‑effect‑free helpers — the same "extract the testable logic, exercise it without the network" pattern used by [`charityNavigator.unit.test.ts`](charityNavigator.unit.test.ts) and [`givingTuesday.unit.test.ts`](givingTuesday.unit.test.ts): + +- **`csvField`** — leaves plain/empty values unquoted; quotes on comma, embedded quote (doubled), and newline. +- **`formatCount`** — thousands separators, `0` (not an em dash), and `null` → `—`. +- **`summarize`** — ok/failed counts and item total; empty list; excludes non‑ok counts from the total. +- **`renderCsv`** — header row, an ok row, a failed row (empty count field + note), and comma‑in‑note quoting. +- **`renderTable`** — all four column headers, slash‑prefixed paths with formatted counts, em‑dash + note for unavailable endpoints, and right‑aligned counts. +- **`renderJson`** — valid JSON carrying `generatedAt`, the `summary` block, and the `metrics` array. +- **`sortMetrics`** — alphabetical-by-path ordering, input not mutated, empty list unchanged. +- **`resolveFullCount`** — `max(total, entries)` logic: missing total, page-scoped total, truncated entries, empty collection, and the fetch-ceiling floor note. + +The network‑facing functions (`getEndpointCount`, `collectMetrics`) and their `EndpointMetric`/`MetricStatus` types are also exported for integration testing with a mocked Axios `client`, and `authenticateInteractively` is deliberately isolated from metric collection so the counting logic can be exercised with a plain token. + +--- + +## 11. Notable constants & TODOs + +- `PDC_ENDPOINTS` — the editable catalogue of collections to count. +- `DEFAULT_OIDC_CLIENT_ID = 'pdc-metrics'` — the PDC web app's public client (see the loopback‑redirect caveat in [§3](#3-authentication-the-interactive-browser-flow)). +- `DEFAULT_CALLBACK_PORT = 9736` — local OAuth redirect port. +- `AUTH_TIMEOUT_MS = 300_000` — browser‑login timeout. +- `PROBE_PAGE = '1'` / `PROBE_COUNT = '1'` — the cheap single‑item probe. +- `HTTP_STATUS_*` — with the same comment as the other scripts noting a shared `@pdc/http-status-codes` package should replace these once available. diff --git a/docs/givingTuesday.md b/docs/givingTuesday.md new file mode 100644 index 0000000..91b2280 --- /dev/null +++ b/docs/givingTuesday.md @@ -0,0 +1,233 @@ +# GivingTuesday Integration — Technical Report + +**Source file:** [`../src/givingTuesday.ts`](../src/givingTuesday.ts) +**Generated:** 2026-08-06 +**Purpose:** Fetch IRS Business Master File (BMF) nonprofit data from the GivingTuesday 990 Data API and import it into the Philanthropy Data Commons (PDC) as changemaker field values. + +--- + +## 1. Overview + +`givingTuesday.ts` defines a [yargs](https://yargs.js.org/) command module (`givingTuesday`) registered in [`../src/index.ts`](../src/index.ts) as part of the `data-scripts` CLI. It exposes three subcommands: + +| Subcommand | Purpose | Writes to PDC? | Needs PDC auth? | +| --------------- | --------------------------------------------------------------------------------------------------------- | -------------- | -------------------------- | +| `lookup` | Fetch GivingTuesday BMF data for an explicit list of EINs and print or save it | No | No | +| `lookupFromPdc` | Pull all changemaker EINs from PDC, look them up in GivingTuesday, print/save the result (dry-run style) | No | No (reads PDC anonymously) | +| `updateAll` | Full sync: read PDC changemakers → look up in GivingTuesday → write results back into PDC as field values | **Yes** | **Yes (OIDC)** | + +Invocation examples: + +```bash +data-scripts givingTuesday lookup --eins 84-2929872 +``` + +```bash +data-scripts givingTuesday updateAll --pdc-api-base-url --oidc-base-url --oidc-client-id --oidc-client-secret +``` + +> Unlike the Charity Navigator integration, the GivingTuesday API is **open-access and unauthenticated** — no API key is required for the source system. Authentication (OIDC) is only needed to _write_ into PDC via `updateAll`. + +--- + +## 2. External Systems + +### GivingTuesday 990 Data API (source) + +- **Base URL:** `https://990-infrastructure.gtdata.org` (constant `API_BASE_URL`) +- **Endpoint:** `/irs-data/bmf` (constant `BMF_PATH`) — the IRS Business Master File. _(Code comment notes the published docs render `/irs_data/` but the live API uses the hyphenated `/irs-data/` path.)_ +- **Client:** the shared Axios `client` ([`../src/client.ts`](../src/client.ts)) — plain REST GET, **one request per EIN**, passed as a `?ein=` query parameter +- **Auth:** none (open-access) +- **Rate limit:** 300 requests / 5 minutes (~1/sec). The code sleeps `RATE_LIMIT_DELAY_MS = 1100` ms between requests rather than implementing 429 backoff (matching the Candid approach). + +### PDC API (destination) + +- **Client:** Axios wrapper in [`../src/pdc-api.ts`](../src/pdc-api.ts) / [`../src/client.ts`](../src/client.ts) +- **Auth:** OIDC `client_credentials` grant via [`../src/oidc.ts`](../src/oidc.ts) (`getToken`). Only `updateAll` authenticates; reads of `/changemakers` are anonymous. +- **Relevant endpoints:** + - `GET /changemakers` — list all changemakers (shallow, anonymous) + - `GET /sources` — find the GivingTuesday source + - `POST /sources` — create the source (admin-only; typically fails for non-admins) + - `POST /changemakerFieldValueBatches` — open a batch + - `POST /changemakerFieldValues` — write one field value + +--- + +## 3. Data Flow (the `updateAll` path) + +``` +PDC /changemakers ──► extract taxId (EIN) list + │ + ▼ +Validate EINs (isValidEin) ──► split valid / invalid + │ │ + │ (invalid logged as warning, skipped) + ▼ +GivingTuesday BMF API (getGivingTuesdayProfiles) + └─ one GET per EIN, normalized via toGivingTuesdayEin + (hyphen-stripped, zero-padded to 9 digits) + └─ sleep 1100ms between requests (rate limit) + └─ per-EIN failure logged & skipped ──► BmfRecord[] results + │ + ▼ +Authenticate to PDC (OIDC client_credentials) ──► access token + │ + ▼ +Find or create PDC Source (dataProviderShortCode = "givingtuesday") + │ + ▼ +Open a ChangemakerFieldValueBatch (sourceId + notes with timestamp) + │ + ▼ +For each BMF record: + match to a PDC changemaker by normalized EIN (getChangemakerByEin) + derive goodAsOf from Date_Released (parseGivingTuesdayDate) + For each mapped field (baseFieldMap): + if the BMF attribute is present & non-empty ──► POST /changemakerFieldValues + (batchId, changemakerId, baseFieldShortCode, value, goodAsOf) + └─ 403 Forbidden ──► warn + record changemakerId, continue +``` + +### Key processing details + +- **EIN normalization (`toGivingTuesdayEin`):** GivingTuesday requires **zero-padded, 9-digit, hyphen-free** EINs. The helper strips a hyphen and left-pads to 9 characters (`ein.replace('-', '').padStart(9, '0')`). This same normalization is applied on both sides of the match in `getChangemakerByEin`, so PDC tax IDs and the EINs GivingTuesday echoes back compare correctly. +- **EIN validation:** `isValidEin` ([`../src/ein.ts`](../src/ein.ts)) accepts `^\d{2}-?\d{7}$`. Invalid EINs are logged and skipped; valid ones proceed. (Note: unlike Charity Navigator, hyphens are _not_ stripped before validation — validation runs on the raw `taxId`, and normalization happens later per-request.) +- **EIN → changemaker matching:** `getChangemakerByEin` returns a changemaker only when **exactly one** matches. Zero → logged at info and skipped; more than one → warning and skipped (ambiguous, nothing written). +- **One request per EIN:** Unlike Charity Navigator's single paginated GraphQL query filtered by a set of EINs, GivingTuesday is queried **individually per EIN**. `getGivingTuesdayProfiles` loops sequentially, sleeping between calls. +- **Per-EIN fault tolerance:** A failure for one EIN is caught, logged via `logger.error`, and skipped so a single bad lookup doesn't abort the whole run. +- **Response validation:** `extractResultsFromResponse` throws a clear, EIN-tagged error when the response is `null`/`undefined`, when `body` is missing, or when `body.results` is not an array — so a malformed lookup is never silently treated as "no records found". +- **Runtime type guard:** `isBmfRecord` verifies both `ein` and `primary_name_of_organization` are strings before an untyped result is treated as a valid `BmfRecord`. +- **`goodAsOf` derivation (`parseGivingTuesdayDate`):** GivingTuesday's `Date_Released` arrives as `YYYY_MM_DD` (month/day not necessarily zero-padded). The helper converts it to an ISO `YYYY-MM-DD` string, zero-padding month/day, and returns `null` when the input is missing or unparseable (`goodAsOf` is nullable in PDC). +- **Empty-value handling:** A field is only posted when the BMF attribute is not `undefined`, not `null`, **and not the empty string `''`** (a slightly stricter check than the Charity Navigator script, which does not exclude empty strings). Values are coerced with `.toString()` — relevant because several IRS codes arrive as `number | string`. +- **Sequential writes:** Field values are POSTed one at a time (not `Promise.all`) because the PDC API times out under concurrent POSTs to `/changemakerFieldValues`. +- **403 handling:** `postChangemakerFieldValueWarnOnForbidden` swallows HTTP 403 (logs a warning, records the `changemakerId` for an end-of-run summary) and re-throws any other error. +- **Source resolution:** `getOrCreateSource` finds an existing source with `dataProviderShortCode === "givingtuesday"`; otherwise attempts to create one, warning that this usually requires a `pdc-admin`. + +--- + +## 4. Field Mapping: GivingTuesday BMF → PDC + +The mapping is defined by the `baseFieldMap` constant in [`../src/givingTuesday.ts:67`](../src/givingTuesday.ts). Each GivingTuesday `BmfRecord` attribute is written to a PDC **base field** identified by its short code. This is a substantially richer mapping than the Charity Navigator integration (14 fields vs. 4). + +### 4.1 Fields written to PDC + +| GivingTuesday BMF attribute (`BmfRecord`) | PDC base field short code | +| ------------------------------------------------ | ----------------------------------------- | +| `primary_name_of_organization` | `organization_irs_name` | +| `street_address` | `organization_irs_address` | +| `city` | `organization_irs_city` | +| `state` | `organization_irs_state` | +| `zip_code` | `organization_irs_zip` | +| `subsection_descrip` | `organization_irs_subsection` | +| `classification_codes` | `organization_irs_classification` | +| `foundation_descrip` | `organization_irs_foundation_information` | +| `foundation_code` | `organization_foundation_code` | +| `national_taxonomy_of_exempt_entities_ntee_code` | `organization_ntee_code` | +| `deductibility_code` | `organization_deductibility_code` | +| `deductability_descrip` | `organization_deductibility_status` | +| `ruling_date` | `organization_ruling_date` | +| `tax_period` | `organization_tax_period` | + +Each written value becomes a **`ChangemakerFieldValue`** with this shape: + +| PDC field value property | Value / source | +| ------------------------ | ------------------------------------------------------------------------------ | +| `changemakerId` | Resolved via `getChangemakerByEin` (normalized EIN match) | +| `batchId` | ID of the batch opened for this run | +| `baseFieldShortCode` | From the mapping table above | +| `value` | The BMF attribute, coerced with `.toString()` | +| `goodAsOf` | Derived from `Date_Released` via `parseGivingTuesdayDate` (ISO date or `null`) | + +### 4.2 EIN — used for matching, not stored as a field + +`ein` is the join key between the two systems. It is used to match a BMF record to a PDC changemaker (after normalization) but is **not** itself written as a field value. + +### 4.3 Fields present on `BmfRecord` but NOT mapped/imported + +The `BmfRecord` interface declares a few attributes that are **not** in `baseFieldMap` and are therefore never written to PDC (though they appear in the raw output of `lookup`/`lookupFromPdc` when `--output-file` is used): + +| BMF attribute | Currently imported? | +| ---------------- | ------------------------------------------------------------------ | +| `Date_Released` | Not a field value, but reused as `goodAsOf` on every written field | +| `Date_Processed` | No | + +> Note: the IRS BMF endpoint may return additional attributes beyond those declared on `BmfRecord`; only the fields explicitly listed in `baseFieldMap` are ever written to PDC. + +--- + +## 5. Command Reference + +### `lookup` + +- **Args:** `--eins` (array, validated by `isValidEin`), `--output-file`/`--write` (optional). +- **Behavior:** Calls `getGivingTuesdayProfiles` for the given EINs (one rate-limited request each). Writes JSON to the output file if given, otherwise logs the result. No PDC interaction. No API key required. + +### `lookupFromPdc` + +- **Args:** `--pdc-api-base-url` (required), `--output-file`. +- **Behavior:** Reads all PDC changemakers, extracts + validates their EINs, looks each up in GivingTuesday. If no output file, logs which changemaker IDs were found in GivingTuesday; otherwise writes the raw response to file. **Read-only** with respect to PDC. + +### `updateAll` + +- **Args:** `--pdc-api-base-url` (required) and all `oidcOptions` (`--oidc-base-url`, `--oidc-client-id`, `--oidc-client-secret`, all required). +- **Behavior:** The full sync described in Section 3. The only subcommand that writes to PDC. + +--- + +## 6. Error Handling & Resilience Summary + +| Concern | Handling | +| ------------------------------------------ | ---------------------------------------------------------------------- | +| Invalid EINs in PDC | Filtered out and logged as a warning; valid EINs still processed | +| Rate limiting | 1100 ms sleep between per-EIN requests (no 429 backoff logic) | +| Per-EIN request failure | Caught, logged, and skipped — the run continues | +| Null/undefined or malformed response | `extractResultsFromResponse` throws a clear, EIN-tagged error | +| Untyped results | `isBmfRecord` runtime type guard (requires string `ein` + org name) | +| Ambiguous EIN → changemaker (>1 match) | Skipped with a warning; nothing written | +| Null / undefined / empty-string attributes | Skipped (not posted) | +| Unparseable / missing `Date_Released` | `goodAsOf` set to `null` | +| Numeric-or-string IRS codes | Coerced via `.toString()` before posting | +| HTTP 403 on write | Warned + changemakerId recorded; run continues; summary warning at end | +| PDC concurrency timeouts | Field values POSTed sequentially | +| Missing GivingTuesday source | Attempt to create (usually admin-only); warns it may fail | + +--- + +## 7. Testing + +Unit tests in [`../src/givingTuesday.unit.test.ts`](../src/givingTuesday.unit.test.ts) cover the four exported pure helpers: + +- **`toGivingTuesdayEin`** — strips a hyphen, leaves an already-normalized EIN unchanged, zero-pads a short EIN to nine digits. +- **`parseGivingTuesdayDate`** — converts zero-padded and single-digit `YYYY_MM_DD` to ISO; returns `null` for `null`/`undefined`/unparseable input. +- **`isBmfRecord`** — accepts a record with a string `ein` and org name; rejects a missing org name or a non-string `ein`. +- **`extractResultsFromResponse`** — returns the results array for well-formed and empty responses; throws clear, EIN-tagged errors for `null`/`undefined` responses, a missing `body`, or a non-array `results`. + +The network-facing functions (`getGivingTuesdayBmfRecords`, `getGivingTuesdayProfiles`, the command handlers, and all PDC writes) are not directly unit-tested; testability is achieved by extracting the pure normalization/parsing/extraction logic into the four exported helpers. + +--- + +## 8. Notable Constants & TODOs in the Code + +- `GT_SHORT_CODE = 'givingtuesday'` — the PDC data provider short code for GivingTuesday. +- `RATE_LIMIT_DELAY_MS = 1100` — inter-request sleep to stay under 300 requests / 5 min. +- `EIN_LENGTH = 9` — zero-pad target for normalized EINs. +- `DATE_PART_LENGTH = 2` — zero-pad width for ISO month/day parts. +- `API_BASE_URL` / `BMF_PATH` — with a code comment noting the docs' `/irs_data/` vs. the live API's `/irs-data/` path discrepancy. +- `HTTP_STATUS_FORBIDDEN = 403` — with a comment noting a shared `@pdc/http-status-codes` package should replace this once available. + +--- + +## 9. Comparison with the Charity Navigator Integration + +The two scripts share an almost identical architecture (three subcommands, source resolution, batch + sequential field posting, 403 tolerance, EIN → changemaker matching). Key differences: + +| Aspect | Charity Navigator | GivingTuesday | +| -------------------- | --------------------------------------------- | ----------------------------------------- | +| Source protocol | GraphQL (Apollo) | REST (Axios GET) | +| Source auth | Bearer API key required | None (open-access) | +| Query strategy | One paginated query filtered by a set of EINs | One request **per EIN** | +| Rate limiting | None | 1100 ms sleep between requests | +| EIN normalization | Strip hyphen | Strip hyphen **+ zero-pad to 9 digits** | +| `goodAsOf` source | `updatedAt` (as-is) | `Date_Released` parsed `YYYY_MM_DD` → ISO | +| Empty-value skip | `undefined` / `null` | `undefined` / `null` / `''` | +| Fields mapped to PDC | 4 (name, website, phone, mission) | 14 (IRS BMF address, codes, dates, etc.) | diff --git a/package.json b/package.json index 71832c5..531b612 100644 --- a/package.json +++ b/package.json @@ -21,7 +21,9 @@ "generateBaseFieldsInserts": "node --import ./register.mjs src/generateBaseFieldsInserts.ts", "generateApplicationFormJson": "node --import ./register.mjs src/generateApplicationFormJson.ts", "postProposalVersions": "node --import ./register.mjs src/postProposalVersions.ts", - "charityNavigator": "node --import ./register.mjs src/index.ts charityNavigator" + "charityNavigator": "node --import ./register.mjs src/index.ts charityNavigator", + "givingTuesday": "node --import ./register.mjs src/index.ts givingTuesday", + "getMetrics": "node --import ./register.mjs src/index.ts getMetrics" }, "author": "Open Tech Strategies", "license": "AGPL-3.0-or-later", diff --git a/src/getMetrics.ts b/src/getMetrics.ts new file mode 100644 index 0000000..914fccb --- /dev/null +++ b/src/getMetrics.ts @@ -0,0 +1,569 @@ +import { writeFile } from 'node:fs/promises'; +import { exec } from 'node:child_process'; +import { createServer, type IncomingMessage, type Server, type ServerResponse } from 'node:http'; +import { AxiosError } from 'axios'; +import { Issuer, generators } from 'openid-client'; +import { client } from './client.js'; +import { logger } from './logger.js'; +import { getToken } from './oidc.js'; +import type { CommandModule } from 'yargs'; + +// --------------------------------------------------------------------------- +// Defaults & constants +// --------------------------------------------------------------------------- + +const DEFAULT_PDC_API_BASE_URL = 'https://api.philanthropydatacommons.org/'; +const DEFAULT_OIDC_BASE_URL = 'https://auth.philanthropydatacommons.org/realms/pdc'; +// The PDC web application ("pdc metrics") is a public OIDC client. It is used +// here as the default for the interactive authorization-code flow. NOTE: for +// the browser flow to succeed, this client (or whichever `--oidc-client-id` you +// pass) MUST have the loopback redirect URI `http://localhost:/callback` +// registered in Keycloak. If it does not, either register one (a one-time admin +// task) or skip the browser flow entirely by passing `--access-token` / setting +// DS_ACCESS_TOKEN. See docs/getMetrics-report.md for details. +const DEFAULT_OIDC_CLIENT_ID = 'pdc-metrics'; +const DEFAULT_CALLBACK_PORT = 9736; +const CALLBACK_PATH = '/callback'; +// How long to wait for the caller to complete the browser login before giving up. +const AUTH_TIMEOUT_MS = 300_000; +const JSON_SPACES = 2; +// First probe requests a single item: well-behaved endpoints report the full +// `total` even at `_count=1`, so a page size of 1 keeps that common case cheap. +const PROBE_PAGE = '1'; +const PROBE_COUNT = '1'; +// A `total` of 0 or 1 is indistinguishable from a page-scoped value at +// `_count=1`, so it is treated as ambiguous and triggers a full fetch. +const AMBIGUOUS_TOTAL_MAX = 1; +// Fallback page size for the full fetch — large enough to pull every item in any +// current PDC collection in one request (mirrors the other data-scripts, which +// read entire collections with counts in the millions). +const FETCH_COUNT = '1000000'; +const FETCH_COUNT_CEILING = 1_000_000; + +// When `@pdc/http-status-codes` is ready (issues 18-20 solved), use it instead. +const HTTP_STATUS_UNAUTHORIZED = 401; +const HTTP_STATUS_FORBIDDEN = 403; +const HTTP_STATUS_NOT_FOUND = 404; +const HTTP_STATUS_OK = 200; + +// --------------------------------------------------------------------------- +// Endpoint catalogue +// --------------------------------------------------------------------------- + +interface PdcEndpoint { + /** Collection path, relative to the API base URL (no leading slash needed). */ + path: string; + /** Human-friendly label for the report. */ + label: string; + /** + * Whether the endpoint is readable anonymously. `baseFields` and + * `changemakers` are public; everything else requires a bearer token. This is + * informational only — the script reports the actual outcome regardless. + */ + public: boolean; +} + +/** + * Top-level PDC collection endpoints. Each returns a "bundle" + * (`{ entries: [...], total: }`), so the item count is read straight + * from `total`. This list mirrors the bundle types exposed by `@pdc/sdk` that + * resolve to a real top-level route on the live API; add or remove entries here + * as the API surface changes. + */ +const PDC_ENDPOINTS: PdcEndpoint[] = [ + { path: 'baseFields', label: 'Base Fields', public: true }, + { path: 'changemakers', label: 'Changemakers', public: true }, + { path: 'proposals', label: 'Proposals', public: false }, + { path: 'sources', label: 'Sources', public: false }, + { path: 'dataProviders', label: 'Data Providers', public: false }, + { path: 'funders', label: 'Funders', public: false }, + { path: 'opportunities', label: 'Opportunities', public: false }, + { path: 'users', label: 'Users', public: false }, + { path: 'changemakerFieldValues', label: 'Changemaker Field Values', public: false }, + { path: 'changemakerFieldValueBatches', label: 'Changemaker Field Value Batches', public: false }, + { path: 'changemakerProposals', label: 'Changemaker–Proposal Links', public: false }, + { path: 'applicationForms', label: 'Application Forms', public: false }, + { path: 'terminologySets', label: 'Terminology Sets', public: false }, + { path: 'permissionGrants', label: 'Permission Grants', public: false }, + { path: 'files', label: 'Files', public: false }, +]; + +// --------------------------------------------------------------------------- +// Metric collection +// --------------------------------------------------------------------------- + +type MetricStatus = 'ok' | 'unauthorized' | 'forbidden' | 'not_found' | 'error'; + +interface EndpointMetric { + path: string; + label: string; + /** Item count from the endpoint's `total`, or null when it could not be read. */ + count: number | null; + status: MetricStatus; + note: string; +} + +/** The shape we care about in a bundle response; runtime data is untyped. */ +interface CountableBundle { + total?: number; + entries?: unknown[]; +} + +/** Extract the numeric `total` and the entry-array length from a bundle, or null when absent. */ +const readBundle = (data: CountableBundle): { total: number | null; entriesLength: number | null } => ({ + total: typeof data.total === 'number' ? data.total : null, + entriesLength: Array.isArray(data.entries) ? data.entries.length : null, +}); + +/** Map an HTTP status from a failed request to a metric status + note. */ +const classifyHttpError = (status: number | undefined): { status: MetricStatus; note: string } => { + if (status === HTTP_STATUS_UNAUTHORIZED) { + return { status: 'unauthorized', note: 'Authentication required (no valid token supplied)' }; + } + if (status === HTTP_STATUS_FORBIDDEN) { + return { status: 'forbidden', note: 'Token lacks permission to read this collection' }; + } + if (status === HTTP_STATUS_NOT_FOUND) { + return { status: 'not_found', note: 'Endpoint not found (route may have moved or been removed)' }; + } + return { status: 'error', note: `Unexpected HTTP status ${String(status)}` }; +}; + +/** + * Turn a fully-fetched bundle's `total` / entry count into a count + status + + * note. The reported count is the larger of `total` and the number of entries + * returned, so neither a missing/short `total` nor a truncated page undercounts. + */ +const resolveFullCount = ( + total: number | null, + entriesLength: number | null, +): { count: number | null; status: MetricStatus; note: string } => { + if (total === null && entriesLength === null) { + return { count: null, status: 'error', note: 'Response had neither a `total` nor an `entries` array' }; + } + const count = Math.max(total ?? 0, entriesLength ?? 0); + if (entriesLength !== null && entriesLength >= FETCH_COUNT_CEILING) { + return { + count, + status: 'ok', + note: `Count is a floor; collection has at least ${FETCH_COUNT_CEILING.toLocaleString('en-US')} items`, + }; + } + const note = total === null ? 'No numeric `total` in response; counted the entries returned' : ''; + return { count, status: 'ok', note }; +}; + +/** + * Read the item count for a single endpoint. + * + * A cheap `_count=1` probe is enough for well-behaved endpoints: they report the + * full `total` even on a one-item page, so any `total > 1` is taken as-is. When + * the probe's `total` is missing, 0, or 1 — indistinguishable from a page-scoped + * value — the whole collection is fetched (`_count=1000000`) and interpreted by + * `resolveFullCount`. + * + * Returns a fully-populated metric, never throws: transport and HTTP errors are + * folded into the metric's status. + */ +const getEndpointCount = async ( + baseUrl: string, + endpoint: PdcEndpoint, + accessToken: string | undefined, +): Promise => { + const url = new URL(endpoint.path, baseUrl).toString(); + const headers = accessToken === undefined ? {} : { authorization: `Bearer ${accessToken}` }; + const metric = (count: number | null, status: MetricStatus, note: string): EndpointMetric => ({ + path: endpoint.path, + label: endpoint.label, + count, + status, + note, + }); + try { + const probe = await client.get(url, { + headers, + params: { _page: PROBE_PAGE, _count: PROBE_COUNT }, + }); + const probeRead = readBundle(probe.data); + logger.debug( + { endpoint: endpoint.path, total: probeRead.total, entries: probeRead.entriesLength }, + 'probe response (_count=1)', + ); + if (probeRead.total !== null && probeRead.total > AMBIGUOUS_TOTAL_MAX) { + // Unambiguous grand total (a one-item page could never report >1), trust it. + return metric(probeRead.total, 'ok', ''); + } + + // Ambiguous or missing total: fetch the whole collection and count directly. + const full = await client.get(url, { + headers, + params: { _page: PROBE_PAGE, _count: FETCH_COUNT }, + }); + const { total, entriesLength } = readBundle(full.data); + logger.debug({ endpoint: endpoint.path, total, entries: entriesLength }, `full response (_count=${FETCH_COUNT})`); + const resolved = resolveFullCount(total, entriesLength); + return metric(resolved.count, resolved.status, resolved.note); + } catch (error: unknown) { + if (error instanceof AxiosError) { + const { status, note } = classifyHttpError(error.response?.status); + return metric(null, status, note); + } + const message = error instanceof Error ? error.message : String(error); + return metric(null, 'error', message); + } +}; + +/** + * Collect counts for every endpoint. Requests are issued sequentially rather + * than via `Promise.all`, matching the other data-scripts' gentleness toward + * the PDC API. + */ +const collectMetrics = async ( + baseUrl: string, + endpoints: PdcEndpoint[], + accessToken: string | undefined, +): Promise => { + const metrics: EndpointMetric[] = []; + /* eslint-disable no-await-in-loop -- sequential reads to avoid hammering the + PDC API with concurrent requests. */ + for (const endpoint of endpoints) { + const metric = await getEndpointCount(baseUrl, endpoint, accessToken); + logger.info(`${endpoint.label}: ${metric.count ?? metric.status}`); + metrics.push(metric); + } + /* eslint-enable no-await-in-loop */ + return metrics; +}; + +/** + * Return a copy of the metrics sorted alphabetically (case-insensitively) by + * endpoint path — the primary column shown in every output format. + */ +const sortMetrics = (metrics: EndpointMetric[]): EndpointMetric[] => + [...metrics].sort((a, b) => a.path.localeCompare(b.path, 'en-US', { sensitivity: 'base' })); + +// --------------------------------------------------------------------------- +// Interactive (browser) authentication — OAuth 2.0 authorization code + PKCE +// --------------------------------------------------------------------------- + +/** Best-effort cross-platform "open this URL in the default browser". */ +const openBrowser = (url: string): void => { + const command = + process.platform === 'win32' + ? `start "" "${url}"` + : process.platform === 'darwin' + ? `open "${url}"` + : `xdg-open "${url}"`; + exec(command, (error) => { + if (error !== null) { + logger.warn(`Could not open a browser automatically. Please open this URL manually:\n${url}`); + } + }); +}; + +/** HTML shown in the browser tab once the callback has been received. */ +const CALLBACK_SUCCESS_HTML = + 'PDC authentication' + + '

Authentication complete

' + + '

You may close this tab and return to the terminal.

'; + +/** + * Run the OAuth 2.0 authorization-code flow (with PKCE) against the PDC + * Keycloak realm, using the caller's browser. Spins up a throwaway localhost + * HTTP server to catch the redirect, exchanges the code for a token, and + * resolves with the access token. Rejects on timeout or auth error. + */ +const authenticateInteractively = async (oidcBaseUrl: string, clientId: string, port: number): Promise => { + const issuer = await Issuer.discover(oidcBaseUrl); + logger.debug(`Discovered OIDC issuer at ${issuer.metadata.issuer}`); + const redirectUri = `http://localhost:${String(port)}${CALLBACK_PATH}`; + + const oidcClient = new issuer.Client({ + client_id: clientId, + redirect_uris: [redirectUri], + response_types: ['code'], + // Public client: no secret, PKCE protects the exchange. + token_endpoint_auth_method: 'none', + }); + + const codeVerifier = generators.codeVerifier(); + const codeChallenge = generators.codeChallenge(codeVerifier); + const state = generators.state(); + const authorizationUrl = oidcClient.authorizationUrl({ + scope: 'openid', + code_challenge: codeChallenge, + code_challenge_method: 'S256', + state, + }); + + // A deferred lets the request handler (an event callback) settle the outcome. + let resolveToken: (token: string) => void = () => undefined; + let rejectToken: (error: Error) => void = () => undefined; + /* eslint-disable-next-line promise/avoid-new -- bridging Node's event-driven + HTTP server, a timeout, and the token exchange into a single awaitable. */ + const tokenPromise = new Promise((resolve, reject) => { + resolveToken = resolve; + rejectToken = reject; + }); + + const server: Server = createServer((req: IncomingMessage, res: ServerResponse) => { + const requestUrl = new URL(req.url ?? '', redirectUri); + if (requestUrl.pathname !== CALLBACK_PATH) { + res.writeHead(HTTP_STATUS_NOT_FOUND); + res.end(); + return; + } + const params = oidcClient.callbackParams(req); + oidcClient + .callback(redirectUri, params, { code_verifier: codeVerifier, state }) + .then((tokenSet) => { + res.writeHead(HTTP_STATUS_OK, { 'content-type': 'text/html' }); + res.end(CALLBACK_SUCCESS_HTML); + const { access_token: accessToken } = tokenSet; + if (accessToken === undefined) { + rejectToken(new Error('Authorization succeeded but no access token was returned')); + } else { + resolveToken(accessToken); + } + }) + .catch((err: unknown) => { + res.writeHead(HTTP_STATUS_OK, { 'content-type': 'text/html' }); + res.end('

Authentication failed. Check the terminal for details.

'); + rejectToken(err instanceof Error ? err : new Error(String(err))); + }); + }); + + const timer = setTimeout(() => { + rejectToken(new Error(`Timed out after ${String(AUTH_TIMEOUT_MS)}ms waiting for the browser login to complete`)); + }, AUTH_TIMEOUT_MS); + server.on('error', (err) => { + rejectToken(err); + }); + server.listen(port, () => { + logger.info(`Waiting for you to sign in. Opening your browser to:\n${authorizationUrl}`); + openBrowser(authorizationUrl); + }); + + try { + return await tokenPromise; + } finally { + clearTimeout(timer); + server.close(); + } +}; + +// --------------------------------------------------------------------------- +// Report rendering +// --------------------------------------------------------------------------- + +/** Format an item count for display; failed reads render as an em dash. */ +const formatCount = (count: number | null): string => (count === null ? '—' : count.toLocaleString('en-US')); + +/** Quote a CSV field when it contains a comma, quote, or newline (RFC 4180). */ +const csvField = (value: string): string => (/[",\n\r]/v.test(value) ? `"${value.replace(/"/gv, '""')}"` : value); + +const CSV_HEADER = ['endpoint', 'label', 'count', 'status', 'note']; + +// Label for the header line that names the environment a report was run against. +const REPORT_HEADER_LABEL = 'PDC API base URL'; + +/** Render metrics as RFC-4180 CSV, with a leading `#` comment naming the environment. */ +const renderCsv = (metrics: EndpointMetric[], baseUrl: string): string => { + const rows = metrics.map((m) => + [m.path, m.label, m.count === null ? '' : String(m.count), m.status, m.note].map(csvField).join(','), + ); + return [`# ${REPORT_HEADER_LABEL}: ${baseUrl}`, CSV_HEADER.join(','), ...rows].join('\n'); +}; + +/** Render metrics as a plain-text, column-aligned table, with a header line naming the environment. */ +const renderTable = (metrics: EndpointMetric[], baseUrl: string): string => { + const header = { endpoint: 'ENDPOINT', count: 'COUNT', status: 'STATUS', note: 'NOTE' }; + const rows = metrics.map((m) => ({ + endpoint: `/${m.path}`, + count: formatCount(m.count), + status: m.status, + note: m.note, + })); + const all = [header, ...rows]; + const width = (key: keyof typeof header): number => Math.max(...all.map((r) => r[key].length)); + const endpointWidth = width('endpoint'); + const countWidth = width('count'); + const statusWidth = width('status'); + const line = (r: (typeof all)[number]): string => + `${r.endpoint.padEnd(endpointWidth)} ${r.count.padStart(countWidth)} ${r.status.padEnd(statusWidth)} ${r.note}`.trimEnd(); + return [`${REPORT_HEADER_LABEL}: ${baseUrl}`, '', ...all.map(line)].join('\n'); +}; + +/** Summary counters describing a metrics run. */ +interface MetricsSummary { + endpointCount: number; + okCount: number; + failedCount: number; + itemTotal: number; +} + +const summarize = (metrics: EndpointMetric[]): MetricsSummary => { + const ok = metrics.filter((m) => m.status === 'ok'); + return { + endpointCount: metrics.length, + okCount: ok.length, + failedCount: metrics.length - ok.length, + itemTotal: ok.reduce((sum, m) => sum + (m.count ?? 0), 0), + }; +}; + +/** Render metrics as JSON, including the environment URL and a summary block. */ +const renderJson = (metrics: EndpointMetric[], baseUrl: string): string => + JSON.stringify( + { pdcApiBaseUrl: baseUrl, generatedAt: new Date().toISOString(), summary: summarize(metrics), metrics }, + null, + JSON_SPACES, + ); + +type OutputFormat = 'table' | 'csv' | 'json'; + +const renderReport = (metrics: EndpointMetric[], format: OutputFormat, baseUrl: string): string => { + if (format === 'csv') { + return renderCsv(metrics, baseUrl); + } + if (format === 'json') { + return renderJson(metrics, baseUrl); + } + return renderTable(metrics, baseUrl); +}; + +// --------------------------------------------------------------------------- +// Command module +// --------------------------------------------------------------------------- + +interface GetMetricsCommandArgs { + 'pdc-api-base-url': string; + 'oidc-base-url': string; + 'oidc-client-id': string; + 'oidc-client-secret'?: string; + format: OutputFormat; + 'callback-port': number; + 'access-token'?: string; + 'skip-auth': boolean; + outputFile?: string; +} + +/** + * Decide which access token (if any) to use, in priority order: + * 1. `--skip-auth` → no token (public endpoints only). + * 2. `--access-token`/DS_ACCESS_TOKEN → use it as-is. + * 3. `--oidc-client-secret` present → non-interactive OIDC client-credentials + * grant (reuses `getToken` from oidc.ts) — a single-command, headless login. + * 4. otherwise → the interactive browser (authorization-code + PKCE) login. + */ +const resolveAccessToken = async (args: { + skipAuth: boolean; + accessToken?: string; + oidcBaseUrl: string; + oidcClientId: string; + oidcClientSecret?: string; + callbackPort: number; +}): Promise => { + if (args.skipAuth) { + logger.warn('Running with --skip-auth: only public endpoints will report a count'); + return undefined; + } + const { accessToken, oidcClientSecret } = args; + if (accessToken !== undefined && accessToken !== '') { + logger.info('Using the supplied access token (skipping interactive login)'); + return accessToken; + } + if (oidcClientSecret !== undefined && oidcClientSecret !== '') { + logger.info('Authenticating with the OIDC client-credentials grant (no browser)'); + const token = await getToken(args.oidcBaseUrl, args.oidcClientId, oidcClientSecret); + return token.access_token; + } + const token = await authenticateInteractively(args.oidcBaseUrl, args.oidcClientId, args.callbackPort); + logger.info('Authentication successful'); + return token; +}; + +const getMetrics: CommandModule = { + command: 'getMetrics', + describe: 'Read the PDC API and report a count of items per endpoint', + builder: (y) => + y + .option('pdc-api-base-url', { + describe: 'Location of the PDC API', + default: DEFAULT_PDC_API_BASE_URL, + type: 'string', + }) + .option('oidc-base-url', { + describe: 'OpenID Connect authority (realm) base URL', + default: DEFAULT_OIDC_BASE_URL, + type: 'string', + }) + .option('oidc-client-id', { + describe: 'OIDC client ID for the interactive browser login or the client-credentials grant', + default: DEFAULT_OIDC_CLIENT_ID, + type: 'string', + }) + .option('oidc-client-secret', { + describe: + 'OIDC client secret; when set, authenticate non-interactively via the client-credentials grant instead of the browser (can also be set via DS_OIDC_CLIENT_SECRET)', + type: 'string', + }) + .option('format', { + describe: 'Output format for the report', + choices: ['table', 'csv', 'json'] as const, + default: 'table' as const, + }) + .option('callback-port', { + describe: 'Local port for the OAuth redirect (loopback) listener', + default: DEFAULT_CALLBACK_PORT, + type: 'number', + }) + .option('access-token', { + describe: + 'Use this bearer token instead of the interactive browser login (can also be set via DS_ACCESS_TOKEN)', + type: 'string', + }) + .option('skip-auth', { + describe: 'Do not authenticate; only public endpoints will report a count', + default: false, + type: 'boolean', + }) + .option('output-file', { + alias: 'write', + describe: 'Write the report to this file instead of logging it', + normalize: true, + type: 'string', + }), + handler: async (args) => { + const accessToken = await resolveAccessToken(args); + const metrics = sortMetrics(await collectMetrics(args.pdcApiBaseUrl, PDC_ENDPOINTS, accessToken)); + const summary = summarize(metrics); + const report = renderReport(metrics, args.format, args.pdcApiBaseUrl); + + if (args.outputFile === undefined || args.outputFile === '') { + logger.info(`PDC metrics report (${args.format}):\n${report}`); + } else { + await writeFile(args.outputFile, report); + logger.info(`Wrote PDC metrics report to ${args.outputFile}`); + } + logger.info( + `Counted ${String(summary.itemTotal)} items across ${String(summary.okCount)}/${String(summary.endpointCount)} endpoints (${String(summary.failedCount)} unavailable)`, + ); + }, +}; + +export { + type EndpointMetric, + type MetricStatus, + collectMetrics, + csvField, + formatCount, + getEndpointCount, + getMetrics, + renderCsv, + renderJson, + renderTable, + resolveFullCount, + sortMetrics, + summarize, +}; diff --git a/src/getMetrics.unit.test.ts b/src/getMetrics.unit.test.ts new file mode 100644 index 0000000..d80951f --- /dev/null +++ b/src/getMetrics.unit.test.ts @@ -0,0 +1,240 @@ +import { describe, expect, it } from '@jest/globals'; +import { + type EndpointMetric, + csvField, + formatCount, + renderCsv, + renderJson, + renderTable, + resolveFullCount, + sortMetrics, + summarize, +} from './getMetrics.js'; + +// A small, representative set of metrics: two successful reads and one that +// failed authentication (so `count` is null and there is a note). +const sampleMetrics: EndpointMetric[] = [ + { path: 'baseFields', label: 'Base Fields', count: 282, status: 'ok', note: '' }, + { path: 'changemakers', label: 'Changemakers', count: 17, status: 'ok', note: '' }, + { + path: 'proposals', + label: 'Proposals', + count: null, + status: 'unauthorized', + note: 'Authentication required (no valid token supplied)', + }, +]; + +const BASE_URL = 'https://api.philanthropydatacommons.org/'; + +describe('csvField', () => { + it('leaves a plain value unquoted', () => { + expect(csvField('Base Fields')).toBe('Base Fields'); + }); + + it('quotes a value containing a comma', () => { + expect(csvField('a,b')).toBe('"a,b"'); + }); + + it('quotes and doubles an embedded double quote', () => { + expect(csvField('a"b')).toBe('"a""b"'); + }); + + it('quotes a value containing a newline', () => { + expect(csvField('a\nb')).toBe('"a\nb"'); + }); + + it('leaves an empty string unquoted', () => { + expect(csvField('')).toBe(''); + }); +}); + +describe('formatCount', () => { + it('formats a number with thousands separators', () => { + expect(formatCount(1234567)).toBe('1,234,567'); + }); + + it('formats a small number without separators', () => { + expect(formatCount(17)).toBe('17'); + }); + + it('renders null as an em dash', () => { + expect(formatCount(null)).toBe('—'); + }); + + it('renders zero as "0", not an em dash', () => { + expect(formatCount(0)).toBe('0'); + }); +}); + +describe('summarize', () => { + it('counts ok / failed endpoints and sums the item totals', () => { + expect(summarize(sampleMetrics)).toStrictEqual({ + endpointCount: 3, + okCount: 2, + failedCount: 1, + itemTotal: 299, + }); + }); + + it('reports zeroes for an empty metrics list', () => { + expect(summarize([])).toStrictEqual({ + endpointCount: 0, + okCount: 0, + failedCount: 0, + itemTotal: 0, + }); + }); + + it('ignores the count of non-ok endpoints in the item total', () => { + const metrics: EndpointMetric[] = [ + { path: 'baseFields', label: 'Base Fields', count: 10, status: 'ok', note: '' }, + { path: 'files', label: 'Files', count: null, status: 'forbidden', note: 'nope' }, + ]; + expect(summarize(metrics).itemTotal).toBe(10); + }); +}); + +describe('sortMetrics', () => { + const unsorted: EndpointMetric[] = [ + { path: 'proposals', label: 'Proposals', count: 3, status: 'ok', note: '' }, + { path: 'baseFields', label: 'Base Fields', count: 282, status: 'ok', note: '' }, + { path: 'applicationForms', label: 'Application Forms', count: 11, status: 'ok', note: '' }, + ]; + + it('orders metrics alphabetically by path', () => { + expect(sortMetrics(unsorted).map((m) => m.path)).toStrictEqual(['applicationForms', 'baseFields', 'proposals']); + }); + + it('does not mutate the input array', () => { + const before = unsorted.map((m) => m.path); + sortMetrics(unsorted); + expect(unsorted.map((m) => m.path)).toStrictEqual(before); + }); + + it('returns an empty array unchanged', () => { + expect(sortMetrics([])).toStrictEqual([]); + }); +}); + +describe('resolveFullCount', () => { + it('errors when both total and entries are absent', () => { + expect(resolveFullCount(null, null)).toStrictEqual({ + count: null, + status: 'error', + note: 'Response had neither a `total` nor an `entries` array', + }); + }); + + it('counts entries when total is absent (the applicationForms case)', () => { + // total missing, 11 entries fetched → reports 11, not 1. + expect(resolveFullCount(null, 11)).toStrictEqual({ + count: 11, + status: 'ok', + note: 'No numeric `total` in response; counted the entries returned', + }); + }); + + it('prefers a larger entries count over a short/page-scoped total', () => { + // total reported as 1 but 11 entries were actually returned → 11 wins. + expect(resolveFullCount(1, 11)).toStrictEqual({ count: 11, status: 'ok', note: '' }); + }); + + it('prefers a larger total when entries were truncated below it', () => { + expect(resolveFullCount(500, 200)).toStrictEqual({ count: 500, status: 'ok', note: '' }); + }); + + it('uses total as-is when it matches the entries returned', () => { + expect(resolveFullCount(282, 282)).toStrictEqual({ count: 282, status: 'ok', note: '' }); + }); + + it('reports zero for a genuinely empty collection', () => { + expect(resolveFullCount(0, 0)).toStrictEqual({ count: 0, status: 'ok', note: '' }); + }); + + it('flags the count as a floor when entries hit the fetch ceiling', () => { + const result = resolveFullCount(null, 1_000_000); + expect(result.count).toBe(1_000_000); + expect(result.status).toBe('ok'); + expect(result.note).toMatch(/at least 1,000,000 items/v); + }); +}); + +describe('renderCsv', () => { + it('starts with a comment line naming the environment', () => { + expect(renderCsv(sampleMetrics, BASE_URL).split('\n')[0]).toBe(`# PDC API base URL: ${BASE_URL}`); + }); + + it('puts the CSV header row directly after the comment line', () => { + expect(renderCsv(sampleMetrics, BASE_URL).split('\n')[1]).toBe('endpoint,label,count,status,note'); + }); + + it('renders an ok row with its count and an empty note', () => { + const lines = renderCsv(sampleMetrics, BASE_URL).split('\n'); + expect(lines[2]).toBe('baseFields,Base Fields,282,ok,'); + }); + + it('renders a failed row with an empty count field and its note', () => { + const lines = renderCsv(sampleMetrics, BASE_URL).split('\n'); + expect(lines[4]).toBe('proposals,Proposals,,unauthorized,Authentication required (no valid token supplied)'); + }); + + it('quotes a note that contains a comma', () => { + const metrics: EndpointMetric[] = [ + { path: 'files', label: 'Files', count: null, status: 'error', note: 'boom, it broke' }, + ]; + expect(renderCsv(metrics, BASE_URL).split('\n')[2]).toBe('files,Files,,error,"boom, it broke"'); + }); +}); + +describe('renderTable', () => { + it('starts with a header line naming the environment, then a blank line', () => { + const lines = renderTable(sampleMetrics, BASE_URL).split('\n'); + expect(lines[0]).toBe(`PDC API base URL: ${BASE_URL}`); + expect(lines[1]).toBe(''); + }); + + it('includes a column header with all four columns', () => { + const [, , columnHeader] = renderTable(sampleMetrics, BASE_URL).split('\n'); + expect(columnHeader).toContain('ENDPOINT'); + expect(columnHeader).toContain('COUNT'); + expect(columnHeader).toContain('STATUS'); + expect(columnHeader).toContain('NOTE'); + }); + + it('prefixes each endpoint path with a slash and shows its formatted count', () => { + const table = renderTable(sampleMetrics, BASE_URL); + expect(table).toContain('/baseFields'); + expect(table).toContain('282'); + expect(table).toContain('ok'); + }); + + it('renders an unavailable endpoint with an em dash and its note', () => { + const table = renderTable(sampleMetrics, BASE_URL); + expect(table).toContain('/proposals'); + expect(table).toContain('—'); + expect(table).toContain('unauthorized'); + }); + + it('left-pads the count column so values are right-aligned', () => { + // "282" is width 3 (the widest count once the header "COUNT" is considered), + // so the single-item "17" must be right-aligned under it. + const line = renderTable(sampleMetrics, BASE_URL) + .split('\n') + .find((l) => l.includes('/changemakers')); + expect(line).toBeDefined(); + expect(line).toContain(' 17 '); + }); +}); + +describe('renderJson', () => { + it('produces valid JSON with the environment URL, summary, and metrics', () => { + const parsed: unknown = JSON.parse(renderJson(sampleMetrics, BASE_URL)); + expect(parsed).toStrictEqual({ + pdcApiBaseUrl: BASE_URL, + generatedAt: expect.any(String), + summary: { endpointCount: 3, okCount: 2, failedCount: 1, itemTotal: 299 }, + metrics: sampleMetrics, + }); + }); +}); diff --git a/src/givingTuesday.ts b/src/givingTuesday.ts new file mode 100644 index 0000000..d5a1315 --- /dev/null +++ b/src/givingTuesday.ts @@ -0,0 +1,418 @@ +import { writeFile } from 'node:fs/promises'; +import { setTimeout } from 'node:timers/promises'; +import { AxiosError } from 'axios'; +import { client } from './client.js'; +import { isValidEin } from './ein.js'; +import { logger } from './logger.js'; +import { type AccessTokenSet, getToken, oidcOptions } from './oidc.js'; +import { + getChangemakers, + getSources, + postChangemakerFieldValue, + postChangemakerFieldValueBatch, + postSource, + type WritableChangemakerFieldValue, +} from './pdc-api.js'; +import type { CommandModule } from 'yargs'; +import type { Changemaker, ChangemakerBundle, Source } from '@pdc/sdk'; + +const GT_SHORT_CODE = 'givingtuesday'; +const JSON_SPACES = 2; +// When `@pdc/http-status-codes` is ready (issues 18-20 solved), use it instead. +const HTTP_STATUS_FORBIDDEN = 403; +// GivingTuesday's open-access API allows 300 requests per 5 minutes (1/sec). +// Sleep between per-EIN requests rather than implementing 429 backoff, matching +// the approach used for Candid. +const RATE_LIMIT_DELAY_MS = 1100; +// GivingTuesday requires zero-padded, 9-digit EINs with no hyphens. +const EIN_LENGTH = 9; +// Month and day components are padded to two characters for ISO dates. +const DATE_PART_LENGTH = 2; + +const API_BASE_URL = 'https://990-infrastructure.gtdata.org'; +// The published endpoint table renders `/irs_data/`, but the working sample +// requests (and the live API) use the hyphenated `/irs-data/` path. +const BMF_PATH = '/irs-data/bmf'; + +/** + * A single record from the IRS Business Master File (BMF) endpoint. All fields + * beyond `ein` are optional and, at runtime, may be null: the IRS data is + * sparse and the API is untyped. Numeric IRS codes (ruling_date, tax_period, + * classification_codes, ...) arrive as either numbers or strings depending on + * the field, so they are typed as `number | string` and stringified on the way + * into the PDC. + */ +interface BmfRecord { + ein: string; + primary_name_of_organization?: string | null; + street_address?: string | null; + city?: string | null; + state?: string | null; + zip_code?: string | null; + subsection_descrip?: string | null; + classification_codes?: number | string | null; + foundation_code?: string | null; + foundation_descrip?: string | null; + national_taxonomy_of_exempt_entities_ntee_code?: string | null; + deductibility_code?: number | string | null; + deductability_descrip?: string | null; + ruling_date?: number | string | null; + tax_period?: number | string | null; + // `YYYY_MM_DD`, e.g. `2024_02_13`; used to derive each field value's goodAsOf. + Date_Released?: string | null; + Date_Processed?: string | null; +} + +/** Map from GivingTuesday BMF attribute name to PDC base field short code */ +const baseFieldMap: Array<[keyof BmfRecord, string]> = [ + ['primary_name_of_organization', 'organization_irs_name'], + ['street_address', 'organization_irs_address'], + ['city', 'organization_irs_city'], + ['state', 'organization_irs_state'], + ['zip_code', 'organization_irs_zip'], + ['subsection_descrip', 'organization_irs_subsection'], + ['classification_codes', 'organization_irs_classification'], + ['foundation_descrip', 'organization_irs_foundation_information'], + ['foundation_code', 'organization_foundation_code'], + ['national_taxonomy_of_exempt_entities_ntee_code', 'organization_ntee_code'], + ['deductibility_code', 'organization_deductibility_code'], + ['deductability_descrip', 'organization_deductibility_status'], + ['ruling_date', 'organization_ruling_date'], + ['tax_period', 'organization_tax_period'], +]; + +interface GivingTuesdayResponseBody { + query: string; + no_results: number; + results: T[]; +} + +interface GivingTuesdayResponse { + statusCode: number; + body: GivingTuesdayResponseBody; +} + +const isBmfRecord = (result: object): result is BmfRecord => { + /* eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- + Defensive runtime validation of untyped REST response data requires asserting + a generic object to a record to inspect its properties. */ + const obj = result as Record; + return typeof obj.ein === 'string' && typeof obj.primary_name_of_organization === 'string'; +}; + +/** + * Convert GivingTuesday's `Date_Released`/`Date_Processed` format (`YYYY_MM_DD`, + * with month/day not necessarily zero-padded) to an ISO `YYYY-MM-DD` date for + * use as a field value's goodAsOf. Returns null when the input is missing or + * cannot be parsed, since goodAsOf is nullable in the PDC. + */ +const parseGivingTuesdayDate = (value: string | null | undefined): string | null => { + if (value === null || value === undefined) { + return null; + } + const match = /^(?\d{4})_(?\d{1,2})_(?\d{1,2})$/v.exec(value); + const { groups } = match ?? {}; + if (groups === undefined) { + return null; + } + const { year, month, day } = groups; + if (year === undefined || month === undefined || day === undefined) { + return null; + } + return `${year}-${month.padStart(DATE_PART_LENGTH, '0')}-${day.padStart(DATE_PART_LENGTH, '0')}`; +}; + +/** Normalize an EIN to the zero-padded, hyphen-free 9-digit form GivingTuesday expects. */ +const toGivingTuesdayEin = (ein: string): string => ein.replace('-', '').padStart(EIN_LENGTH, '0'); + +/** + * Validate an untyped GivingTuesday response and return its `results` array. + * Apollo-style, the transport types the payload as parsed, but a partial or + * errored response can carry a missing body/results; throw a clear error so a + * malformed lookup is never mistaken for "no records found". + */ +const extractResultsFromResponse = (response: GivingTuesdayResponse | null | undefined, ein: string): T[] => { + if (response === undefined || response === null) { + throw new Error(`GivingTuesday query returned no data for EIN ${ein}`); + } + const { body } = response; + /* eslint-disable-next-line @typescript-eslint/no-unnecessary-condition -- + body is typed as present but the untyped runtime response can omit it. */ + if (body === undefined || body === null || !Array.isArray(body.results)) { + throw new Error(`GivingTuesday returned a malformed response for EIN ${ein}: ${JSON.stringify(response)}`); + } + return body.results; +}; + +/** Fetch the BMF record(s) for a single EIN from the GivingTuesday API. */ +const getGivingTuesdayBmfRecords = async (ein: string): Promise => { + const gtEin = toGivingTuesdayEin(ein); + logger.info(`Looking up EIN ${gtEin} in GivingTuesday BMF API`); + const { data } = await client.get>(`${API_BASE_URL}${BMF_PATH}`, { + params: { ein: gtEin }, + }); + return extractResultsFromResponse(data, gtEin); +}; + +/** + * Fetch BMF records for many EINs, one request per EIN, sleeping between + * requests to stay under GivingTuesday's rate limit. A failure for one EIN is + * logged and skipped so a single bad lookup doesn't abort the whole run. + */ +const getGivingTuesdayProfiles = async (eins: string[]): Promise<{ data: { results: BmfRecord[] } }> => { + const results: BmfRecord[] = []; + /* eslint-disable no-await-in-loop -- sequential, rate-limited requests: the + GivingTuesday API is one EIN per request and capped at 300 requests / 5 min. */ + for (const [i, ein] of eins.entries()) { + try { + const records = await getGivingTuesdayBmfRecords(ein); + results.push(...records); + logger.debug(`[${i + 1}/${eins.length}] Fetched ${records.length} BMF record(s) for ${ein}`); + } catch (error: unknown) { + logger.error({ error }, `Error loading GivingTuesday data for ${ein}`); + } + await setTimeout(RATE_LIMIT_DELAY_MS); + } + /* eslint-enable no-await-in-loop */ + return { data: { results } }; +}; + +interface LookupCommandArgs { + eins: string[]; + outputFile?: string; +} + +interface LookupFromPdcCommandArgs { + 'pdc-api-base-url': string; + outputFile?: string; +} + +interface UpdateAllCommandArgs { + 'oidc-base-url': string; + 'oidc-client-id': string; + 'oidc-client-secret': string; + 'pdc-api-base-url': string; +} + +const lookupCommand: CommandModule = { + command: 'lookup', + describe: 'Fetch and display GivingTuesday BMF information about organizations by EIN', + builder: (y) => + y + .option('output-file', { + alias: 'write', + describe: 'Write organization information to the specified JSON file', + normalize: true, + type: 'string', + }) + .option('eins', { + string: true, + describe: 'US tax IDs of organizations to look up', + type: 'array', + default: [], + }) + .check(({ eins }) => !new Set(eins.map(isValidEin)).has(false)), + handler: async (args) => { + const result = await getGivingTuesdayProfiles(args.eins).catch((err: unknown) => { + logger.error(err, 'error calling GivingTuesday api'); + throw err; + }); + + if (args.outputFile === undefined || args.outputFile === '') { + logger.info({ result }, 'GivingTuesday result'); + } else { + await writeFile(args.outputFile, JSON.stringify(result, null, JSON_SPACES)); + logger.info(`Wrote GivingTuesday data for ${JSON.stringify(args.eins)} to ${JSON.stringify(args.outputFile)}`); + } + }, +}; + +const getChangemakerByEin = (ein: string, changemakers: ChangemakerBundle): Changemaker | null => { + // Make the comparison with hyphens stripped and zero-padded to match the + // normalized EIN GivingTuesday echoes back in its records. + const normalized = toGivingTuesdayEin(ein); + const matches = changemakers.entries.filter((c) => toGivingTuesdayEin(c.taxId) === normalized); + if (matches.length > 1) { + logger.warn(`Found multiple changemakers with EIN ${ein}, not returning any.`); + return null; + } + if (matches.length < 1) { + logger.info(`Found no changemaker with EIN ${ein}`); + return null; + } + if (matches.length === 1 && matches[0] !== undefined) { + return matches[0]; + } + throw new Error('How could this have happened?'); +}; + +/** Light wrapper around `postChangemakerFieldValue` that logs warning on HTTP 403 */ +const postChangemakerFieldValueWarnOnForbidden = async ( + baseUrl: string, + token: AccessTokenSet, + data: WritableChangemakerFieldValue, + warnedChangemakers: Set, // Mutated! This is for observation/logs, not control! +): Promise => { + try { + const fieldValue = await postChangemakerFieldValue(baseUrl, token, data); + logger.info(`Added changemaker field value: ${JSON.stringify(fieldValue)}`); + } catch (e: unknown) { + if (e instanceof AxiosError && e.status === HTTP_STATUS_FORBIDDEN) { + logger.warn(`No permission (403) to create ${JSON.stringify(data)}`); + warnedChangemakers.add(data.changemakerId); + } else { + throw e; + } + } +}; + +const lookupFromPdcCommand: CommandModule = { + command: 'lookupFromPdc', + describe: 'Fetch and display GivingTuesday information about organizations present in PDC', + builder: (y) => + y + .option('output-file', { + alias: 'write', + describe: 'Write organization information to the specified JSON file', + normalize: true, + type: 'string', + }) + .option('pdc-api-base-url', { + describe: 'Location of PDC API', + demandOption: true, + type: 'string', + }), + handler: async (args) => { + const { pdcApiBaseUrl } = args; + if (pdcApiBaseUrl === '') { + throw new Error('Missing required argument: pdc-api-base-url'); + } + const changemakers = await getChangemakers(pdcApiBaseUrl); + const eins = changemakers.entries.flatMap((c) => c.taxId); + const validEins = eins.filter(isValidEin); + const invalidEins = eins.filter((e) => !isValidEin(e)); + if (invalidEins.length > 0) { + logger.warn(invalidEins, 'These EINs in PDC are invalid and will not be queried'); + } + logger.info(validEins, 'Found these valid EINs which will be requested from GivingTuesday'); + const givingTuesdayResponse = await getGivingTuesdayProfiles(validEins); + if (args.outputFile === undefined || args.outputFile === '') { + logger.info({ givingTuesdayResponse }, 'GivingTuesday result'); + const { + data: { results }, + } = givingTuesdayResponse; + const nonprofits = results.filter((r): r is BmfRecord => isBmfRecord(r)); + const changemakerIds = nonprofits + .map((r) => getChangemakerByEin(r.ein, changemakers)) + .filter((c) => c !== null) + .map((c) => c.id); + logger.info({ changemakerIds }, 'Changemaker IDs present in GivingTuesday'); + } else { + await writeFile(args.outputFile, JSON.stringify(givingTuesdayResponse, null, JSON_SPACES)); + logger.info(`Wrote GivingTuesday data for ${JSON.stringify(validEins)} to ${JSON.stringify(args.outputFile)}`); + } + }, +}; + +const getOrCreateSource = async (baseUrl: string, token: AccessTokenSet): Promise => { + const sources = await getSources(baseUrl, token); + const filteredSources = sources.entries.filter((s) => s.dataProviderShortCode === GT_SHORT_CODE); + if (filteredSources.length === 1 && filteredSources[0] !== undefined) { + // Hurray, an existing GivingTuesday Source was found, return it! + return filteredSources[0]; + } + // Create the GivingTuesday Source, we expect/require the Data Provider to exist. + logger.warn('Have a `pdc-admin` create a source because only administrators may be able.'); + // The following may not succeed, doesn't succeed as of this writing. + return await postSource(baseUrl, token, { + dataProviderShortCode: GT_SHORT_CODE, + label: 'GivingTuesday', + }); +}; + +const updateAllCommand: CommandModule = { + command: 'updateAll', + describe: 'For each changemaker present in the PDC, get GivingTuesday data and upload it to PDC.', + builder: { + ...oidcOptions, + 'pdc-api-base-url': { + describe: 'Location of PDC API', + demandOption: true, + type: 'string', + }, + }, + handler: async (args) => { + const changemakers = await getChangemakers(args.pdcApiBaseUrl); + const eins = changemakers.entries.flatMap((c) => c.taxId); + const validEins = eins.filter(isValidEin); + const invalidEins = eins.filter((e) => !isValidEin(e)); + if (invalidEins.length > 0) { + logger.warn(invalidEins, 'These EINs in PDC are invalid and will not be queried'); + } + logger.info(validEins, 'Found these valid EINs which will be requested from GivingTuesday'); + const givingTuesdayResponse = await getGivingTuesdayProfiles(validEins); + logger.info({ givingTuesdayResponse }, 'GivingTuesday result'); + // Up to this point we didn't need PDC authentication. Now we do. + const token = await getToken(args.oidcBaseUrl, args.oidcClientId, args.oidcClientSecret); + // First, find the existing source. As of this writing, it cannot be created by non-admins. + const source = await getOrCreateSource(args.pdcApiBaseUrl, token); + logger.info(source, 'The PDC Source for GivingTuesday was found'); + // Second, collect the well-formed nonprofit records. + const { + data: { results }, + } = givingTuesdayResponse; + const nonprofits = results.filter((r): r is BmfRecord => isBmfRecord(r)); + logger.info(nonprofits, 'Found these nonprofits'); + // Third, register a batch of changemaker fields to be posted. + const fieldBatch = await postChangemakerFieldValueBatch(args.pdcApiBaseUrl, token, { + sourceId: source.id, + notes: `data-scripts givingTuesday.ts execution ${Date.now()}`, + }); + const missingPermissionChangemakerIds: Set = new Set(); + // Last, for each nonprofit, for each field, post the field. These are + // issued sequentially rather than via Promise.all because the PDC API + // times out under concurrent POSTs to /changemakerFieldValues. + /* eslint-disable no-await-in-loop -- sequential POSTs avoid PDC API + connection saturation. */ + for (const record of nonprofits) { + const changemaker = getChangemakerByEin(record.ein, changemakers); + if (changemaker !== null) { + const goodAsOf = parseGivingTuesdayDate(record.Date_Released); + for (const [gtAttributeName, baseFieldShortCode] of baseFieldMap) { + const { [gtAttributeName]: gtAttribute } = record; + if (gtAttribute !== undefined && gtAttribute !== null && gtAttribute !== '') { + const fieldValue = { + changemakerId: changemaker.id, + batchId: fieldBatch.id, + baseFieldShortCode, + value: gtAttribute.toString(), + goodAsOf, + }; + await postChangemakerFieldValueWarnOnForbidden( + args.pdcApiBaseUrl, + token, + fieldValue, + missingPermissionChangemakerIds, + ); + } + } + } + } + /* eslint-enable no-await-in-loop */ + if (missingPermissionChangemakerIds.size > 0) { + logger.warn( + `No permission for at least one field in each of these changemakers (so not updated): ${JSON.stringify([...missingPermissionChangemakerIds])}`, + ); + } + }, +}; + +const givingTuesday: CommandModule = { + command: 'givingTuesday', + describe: 'Interact with the GivingTuesday 990 Data API', + builder: (y) => y.command(lookupCommand).command(lookupFromPdcCommand).command(updateAllCommand).demandCommand(1), + /* eslint-disable-next-line @typescript-eslint/no-empty-function -- yargs demandCommand handles routing to subcommands */ + handler: () => {}, +}; + +export { extractResultsFromResponse, givingTuesday, isBmfRecord, parseGivingTuesdayDate, toGivingTuesdayEin }; diff --git a/src/givingTuesday.unit.test.ts b/src/givingTuesday.unit.test.ts new file mode 100644 index 0000000..ea64587 --- /dev/null +++ b/src/givingTuesday.unit.test.ts @@ -0,0 +1,103 @@ +import { describe, expect, it } from '@jest/globals'; +import { + extractResultsFromResponse, + isBmfRecord, + parseGivingTuesdayDate, + toGivingTuesdayEin, +} from './givingTuesday.js'; + +describe('toGivingTuesdayEin', () => { + it('strips a hyphen', () => { + expect(toGivingTuesdayEin('84-2929872')).toBe('842929872'); + }); + + it('leaves an already-normalized EIN unchanged', () => { + expect(toGivingTuesdayEin('842929872')).toBe('842929872'); + }); + + it('zero-pads a short EIN to nine digits', () => { + expect(toGivingTuesdayEin('100514')).toBe('000100514'); + }); +}); + +describe('parseGivingTuesdayDate', () => { + it('converts a zero-padded YYYY_MM_DD value to ISO', () => { + expect(parseGivingTuesdayDate('2024_02_13')).toBe('2024-02-13'); + }); + + it('zero-pads single-digit month and day', () => { + expect(parseGivingTuesdayDate('2024_4_4')).toBe('2024-04-04'); + }); + + it('returns null for null input', () => { + expect(parseGivingTuesdayDate(null)).toBeNull(); + }); + + it('returns null for undefined input', () => { + expect(parseGivingTuesdayDate(undefined)).toBeNull(); + }); + + it('returns null for an unparseable value', () => { + expect(parseGivingTuesdayDate('not-a-date')).toBeNull(); + }); +}); + +describe('isBmfRecord', () => { + it('accepts a record with a string ein and organization name', () => { + expect(isBmfRecord({ ein: '842929872', primary_name_of_organization: 'GIVING TUESDAY INC' })).toBe(true); + }); + + it('rejects a record missing the organization name', () => { + expect(isBmfRecord({ ein: '842929872' })).toBe(false); + }); + + it('rejects a record with a non-string ein', () => { + expect(isBmfRecord({ ein: 842929872, primary_name_of_organization: 'GIVING TUESDAY INC' })).toBe(false); + }); +}); + +describe('extractResultsFromResponse', () => { + const record = { ein: '842929872', primary_name_of_organization: 'GIVING TUESDAY INC' }; + const validResponse = { + statusCode: 200, + body: { query: '842929872', no_results: 1, results: [record] }, + }; + + it('returns the results array when the response is well-formed', () => { + expect(extractResultsFromResponse(validResponse, '842929872')).toStrictEqual([record]); + }); + + it('returns an empty array when there are no results', () => { + const empty = { statusCode: 200, body: { query: '000000001', no_results: 0, results: [] } }; + expect(extractResultsFromResponse(empty, '000000001')).toStrictEqual([]); + }); + + it('throws a clear error when the response is null', () => { + expect(() => extractResultsFromResponse(null, '842929872')).toThrow( + /GivingTuesday query returned no data for EIN 842929872/v, + ); + }); + + it('throws a clear error when the response is undefined', () => { + expect(() => extractResultsFromResponse(undefined, '842929872')).toThrow( + /GivingTuesday query returned no data for EIN 842929872/v, + ); + }); + + it('throws a clear error when the body is missing', () => { + /* eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- + simulate a malformed runtime response where body is absent. */ + const malformed = { statusCode: 502 } as unknown as Parameters[0]; + expect(() => extractResultsFromResponse(malformed, '842929872')).toThrow(/malformed response for EIN 842929872/v); + }); + + it('throws a clear error when results is not an array', () => { + /* eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- + simulate a malformed runtime response where results is not an array. */ + const malformed = { + statusCode: 200, + body: { query: '842929872', no_results: 1, results: null }, + } as unknown as Parameters[0]; + expect(() => extractResultsFromResponse(malformed, '842929872')).toThrow(/malformed response for EIN 842929872/v); + }); +}); diff --git a/src/index.ts b/src/index.ts index 327b428..96e61c8 100644 --- a/src/index.ts +++ b/src/index.ts @@ -3,6 +3,8 @@ import yargs from 'yargs/yargs'; import { hideBin } from 'yargs/helpers'; import { candid } from './candid.js'; import { charityNavigator } from './charityNavigator.js'; +import { getMetrics } from './getMetrics.js'; +import { givingTuesday } from './givingTuesday.js'; import { logger } from './logger.js'; import { getTokenCommand } from './oidc.js'; @@ -33,6 +35,8 @@ const main = async (argv: string[]): Promise => { .command(getTokenCommand) .command(candid) .command(charityNavigator) + .command(getMetrics) + .command(givingTuesday) .demandCommand() .parse(); };