Modernize site build, model catalog, and content structure

.dockerignore

@@ -0,0 +1,9 @@
+.agent/
+.git/
+.hugo_build.lock
+.hugo_cache/
+data/publications.json
+assets/data/models.csv
+node_modules/
+public/
+resources/_gen/

.github/dependabot.yml

@@ -1,19 +1,16 @@
-# To get started with Dependabot version updates, you'll need to specify which
-# package ecosystems to update and where the package manifests are located.
-# Please see the documentation for all configuration options:
-# https://docs.github.com/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file
-
 version: 2
 updates:
-  - package-ecosystem: "npm" # See documentation for possible values
-    directory: "/" # Location of package manifests
+  - package-ecosystem: "npm"
+    directory: "/"
     schedule:
       interval: "weekly"
 
-# Enable version updates for Docker
   - package-ecosystem: "docker"
-    # Look for a `Dockerfile` in the `root` directory
     directory: "/"
-    # Check for updates once a week
+    schedule:
+      interval: "weekly"
+
+  - package-ecosystem: "github-actions"
+    directory: "/"
     schedule:
       interval: "weekly"

.github/scripts/bibtex-to-json.mjs

@@ -0,0 +1,56 @@
+import { mkdir, readFile, writeFile } from "node:fs/promises";
+import { dirname } from "node:path";
+
+import { parse } from "@retorquere/bibtex-parser";
+
+const inputPath = process.argv[2] ?? "assets/bibliographies/publications.bib";
+const outputPath = process.argv[3] ?? "data/publications.json";
+
+const formatName = (name) =>
+  [name.firstName, name.prefix, name.lastName, name.suffix]
+    .filter(Boolean)
+    .join(" ");
+
+const formatValue = (value) => {
+  if (!Array.isArray(value)) return String(value);
+  if (value.every((item) => typeof item === "string")) return value.join(", ");
+  return value.map(formatName).join(" and ");
+};
+
+const source = await readFile(inputPath, "utf8");
+const result = parse(source);
+
+if (result.errors.length > 0) {
+  const errors = result.errors.map(({ error }) => error).join("\n");
+  throw new Error(`Invalid BibTeX in ${inputPath}:\n${errors}`);
+}
+
+const keys = new Set();
+const publications = result.entries.map((entry) => {
+  if (keys.has(entry.key)) throw new Error(`Duplicate BibTeX key: ${entry.key}`);
+  keys.add(entry.key);
+
+  const fields = Object.entries(entry.fields).map(([name, value]) => ({
+    name,
+    value: formatValue(value),
+  }));
+  const fieldMap = Object.fromEntries(fields.map(({ name, value }) => [name, value]));
+
+  return {
+    key: entry.key,
+    type: entry.type,
+    sortYear: Number.parseInt(fieldMap.year, 10) || 0,
+    fields,
+    fieldMap,
+    authorList: (entry.fields.author ?? []).map(formatName),
+  };
+});
+
+publications.sort(
+  (left, right) => right.sortYear - left.sortYear || left.key.localeCompare(right.key),
+);
+
+await mkdir(dirname(outputPath), { recursive: true });
+await writeFile(outputPath, `${JSON.stringify(publications, null, 2)}\n`, "utf8");
+
+console.log(`Generated ${outputPath} from ${publications.length} BibTeX entries.`);

.github/scripts/build-site.sh

@@ -0,0 +1,44 @@
+#!/bin/sh
+
+set -eu
+
+export HUGO_ENV="${HUGO_ENV:-production}"
+
+HUGO_CACHEDIR="${HUGO_CACHEDIR:-/src/.hugo_cache}"
+OUTPUT_DIR="${OUTPUT_DIR:-/src/public}"
+BASE_URL="${BASE_URL:-}"
+
+require_absolute_path() {
+  value="$1"
+  name="$2"
+  case "$value" in
+    /*) ;;
+    *)
+      echo "ERROR: ${name} must be an absolute path, got: ${value}" >&2
+      exit 1
+      ;;
+  esac
+}
+
+require_absolute_path "$HUGO_CACHEDIR" HUGO_CACHEDIR
+require_absolute_path "$OUTPUT_DIR" OUTPUT_DIR
+
+npm run models:fetch
+npm run bibliography:check
+npm run bibliography
+
+mkdir -p "$HUGO_CACHEDIR" "$OUTPUT_DIR"
+
+set -- build \
+  --gc \
+  --minify \
+  --cacheDir "$HUGO_CACHEDIR" \
+  -d "$OUTPUT_DIR"
+
+if [ -n "$BASE_URL" ]; then
+  trimmed_base_url=${BASE_URL%/}
+  set -- "$@" --baseURL "${trimmed_base_url}/"
+fi
+
+echo "Running: hugo $*" >&2
+exec hugo "$@"

.github/scripts/check-bibliography-sync.mjs

@@ -0,0 +1,185 @@
+import { readFile, rename, rm, writeFile } from "node:fs/promises";
+
+import { parse } from "@retorquere/bibtex-parser";
+
+const bibliographyPath = "assets/bibliographies/publications.bib";
+const modelsCsvPath = "assets/data/models.csv";
+const modelsJsonPath = "data/models.json";
+const expectedHeaders = [
+  "publication_citation",
+  "domain",
+  "available_code",
+  "license",
+  "doi",
+  "documentation",
+  "clean_code",
+  "status",
+  "issue_link",
+  "name_short",
+  "article_doi",
+  "doi_link",
+  "citation_nodoi",
+];
+
+// models.csv currently reuses santos-etal-2006 for two different publications.
+const keyOverrides = new Map([
+  ["10.1098/rspb.2005.3272", "santos-rodrigues-pacheco-2006"],
+]);
+const allowedValues = {
+  domain: new Set(["Cooperation", "Crowd Dynamics", "Ecological Processes", "Land Use"]),
+  available_code: new Set(["N", "Y"]),
+  license: new Set(["N", "Y"]),
+  doi: new Set(["N", "Y"]),
+  documentation: new Set(["", "A", "B", "C", "D", "E"]),
+  clean_code: new Set(["", "A", "B", "C", "D", "E"]),
+  status: new Set([
+    "Not yet started",
+    "Looking for collaborators",
+    "In progress",
+    "Meets FAIR criteria!",
+  ]),
+};
+
+const normalizeDoi = (value) =>
+  value
+    .trim()
+    .replace(/^https?:\/\/(dx\.)?doi\.org\//i, "")
+    .toLowerCase();
+
+const parseCsv = (source) => {
+  const rows = [];
+  let row = [];
+  let field = "";
+  let quoted = false;
+
+  for (let index = 0; index < source.length; index += 1) {
+    const character = source[index];
+    if (quoted) {
+      if (character === '"' && source[index + 1] === '"') {
+        field += '"';
+        index += 1;
+      } else if (character === '"') {
+        quoted = false;
+      } else {
+        field += character;
+      }
+    } else if (character === '"') {
+      quoted = true;
+    } else if (character === ",") {
+      row.push(field);
+      field = "";
+    } else if (character === "\n") {
+      row.push(field.replace(/\r$/, ""));
+      rows.push(row);
+      row = [];
+      field = "";
+    } else {
+      field += character;
+    }
+  }
+
+  if (quoted) throw new Error(`${modelsCsvPath} contains an unterminated quoted field.`);
+
+  if (field || row.length > 0) {
+    row.push(field);
+    rows.push(row);
+  }
+
+  const headers = rows.shift() ?? [];
+  if (headers.length !== expectedHeaders.length ||
+      headers.some((header, index) => header !== expectedHeaders[index])) {
+    throw new Error(
+      `${modelsCsvPath} schema changed. Expected headers:\n${expectedHeaders.join(",")}`,
+    );
+  }
+
+  return rows.filter((values) => values.some(Boolean)).map((values, rowIndex) => {
+    if (values.length !== headers.length) {
+      throw new Error(
+        `${modelsCsvPath} row ${rowIndex + 2} has ${values.length} fields; expected ${headers.length}.`,
+      );
+    }
+    return Object.fromEntries(headers.map((header, index) => [header, values[index]]));
+  });
+};
+
+const models = parseCsv(await readFile(modelsCsvPath, "utf8"));
+const bibliography = parse(await readFile(bibliographyPath, "utf8"));
+if (bibliography.errors.length > 0) {
+  throw new Error(
+    `Invalid BibTeX:\n${bibliography.errors.map(({ error }) => error).join("\n")}`,
+  );
+}
+
+const problems = [];
+const modelsByDoi = new Map();
+const modelKeys = new Set();
+for (const model of models) {
+  const doi = normalizeDoi(model.article_doi);
+  const expectedKey = keyOverrides.get(doi) ?? model.name_short.trim();
+  if (!model.publication_citation.trim()) {
+    problems.push(`models.csv row has no publication_citation: ${expectedKey || doi}`);
+  }
+  if (!model.domain.trim()) problems.push(`models.csv row has no domain: ${expectedKey || doi}`);
+  if (!doi) problems.push(`models.csv row has no article_doi: ${model.name_short}`);
+  if (!expectedKey) problems.push(`models.csv row has no name_short: ${doi}`);
+  for (const [field, allowed] of Object.entries(allowedValues)) {
+    if (!allowed.has(model[field])) {
+      problems.push(`models.csv ${expectedKey || doi} has invalid ${field}: ${model[field]}`);
+    }
+  }
+  if (model.issue_link &&
+      !/^https:\/\/github\.com\/make-models-fair\/coordination\/issues\/\d+$/.test(model.issue_link)) {
+    problems.push(`models.csv ${expectedKey || doi} has invalid issue_link: ${model.issue_link}`);
+  }
+  if (modelsByDoi.has(doi)) problems.push(`models.csv contains duplicate DOI: ${doi}`);
+  if (modelKeys.has(expectedKey)) problems.push(`models.csv resolves to duplicate key: ${expectedKey}`);
+  modelKeys.add(expectedKey);
+  modelsByDoi.set(doi, { expectedKey });
+}
+
+const bibliographyByDoi = new Map();
+const bibliographyKeys = new Set();
+for (const entry of bibliography.entries) {
+  const doi = normalizeDoi(entry.fields.doi ?? "");
+  for (const field of ["title", "author", "year", "doi"]) {
+    const value = entry.fields[field];
+    if (!value || (Array.isArray(value) && value.length === 0)) {
+      problems.push(`BibTeX entry ${entry.key} has no ${field}`);
+    }
+  }
+  if (!doi) problems.push(`BibTeX entry has no DOI: ${entry.key}`);
+  if (bibliographyKeys.has(entry.key)) problems.push(`BibTeX contains duplicate key: ${entry.key}`);
+  if (bibliographyByDoi.has(doi)) problems.push(`BibTeX contains duplicate DOI: ${doi}`);
+  bibliographyKeys.add(entry.key);
+  bibliographyByDoi.set(doi, entry);
+}
+
+for (const [doi, { expectedKey }] of modelsByDoi) {
+  const entry = bibliographyByDoi.get(doi);
+  if (!entry) {
+    problems.push(`Missing BibTeX entry for ${expectedKey} (${doi})`);
+  } else if (entry.key !== expectedKey) {
+    problems.push(`BibTeX key for ${doi} is ${entry.key}; expected ${expectedKey}`);
+  }
+}
+
+for (const [doi, entry] of bibliographyByDoi) {
+  if (!modelsByDoi.has(doi)) problems.push(`BibTeX entry is not in models.csv: ${entry.key} (${doi})`);
+}
+
+if (problems.length > 0) {
+  throw new Error(`Bibliography is out of sync:\n- ${problems.join("\n- ")}`);
+}
+
+const temporaryModelsJsonPath = `${modelsJsonPath}.${process.pid}.tmp`;
+try {
+  await writeFile(temporaryModelsJsonPath, `${JSON.stringify(models, null, 2)}\n`, "utf8");
+  await rename(temporaryModelsJsonPath, modelsJsonPath);
+} finally {
+  await rm(temporaryModelsJsonPath, { force: true });
+}
+
+console.log(
+  `Bibliography is synchronized: ${bibliography.entries.length} entries match ${models.length} models.`,
+);

.github/scripts/fetch-models.mjs

@@ -0,0 +1,65 @@
+import { mkdir, readFile, rename, rm, writeFile } from "node:fs/promises";
+import { dirname } from "node:path";
+import { createHash } from "node:crypto";
+
+const lockPath = "model-catalog.lock.json";
+const outputPath = "assets/data/models.csv";
+const lock = JSON.parse(await readFile(lockPath, "utf8"));
+
+if (!/^[\w.-]+\/[\w.-]+$/.test(lock.repository)) {
+  throw new Error(`${lockPath} repository must be an owner/repository name.`);
+}
+if (!lock.path || lock.path.startsWith("/") || lock.path.split("/").includes("..")) {
+  throw new Error(`${lockPath} path must be relative and cannot contain '..'.`);
+}
+if (!/^[0-9a-f]{40}$/.test(lock.commit)) {
+  throw new Error(`${lockPath} commit must be one full lowercase commit SHA.`);
+}
+if (!/^[0-9a-f]{64}$/.test(lock.sha256)) {
+  throw new Error(`${lockPath} sha256 must be one lowercase SHA-256 digest.`);
+}
+
+const sourcePath = lock.path.split("/").map(encodeURIComponent).join("/");
+const sourceUrl = `https://raw.githubusercontent.com/${lock.repository}/${lock.commit}/${sourcePath}`;
+
+let sourceBytes;
+for (let attempt = 1; attempt <= 3; attempt += 1) {
+  try {
+    const response = await fetch(sourceUrl, {
+      headers: {
+        "User-Agent": "make-models-fair.github.io model snapshot fetcher",
+      },
+      signal: AbortSignal.timeout(15_000),
+    });
+    if (!response.ok) {
+      throw new Error(`${response.status} ${response.statusText}`);
+    }
+    sourceBytes = Buffer.from(await response.arrayBuffer());
+    if (sourceBytes.length === 0) throw new Error("response was empty");
+    if (sourceBytes.length > 5_000_000) throw new Error("response exceeded 5 MB");
+    break;
+  } catch (error) {
+    if (attempt === 3) {
+      throw new Error(`Could not fetch ${lock.path} at ${lock.commit}: ${error.message}`);
+    }
+    await new Promise((resolve) => setTimeout(resolve, attempt * 1_000));
+  }
+}
+
+const checksum = createHash("sha256").update(sourceBytes).digest("hex");
+if (checksum !== lock.sha256) {
+  throw new Error(
+    `${lock.path} checksum mismatch at ${lock.commit}: expected ${lock.sha256}, received ${checksum}`,
+  );
+}
+
+await mkdir(dirname(outputPath), { recursive: true });
+const temporaryPath = `${outputPath}.${process.pid}.tmp`;
+try {
+  await writeFile(temporaryPath, sourceBytes);
+  await rename(temporaryPath, outputPath);
+} finally {
+  await rm(temporaryPath, { force: true });
+}
+
+console.log(`Fetched ${lock.repository}/${lock.path} at ${lock.commit} (${checksum}).`);