From e36e110857e0c223718867f1950f4eb2305bff59 Mon Sep 17 00:00:00 2001 From: Eric McGinnis Date: Mon, 27 Jul 2026 09:57:40 -0700 Subject: [PATCH 01/14] renovate the attack data archive production process. --- .github/workflows/build_dataset_archive.yml | 36 +++ bin/build_dataset_archive.py | 265 ++++++++++++++++++++ bin/requirements.txt | 4 +- 3 files changed, 304 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/build_dataset_archive.yml create mode 100755 bin/build_dataset_archive.py diff --git a/.github/workflows/build_dataset_archive.yml b/.github/workflows/build_dataset_archive.yml new file mode 100644 index 000000000..5c8209632 --- /dev/null +++ b/.github/workflows/build_dataset_archive.yml @@ -0,0 +1,36 @@ +name: build-dataset-archive-on-merge-to-default-branch + +on: + push: + branches: + - master + +jobs: + build-archive: + runs-on: + group: attack-data-runners + steps: + - name: Checkout Repo + uses: actions/checkout@v6 + with: + lfs: true + fetch-depth: 0 # full history needed to compute per-file last-updated timestamps + + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: '3.14' + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r bin/requirements.txt + + - name: Build datasets archive + run: python bin/build_dataset_archive.py -o attack_data_archive.zip + + - name: Upload archive as workflow artifact + uses: actions/upload-artifact@v4 + with: + name: attack_data_archive + path: attack_data_archive.zip diff --git a/bin/build_dataset_archive.py b/bin/build_dataset_archive.py new file mode 100755 index 000000000..eeb90e118 --- /dev/null +++ b/bin/build_dataset_archive.py @@ -0,0 +1,265 @@ +#!/usr/bin/env python3 +""" +Build a compressed (ZIP_ZSTANDARD) archive of the datasets/ folder. + +The archive contains: + - datasets/... the datasets folder, unchanged + - metadata.yml generation time, git ref, size, etc. + - url_to_file_mappings.yml LFS vs. non-LFS file listing + +metadata.yml and url_to_file_mappings.yml are also written as standalone +files next to the archive, in addition to being embedded inside it. + +Requires Python 3.14+ (zipfile.ZIP_ZSTANDARD support), pydantic, and pydantic-cli. +""" + +import os +import re +import subprocess +import sys +import zipfile +from datetime import datetime, timezone +from pathlib import Path +from typing import Dict, Iterator, List, Literal, Optional, Tuple + +try: + import yaml + from pydantic import BaseModel, Field, field_serializer + from pydantic_cli import Cmd, run_and_exit +except ImportError as exc: + sys.exit(f"Error: missing dependency ({exc}). Install with: pip install -r bin/requirements.txt") + +MIN_PYTHON = (3, 14) + +if sys.version_info < MIN_PYTHON: + sys.exit( + f"Error: this script requires Python {MIN_PYTHON[0]}.{MIN_PYTHON[1]}+ " + f"(zipfile.ZIP_ZSTANDARD support). Running {sys.version_info.major}.{sys.version_info.minor}." + ) + + +def _as_utc_iso(dt: Optional[datetime]) -> Optional[str]: + """Format a datetime as a UTC ISO-8601 string ending in 'Z', or None if dt is None.""" + return dt.astimezone(timezone.utc).isoformat().replace("+00:00", "Z") if dt else None + + +class Metadata(BaseModel): + """Contents of metadata.yml: when and from what git state the archive was built.""" + + generated_at_utc: datetime + file_count: int + gitref: str + github_url: str + total_uncompressed_size_bytes: int + + @field_serializer("generated_at_utc", when_used="json") + def _serialize_generated_at_utc(self, value: datetime) -> str: + """Serialize generated_at_utc as a UTC ISO-8601 string.""" + return _as_utc_iso(value) + + +class LfsFileEntry(BaseModel): + """Details for a single git-lfs-tracked file, keyed by its download URL in url_to_file_mappings.yml.""" + + relative_path: str + uncompressed_size: int + last_updated: Optional[datetime] = Field(default=None, alias="last-updated") + + model_config = {"populate_by_name": True} + + @field_serializer("last_updated", when_used="json") + def _serialize_last_updated(self, value: Optional[datetime]) -> Optional[str]: + """Serialize last_updated as a UTC ISO-8601 string.""" + return _as_utc_iso(value) + + +class UrlToFileMappings(BaseModel): + """Contents of url_to_file_mappings.yml: LFS files keyed by URL, plus a flat list of non-LFS file paths.""" + + lfs_files: Dict[str, LfsFileEntry] = Field(default_factory=dict, alias="lfs-files") + non_lfs_files: List[str] = Field(default_factory=list, alias="non-lfs-files") + + model_config = {"populate_by_name": True} + + +def to_yaml(model: BaseModel) -> str: + """Serialize a pydantic model to a YAML document, using its field aliases as keys.""" + return yaml.safe_dump(model.model_dump(mode="json", by_alias=True), sort_keys=False, default_flow_style=False) + + +def run_git(args: List[str], cwd: "Path | str") -> str: + """Run a git command in cwd and return its stripped stdout, raising CalledProcessError on failure.""" + result = subprocess.run(["git", *args], cwd=cwd, capture_output=True, text=True, check=True) + return result.stdout.strip() + + +def parse_github_owner_repo(remote_url: str) -> Tuple[str, str]: + """Extract the (owner, repo) pair from a GitHub remote URL, in https or ssh form.""" + match = re.search(r"github\.com[:/]([^/]+)/([^/.]+?)(?:\.git)?/?$", remote_url) + if not match: + raise ValueError(f"Could not parse a GitHub owner/repo from remote URL: {remote_url}") + return match.group(1), match.group(2) + + +def determine_ref( + repo_root: Path, ref_override: Optional[str], ref_type_override: Optional[str] +) -> Tuple[str, str]: + """Resolve the (ref_name, ref_type) to embed in generated URLs. + + Preference order: an explicit override, then the GitHub Actions + GITHUB_REF_NAME/GITHUB_REF_TYPE env vars, then the current local branch. + """ + if ref_override: + return ref_override, ref_type_override or "branch" + + ref_name = os.environ.get("GITHUB_REF_NAME") + ref_type = os.environ.get("GITHUB_REF_TYPE") + if ref_name and ref_type: + return ref_name, ref_type + + try: + branch = run_git(["symbolic-ref", "--short", "HEAD"], cwd=repo_root) + if branch: + return branch, "branch" + except subprocess.CalledProcessError: + pass + + raise RuntimeError( + "Could not determine a branch/tag name for this checkout (HEAD is " + "detached and GITHUB_REF_NAME is not set). Pass --ref/--ref-type explicitly." + ) + + +def collect_last_updated(repo_root: Path) -> Dict[str, datetime]: + """Map {relative_path: last commit datetime} for every file ever touched under datasets/.""" + raw = run_git( + ["log", "--name-only", "--pretty=format:%x00%H%x01%cI", "--", "datasets"], + cwd=repo_root, + ) + last_updated: Dict[str, datetime] = {} + for chunk in raw.split("\x00")[1:]: + header, _, files_block = chunk.partition("\n") + _commit_hash, _, date_str = header.partition("\x01") + commit_dt = datetime.fromisoformat(date_str) + for line in files_block.splitlines(): + line = line.strip() + if line and line not in last_updated: + # git log is newest-first, so the first hit for a path is its most recent commit. + last_updated[line] = commit_dt + return last_updated + + +def iter_dataset_files(datasets_dir: Path) -> Iterator[Path]: + """Yield every regular file under datasets_dir, in sorted order.""" + for path in sorted(datasets_dir.rglob("*")): + if path.is_file(): + yield path + + +def build_archive( + repo_root: Path, + datasets_dir: Path, + output_path: str, + compresslevel: int, + ref_name: str, + ref_type: str, +) -> Tuple[int, int, int, int]: + """Write a ZIP_ZSTANDARD archive of datasets_dir plus metadata.yml and url_to_file_mappings.yml. + + metadata.yml and url_to_file_mappings.yml are also written as standalone + files alongside output_path, in addition to being embedded in the archive. + + Returns (file_count, total_uncompressed_size_bytes, lfs_file_count, non_lfs_file_count). + """ + owner, repo = parse_github_owner_repo(run_git(["remote", "get-url", "origin"], cwd=repo_root)) + ref_segment = "heads" if ref_type == "branch" else "tags" + git_hash = run_git(["rev-parse", "HEAD"], cwd=repo_root) + + lfs_paths = { + line.strip() + for line in run_git(["lfs", "ls-files", "-n"], cwd=repo_root).splitlines() + if line.strip() + } + last_updated_map = collect_last_updated(repo_root) + + files = list(iter_dataset_files(datasets_dir)) + total_size = 0 + mappings = UrlToFileMappings() + + with zipfile.ZipFile(output_path, "w") as zf: + for index, path in enumerate(files, start=1): + rel = path.relative_to(repo_root).as_posix() # e.g. datasets/foo/bar.log + size = path.stat().st_size + total_size += size + + zf.write(path, arcname=rel, compress_type=zipfile.ZIP_ZSTANDARD, compresslevel=compresslevel) + + if index % 100 == 0 or index == len(files): + print(f" added {index}/{len(files)} files...") + + if rel in lfs_paths: + url = f"https://media.githubusercontent.com/media/{owner}/{repo}/refs/{ref_segment}/{ref_name}/{rel}" + mappings.lfs_files[url] = LfsFileEntry( + relative_path=rel, + uncompressed_size=size, + **{"last-updated": last_updated_map.get(rel)}, + ) + else: + mappings.non_lfs_files.append(rel) + + metadata = Metadata( + generated_at_utc=datetime.now(timezone.utc), + file_count=len(files), + gitref=git_hash, + github_url=f"https://github.com/{owner}/{repo}/tree/{ref_name}", + total_uncompressed_size_bytes=total_size, + ) + metadata_yaml = to_yaml(metadata) + mappings_yaml = to_yaml(mappings) + zf.writestr("metadata.yml", metadata_yaml) + zf.writestr("url_to_file_mappings.yml", mappings_yaml) + + output_dir = Path(output_path).resolve().parent + (output_dir / "metadata.yml").write_text(metadata_yaml) + (output_dir / "url_to_file_mappings.yml").write_text(mappings_yaml) + + return len(files), total_size, len(mappings.lfs_files), len(mappings.non_lfs_files) + + +class Options(Cmd): + """CLI options for building the datasets archive.""" + + output: str = Field("attack_data_archive.zip", cli=("-o", "--output"), description="Path to the archive to create") + compresslevel: int = Field(9, cli=("--compresslevel",), description="Zstandard compression level (default: 9)") + ref: Optional[str] = Field( + None, cli=("--ref",), description="Override the git branch/tag name used to build GitHub URLs" + ) + ref_type: Optional[Literal["branch", "tag"]] = Field( + None, cli=("--ref-type",), description="Whether --ref is a branch or a tag (default: branch)" + ) + + def run(self) -> None: + """Build the archive using the parsed CLI options and print a summary.""" + repo_root = Path(run_git(["rev-parse", "--show-toplevel"], cwd=os.getcwd())).resolve() + datasets_dir = repo_root / "datasets" + if not datasets_dir.is_dir(): + sys.exit(f"Error: {datasets_dir} does not exist") + + ref_name, ref_type = determine_ref(repo_root, self.ref, self.ref_type) + + file_count, total_size, lfs_count, non_lfs_count = build_archive( + repo_root, datasets_dir, self.output, self.compresslevel, ref_name, ref_type + ) + + output_dir = Path(self.output).resolve().parent + print(f"Wrote {self.output}") + print(f"Wrote {output_dir / 'metadata.yml'}") + print(f"Wrote {output_dir / 'url_to_file_mappings.yml'}") + print(f" files: {file_count} ({total_size:,} bytes uncompressed)") + print(f" lfs files: {lfs_count}") + print(f" non-lfs files: {non_lfs_count}") + print(f" ref: {ref_name} ({ref_type})") + + +if __name__ == "__main__": + run_and_exit(Options, description=__doc__, version="1.0.0") diff --git a/bin/requirements.txt b/bin/requirements.txt index f15921c97..796907ac7 100644 --- a/bin/requirements.txt +++ b/bin/requirements.txt @@ -5,4 +5,6 @@ splunk-sdk gitpython tqdm colorama -jsonschema \ No newline at end of file +jsonschema +pydantic +pydantic-cli \ No newline at end of file From 62e055e99b85dbdf48cfb82e76f7c5e48f0e2fe9 Mon Sep 17 00:00:00 2001 From: Eric McGinnis Date: Mon, 3 Aug 2026 14:29:49 -0700 Subject: [PATCH 02/14] include info about what the attack_data_cache file will hold --- README_ATTACK_DATA_CACHE.md | 39 +++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 README_ATTACK_DATA_CACHE.md diff --git a/README_ATTACK_DATA_CACHE.md b/README_ATTACK_DATA_CACHE.md new file mode 100644 index 000000000..fccf49914 --- /dev/null +++ b/README_ATTACK_DATA_CACHE.md @@ -0,0 +1,39 @@ +# Attack Data Archive Cache + +This directory holds a local snapshot produced by `bin/build_dataset_archive.py`: +a Zstandard-compressed archive of the `datasets/` folder from +[splunk/attack_data](https://github.com/splunk/attack_data), plus two +standalone metadata files describing exactly what went into it. + +## Files + +### `metadata.yml` +Summary of the archive build: + +- `generated_at_utc` — when the archive was built +- `file_count` — total number of files included +- `gitref` — the exact commit hash the snapshot was built from +- `github_url` — link to the source branch/tag on GitHub +- `total_uncompressed_size_bytes` — combined size of all files before compression + +### `url_to_file_mappings.yml` +Maps every file in the archive back to its source, split into two sections: + +- `lfs-files` — a map keyed by the file's Git LFS download URL + (`media.githubusercontent.com/...`), with: + - `relative_path` — path within `datasets/` + - `uncompressed_size` — size in bytes + - `last-updated` — timestamp of the most recent commit that touched the file +- `non-lfs-files` — a flat list of relative paths for files stored directly + in git (not LFS-tracked) + +Use this file to fetch an individual dataset file directly from GitHub without +downloading the full archive. + +## Notes + +- Both files are also embedded inside the `.zip` archive itself, so they travel + with it even if separated from these standalone copies. +- `gitref` in `metadata.yml` pins the exact commit; re-running the build script + against a later commit will produce different contents even if `datasets/` + is otherwise unchanged (e.g. `last-updated` timestamps). From c0e14d9bba16aa174eca4455151c592b14c337dc Mon Sep 17 00:00:00 2001 From: Eric McGinnis Date: Mon, 3 Aug 2026 15:44:12 -0700 Subject: [PATCH 03/14] combine metadata and mappings into single file --- README_ATTACK_DATA_CACHE.md | 24 ++++++--------- bin/build_dataset_archive.py | 60 ++++++++++++++++-------------------- 2 files changed, 36 insertions(+), 48 deletions(-) diff --git a/README_ATTACK_DATA_CACHE.md b/README_ATTACK_DATA_CACHE.md index fccf49914..e68440cd3 100644 --- a/README_ATTACK_DATA_CACHE.md +++ b/README_ATTACK_DATA_CACHE.md @@ -2,23 +2,19 @@ This directory holds a local snapshot produced by `bin/build_dataset_archive.py`: a Zstandard-compressed archive of the `datasets/` folder from -[splunk/attack_data](https://github.com/splunk/attack_data), plus two -standalone metadata files describing exactly what went into it. +[splunk/attack_data](https://github.com/splunk/attack_data), plus a +standalone metadata file describing exactly what went into it. ## Files ### `metadata.yml` -Summary of the archive build: +Describes the archive build and maps every file in it back to its source: - `generated_at_utc` — when the archive was built - `file_count` — total number of files included - `gitref` — the exact commit hash the snapshot was built from - `github_url` — link to the source branch/tag on GitHub - `total_uncompressed_size_bytes` — combined size of all files before compression - -### `url_to_file_mappings.yml` -Maps every file in the archive back to its source, split into two sections: - - `lfs-files` — a map keyed by the file's Git LFS download URL (`media.githubusercontent.com/...`), with: - `relative_path` — path within `datasets/` @@ -27,13 +23,13 @@ Maps every file in the archive back to its source, split into two sections: - `non-lfs-files` — a flat list of relative paths for files stored directly in git (not LFS-tracked) -Use this file to fetch an individual dataset file directly from GitHub without -downloading the full archive. +Use the `lfs-files`/`non-lfs-files` sections to fetch an individual dataset +file directly from GitHub without downloading the full archive. ## Notes -- Both files are also embedded inside the `.zip` archive itself, so they travel - with it even if separated from these standalone copies. -- `gitref` in `metadata.yml` pins the exact commit; re-running the build script - against a later commit will produce different contents even if `datasets/` - is otherwise unchanged (e.g. `last-updated` timestamps). +- `metadata.yml` is also embedded inside the `.zip` archive itself, so it + travels with it even if separated from this standalone copy. +- `gitref` pins the exact commit; re-running the build script against a + later commit will produce different contents even if `datasets/` is + otherwise unchanged (e.g. `last-updated` timestamps). diff --git a/bin/build_dataset_archive.py b/bin/build_dataset_archive.py index eeb90e118..7dc85b691 100755 --- a/bin/build_dataset_archive.py +++ b/bin/build_dataset_archive.py @@ -3,12 +3,11 @@ Build a compressed (ZIP_ZSTANDARD) archive of the datasets/ folder. The archive contains: - - datasets/... the datasets folder, unchanged - - metadata.yml generation time, git ref, size, etc. - - url_to_file_mappings.yml LFS vs. non-LFS file listing + - datasets/... the datasets folder, unchanged + - metadata.yml generation info plus the LFS vs. non-LFS file listing -metadata.yml and url_to_file_mappings.yml are also written as standalone -files next to the archive, in addition to being embedded inside it. +metadata.yml is also written as a standalone file next to the archive, in +addition to being embedded inside it. Requires Python 3.14+ (zipfile.ZIP_ZSTANDARD support), pydantic, and pydantic-cli. """ @@ -43,23 +42,8 @@ def _as_utc_iso(dt: Optional[datetime]) -> Optional[str]: return dt.astimezone(timezone.utc).isoformat().replace("+00:00", "Z") if dt else None -class Metadata(BaseModel): - """Contents of metadata.yml: when and from what git state the archive was built.""" - - generated_at_utc: datetime - file_count: int - gitref: str - github_url: str - total_uncompressed_size_bytes: int - - @field_serializer("generated_at_utc", when_used="json") - def _serialize_generated_at_utc(self, value: datetime) -> str: - """Serialize generated_at_utc as a UTC ISO-8601 string.""" - return _as_utc_iso(value) - - class LfsFileEntry(BaseModel): - """Details for a single git-lfs-tracked file, keyed by its download URL in url_to_file_mappings.yml.""" + """Details for a single git-lfs-tracked file, keyed by its download URL in metadata.yml.""" relative_path: str uncompressed_size: int @@ -73,14 +57,24 @@ def _serialize_last_updated(self, value: Optional[datetime]) -> Optional[str]: return _as_utc_iso(value) -class UrlToFileMappings(BaseModel): - """Contents of url_to_file_mappings.yml: LFS files keyed by URL, plus a flat list of non-LFS file paths.""" +class Metadata(BaseModel): + """Contents of metadata.yml: generation info, and the LFS/non-LFS file listing.""" + generated_at_utc: datetime + file_count: int + gitref: str + github_url: str + total_uncompressed_size_bytes: int lfs_files: Dict[str, LfsFileEntry] = Field(default_factory=dict, alias="lfs-files") non_lfs_files: List[str] = Field(default_factory=list, alias="non-lfs-files") model_config = {"populate_by_name": True} + @field_serializer("generated_at_utc", when_used="json") + def _serialize_generated_at_utc(self, value: datetime) -> str: + """Serialize generated_at_utc as a UTC ISO-8601 string.""" + return _as_utc_iso(value) + def to_yaml(model: BaseModel) -> str: """Serialize a pydantic model to a YAML document, using its field aliases as keys.""" @@ -164,10 +158,10 @@ def build_archive( ref_name: str, ref_type: str, ) -> Tuple[int, int, int, int]: - """Write a ZIP_ZSTANDARD archive of datasets_dir plus metadata.yml and url_to_file_mappings.yml. + """Write a ZIP_ZSTANDARD archive of datasets_dir plus metadata.yml. - metadata.yml and url_to_file_mappings.yml are also written as standalone - files alongside output_path, in addition to being embedded in the archive. + metadata.yml is also written as a standalone file alongside output_path, + in addition to being embedded in the archive. Returns (file_count, total_uncompressed_size_bytes, lfs_file_count, non_lfs_file_count). """ @@ -184,7 +178,8 @@ def build_archive( files = list(iter_dataset_files(datasets_dir)) total_size = 0 - mappings = UrlToFileMappings() + lfs_files: Dict[str, LfsFileEntry] = {} + non_lfs_files: List[str] = [] with zipfile.ZipFile(output_path, "w") as zf: for index, path in enumerate(files, start=1): @@ -199,13 +194,13 @@ def build_archive( if rel in lfs_paths: url = f"https://media.githubusercontent.com/media/{owner}/{repo}/refs/{ref_segment}/{ref_name}/{rel}" - mappings.lfs_files[url] = LfsFileEntry( + lfs_files[url] = LfsFileEntry( relative_path=rel, uncompressed_size=size, **{"last-updated": last_updated_map.get(rel)}, ) else: - mappings.non_lfs_files.append(rel) + non_lfs_files.append(rel) metadata = Metadata( generated_at_utc=datetime.now(timezone.utc), @@ -213,17 +208,15 @@ def build_archive( gitref=git_hash, github_url=f"https://github.com/{owner}/{repo}/tree/{ref_name}", total_uncompressed_size_bytes=total_size, + **{"lfs-files": lfs_files, "non-lfs-files": non_lfs_files}, ) metadata_yaml = to_yaml(metadata) - mappings_yaml = to_yaml(mappings) zf.writestr("metadata.yml", metadata_yaml) - zf.writestr("url_to_file_mappings.yml", mappings_yaml) output_dir = Path(output_path).resolve().parent (output_dir / "metadata.yml").write_text(metadata_yaml) - (output_dir / "url_to_file_mappings.yml").write_text(mappings_yaml) - return len(files), total_size, len(mappings.lfs_files), len(mappings.non_lfs_files) + return len(files), total_size, len(lfs_files), len(non_lfs_files) class Options(Cmd): @@ -254,7 +247,6 @@ def run(self) -> None: output_dir = Path(self.output).resolve().parent print(f"Wrote {self.output}") print(f"Wrote {output_dir / 'metadata.yml'}") - print(f"Wrote {output_dir / 'url_to_file_mappings.yml'}") print(f" files: {file_count} ({total_size:,} bytes uncompressed)") print(f" lfs files: {lfs_count}") print(f" non-lfs files: {non_lfs_count}") From a082670a6fb84fa473bae5f2359dd974f0d61e5c Mon Sep 17 00:00:00 2001 From: Eric McGinnis Date: Thu, 6 Aug 2026 13:50:52 -0700 Subject: [PATCH 04/14] improved dataset building to directory. --- .github/workflows/build_dataset_archive.yml | 4 +- bin/build_dataset_archive.py | 47 ++++++++++++--------- bin/requirements.txt | 2 +- 3 files changed, 31 insertions(+), 22 deletions(-) diff --git a/.github/workflows/build_dataset_archive.yml b/.github/workflows/build_dataset_archive.yml index 5c8209632..83ec45820 100644 --- a/.github/workflows/build_dataset_archive.yml +++ b/.github/workflows/build_dataset_archive.yml @@ -27,10 +27,10 @@ jobs: pip install -r bin/requirements.txt - name: Build datasets archive - run: python bin/build_dataset_archive.py -o attack_data_archive.zip + run: python bin/build_dataset_archive.py - name: Upload archive as workflow artifact uses: actions/upload-artifact@v4 with: name: attack_data_archive - path: attack_data_archive.zip + path: attack_data_archive/attack_data_archive.zip diff --git a/bin/build_dataset_archive.py b/bin/build_dataset_archive.py index 7dc85b691..9f14dea0c 100755 --- a/bin/build_dataset_archive.py +++ b/bin/build_dataset_archive.py @@ -6,10 +6,12 @@ - datasets/... the datasets folder, unchanged - metadata.yml generation info plus the LFS vs. non-LFS file listing -metadata.yml is also written as a standalone file next to the archive, in -addition to being embedded inside it. +Output is always written to attack_data_archive/ (created if missing): + - attack_data_archive/attack_data_archive.zip + - attack_data_archive/metadata.yml (the same metadata.yml, also + written standalone alongside the archive) -Requires Python 3.14+ (zipfile.ZIP_ZSTANDARD support), pydantic, and pydantic-cli. +Requires Python 3.14+ (zipfile.ZIP_ZSTANDARD support), pydantic, and pydantic-settings. """ import os @@ -24,12 +26,15 @@ try: import yaml from pydantic import BaseModel, Field, field_serializer - from pydantic_cli import Cmd, run_and_exit + from pydantic_settings import BaseSettings, CliApp, SettingsConfigDict except ImportError as exc: sys.exit(f"Error: missing dependency ({exc}). Install with: pip install -r bin/requirements.txt") MIN_PYTHON = (3, 14) +OUTPUT_DIR_NAME = "attack_data_archive" +ARCHIVE_FILE_NAME = "attack_data_archive.zip" + if sys.version_info < MIN_PYTHON: sys.exit( f"Error: this script requires Python {MIN_PYTHON[0]}.{MIN_PYTHON[1]}+ " @@ -153,18 +158,21 @@ def iter_dataset_files(datasets_dir: Path) -> Iterator[Path]: def build_archive( repo_root: Path, datasets_dir: Path, - output_path: str, + output_dir: Path, compresslevel: int, ref_name: str, ref_type: str, ) -> Tuple[int, int, int, int]: - """Write a ZIP_ZSTANDARD archive of datasets_dir plus metadata.yml. + """Write a ZIP_ZSTANDARD archive of datasets_dir plus metadata.yml into output_dir. - metadata.yml is also written as a standalone file alongside output_path, + metadata.yml is also written as a standalone file alongside the archive, in addition to being embedded in the archive. Returns (file_count, total_uncompressed_size_bytes, lfs_file_count, non_lfs_file_count). """ + output_dir.mkdir(parents=True, exist_ok=True) + output_path = output_dir / ARCHIVE_FILE_NAME + owner, repo = parse_github_owner_repo(run_git(["remote", "get-url", "origin"], cwd=repo_root)) ref_segment = "heads" if ref_type == "branch" else "tags" git_hash = run_git(["rev-parse", "HEAD"], cwd=repo_root) @@ -213,39 +221,40 @@ def build_archive( metadata_yaml = to_yaml(metadata) zf.writestr("metadata.yml", metadata_yaml) - output_dir = Path(output_path).resolve().parent (output_dir / "metadata.yml").write_text(metadata_yaml) return len(files), total_size, len(lfs_files), len(non_lfs_files) -class Options(Cmd): +class Options(BaseSettings): """CLI options for building the datasets archive.""" - output: str = Field("attack_data_archive.zip", cli=("-o", "--output"), description="Path to the archive to create") - compresslevel: int = Field(9, cli=("--compresslevel",), description="Zstandard compression level (default: 9)") - ref: Optional[str] = Field( - None, cli=("--ref",), description="Override the git branch/tag name used to build GitHub URLs" + model_config = SettingsConfigDict( + cli_prog_name="build_dataset_archive.py", + cli_kebab_case=True, ) + + compresslevel: int = Field(9, description="Zstandard compression level (default: 9)") + ref: Optional[str] = Field(None, description="Override the git branch/tag name used to build GitHub URLs") ref_type: Optional[Literal["branch", "tag"]] = Field( - None, cli=("--ref-type",), description="Whether --ref is a branch or a tag (default: branch)" + None, description="Whether --ref is a branch or a tag (default: branch)" ) - def run(self) -> None: + def cli_cmd(self) -> None: """Build the archive using the parsed CLI options and print a summary.""" repo_root = Path(run_git(["rev-parse", "--show-toplevel"], cwd=os.getcwd())).resolve() datasets_dir = repo_root / "datasets" if not datasets_dir.is_dir(): sys.exit(f"Error: {datasets_dir} does not exist") + output_dir = repo_root / OUTPUT_DIR_NAME ref_name, ref_type = determine_ref(repo_root, self.ref, self.ref_type) file_count, total_size, lfs_count, non_lfs_count = build_archive( - repo_root, datasets_dir, self.output, self.compresslevel, ref_name, ref_type + repo_root, datasets_dir, output_dir, self.compresslevel, ref_name, ref_type ) - output_dir = Path(self.output).resolve().parent - print(f"Wrote {self.output}") + print(f"Wrote {output_dir / ARCHIVE_FILE_NAME}") print(f"Wrote {output_dir / 'metadata.yml'}") print(f" files: {file_count} ({total_size:,} bytes uncompressed)") print(f" lfs files: {lfs_count}") @@ -254,4 +263,4 @@ def run(self) -> None: if __name__ == "__main__": - run_and_exit(Options, description=__doc__, version="1.0.0") + CliApp.run(Options) diff --git a/bin/requirements.txt b/bin/requirements.txt index 796907ac7..7215b32eb 100644 --- a/bin/requirements.txt +++ b/bin/requirements.txt @@ -7,4 +7,4 @@ tqdm colorama jsonschema pydantic -pydantic-cli \ No newline at end of file +pydantic-settings \ No newline at end of file From a40947876e9189a9f9d54423a2553920248f7d0a Mon Sep 17 00:00:00 2001 From: Eric McGinnis Date: Thu, 6 Aug 2026 14:22:47 -0700 Subject: [PATCH 05/14] Provide fixture for seeing if files exist in archive --- bin/attack_data_archive_models.py | 70 ++++++++++++++++ bin/attack_data_archive_reader.py | 130 ++++++++++++++++++++++++++++++ bin/build_dataset_archive.py | 65 +++------------ 3 files changed, 213 insertions(+), 52 deletions(-) create mode 100644 bin/attack_data_archive_models.py create mode 100755 bin/attack_data_archive_reader.py diff --git a/bin/attack_data_archive_models.py b/bin/attack_data_archive_models.py new file mode 100644 index 000000000..954c5baa3 --- /dev/null +++ b/bin/attack_data_archive_models.py @@ -0,0 +1,70 @@ +#!/usr/bin/env python3 +""" +Pydantic models and shared constants for the datasets archive (attack_data_archive/). + +Used by build_dataset_archive.py to generate metadata.yml, and by any tool +that reads the archive back (e.g. fetch_archived_dataset.py). +""" + +import sys +from datetime import datetime, timezone +from typing import Dict, List, Optional + +try: + import yaml + from pydantic import BaseModel, Field, field_serializer +except ImportError as exc: + sys.exit(f"Error: missing dependency ({exc}). Install with: pip install -r bin/requirements.txt") + +OUTPUT_DIR_NAME = "attack_data_archive" +ARCHIVE_FILE_NAME = "attack_data_archive.zip" +METADATA_FILE_NAME = "metadata.yml" + + +def _as_utc_iso(dt: Optional[datetime]) -> Optional[str]: + """Format a datetime as a UTC ISO-8601 string ending in 'Z', or None if dt is None.""" + return dt.astimezone(timezone.utc).isoformat().replace("+00:00", "Z") if dt else None + + +class LfsFileEntry(BaseModel): + """Details for a single git-lfs-tracked file, keyed by its download URL in metadata.yml.""" + + relative_path: str + uncompressed_size: int + last_updated: Optional[datetime] = Field(default=None, alias="last-updated") + + model_config = {"populate_by_name": True} + + @field_serializer("last_updated", when_used="json") + def _serialize_last_updated(self, value: Optional[datetime]) -> Optional[str]: + """Serialize last_updated as a UTC ISO-8601 string.""" + return _as_utc_iso(value) + + +class Metadata(BaseModel): + """Contents of metadata.yml: generation info, and the LFS/non-LFS file listing.""" + + generated_at_utc: datetime + file_count: int + gitref: str + github_url: str + total_uncompressed_size_bytes: int + lfs_files: Dict[str, LfsFileEntry] = Field(default_factory=dict, alias="lfs-files") + non_lfs_files: List[str] = Field(default_factory=list, alias="non-lfs-files") + + model_config = {"populate_by_name": True} + + @field_serializer("generated_at_utc", when_used="json") + def _serialize_generated_at_utc(self, value: datetime) -> str: + """Serialize generated_at_utc as a UTC ISO-8601 string.""" + return _as_utc_iso(value) + + +def to_yaml(model: BaseModel) -> str: + """Serialize a pydantic model to a YAML document, using its field aliases as keys.""" + return yaml.safe_dump(model.model_dump(mode="json", by_alias=True), sort_keys=False, default_flow_style=False) + + +def parse_metadata_yaml(text: str) -> Metadata: + """Parse a metadata.yml document (as text) into a Metadata model.""" + return Metadata(**yaml.safe_load(text)) diff --git a/bin/attack_data_archive_reader.py b/bin/attack_data_archive_reader.py new file mode 100755 index 000000000..2e58d6e55 --- /dev/null +++ b/bin/attack_data_archive_reader.py @@ -0,0 +1,130 @@ +#!/usr/bin/env python3 +""" +Fetch a single LFS-tracked dataset file's bytes out of the local attack_data_archive/ +folder produced by build_dataset_archive.py. + +Given an LFS file's download URL (the key under lfs-files in metadata.yml), this: + 1. Verifies the attack_data_archive/ folder exists. + 2. Verifies attack_data_archive.zip and metadata.yml exist inside it. + 3. Parses metadata.yml. + 4. Verifies the URL is a known LFS file in metadata.yml. + 5. Verifies that file's relative path is actually present in the zip. + 6. Returns that file's bytes. + +Can be used as a CLI, or imported and called as a function +(see get_lfs_file_bytes / load_metadata). + +Requires Python 3.14+ (zipfile.ZIP_ZSTANDARD support), pydantic, and pydantic-settings. +""" + +import sys +import zipfile +from pathlib import Path +from typing import Optional + +try: + from pydantic import Field + from pydantic_settings import BaseSettings, CliApp, CliPositionalArg, SettingsConfigDict +except ImportError as exc: + sys.exit(f"Error: missing dependency ({exc}). Install with: pip install -r bin/requirements.txt") + +from attack_data_archive_models import ( + ARCHIVE_FILE_NAME, + METADATA_FILE_NAME, + OUTPUT_DIR_NAME, + Metadata, + parse_metadata_yaml, +) + +MIN_PYTHON = (3, 14) + +if sys.version_info < MIN_PYTHON: + sys.exit( + f"Error: this script requires Python {MIN_PYTHON[0]}.{MIN_PYTHON[1]}+ " + f"(zipfile.ZIP_ZSTANDARD support). Running {sys.version_info.major}.{sys.version_info.minor}." + ) + + +class ArchiveVerificationError(Exception): + """Raised when the attack_data_archive folder, its files, or a requested LFS entry fail verification.""" + + +def default_archive_dir() -> Path: + """The attack_data_archive/ folder that build_dataset_archive.py writes, alongside this repo's bin/ folder.""" + return Path(__file__).resolve().parent.parent / OUTPUT_DIR_NAME + + +def load_metadata(archive_dir: Path) -> Metadata: + """Verify archive_dir and its zip/yml files exist, then parse and return metadata.yml. + + Raises ArchiveVerificationError if the folder or either file is missing. + """ + if not archive_dir.is_dir(): + raise ArchiveVerificationError(f"Archive folder not found: {archive_dir}") + + zip_path = archive_dir / ARCHIVE_FILE_NAME + if not zip_path.is_file(): + raise ArchiveVerificationError(f"Archive zip not found: {zip_path}") + + yml_path = archive_dir / METADATA_FILE_NAME + if not yml_path.is_file(): + raise ArchiveVerificationError(f"Archive metadata not found: {yml_path}") + + return parse_metadata_yaml(yml_path.read_text()) + + +def get_lfs_file_bytes(url: str, archive_dir: Optional[Path] = None) -> bytes: + """Look up an LFS file by its download URL and return its bytes from the local archive. + + archive_dir defaults to the attack_data_archive/ folder alongside this repo's bin/ folder. + + Raises ArchiveVerificationError if the archive folder/files are missing, the URL is not + a known LFS file in metadata.yml, or the file it points to is missing from the zip. + """ + archive_dir = Path(archive_dir) if archive_dir is not None else default_archive_dir() + metadata = load_metadata(archive_dir) + + entry = metadata.lfs_files.get(url) + if entry is None: + raise ArchiveVerificationError(f"URL not found in {METADATA_FILE_NAME}'s lfs-files: {url}") + + zip_path = archive_dir / ARCHIVE_FILE_NAME + with zipfile.ZipFile(zip_path) as zf: + if entry.relative_path not in zf.namelist(): + raise ArchiveVerificationError( + f"{entry.relative_path} is listed in {METADATA_FILE_NAME} but missing from {zip_path}" + ) + return zf.read(entry.relative_path) + + +class Options(BaseSettings): + """CLI options for fetching one LFS file's bytes from the local attack_data_archive.""" + + model_config = SettingsConfigDict( + cli_prog_name="attack_data_archive_reader.py", + cli_kebab_case=True, + cli_shortcuts={"output": "o", "archive_dir": "d"}, + ) + + url: CliPositionalArg[str] = Field(description="LFS download URL to look up under lfs-files in metadata.yml") + output: Optional[str] = Field(None, description="Write the file's bytes here instead of stdout") + archive_dir: Optional[str] = Field( + None, description="Path to the attack_data_archive folder (default: alongside this repo's bin/ folder)" + ) + + def cli_cmd(self) -> None: + """Fetch the requested LFS file's bytes and write them to --output or stdout.""" + try: + data = get_lfs_file_bytes(self.url, Path(self.archive_dir) if self.archive_dir else None) + except ArchiveVerificationError as exc: + sys.exit(f"Error: {exc}") + + if self.output: + Path(self.output).write_bytes(data) + print(f"Wrote {len(data):,} bytes to {self.output}") + else: + sys.stdout.buffer.write(data) + + +if __name__ == "__main__": + CliApp.run(Options) diff --git a/bin/build_dataset_archive.py b/bin/build_dataset_archive.py index 9f14dea0c..9eb27b6dd 100755 --- a/bin/build_dataset_archive.py +++ b/bin/build_dataset_archive.py @@ -24,16 +24,21 @@ from typing import Dict, Iterator, List, Literal, Optional, Tuple try: - import yaml - from pydantic import BaseModel, Field, field_serializer + from pydantic import Field from pydantic_settings import BaseSettings, CliApp, SettingsConfigDict except ImportError as exc: sys.exit(f"Error: missing dependency ({exc}). Install with: pip install -r bin/requirements.txt") -MIN_PYTHON = (3, 14) +from attack_data_archive_models import ( + ARCHIVE_FILE_NAME, + METADATA_FILE_NAME, + OUTPUT_DIR_NAME, + LfsFileEntry, + Metadata, + to_yaml, +) -OUTPUT_DIR_NAME = "attack_data_archive" -ARCHIVE_FILE_NAME = "attack_data_archive.zip" +MIN_PYTHON = (3, 14) if sys.version_info < MIN_PYTHON: sys.exit( @@ -42,50 +47,6 @@ ) -def _as_utc_iso(dt: Optional[datetime]) -> Optional[str]: - """Format a datetime as a UTC ISO-8601 string ending in 'Z', or None if dt is None.""" - return dt.astimezone(timezone.utc).isoformat().replace("+00:00", "Z") if dt else None - - -class LfsFileEntry(BaseModel): - """Details for a single git-lfs-tracked file, keyed by its download URL in metadata.yml.""" - - relative_path: str - uncompressed_size: int - last_updated: Optional[datetime] = Field(default=None, alias="last-updated") - - model_config = {"populate_by_name": True} - - @field_serializer("last_updated", when_used="json") - def _serialize_last_updated(self, value: Optional[datetime]) -> Optional[str]: - """Serialize last_updated as a UTC ISO-8601 string.""" - return _as_utc_iso(value) - - -class Metadata(BaseModel): - """Contents of metadata.yml: generation info, and the LFS/non-LFS file listing.""" - - generated_at_utc: datetime - file_count: int - gitref: str - github_url: str - total_uncompressed_size_bytes: int - lfs_files: Dict[str, LfsFileEntry] = Field(default_factory=dict, alias="lfs-files") - non_lfs_files: List[str] = Field(default_factory=list, alias="non-lfs-files") - - model_config = {"populate_by_name": True} - - @field_serializer("generated_at_utc", when_used="json") - def _serialize_generated_at_utc(self, value: datetime) -> str: - """Serialize generated_at_utc as a UTC ISO-8601 string.""" - return _as_utc_iso(value) - - -def to_yaml(model: BaseModel) -> str: - """Serialize a pydantic model to a YAML document, using its field aliases as keys.""" - return yaml.safe_dump(model.model_dump(mode="json", by_alias=True), sort_keys=False, default_flow_style=False) - - def run_git(args: List[str], cwd: "Path | str") -> str: """Run a git command in cwd and return its stripped stdout, raising CalledProcessError on failure.""" result = subprocess.run(["git", *args], cwd=cwd, capture_output=True, text=True, check=True) @@ -219,9 +180,9 @@ def build_archive( **{"lfs-files": lfs_files, "non-lfs-files": non_lfs_files}, ) metadata_yaml = to_yaml(metadata) - zf.writestr("metadata.yml", metadata_yaml) + zf.writestr(METADATA_FILE_NAME, metadata_yaml) - (output_dir / "metadata.yml").write_text(metadata_yaml) + (output_dir / METADATA_FILE_NAME).write_text(metadata_yaml) return len(files), total_size, len(lfs_files), len(non_lfs_files) @@ -255,7 +216,7 @@ def cli_cmd(self) -> None: ) print(f"Wrote {output_dir / ARCHIVE_FILE_NAME}") - print(f"Wrote {output_dir / 'metadata.yml'}") + print(f"Wrote {output_dir / METADATA_FILE_NAME}") print(f" files: {file_count} ({total_size:,} bytes uncompressed)") print(f" lfs files: {lfs_count}") print(f" non-lfs files: {non_lfs_count}") From 37892270f7f68d6da4ac03114a765ca23c505f0e Mon Sep 17 00:00:00 2001 From: Eric McGinnis Date: Thu, 6 Aug 2026 15:03:33 -0700 Subject: [PATCH 06/14] Fix it so that only data from master branch can go into the archive. --- bin/build_dataset_archive.py | 49 ++++++------------------------------ 1 file changed, 7 insertions(+), 42 deletions(-) diff --git a/bin/build_dataset_archive.py b/bin/build_dataset_archive.py index 9eb27b6dd..4cfd39903 100755 --- a/bin/build_dataset_archive.py +++ b/bin/build_dataset_archive.py @@ -21,7 +21,7 @@ import zipfile from datetime import datetime, timezone from pathlib import Path -from typing import Dict, Iterator, List, Literal, Optional, Tuple +from typing import Dict, Iterator, List, Tuple try: from pydantic import Field @@ -40,6 +40,8 @@ MIN_PYTHON = (3, 14) +REF_NAME = "master" + if sys.version_info < MIN_PYTHON: sys.exit( f"Error: this script requires Python {MIN_PYTHON[0]}.{MIN_PYTHON[1]}+ " @@ -61,35 +63,6 @@ def parse_github_owner_repo(remote_url: str) -> Tuple[str, str]: return match.group(1), match.group(2) -def determine_ref( - repo_root: Path, ref_override: Optional[str], ref_type_override: Optional[str] -) -> Tuple[str, str]: - """Resolve the (ref_name, ref_type) to embed in generated URLs. - - Preference order: an explicit override, then the GitHub Actions - GITHUB_REF_NAME/GITHUB_REF_TYPE env vars, then the current local branch. - """ - if ref_override: - return ref_override, ref_type_override or "branch" - - ref_name = os.environ.get("GITHUB_REF_NAME") - ref_type = os.environ.get("GITHUB_REF_TYPE") - if ref_name and ref_type: - return ref_name, ref_type - - try: - branch = run_git(["symbolic-ref", "--short", "HEAD"], cwd=repo_root) - if branch: - return branch, "branch" - except subprocess.CalledProcessError: - pass - - raise RuntimeError( - "Could not determine a branch/tag name for this checkout (HEAD is " - "detached and GITHUB_REF_NAME is not set). Pass --ref/--ref-type explicitly." - ) - - def collect_last_updated(repo_root: Path) -> Dict[str, datetime]: """Map {relative_path: last commit datetime} for every file ever touched under datasets/.""" raw = run_git( @@ -121,8 +94,6 @@ def build_archive( datasets_dir: Path, output_dir: Path, compresslevel: int, - ref_name: str, - ref_type: str, ) -> Tuple[int, int, int, int]: """Write a ZIP_ZSTANDARD archive of datasets_dir plus metadata.yml into output_dir. @@ -135,7 +106,6 @@ def build_archive( output_path = output_dir / ARCHIVE_FILE_NAME owner, repo = parse_github_owner_repo(run_git(["remote", "get-url", "origin"], cwd=repo_root)) - ref_segment = "heads" if ref_type == "branch" else "tags" git_hash = run_git(["rev-parse", "HEAD"], cwd=repo_root) lfs_paths = { @@ -162,7 +132,7 @@ def build_archive( print(f" added {index}/{len(files)} files...") if rel in lfs_paths: - url = f"https://media.githubusercontent.com/media/{owner}/{repo}/refs/{ref_segment}/{ref_name}/{rel}" + url = f"https://media.githubusercontent.com/media/{owner}/{repo}/{REF_NAME}/{rel}" lfs_files[url] = LfsFileEntry( relative_path=rel, uncompressed_size=size, @@ -175,7 +145,7 @@ def build_archive( generated_at_utc=datetime.now(timezone.utc), file_count=len(files), gitref=git_hash, - github_url=f"https://github.com/{owner}/{repo}/tree/{ref_name}", + github_url=f"https://github.com/{owner}/{repo}/tree/{REF_NAME}", total_uncompressed_size_bytes=total_size, **{"lfs-files": lfs_files, "non-lfs-files": non_lfs_files}, ) @@ -196,10 +166,6 @@ class Options(BaseSettings): ) compresslevel: int = Field(9, description="Zstandard compression level (default: 9)") - ref: Optional[str] = Field(None, description="Override the git branch/tag name used to build GitHub URLs") - ref_type: Optional[Literal["branch", "tag"]] = Field( - None, description="Whether --ref is a branch or a tag (default: branch)" - ) def cli_cmd(self) -> None: """Build the archive using the parsed CLI options and print a summary.""" @@ -209,10 +175,9 @@ def cli_cmd(self) -> None: sys.exit(f"Error: {datasets_dir} does not exist") output_dir = repo_root / OUTPUT_DIR_NAME - ref_name, ref_type = determine_ref(repo_root, self.ref, self.ref_type) file_count, total_size, lfs_count, non_lfs_count = build_archive( - repo_root, datasets_dir, output_dir, self.compresslevel, ref_name, ref_type + repo_root, datasets_dir, output_dir, self.compresslevel ) print(f"Wrote {output_dir / ARCHIVE_FILE_NAME}") @@ -220,7 +185,7 @@ def cli_cmd(self) -> None: print(f" files: {file_count} ({total_size:,} bytes uncompressed)") print(f" lfs files: {lfs_count}") print(f" non-lfs files: {non_lfs_count}") - print(f" ref: {ref_name} ({ref_type})") + print(f" ref: {REF_NAME} (branch)") if __name__ == "__main__": From 07de79593d5f165f1ace6e937a8e24118074e1e7 Mon Sep 17 00:00:00 2001 From: Eric McGinnis Date: Thu, 6 Aug 2026 15:09:26 -0700 Subject: [PATCH 07/14] add drift detection when metadata file outside archive does not match metadata file inside archive --- bin/attack_data_archive_reader.py | 48 +++++++++++++++++++++++++++---- 1 file changed, 42 insertions(+), 6 deletions(-) diff --git a/bin/attack_data_archive_reader.py b/bin/attack_data_archive_reader.py index 2e58d6e55..2900e96a2 100755 --- a/bin/attack_data_archive_reader.py +++ b/bin/attack_data_archive_reader.py @@ -6,10 +6,11 @@ Given an LFS file's download URL (the key under lfs-files in metadata.yml), this: 1. Verifies the attack_data_archive/ folder exists. 2. Verifies attack_data_archive.zip and metadata.yml exist inside it. - 3. Parses metadata.yml. - 4. Verifies the URL is a known LFS file in metadata.yml. - 5. Verifies that file's relative path is actually present in the zip. - 6. Returns that file's bytes. + 3. Verifies the standalone metadata.yml matches the copy embedded in the zip. + 4. Parses metadata.yml. + 5. Verifies the URL is a known LFS file in metadata.yml. + 6. Verifies that file's relative path is actually present in the zip. + 7. Returns that file's bytes. Can be used as a CLI, or imported and called as a function (see get_lfs_file_bytes / load_metadata). @@ -49,6 +50,23 @@ class ArchiveVerificationError(Exception): """Raised when the attack_data_archive folder, its files, or a requested LFS entry fail verification.""" +class MetadataFilesDiffer(ArchiveVerificationError): + """Raised when the standalone metadata.yml and the copy embedded in the zip are not byte-identical.""" + + +def _first_differing_line(a: str, b: str) -> int: + """Return the 1-indexed line number of the first line where a and b differ. + + If one text is a prefix of the other, returns the line just past the shorter text's end. + """ + a_lines = a.splitlines() + b_lines = b.splitlines() + for i, (a_line, b_line) in enumerate(zip(a_lines, b_lines), start=1): + if a_line != b_line: + return i + return min(len(a_lines), len(b_lines)) + 1 + + def default_archive_dir() -> Path: """The attack_data_archive/ folder that build_dataset_archive.py writes, alongside this repo's bin/ folder.""" return Path(__file__).resolve().parent.parent / OUTPUT_DIR_NAME @@ -57,7 +75,11 @@ def default_archive_dir() -> Path: def load_metadata(archive_dir: Path) -> Metadata: """Verify archive_dir and its zip/yml files exist, then parse and return metadata.yml. - Raises ArchiveVerificationError if the folder or either file is missing. + Also verifies the standalone metadata.yml is byte-identical to the copy embedded + in the zip, so the two can never silently drift apart. + + Raises ArchiveVerificationError if the folder or either file is missing, or + MetadataFilesDiffer if the standalone and embedded metadata.yml contents differ. """ if not archive_dir.is_dir(): raise ArchiveVerificationError(f"Archive folder not found: {archive_dir}") @@ -70,7 +92,21 @@ def load_metadata(archive_dir: Path) -> Metadata: if not yml_path.is_file(): raise ArchiveVerificationError(f"Archive metadata not found: {yml_path}") - return parse_metadata_yaml(yml_path.read_text()) + standalone_text = yml_path.read_text() + + with zipfile.ZipFile(zip_path) as zf: + try: + embedded_text = zf.read(METADATA_FILE_NAME).decode() + except KeyError: + raise ArchiveVerificationError(f"{METADATA_FILE_NAME} not found inside {zip_path}") from None + + if standalone_text != embedded_text: + line = _first_differing_line(standalone_text, embedded_text) + raise MetadataFilesDiffer( + f"{yml_path} and the {METADATA_FILE_NAME} embedded in {zip_path} differ, first at line {line}" + ) + + return parse_metadata_yaml(standalone_text) def get_lfs_file_bytes(url: str, archive_dir: Optional[Path] = None) -> bytes: From 020fa2a1697d44b85a294484f9a2eee12a17f1bd Mon Sep 17 00:00:00 2001 From: Eric McGinnis Date: Thu, 6 Aug 2026 15:26:18 -0700 Subject: [PATCH 08/14] attack data resolver and returner object that can be reused --- bin/attack_data_archive_reader.py | 150 ++++++++++++++++++------------ 1 file changed, 93 insertions(+), 57 deletions(-) diff --git a/bin/attack_data_archive_reader.py b/bin/attack_data_archive_reader.py index 2900e96a2..5625e576b 100755 --- a/bin/attack_data_archive_reader.py +++ b/bin/attack_data_archive_reader.py @@ -3,17 +3,20 @@ Fetch a single LFS-tracked dataset file's bytes out of the local attack_data_archive/ folder produced by build_dataset_archive.py. -Given an LFS file's download URL (the key under lfs-files in metadata.yml), this: - 1. Verifies the attack_data_archive/ folder exists. - 2. Verifies attack_data_archive.zip and metadata.yml exist inside it. - 3. Verifies the standalone metadata.yml matches the copy embedded in the zip. - 4. Parses metadata.yml. - 5. Verifies the URL is a known LFS file in metadata.yml. - 6. Verifies that file's relative path is actually present in the zip. - 7. Returns that file's bytes. - -Can be used as a CLI, or imported and called as a function -(see get_lfs_file_bytes / load_metadata). +AttackDataArchiveResolver does the one-time verification (archive folder exists, +attack_data_archive.zip and metadata.yml exist inside it, and the standalone +metadata.yml matches the copy embedded in the zip) once, at construction. Reuse the +same instance across many calls to verify_lfs_file_existence / get_lfs_file_bytes +to avoid re-verifying and re-reading metadata.yml every time. + +Given an LFS file's download URL (the key under lfs-files in metadata.yml), +verify_lfs_file_existence: + 1. Verifies the URL is a known LFS file in metadata.yml. + 2. Verifies that file's relative path is actually present in the zip. + +get_lfs_file_bytes calls verify_lfs_file_existence, then returns that file's bytes. + +Can be used as a CLI, or imported and used as a class (see AttackDataArchiveResolver). Requires Python 3.14+ (zipfile.ZIP_ZSTANDARD support), pydantic, and pydantic-settings. """ @@ -24,7 +27,7 @@ from typing import Optional try: - from pydantic import Field + from pydantic import BaseModel, Field, PrivateAttr from pydantic_settings import BaseSettings, CliApp, CliPositionalArg, SettingsConfigDict except ImportError as exc: sys.exit(f"Error: missing dependency ({exc}). Install with: pip install -r bin/requirements.txt") @@ -72,65 +75,96 @@ def default_archive_dir() -> Path: return Path(__file__).resolve().parent.parent / OUTPUT_DIR_NAME -def load_metadata(archive_dir: Path) -> Metadata: - """Verify archive_dir and its zip/yml files exist, then parse and return metadata.yml. +class AttackDataArchiveResolver(BaseModel): + """Verifies the local attack_data_archive/ once, then resolves many LFS URLs against it cheaply. - Also verifies the standalone metadata.yml is byte-identical to the copy embedded - in the zip, so the two can never silently drift apart. + Construction verifies that archive_dir exists, that attack_data_archive.zip and metadata.yml + exist inside it, and that the standalone metadata.yml is byte-identical to the copy embedded + in the zip. metadata.yml is parsed once and cached on the instance. + + Reuse the same instance across many verify_lfs_file_existence / get_lfs_file_bytes calls to + avoid re-verifying and re-parsing metadata.yml on every call. Raises ArchiveVerificationError if the folder or either file is missing, or MetadataFilesDiffer if the standalone and embedded metadata.yml contents differ. """ - if not archive_dir.is_dir(): - raise ArchiveVerificationError(f"Archive folder not found: {archive_dir}") - zip_path = archive_dir / ARCHIVE_FILE_NAME - if not zip_path.is_file(): - raise ArchiveVerificationError(f"Archive zip not found: {zip_path}") + archive_dir: Path = Field(default_factory=default_archive_dir) - yml_path = archive_dir / METADATA_FILE_NAME - if not yml_path.is_file(): - raise ArchiveVerificationError(f"Archive metadata not found: {yml_path}") + _metadata: Metadata = PrivateAttr() - standalone_text = yml_path.read_text() + def model_post_init(self, __context: object) -> None: + """Verify the archive folder/files exist and are consistent, then cache parsed metadata.yml.""" + archive_dir = self.archive_dir - with zipfile.ZipFile(zip_path) as zf: - try: - embedded_text = zf.read(METADATA_FILE_NAME).decode() - except KeyError: - raise ArchiveVerificationError(f"{METADATA_FILE_NAME} not found inside {zip_path}") from None + if not archive_dir.is_dir(): + raise ArchiveVerificationError(f"Archive folder not found: {archive_dir}") - if standalone_text != embedded_text: - line = _first_differing_line(standalone_text, embedded_text) - raise MetadataFilesDiffer( - f"{yml_path} and the {METADATA_FILE_NAME} embedded in {zip_path} differ, first at line {line}" - ) + zip_path = self.zip_path + if not zip_path.is_file(): + raise ArchiveVerificationError(f"Archive zip not found: {zip_path}") - return parse_metadata_yaml(standalone_text) + yml_path = self.yml_path + if not yml_path.is_file(): + raise ArchiveVerificationError(f"Archive metadata not found: {yml_path}") + standalone_text = yml_path.read_text() -def get_lfs_file_bytes(url: str, archive_dir: Optional[Path] = None) -> bytes: - """Look up an LFS file by its download URL and return its bytes from the local archive. + with zipfile.ZipFile(zip_path) as zf: + try: + embedded_text = zf.read(METADATA_FILE_NAME).decode() + except KeyError: + raise ArchiveVerificationError(f"{METADATA_FILE_NAME} not found inside {zip_path}") from None - archive_dir defaults to the attack_data_archive/ folder alongside this repo's bin/ folder. - - Raises ArchiveVerificationError if the archive folder/files are missing, the URL is not - a known LFS file in metadata.yml, or the file it points to is missing from the zip. - """ - archive_dir = Path(archive_dir) if archive_dir is not None else default_archive_dir() - metadata = load_metadata(archive_dir) - - entry = metadata.lfs_files.get(url) - if entry is None: - raise ArchiveVerificationError(f"URL not found in {METADATA_FILE_NAME}'s lfs-files: {url}") - - zip_path = archive_dir / ARCHIVE_FILE_NAME - with zipfile.ZipFile(zip_path) as zf: - if entry.relative_path not in zf.namelist(): - raise ArchiveVerificationError( - f"{entry.relative_path} is listed in {METADATA_FILE_NAME} but missing from {zip_path}" + if standalone_text != embedded_text: + line = _first_differing_line(standalone_text, embedded_text) + raise MetadataFilesDiffer( + f"{yml_path} and the {METADATA_FILE_NAME} embedded in {zip_path} differ, first at line {line}" ) - return zf.read(entry.relative_path) + + self._metadata = parse_metadata_yaml(standalone_text) + + @property + def zip_path(self) -> Path: + """Path to attack_data_archive.zip inside archive_dir.""" + return self.archive_dir / ARCHIVE_FILE_NAME + + @property + def yml_path(self) -> Path: + """Path to the standalone metadata.yml inside archive_dir.""" + return self.archive_dir / METADATA_FILE_NAME + + @property + def metadata(self) -> Metadata: + """The parsed metadata.yml, cached at construction time.""" + return self._metadata + + def verify_lfs_file_existence(self, url: str) -> str: + """Verify url is a known LFS file in metadata.yml and that its file is present in the zip. + + Returns the file's relative path within the zip on success. + + Raises ArchiveVerificationError if the URL is not a known LFS file in metadata.yml, + or the file it points to is missing from the zip. + """ + entry = self._metadata.lfs_files.get(url) + if entry is None: + raise ArchiveVerificationError(f"URL not found in {METADATA_FILE_NAME}'s lfs-files: {url}") + + with zipfile.ZipFile(self.zip_path) as zf: + if entry.relative_path not in zf.namelist(): + raise ArchiveVerificationError( + f"{entry.relative_path} is listed in {METADATA_FILE_NAME} but missing from {self.zip_path}" + ) + + return entry.relative_path + + def get_lfs_file_bytes(self, url: str) -> bytes: + """Look up an LFS file by its download URL and return its bytes from the local archive.""" + relative_path = self.verify_lfs_file_existence(url) + + with zipfile.ZipFile(self.zip_path) as zf: + return zf.read(relative_path) class Options(BaseSettings): @@ -151,7 +185,9 @@ class Options(BaseSettings): def cli_cmd(self) -> None: """Fetch the requested LFS file's bytes and write them to --output or stdout.""" try: - data = get_lfs_file_bytes(self.url, Path(self.archive_dir) if self.archive_dir else None) + kwargs = {"archive_dir": Path(self.archive_dir)} if self.archive_dir else {} + resolver = AttackDataArchiveResolver(**kwargs) + data = resolver.get_lfs_file_bytes(self.url) except ArchiveVerificationError as exc: sys.exit(f"Error: {exc}") From 375ebedb7b825f8a22ac58324cb1d449706c71a2 Mon Sep 17 00:00:00 2001 From: Eric McGinnis Date: Thu, 6 Aug 2026 17:00:08 -0700 Subject: [PATCH 09/14] common code path for head and get resolution --- bin/attack_data_archive_reader.py | 227 +++++++++++++++++++++++------- 1 file changed, 175 insertions(+), 52 deletions(-) diff --git a/bin/attack_data_archive_reader.py b/bin/attack_data_archive_reader.py index 5625e576b..dc82cfed4 100755 --- a/bin/attack_data_archive_reader.py +++ b/bin/attack_data_archive_reader.py @@ -1,20 +1,24 @@ #!/usr/bin/env python3 """ -Fetch a single LFS-tracked dataset file's bytes out of the local attack_data_archive/ -folder produced by build_dataset_archive.py. +Fetch a single dataset file's bytes, either from the local attack_data_archive/ +folder produced by build_dataset_archive.py or, for LFS URLs not present in that +archive, straight from GitHub. AttackDataArchiveResolver does the one-time verification (archive folder exists, attack_data_archive.zip and metadata.yml exist inside it, and the standalone metadata.yml matches the copy embedded in the zip) once, at construction. Reuse the -same instance across many calls to verify_lfs_file_existence / get_lfs_file_bytes -to avoid re-verifying and re-reading metadata.yml every time. - -Given an LFS file's download URL (the key under lfs-files in metadata.yml), -verify_lfs_file_existence: - 1. Verifies the URL is a known LFS file in metadata.yml. - 2. Verifies that file's relative path is actually present in the zip. - -get_lfs_file_bytes calls verify_lfs_file_existence, then returns that file's bytes. +same instance across many calls to verify_path / get_data to avoid re-verifying and +re-reading metadata.yml every time. + +verify_path and get_data are the only public entry points; everything else on +AttackDataArchiveResolver is a private implementation detail. Both accept either a +FilePath (an existing local file) or an HttpUrl (an LFS download URL): + - verify_path checks that a FilePath exists, or that an HttpUrl is a known, + present-in-the-zip LFS file (i.e. it's in the cache) — or, if + check_existence_with_head_request is enabled, that an uncached HttpUrl is + reachable via a HEAD request. + - get_data calls verify_path, then returns the bytes: read from disk for a + FilePath, from the cache for a cached HttpUrl, or via a GET request otherwise. Can be used as a CLI, or imported and used as a class (see AttackDataArchiveResolver). @@ -23,14 +27,27 @@ import sys import zipfile +from functools import cached_property from pathlib import Path -from typing import Optional - -try: - from pydantic import BaseModel, Field, PrivateAttr - from pydantic_settings import BaseSettings, CliApp, CliPositionalArg, SettingsConfigDict -except ImportError as exc: - sys.exit(f"Error: missing dependency ({exc}). Install with: pip install -r bin/requirements.txt") +from typing import Callable, Optional, Union + +import requests +from pydantic import ( + BaseModel, + ConfigDict, + Field, + FilePath, + HttpUrl, + PrivateAttr, + validate_call, +) +from pydantic_settings import ( + BaseSettings, + CliApp, + CliImplicitFlag, + CliPositionalArg, + SettingsConfigDict, +) from attack_data_archive_models import ( ARCHIVE_FILE_NAME, @@ -57,11 +74,22 @@ class MetadataFilesDiffer(ArchiveVerificationError): """Raised when the standalone metadata.yml and the copy embedded in the zip are not byte-identical.""" +class UrlUnreachable(ArchiveVerificationError): + """Raised when an HttpUrl not found in the cache fails a HEAD/GET request.""" + + +HTTP_REQUEST_MAX_ATTEMPTS = 3 +HTTP_REQUEST_RETRY_STATUS_CODES = {403, 503} +HTTP_REQUEST_TIMEOUT_SECONDS = 10 + + def _first_differing_line(a: str, b: str) -> int: """Return the 1-indexed line number of the first line where a and b differ. If one text is a prefix of the other, returns the line just past the shorter text's end. """ + x = requests.head + a_lines = a.splitlines() b_lines = b.splitlines() for i, (a_line, b_line) in enumerate(zip(a_lines, b_lines), start=1): @@ -70,26 +98,56 @@ def _first_differing_line(a: str, b: str) -> int: return min(len(a_lines), len(b_lines)) + 1 +def _request_with_retry( + request_fn: Callable[..., requests.Response], method_name: str, url: HttpUrl +) -> "requests.Response": + """Call request_fn(url), retrying up to HTTP_REQUEST_MAX_ATTEMPTS total attempts on 403/503. + + Returns the response on a 200. Raises UrlUnreachable if every attempt returns 403/503, + or on the first response with any other non-200 status. + """ + last_status: Optional[int] = None + for _ in range(1, HTTP_REQUEST_MAX_ATTEMPTS + 1): + response = request_fn(str(url), timeout=HTTP_REQUEST_TIMEOUT_SECONDS) + if response.status_code == 200: + return response + if response.status_code not in HTTP_REQUEST_RETRY_STATUS_CODES: + raise UrlUnreachable(f"{method_name} {url} returned {response.status_code}") + last_status = response.status_code + + raise UrlUnreachable( + f"{method_name} {url} returned {last_status} on all {HTTP_REQUEST_MAX_ATTEMPTS} attempts" + ) + + +def _verify_url_reachable(url: HttpUrl) -> None: + """HEAD-request url, using the same retry/success/failure modes as a GET (see _request_with_retry).""" + _request_with_retry(requests.head, "HEAD", url) + + def default_archive_dir() -> Path: """The attack_data_archive/ folder that build_dataset_archive.py writes, alongside this repo's bin/ folder.""" return Path(__file__).resolve().parent.parent / OUTPUT_DIR_NAME class AttackDataArchiveResolver(BaseModel): - """Verifies the local attack_data_archive/ once, then resolves many LFS URLs against it cheaply. + """Verifies the local attack_data_archive/ once, then resolves many files/URLs against it cheaply. Construction verifies that archive_dir exists, that attack_data_archive.zip and metadata.yml exist inside it, and that the standalone metadata.yml is byte-identical to the copy embedded in the zip. metadata.yml is parsed once and cached on the instance. - Reuse the same instance across many verify_lfs_file_existence / get_lfs_file_bytes calls to - avoid re-verifying and re-parsing metadata.yml on every call. + verify_path and get_data are the only public methods; reuse the same instance across many + calls to them to avoid re-verifying and re-parsing metadata.yml on every call. Raises ArchiveVerificationError if the folder or either file is missing, or MetadataFilesDiffer if the standalone and embedded metadata.yml contents differ. """ + model_config = ConfigDict(frozen=True) + archive_dir: Path = Field(default_factory=default_archive_dir) + check_existence_with_head_request: bool = False _metadata: Metadata = PrivateAttr() @@ -100,11 +158,11 @@ def model_post_init(self, __context: object) -> None: if not archive_dir.is_dir(): raise ArchiveVerificationError(f"Archive folder not found: {archive_dir}") - zip_path = self.zip_path + zip_path = self._zip_path if not zip_path.is_file(): raise ArchiveVerificationError(f"Archive zip not found: {zip_path}") - yml_path = self.yml_path + yml_path = self._yml_path if not yml_path.is_file(): raise ArchiveVerificationError(f"Archive metadata not found: {yml_path}") @@ -114,7 +172,9 @@ def model_post_init(self, __context: object) -> None: try: embedded_text = zf.read(METADATA_FILE_NAME).decode() except KeyError: - raise ArchiveVerificationError(f"{METADATA_FILE_NAME} not found inside {zip_path}") from None + raise ArchiveVerificationError( + f"{METADATA_FILE_NAME} not found inside {zip_path}" + ) from None if standalone_text != embedded_text: line = _first_differing_line(standalone_text, embedded_text) @@ -125,46 +185,87 @@ def model_post_init(self, __context: object) -> None: self._metadata = parse_metadata_yaml(standalone_text) @property - def zip_path(self) -> Path: + def _zip_path(self) -> Path: """Path to attack_data_archive.zip inside archive_dir.""" return self.archive_dir / ARCHIVE_FILE_NAME @property - def yml_path(self) -> Path: + def _yml_path(self) -> Path: """Path to the standalone metadata.yml inside archive_dir.""" return self.archive_dir / METADATA_FILE_NAME - @property - def metadata(self) -> Metadata: - """The parsed metadata.yml, cached at construction time.""" - return self._metadata + @cached_property + def _zip_namelist(self) -> frozenset: + """Set of every relative path stored inside the zip, computed once since the archive is immutable.""" + with zipfile.ZipFile(self._zip_path) as zf: + return frozenset(zf.namelist()) - def verify_lfs_file_existence(self, url: str) -> str: - """Verify url is a known LFS file in metadata.yml and that its file is present in the zip. + def _cached_lfs_relative_path(self, url: str) -> Optional[str]: + """Look up url in the cached metadata.yml and verify its file is present in the zip. - Returns the file's relative path within the zip on success. + Returns the file's relative path within the zip if url is a known, present-in-the-zip + LFS file, or None if url is not a known LFS file at all (i.e. not in the cache). - Raises ArchiveVerificationError if the URL is not a known LFS file in metadata.yml, - or the file it points to is missing from the zip. + Raises ArchiveVerificationError if url is a known LFS file but its file is missing + from the zip. """ entry = self._metadata.lfs_files.get(url) if entry is None: - raise ArchiveVerificationError(f"URL not found in {METADATA_FILE_NAME}'s lfs-files: {url}") + return None - with zipfile.ZipFile(self.zip_path) as zf: - if entry.relative_path not in zf.namelist(): - raise ArchiveVerificationError( - f"{entry.relative_path} is listed in {METADATA_FILE_NAME} but missing from {self.zip_path}" - ) + if entry.relative_path not in self._zip_namelist: + raise ArchiveVerificationError( + f"{entry.relative_path} is listed in {METADATA_FILE_NAME} but missing from {self._zip_path}" + ) return entry.relative_path - def get_lfs_file_bytes(self, url: str) -> bytes: - """Look up an LFS file by its download URL and return its bytes from the local archive.""" - relative_path = self.verify_lfs_file_existence(url) + @validate_call + def verify_path(self, path: Union[FilePath, HttpUrl]) -> None: + """Verify that path exists, whether it's a local file or an HttpUrl. + + A FilePath is verified simply by being accepted as an argument (pydantic's + FilePath validation already requires it to exist on disk). + + An HttpUrl is verified if it's a known, present-in-the-zip LFS file (i.e. it's + in the cache). If it's not in the cache and check_existence_with_head_request + is enabled, falls back to a HEAD request (see _verify_url_reachable). + + Raises ArchiveVerificationError if path is an HttpUrl that's neither in the + cache nor (when enabled) reachable via HEAD request. + """ + if isinstance(path, Path): + return + + if self._cached_lfs_relative_path(str(path)) is not None: + return - with zipfile.ZipFile(self.zip_path) as zf: - return zf.read(relative_path) + if not self.check_existence_with_head_request: + raise ArchiveVerificationError( + f"URL not found in {METADATA_FILE_NAME}'s lfs-files: {path}" + ) + + _verify_url_reachable(path) + + @validate_call + def get_data(self, path: Union[FilePath, HttpUrl]) -> bytes: + """Return the bytes at path, whether it's a local file or an HttpUrl. + + Calls verify_path first. A FilePath's bytes are then read directly from disk. + An HttpUrl's bytes are returned from the cache if it's a known, present-in-the-zip + LFS file; otherwise they're downloaded with a GET request. + """ + self.verify_path(path) + + if isinstance(path, Path): + return path.read_bytes() + + relative_path = self._cached_lfs_relative_path(str(path)) + if relative_path is not None: + with zipfile.ZipFile(self._zip_path) as zf: + return zf.read(relative_path) + + return _request_with_retry(requests.get, "GET", path).content class Options(BaseSettings): @@ -176,18 +277,40 @@ class Options(BaseSettings): cli_shortcuts={"output": "o", "archive_dir": "d"}, ) - url: CliPositionalArg[str] = Field(description="LFS download URL to look up under lfs-files in metadata.yml") - output: Optional[str] = Field(None, description="Write the file's bytes here instead of stdout") + url: CliPositionalArg[str] = Field( + description="LFS download URL to look up under lfs-files in metadata.yml" + ) + output: Optional[str] = Field( + None, description="Write the file's bytes here instead of stdout" + ) archive_dir: Optional[str] = Field( - None, description="Path to the attack_data_archive folder (default: alongside this repo's bin/ folder)" + None, + description="Path to the attack_data_archive folder (default: alongside this repo's bin/ folder)", + ) + resolve: CliImplicitFlag[bool] = Field( + False, + description="Only verify that the URL resolves (is in the cache, or reachable via a HEAD request); don't fetch its bytes", + ) + check_existence_with_head_request: CliImplicitFlag[bool] = Field( + False, + description="When resolving a URL not in the cache, verify it with a HEAD request instead of failing immediately", ) def cli_cmd(self) -> None: - """Fetch the requested LFS file's bytes and write them to --output or stdout.""" + """Fetch the requested LFS file's bytes and write them to --output or stdout, or just verify it resolves.""" try: kwargs = {"archive_dir": Path(self.archive_dir)} if self.archive_dir else {} - resolver = AttackDataArchiveResolver(**kwargs) - data = resolver.get_lfs_file_bytes(self.url) + resolver = AttackDataArchiveResolver( + check_existence_with_head_request=self.check_existence_with_head_request, + **kwargs, + ) + + if self.resolve: + resolver.verify_path(self.url) + print(f"Resolved: {self.url}") + return + + data = resolver.get_data(self.url) except ArchiveVerificationError as exc: sys.exit(f"Error: {exc}") From 8da68016e4e4895b0c9ce2ee94575a49dc95af9b Mon Sep 17 00:00:00 2001 From: Eric McGinnis Date: Thu, 6 Aug 2026 17:06:42 -0700 Subject: [PATCH 10/14] add provenance to describe if data came from file, url, or cache --- bin/attack_data_archive_reader.py | 60 +++++++++++++++++++------------ 1 file changed, 37 insertions(+), 23 deletions(-) diff --git a/bin/attack_data_archive_reader.py b/bin/attack_data_archive_reader.py index dc82cfed4..3d6c60dd4 100755 --- a/bin/attack_data_archive_reader.py +++ b/bin/attack_data_archive_reader.py @@ -12,13 +12,15 @@ verify_path and get_data are the only public entry points; everything else on AttackDataArchiveResolver is a private implementation detail. Both accept either a -FilePath (an existing local file) or an HttpUrl (an LFS download URL): +FilePath (an existing local file) or an HttpUrl (an LFS download URL), and both +return an AttackDataProvenance saying where the data came from (or, for get_data's +URL_GET case, was fetched from): - verify_path checks that a FilePath exists, or that an HttpUrl is a known, present-in-the-zip LFS file (i.e. it's in the cache) — or, if check_existence_with_head_request is enabled, that an uncached HttpUrl is reachable via a HEAD request. - - get_data calls verify_path, then returns the bytes: read from disk for a - FilePath, from the cache for a cached HttpUrl, or via a GET request otherwise. + - get_data calls verify_path, then returns the bytes alongside it: read from disk + for a FilePath, from the cache for a cached HttpUrl, or via a GET request otherwise. Can be used as a CLI, or imported and used as a class (see AttackDataArchiveResolver). @@ -27,9 +29,10 @@ import sys import zipfile +from enum import StrEnum from functools import cached_property from pathlib import Path -from typing import Callable, Optional, Union +from typing import Callable, Optional, Tuple, Union import requests from pydantic import ( @@ -78,6 +81,14 @@ class UrlUnreachable(ArchiveVerificationError): """Raised when an HttpUrl not found in the cache fails a HEAD/GET request.""" +class AttackDataProvenance(StrEnum): + """Where verify_path/get_data found (or would fetch) a path's data.""" + + FILE = "file" + ATTACK_DATA_CACHE = "attack_data_cache" + URL_GET = "url_get" + + HTTP_REQUEST_MAX_ATTEMPTS = 3 HTTP_REQUEST_RETRY_STATUS_CODES = {403, 503} HTTP_REQUEST_TIMEOUT_SECONDS = 10 @@ -221,7 +232,7 @@ def _cached_lfs_relative_path(self, url: str) -> Optional[str]: return entry.relative_path @validate_call - def verify_path(self, path: Union[FilePath, HttpUrl]) -> None: + def verify_path(self, path: Union[FilePath, HttpUrl]) -> AttackDataProvenance: """Verify that path exists, whether it's a local file or an HttpUrl. A FilePath is verified simply by being accepted as an argument (pydantic's @@ -231,14 +242,16 @@ def verify_path(self, path: Union[FilePath, HttpUrl]) -> None: in the cache). If it's not in the cache and check_existence_with_head_request is enabled, falls back to a HEAD request (see _verify_url_reachable). + Returns where the data was found (or, for URL_GET, would be fetched from). + Raises ArchiveVerificationError if path is an HttpUrl that's neither in the cache nor (when enabled) reachable via HEAD request. """ if isinstance(path, Path): - return + return AttackDataProvenance.FILE if self._cached_lfs_relative_path(str(path)) is not None: - return + return AttackDataProvenance.ATTACK_DATA_CACHE if not self.check_existence_with_head_request: raise ArchiveVerificationError( @@ -246,26 +259,27 @@ def verify_path(self, path: Union[FilePath, HttpUrl]) -> None: ) _verify_url_reachable(path) + return AttackDataProvenance.URL_GET @validate_call - def get_data(self, path: Union[FilePath, HttpUrl]) -> bytes: - """Return the bytes at path, whether it's a local file or an HttpUrl. + def get_data(self, path: Union[FilePath, HttpUrl]) -> Tuple[bytes, AttackDataProvenance]: + """Return the bytes at path plus where they came from, whether path is a local file or an HttpUrl. - Calls verify_path first. A FilePath's bytes are then read directly from disk. - An HttpUrl's bytes are returned from the cache if it's a known, present-in-the-zip - LFS file; otherwise they're downloaded with a GET request. + Calls verify_path first, then reuses its result: a FilePath's bytes are read + directly from disk, an HttpUrl's bytes are read from the cache if it's a known, + present-in-the-zip LFS file, or otherwise downloaded with a GET request. """ - self.verify_path(path) + provenance = self.verify_path(path) - if isinstance(path, Path): - return path.read_bytes() + if provenance is AttackDataProvenance.FILE: + return path.read_bytes(), provenance - relative_path = self._cached_lfs_relative_path(str(path)) - if relative_path is not None: + if provenance is AttackDataProvenance.ATTACK_DATA_CACHE: + relative_path = self._cached_lfs_relative_path(str(path)) with zipfile.ZipFile(self._zip_path) as zf: - return zf.read(relative_path) + return zf.read(relative_path), provenance - return _request_with_retry(requests.get, "GET", path).content + return _request_with_retry(requests.get, "GET", path).content, provenance class Options(BaseSettings): @@ -306,17 +320,17 @@ def cli_cmd(self) -> None: ) if self.resolve: - resolver.verify_path(self.url) - print(f"Resolved: {self.url}") + provenance = resolver.verify_path(self.url) + print(f"Resolved ({provenance}): {self.url}") return - data = resolver.get_data(self.url) + data, provenance = resolver.get_data(self.url) except ArchiveVerificationError as exc: sys.exit(f"Error: {exc}") if self.output: Path(self.output).write_bytes(data) - print(f"Wrote {len(data):,} bytes to {self.output}") + print(f"Wrote {len(data):,} bytes ({provenance}) to {self.output}") else: sys.stdout.buffer.write(data) From e7293d81151bffb4a0a1a1d38fcb1dc27206745c Mon Sep 17 00:00:00 2001 From: Eric McGinnis Date: Fri, 7 Aug 2026 12:27:34 -0700 Subject: [PATCH 11/14] track missing files and whether or not they were found via head request --- bin/attack_data_archive_reader.py | 77 +++++++++++++++++++++++++------ 1 file changed, 63 insertions(+), 14 deletions(-) diff --git a/bin/attack_data_archive_reader.py b/bin/attack_data_archive_reader.py index 3d6c60dd4..b94820249 100755 --- a/bin/attack_data_archive_reader.py +++ b/bin/attack_data_archive_reader.py @@ -10,6 +10,10 @@ same instance across many calls to verify_path / get_data to avoid re-verifying and re-reading metadata.yml every time. +By default (require_attack_data_archive=False) the archive is optional: if it's +missing, verify_path/get_data resolve HttpUrls with HEAD/GET requests instead of +raising. Set require_attack_data_archive=True to fail construction if it's missing. + verify_path and get_data are the only public entry points; everything else on AttackDataArchiveResolver is a private implementation detail. Both accept either a FilePath (an existing local file) or an HttpUrl (an LFS download URL), and both @@ -32,7 +36,8 @@ from enum import StrEnum from functools import cached_property from pathlib import Path -from typing import Callable, Optional, Tuple, Union +from types import MappingProxyType +from typing import Callable, Dict, Mapping, Optional, Tuple, Union import requests from pydantic import ( @@ -87,6 +92,7 @@ class AttackDataProvenance(StrEnum): FILE = "file" ATTACK_DATA_CACHE = "attack_data_cache" URL_GET = "url_get" + DOES_NOT_EXIST = "does_not_exist" HTTP_REQUEST_MAX_ATTEMPTS = 3 @@ -151,31 +157,46 @@ class AttackDataArchiveResolver(BaseModel): verify_path and get_data are the only public methods; reuse the same instance across many calls to them to avoid re-verifying and re-parsing metadata.yml on every call. - Raises ArchiveVerificationError if the folder or either file is missing, or - MetadataFilesDiffer if the standalone and embedded metadata.yml contents differ. + require_attack_data_archive controls what happens if archive_dir, attack_data_archive.zip, + or metadata.yml is missing: if True, construction raises ArchiveVerificationError; if False + (the default), construction succeeds with the cache disabled, and verify_path/get_data + resolve every HttpUrl via HEAD/GET requests instead. A present-but-inconsistent archive + (standalone and embedded metadata.yml differing) always raises MetadataFilesDiffer, + regardless of require_attack_data_archive. + + Every HttpUrl that misses the cache is recorded in missing_from_archive_log, keyed by + url, as whichever of URL_GET/DOES_NOT_EXIST it resolved to. """ model_config = ConfigDict(frozen=True) archive_dir: Path = Field(default_factory=default_archive_dir) check_existence_with_head_request: bool = False + require_attack_data_archive: bool = False - _metadata: Metadata = PrivateAttr() + _metadata: Optional[Metadata] = PrivateAttr(default=None) + _missing_from_archive_log: Dict[str, AttackDataProvenance] = PrivateAttr(default_factory=dict) def model_post_init(self, __context: object) -> None: - """Verify the archive folder/files exist and are consistent, then cache parsed metadata.yml.""" + """Verify the archive folder/files, if present, are consistent, then cache parsed metadata.yml.""" archive_dir = self.archive_dir if not archive_dir.is_dir(): - raise ArchiveVerificationError(f"Archive folder not found: {archive_dir}") + if self.require_attack_data_archive: + raise ArchiveVerificationError(f"Archive folder not found: {archive_dir}") + return zip_path = self._zip_path if not zip_path.is_file(): - raise ArchiveVerificationError(f"Archive zip not found: {zip_path}") + if self.require_attack_data_archive: + raise ArchiveVerificationError(f"Archive zip not found: {zip_path}") + return yml_path = self._yml_path if not yml_path.is_file(): - raise ArchiveVerificationError(f"Archive metadata not found: {yml_path}") + if self.require_attack_data_archive: + raise ArchiveVerificationError(f"Archive metadata not found: {yml_path}") + return standalone_text = yml_path.read_text() @@ -195,6 +216,11 @@ def model_post_init(self, __context: object) -> None: self._metadata = parse_metadata_yaml(standalone_text) + @property + def missing_from_archive_log(self) -> Mapping[str, AttackDataProvenance]: + """Read-only view of every HttpUrl that has missed the cache so far, and how it resolved.""" + return MappingProxyType(self._missing_from_archive_log) + @property def _zip_path(self) -> Path: """Path to attack_data_archive.zip inside archive_dir.""" @@ -215,11 +241,15 @@ def _cached_lfs_relative_path(self, url: str) -> Optional[str]: """Look up url in the cached metadata.yml and verify its file is present in the zip. Returns the file's relative path within the zip if url is a known, present-in-the-zip - LFS file, or None if url is not a known LFS file at all (i.e. not in the cache). + LFS file, or None if the archive isn't present (no cache) or url is not a known LFS + file at all (i.e. not in the cache). Raises ArchiveVerificationError if url is a known LFS file but its file is missing from the zip. """ + if self._metadata is None: + return None + entry = self._metadata.lfs_files.get(url) if entry is None: return None @@ -239,13 +269,18 @@ def verify_path(self, path: Union[FilePath, HttpUrl]) -> AttackDataProvenance: FilePath validation already requires it to exist on disk). An HttpUrl is verified if it's a known, present-in-the-zip LFS file (i.e. it's - in the cache). If it's not in the cache and check_existence_with_head_request - is enabled, falls back to a HEAD request (see _verify_url_reachable). + in the cache). If the archive isn't present at all, or it's not in the cache and + check_existence_with_head_request is enabled, falls back to a HEAD request + (see _verify_url_reachable). Returns where the data was found (or, for URL_GET, would be fetched from). + Every HttpUrl that misses the cache is recorded in _missing_from_archive_log, + keyed by url, as either URL_GET or DOES_NOT_EXIST. + Raises ArchiveVerificationError if path is an HttpUrl that's neither in the - cache nor (when enabled) reachable via HEAD request. + cache nor (when the archive is present and check_existence_with_head_request + is disabled) reachable via HEAD request. """ if isinstance(path, Path): return AttackDataProvenance.FILE @@ -253,12 +288,21 @@ def verify_path(self, path: Union[FilePath, HttpUrl]) -> AttackDataProvenance: if self._cached_lfs_relative_path(str(path)) is not None: return AttackDataProvenance.ATTACK_DATA_CACHE - if not self.check_existence_with_head_request: + url = str(path) + + if self._metadata is not None and not self.check_existence_with_head_request: + self._missing_from_archive_log[url] = AttackDataProvenance.DOES_NOT_EXIST raise ArchiveVerificationError( f"URL not found in {METADATA_FILE_NAME}'s lfs-files: {path}" ) - _verify_url_reachable(path) + try: + _verify_url_reachable(path) + except UrlUnreachable: + self._missing_from_archive_log[url] = AttackDataProvenance.DOES_NOT_EXIST + raise + + self._missing_from_archive_log[url] = AttackDataProvenance.URL_GET return AttackDataProvenance.URL_GET @validate_call @@ -309,6 +353,10 @@ class Options(BaseSettings): False, description="When resolving a URL not in the cache, verify it with a HEAD request instead of failing immediately", ) + require_attack_data_archive: CliImplicitFlag[bool] = Field( + False, + description="Fail if the attack_data_archive folder is missing, instead of falling back to HEAD/GET requests for every URL", + ) def cli_cmd(self) -> None: """Fetch the requested LFS file's bytes and write them to --output or stdout, or just verify it resolves.""" @@ -316,6 +364,7 @@ def cli_cmd(self) -> None: kwargs = {"archive_dir": Path(self.archive_dir)} if self.archive_dir else {} resolver = AttackDataArchiveResolver( check_existence_with_head_request=self.check_existence_with_head_request, + require_attack_data_archive=self.require_attack_data_archive, **kwargs, ) From b681d875bed902b0cc6d92c68304c4ca283a05df Mon Sep 17 00:00:00 2001 From: Eric McGinnis Date: Fri, 7 Aug 2026 12:46:43 -0700 Subject: [PATCH 12/14] produce artifacts as yml and md table for file missing from cache --- bin/attack_data_archive_models.py | 30 +++++++++++++++++ bin/attack_data_archive_reader.py | 55 +++++++++++++++++++++---------- 2 files changed, 68 insertions(+), 17 deletions(-) diff --git a/bin/attack_data_archive_models.py b/bin/attack_data_archive_models.py index 954c5baa3..de0316768 100644 --- a/bin/attack_data_archive_models.py +++ b/bin/attack_data_archive_models.py @@ -8,6 +8,7 @@ import sys from datetime import datetime, timezone +from enum import StrEnum from typing import Dict, List, Optional try: @@ -19,6 +20,16 @@ OUTPUT_DIR_NAME = "attack_data_archive" ARCHIVE_FILE_NAME = "attack_data_archive.zip" METADATA_FILE_NAME = "metadata.yml" +MISSING_FROM_CACHE_FILE_NAME = "missing_from_cache.yml" + + +class AttackDataProvenance(StrEnum): + """Where AttackDataArchiveResolver.verify_path/get_data found (or would fetch) a path's data.""" + + FILE = "file" + ATTACK_DATA_CACHE = "attack_data_cache" + URL_GET = "url_get" + DOES_NOT_EXIST = "does_not_exist" def _as_utc_iso(dt: Optional[datetime]) -> Optional[str]: @@ -60,6 +71,18 @@ def _serialize_generated_at_utc(self, value: datetime) -> str: return _as_utc_iso(value) +class MissingFromCache(BaseModel): + """Every HttpUrl that missed the attack_data_archive cache during a run, keyed by url, with how it resolved.""" + + generated_at_utc: datetime + files: Dict[str, AttackDataProvenance] = Field(default_factory=dict) + + @field_serializer("generated_at_utc", when_used="json") + def _serialize_generated_at_utc(self, value: datetime) -> str: + """Serialize generated_at_utc as a UTC ISO-8601 string.""" + return _as_utc_iso(value) + + def to_yaml(model: BaseModel) -> str: """Serialize a pydantic model to a YAML document, using its field aliases as keys.""" return yaml.safe_dump(model.model_dump(mode="json", by_alias=True), sort_keys=False, default_flow_style=False) @@ -68,3 +91,10 @@ def to_yaml(model: BaseModel) -> str: def parse_metadata_yaml(text: str) -> Metadata: """Parse a metadata.yml document (as text) into a Metadata model.""" return Metadata(**yaml.safe_load(text)) + + +def missing_from_cache_to_markdown(model: MissingFromCache) -> str: + """Render a MissingFromCache model as a markdown table, one row per missing URL.""" + lines = ["| URL | Provenance |", "| --- | --- |"] + lines += [f"| {url} | {provenance.value} |" for url, provenance in model.files.items()] + return "\n".join(lines) + "\n" diff --git a/bin/attack_data_archive_reader.py b/bin/attack_data_archive_reader.py index b94820249..95b7816a1 100755 --- a/bin/attack_data_archive_reader.py +++ b/bin/attack_data_archive_reader.py @@ -33,7 +33,7 @@ import sys import zipfile -from enum import StrEnum +from datetime import datetime, timezone from functools import cached_property from pathlib import Path from types import MappingProxyType @@ -60,9 +60,14 @@ from attack_data_archive_models import ( ARCHIVE_FILE_NAME, METADATA_FILE_NAME, + MISSING_FROM_CACHE_FILE_NAME, OUTPUT_DIR_NAME, + AttackDataProvenance, Metadata, + MissingFromCache, + missing_from_cache_to_markdown, parse_metadata_yaml, + to_yaml, ) MIN_PYTHON = (3, 14) @@ -86,15 +91,6 @@ class UrlUnreachable(ArchiveVerificationError): """Raised when an HttpUrl not found in the cache fails a HEAD/GET request.""" -class AttackDataProvenance(StrEnum): - """Where verify_path/get_data found (or would fetch) a path's data.""" - - FILE = "file" - ATTACK_DATA_CACHE = "attack_data_cache" - URL_GET = "url_get" - DOES_NOT_EXIST = "does_not_exist" - - HTTP_REQUEST_MAX_ATTEMPTS = 3 HTTP_REQUEST_RETRY_STATUS_CODES = {403, 503} HTTP_REQUEST_TIMEOUT_SECONDS = 10 @@ -221,6 +217,13 @@ def missing_from_archive_log(self) -> Mapping[str, AttackDataProvenance]: """Read-only view of every HttpUrl that has missed the cache so far, and how it resolved.""" return MappingProxyType(self._missing_from_archive_log) + def missing_from_cache(self) -> MissingFromCache: + """Snapshot missing_from_archive_log as a MissingFromCache model, timestamped with the current time.""" + return MissingFromCache( + generated_at_utc=datetime.now(timezone.utc), + files=dict(self._missing_from_archive_log), + ) + @property def _zip_path(self) -> Path: """Path to attack_data_archive.zip inside archive_dir.""" @@ -360,14 +363,14 @@ class Options(BaseSettings): def cli_cmd(self) -> None: """Fetch the requested LFS file's bytes and write them to --output or stdout, or just verify it resolves.""" - try: - kwargs = {"archive_dir": Path(self.archive_dir)} if self.archive_dir else {} - resolver = AttackDataArchiveResolver( - check_existence_with_head_request=self.check_existence_with_head_request, - require_attack_data_archive=self.require_attack_data_archive, - **kwargs, - ) + kwargs = {"archive_dir": Path(self.archive_dir)} if self.archive_dir else {} + resolver = AttackDataArchiveResolver( + check_existence_with_head_request=self.check_existence_with_head_request, + require_attack_data_archive=self.require_attack_data_archive, + **kwargs, + ) + try: if self.resolve: provenance = resolver.verify_path(self.url) print(f"Resolved ({provenance}): {self.url}") @@ -376,6 +379,8 @@ def cli_cmd(self) -> None: data, provenance = resolver.get_data(self.url) except ArchiveVerificationError as exc: sys.exit(f"Error: {exc}") + finally: + self._write_missing_from_cache(resolver) if self.output: Path(self.output).write_bytes(data) @@ -383,6 +388,22 @@ def cli_cmd(self) -> None: else: sys.stdout.buffer.write(data) + def _write_missing_from_cache(self, resolver: "AttackDataArchiveResolver") -> None: + """If url missed the cache, write missing_from_cache.yml plus a markdown table alongside the archive.""" + if not resolver.missing_from_archive_log: + return + + missing = resolver.missing_from_cache() + resolver.archive_dir.mkdir(parents=True, exist_ok=True) + + yml_path = resolver.archive_dir / MISSING_FROM_CACHE_FILE_NAME + yml_path.write_text(to_yaml(missing)) + + md_path = yml_path.with_suffix(".md") + md_path.write_text(missing_from_cache_to_markdown(missing)) + + print(f"Wrote {yml_path} and {md_path}") + if __name__ == "__main__": CliApp.run(Options) From d4f3dbaaa29492c1e90b590814a3476d39ea888d Mon Sep 17 00:00:00 2001 From: Eric McGinnis Date: Fri, 7 Aug 2026 14:28:30 -0700 Subject: [PATCH 13/14] clean up and abstract logic into single models file and single builder file. remove the reader cli - it will never be used. --- bin/attack_data_archive_models.py | 100 ----- ...rchive_reader.py => attack_data_models.py} | 358 ++++++++---------- bin/build_dataset_archive.py | 13 +- 3 files changed, 158 insertions(+), 313 deletions(-) delete mode 100644 bin/attack_data_archive_models.py rename bin/{attack_data_archive_reader.py => attack_data_models.py} (51%) mode change 100755 => 100644 diff --git a/bin/attack_data_archive_models.py b/bin/attack_data_archive_models.py deleted file mode 100644 index de0316768..000000000 --- a/bin/attack_data_archive_models.py +++ /dev/null @@ -1,100 +0,0 @@ -#!/usr/bin/env python3 -""" -Pydantic models and shared constants for the datasets archive (attack_data_archive/). - -Used by build_dataset_archive.py to generate metadata.yml, and by any tool -that reads the archive back (e.g. fetch_archived_dataset.py). -""" - -import sys -from datetime import datetime, timezone -from enum import StrEnum -from typing import Dict, List, Optional - -try: - import yaml - from pydantic import BaseModel, Field, field_serializer -except ImportError as exc: - sys.exit(f"Error: missing dependency ({exc}). Install with: pip install -r bin/requirements.txt") - -OUTPUT_DIR_NAME = "attack_data_archive" -ARCHIVE_FILE_NAME = "attack_data_archive.zip" -METADATA_FILE_NAME = "metadata.yml" -MISSING_FROM_CACHE_FILE_NAME = "missing_from_cache.yml" - - -class AttackDataProvenance(StrEnum): - """Where AttackDataArchiveResolver.verify_path/get_data found (or would fetch) a path's data.""" - - FILE = "file" - ATTACK_DATA_CACHE = "attack_data_cache" - URL_GET = "url_get" - DOES_NOT_EXIST = "does_not_exist" - - -def _as_utc_iso(dt: Optional[datetime]) -> Optional[str]: - """Format a datetime as a UTC ISO-8601 string ending in 'Z', or None if dt is None.""" - return dt.astimezone(timezone.utc).isoformat().replace("+00:00", "Z") if dt else None - - -class LfsFileEntry(BaseModel): - """Details for a single git-lfs-tracked file, keyed by its download URL in metadata.yml.""" - - relative_path: str - uncompressed_size: int - last_updated: Optional[datetime] = Field(default=None, alias="last-updated") - - model_config = {"populate_by_name": True} - - @field_serializer("last_updated", when_used="json") - def _serialize_last_updated(self, value: Optional[datetime]) -> Optional[str]: - """Serialize last_updated as a UTC ISO-8601 string.""" - return _as_utc_iso(value) - - -class Metadata(BaseModel): - """Contents of metadata.yml: generation info, and the LFS/non-LFS file listing.""" - - generated_at_utc: datetime - file_count: int - gitref: str - github_url: str - total_uncompressed_size_bytes: int - lfs_files: Dict[str, LfsFileEntry] = Field(default_factory=dict, alias="lfs-files") - non_lfs_files: List[str] = Field(default_factory=list, alias="non-lfs-files") - - model_config = {"populate_by_name": True} - - @field_serializer("generated_at_utc", when_used="json") - def _serialize_generated_at_utc(self, value: datetime) -> str: - """Serialize generated_at_utc as a UTC ISO-8601 string.""" - return _as_utc_iso(value) - - -class MissingFromCache(BaseModel): - """Every HttpUrl that missed the attack_data_archive cache during a run, keyed by url, with how it resolved.""" - - generated_at_utc: datetime - files: Dict[str, AttackDataProvenance] = Field(default_factory=dict) - - @field_serializer("generated_at_utc", when_used="json") - def _serialize_generated_at_utc(self, value: datetime) -> str: - """Serialize generated_at_utc as a UTC ISO-8601 string.""" - return _as_utc_iso(value) - - -def to_yaml(model: BaseModel) -> str: - """Serialize a pydantic model to a YAML document, using its field aliases as keys.""" - return yaml.safe_dump(model.model_dump(mode="json", by_alias=True), sort_keys=False, default_flow_style=False) - - -def parse_metadata_yaml(text: str) -> Metadata: - """Parse a metadata.yml document (as text) into a Metadata model.""" - return Metadata(**yaml.safe_load(text)) - - -def missing_from_cache_to_markdown(model: MissingFromCache) -> str: - """Render a MissingFromCache model as a markdown table, one row per missing URL.""" - lines = ["| URL | Provenance |", "| --- | --- |"] - lines += [f"| {url} | {provenance.value} |" for url, provenance in model.files.items()] - return "\n".join(lines) + "\n" diff --git a/bin/attack_data_archive_reader.py b/bin/attack_data_models.py old mode 100755 new mode 100644 similarity index 51% rename from bin/attack_data_archive_reader.py rename to bin/attack_data_models.py index 95b7816a1..39c2a00f9 --- a/bin/attack_data_archive_reader.py +++ b/bin/attack_data_models.py @@ -1,14 +1,14 @@ #!/usr/bin/env python3 """ -Fetch a single dataset file's bytes, either from the local attack_data_archive/ -folder produced by build_dataset_archive.py or, for LFS URLs not present in that -archive, straight from GitHub. +Models, constants, and the reader library for the datasets archive (attack_data_archive/). + +Used by build_dataset_archive.py to generate metadata.yml, and by any tool that reads +the archive back via AttackDataArchiveResolver. AttackDataArchiveResolver does the one-time verification (archive folder exists, -attack_data_archive.zip and metadata.yml exist inside it, and the standalone -metadata.yml matches the copy embedded in the zip) once, at construction. Reuse the -same instance across many calls to verify_path / get_data to avoid re-verifying and -re-reading metadata.yml every time. +attack_data_archive.zip exists inside it) once, at construction, and reads its +embedded metadata.yml. Reuse the same instance across many calls to verify_path / +get_data to avoid re-verifying and re-reading metadata.yml every time. By default (require_attack_data_archive=False) the archive is optional: if it's missing, verify_path/get_data resolve HttpUrls with HEAD/GET requests instead of @@ -26,65 +26,111 @@ - get_data calls verify_path, then returns the bytes alongside it: read from disk for a FilePath, from the cache for a cached HttpUrl, or via a GET request otherwise. -Can be used as a CLI, or imported and used as a class (see AttackDataArchiveResolver). +Every HttpUrl that misses the cache is recorded on the instance (see +missing_from_archive_log / write_missing_from_cache) for downstream tooling to +consume. + +This is a library, meant to be imported; it has no CLI of its own. -Requires Python 3.14+ (zipfile.ZIP_ZSTANDARD support), pydantic, and pydantic-settings. +Requires Python 3.14+ (zipfile.ZIP_ZSTANDARD support), pydantic. """ import sys import zipfile from datetime import datetime, timezone +from enum import StrEnum from functools import cached_property from pathlib import Path from types import MappingProxyType -from typing import Callable, Dict, Mapping, Optional, Tuple, Union - -import requests -from pydantic import ( - BaseModel, - ConfigDict, - Field, - FilePath, - HttpUrl, - PrivateAttr, - validate_call, -) -from pydantic_settings import ( - BaseSettings, - CliApp, - CliImplicitFlag, - CliPositionalArg, - SettingsConfigDict, -) - -from attack_data_archive_models import ( - ARCHIVE_FILE_NAME, - METADATA_FILE_NAME, - MISSING_FROM_CACHE_FILE_NAME, - OUTPUT_DIR_NAME, - AttackDataProvenance, - Metadata, - MissingFromCache, - missing_from_cache_to_markdown, - parse_metadata_yaml, - to_yaml, -) - -MIN_PYTHON = (3, 14) - -if sys.version_info < MIN_PYTHON: - sys.exit( - f"Error: this script requires Python {MIN_PYTHON[0]}.{MIN_PYTHON[1]}+ " - f"(zipfile.ZIP_ZSTANDARD support). Running {sys.version_info.major}.{sys.version_info.minor}." - ) +from typing import Callable, Dict, List, Mapping, Optional, Tuple, Union +try: + import requests + import yaml + from pydantic import BaseModel, Field, FilePath, HttpUrl, field_serializer, validate_call +except ImportError as exc: + sys.exit(f"Error: missing dependency ({exc}). Install with: pip install -r bin/requirements.txt") -class ArchiveVerificationError(Exception): - """Raised when the attack_data_archive folder, its files, or a requested LFS entry fail verification.""" +OUTPUT_DIR_NAME = "attack_data_archive" +ARCHIVE_FILE_NAME = "attack_data_archive.zip" +METADATA_FILE_NAME = "metadata.yml" +MISSING_FROM_CACHE_FILE_NAME = "missing_from_cache.yml" + + +def _now_utc_iso() -> str: + """The current time as a UTC ISO-8601 string ending in 'Z'.""" + return datetime.now().astimezone(timezone.utc).isoformat().replace("+00:00", "Z") + + +def _as_utc_iso(dt: Optional[datetime]) -> Optional[str]: + """Format a datetime as a UTC ISO-8601 string ending in 'Z', or None if dt is None.""" + return dt.astimezone(timezone.utc).isoformat().replace("+00:00", "Z") if dt else None + + +class AttackDataProvenance(StrEnum): + """Where AttackDataArchiveResolver.verify_path/get_data found (or would fetch) a path's data.""" + + FILE = "file" + ATTACK_DATA_CACHE = "attack_data_cache" + URL_GET = "url_get" + DOES_NOT_EXIST = "does_not_exist" + + +class LfsFileEntry(BaseModel): + """Details for a single git-lfs-tracked file, keyed by its download URL in metadata.yml.""" + + relative_path: str + uncompressed_size: int + last_updated: Optional[datetime] = Field(default=None, alias="last-updated") + + model_config = {"populate_by_name": True} + + @field_serializer("last_updated", when_used="json") + def _serialize_last_updated(self, value: Optional[datetime]) -> Optional[str]: + """Serialize last_updated as a UTC ISO-8601 string.""" + return _as_utc_iso(value) + + +class Metadata(BaseModel): + """Contents of metadata.yml: generation info, and the LFS/non-LFS file listing.""" + + generated_at_utc: str = Field(default_factory=_now_utc_iso) + file_count: int + gitref: str + github_url: str + total_uncompressed_size_bytes: int + lfs_files: Dict[str, LfsFileEntry] = Field(default_factory=dict, alias="lfs-files") + non_lfs_files: List[str] = Field(default_factory=list, alias="non-lfs-files") + + model_config = {"populate_by_name": True} + + +class MissingFromCache(BaseModel): + """Every HttpUrl that missed the attack_data_archive cache during a run, keyed by url, with how it resolved.""" + generated_at_utc: str = Field(default_factory=_now_utc_iso) + files: Dict[str, AttackDataProvenance] = Field(default_factory=dict) -class MetadataFilesDiffer(ArchiveVerificationError): - """Raised when the standalone metadata.yml and the copy embedded in the zip are not byte-identical.""" + +def to_yaml(model: BaseModel) -> str: + """Serialize a pydantic model to a YAML document, using its field aliases as keys.""" + return yaml.safe_dump(model.model_dump(mode="json", by_alias=True), sort_keys=False, default_flow_style=False) + + +def parse_metadata_yaml(text: str) -> Metadata: + """Parse a metadata.yml document (as text) into a Metadata model.""" + return Metadata(**yaml.safe_load(text)) + + +def missing_from_cache_to_markdown(model: MissingFromCache) -> str: + """Render a MissingFromCache model as a markdown table, one row per missing URL.""" + lines = ["| URL | Provenance |", "| --- | --- |"] + lines += [f"| {url} | {provenance.value} |" for url, provenance in model.files.items()] + return "\n".join(lines) + "\n" + + +class ArchiveVerificationError(Exception): + """Raised when the attack_data_archive folder, its zip, or a requested LFS entry fail verification.""" class UrlUnreachable(ArchiveVerificationError): @@ -96,21 +142,6 @@ class UrlUnreachable(ArchiveVerificationError): HTTP_REQUEST_TIMEOUT_SECONDS = 10 -def _first_differing_line(a: str, b: str) -> int: - """Return the 1-indexed line number of the first line where a and b differ. - - If one text is a prefix of the other, returns the line just past the shorter text's end. - """ - x = requests.head - - a_lines = a.splitlines() - b_lines = b.splitlines() - for i, (a_line, b_line) in enumerate(zip(a_lines, b_lines), start=1): - if a_line != b_line: - return i - return min(len(a_lines), len(b_lines)) + 1 - - def _request_with_retry( request_fn: Callable[..., requests.Response], method_name: str, url: HttpUrl ) -> "requests.Response": @@ -143,74 +174,66 @@ def default_archive_dir() -> Path: return Path(__file__).resolve().parent.parent / OUTPUT_DIR_NAME -class AttackDataArchiveResolver(BaseModel): +class AttackDataArchiveResolver: """Verifies the local attack_data_archive/ once, then resolves many files/URLs against it cheaply. - Construction verifies that archive_dir exists, that attack_data_archive.zip and metadata.yml - exist inside it, and that the standalone metadata.yml is byte-identical to the copy embedded - in the zip. metadata.yml is parsed once and cached on the instance. - - verify_path and get_data are the only public methods; reuse the same instance across many - calls to them to avoid re-verifying and re-parsing metadata.yml on every call. + Construction verifies that attack_data_archive.zip exists inside archive_dir, and reads its + embedded metadata.yml (the single source of truth for what the archive contains). Reuse the + same instance across many calls to verify_path / get_data to avoid re-verifying and re-parsing + metadata.yml on every call. Treat an instance as read-only after construction — archive_dir + isn't meant to change afterwards. - require_attack_data_archive controls what happens if archive_dir, attack_data_archive.zip, - or metadata.yml is missing: if True, construction raises ArchiveVerificationError; if False - (the default), construction succeeds with the cache disabled, and verify_path/get_data - resolve every HttpUrl via HEAD/GET requests instead. A present-but-inconsistent archive - (standalone and embedded metadata.yml differing) always raises MetadataFilesDiffer, - regardless of require_attack_data_archive. + require_attack_data_archive controls what happens if archive_dir or attack_data_archive.zip is + missing: if True, construction raises ArchiveVerificationError; if False (the default), + construction succeeds with the cache disabled, and verify_path/get_data resolve every HttpUrl + via HEAD/GET requests instead. - Every HttpUrl that misses the cache is recorded in missing_from_archive_log, keyed by - url, as whichever of URL_GET/DOES_NOT_EXIST it resolved to. + Every HttpUrl that misses the cache is recorded in missing_from_archive_log, keyed by url, as + whichever of URL_GET/DOES_NOT_EXIST it resolved to. Call write_missing_from_cache() to persist + that log to disk for other tooling to consume. """ - model_config = ConfigDict(frozen=True) - - archive_dir: Path = Field(default_factory=default_archive_dir) - check_existence_with_head_request: bool = False - require_attack_data_archive: bool = False + def __init__( + self, + archive_dir: Optional[Path] = None, + require_attack_data_archive: bool = False, + ) -> None: + self.archive_dir = Path(archive_dir) if archive_dir is not None else default_archive_dir() + # If an archive directory is supplied, any URLs not in the archive MUST be resolved + # with a HEAD request. + if self.archive_dir is not None: + self.check_existence_with_head_request = True + else: + self.check_existence_with_head_request = False + self.require_attack_data_archive = require_attack_data_archive - _metadata: Optional[Metadata] = PrivateAttr(default=None) - _missing_from_archive_log: Dict[str, AttackDataProvenance] = PrivateAttr(default_factory=dict) + self._metadata: Optional[Metadata] = None + self._missing_from_archive_log: Dict[str, AttackDataProvenance] = {} - def model_post_init(self, __context: object) -> None: - """Verify the archive folder/files, if present, are consistent, then cache parsed metadata.yml.""" - archive_dir = self.archive_dir + self._verify_archive() - if not archive_dir.is_dir(): - if self.require_attack_data_archive: - raise ArchiveVerificationError(f"Archive folder not found: {archive_dir}") - return + def _verify_archive(self) -> None: + """Read metadata.yml out of attack_data_archive.zip, if the zip is present. + If archive_dir or the zip is missing: raises ArchiveVerificationError when + require_attack_data_archive is True, otherwise leaves the cache disabled + (self._metadata stays None). + """ zip_path = self._zip_path if not zip_path.is_file(): if self.require_attack_data_archive: raise ArchiveVerificationError(f"Archive zip not found: {zip_path}") return - yml_path = self._yml_path - if not yml_path.is_file(): - if self.require_attack_data_archive: - raise ArchiveVerificationError(f"Archive metadata not found: {yml_path}") - return - - standalone_text = yml_path.read_text() - with zipfile.ZipFile(zip_path) as zf: try: - embedded_text = zf.read(METADATA_FILE_NAME).decode() + metadata_text = zf.read(METADATA_FILE_NAME).decode() except KeyError: raise ArchiveVerificationError( f"{METADATA_FILE_NAME} not found inside {zip_path}" ) from None - if standalone_text != embedded_text: - line = _first_differing_line(standalone_text, embedded_text) - raise MetadataFilesDiffer( - f"{yml_path} and the {METADATA_FILE_NAME} embedded in {zip_path} differ, first at line {line}" - ) - - self._metadata = parse_metadata_yaml(standalone_text) + self._metadata = parse_metadata_yaml(metadata_text) @property def missing_from_archive_log(self) -> Mapping[str, AttackDataProvenance]: @@ -219,21 +242,32 @@ def missing_from_archive_log(self) -> Mapping[str, AttackDataProvenance]: def missing_from_cache(self) -> MissingFromCache: """Snapshot missing_from_archive_log as a MissingFromCache model, timestamped with the current time.""" - return MissingFromCache( - generated_at_utc=datetime.now(timezone.utc), - files=dict(self._missing_from_archive_log), - ) + return MissingFromCache(files=dict(self._missing_from_archive_log)) + + def write_missing_from_cache(self) -> Optional[Tuple[Path, Path]]: + """Write missing_from_cache.yml plus a markdown table into archive_dir, for other tooling to consume. + + Returns the (yml_path, md_path) written, or None if nothing has missed the cache yet. + """ + if not self._missing_from_archive_log: + return None + + missing = self.missing_from_cache() + self.archive_dir.mkdir(parents=True, exist_ok=True) + + yml_path = self.archive_dir / MISSING_FROM_CACHE_FILE_NAME + yml_path.write_text(to_yaml(missing)) + + md_path = yml_path.with_suffix(".md") + md_path.write_text(missing_from_cache_to_markdown(missing)) + + return yml_path, md_path @property def _zip_path(self) -> Path: """Path to attack_data_archive.zip inside archive_dir.""" return self.archive_dir / ARCHIVE_FILE_NAME - @property - def _yml_path(self) -> Path: - """Path to the standalone metadata.yml inside archive_dir.""" - return self.archive_dir / METADATA_FILE_NAME - @cached_property def _zip_namelist(self) -> frozenset: """Set of every relative path stored inside the zip, computed once since the archive is immutable.""" @@ -327,83 +361,3 @@ def get_data(self, path: Union[FilePath, HttpUrl]) -> Tuple[bytes, AttackDataPro return zf.read(relative_path), provenance return _request_with_retry(requests.get, "GET", path).content, provenance - - -class Options(BaseSettings): - """CLI options for fetching one LFS file's bytes from the local attack_data_archive.""" - - model_config = SettingsConfigDict( - cli_prog_name="attack_data_archive_reader.py", - cli_kebab_case=True, - cli_shortcuts={"output": "o", "archive_dir": "d"}, - ) - - url: CliPositionalArg[str] = Field( - description="LFS download URL to look up under lfs-files in metadata.yml" - ) - output: Optional[str] = Field( - None, description="Write the file's bytes here instead of stdout" - ) - archive_dir: Optional[str] = Field( - None, - description="Path to the attack_data_archive folder (default: alongside this repo's bin/ folder)", - ) - resolve: CliImplicitFlag[bool] = Field( - False, - description="Only verify that the URL resolves (is in the cache, or reachable via a HEAD request); don't fetch its bytes", - ) - check_existence_with_head_request: CliImplicitFlag[bool] = Field( - False, - description="When resolving a URL not in the cache, verify it with a HEAD request instead of failing immediately", - ) - require_attack_data_archive: CliImplicitFlag[bool] = Field( - False, - description="Fail if the attack_data_archive folder is missing, instead of falling back to HEAD/GET requests for every URL", - ) - - def cli_cmd(self) -> None: - """Fetch the requested LFS file's bytes and write them to --output or stdout, or just verify it resolves.""" - kwargs = {"archive_dir": Path(self.archive_dir)} if self.archive_dir else {} - resolver = AttackDataArchiveResolver( - check_existence_with_head_request=self.check_existence_with_head_request, - require_attack_data_archive=self.require_attack_data_archive, - **kwargs, - ) - - try: - if self.resolve: - provenance = resolver.verify_path(self.url) - print(f"Resolved ({provenance}): {self.url}") - return - - data, provenance = resolver.get_data(self.url) - except ArchiveVerificationError as exc: - sys.exit(f"Error: {exc}") - finally: - self._write_missing_from_cache(resolver) - - if self.output: - Path(self.output).write_bytes(data) - print(f"Wrote {len(data):,} bytes ({provenance}) to {self.output}") - else: - sys.stdout.buffer.write(data) - - def _write_missing_from_cache(self, resolver: "AttackDataArchiveResolver") -> None: - """If url missed the cache, write missing_from_cache.yml plus a markdown table alongside the archive.""" - if not resolver.missing_from_archive_log: - return - - missing = resolver.missing_from_cache() - resolver.archive_dir.mkdir(parents=True, exist_ok=True) - - yml_path = resolver.archive_dir / MISSING_FROM_CACHE_FILE_NAME - yml_path.write_text(to_yaml(missing)) - - md_path = yml_path.with_suffix(".md") - md_path.write_text(missing_from_cache_to_markdown(missing)) - - print(f"Wrote {yml_path} and {md_path}") - - -if __name__ == "__main__": - CliApp.run(Options) diff --git a/bin/build_dataset_archive.py b/bin/build_dataset_archive.py index 4cfd39903..c3bc0ccee 100755 --- a/bin/build_dataset_archive.py +++ b/bin/build_dataset_archive.py @@ -19,7 +19,7 @@ import subprocess import sys import zipfile -from datetime import datetime, timezone +from datetime import datetime from pathlib import Path from typing import Dict, Iterator, List, Tuple @@ -29,7 +29,7 @@ except ImportError as exc: sys.exit(f"Error: missing dependency ({exc}). Install with: pip install -r bin/requirements.txt") -from attack_data_archive_models import ( +from attack_data_models import ( ARCHIVE_FILE_NAME, METADATA_FILE_NAME, OUTPUT_DIR_NAME, @@ -38,16 +38,8 @@ to_yaml, ) -MIN_PYTHON = (3, 14) - REF_NAME = "master" -if sys.version_info < MIN_PYTHON: - sys.exit( - f"Error: this script requires Python {MIN_PYTHON[0]}.{MIN_PYTHON[1]}+ " - f"(zipfile.ZIP_ZSTANDARD support). Running {sys.version_info.major}.{sys.version_info.minor}." - ) - def run_git(args: List[str], cwd: "Path | str") -> str: """Run a git command in cwd and return its stripped stdout, raising CalledProcessError on failure.""" @@ -142,7 +134,6 @@ def build_archive( non_lfs_files.append(rel) metadata = Metadata( - generated_at_utc=datetime.now(timezone.utc), file_count=len(files), gitref=git_hash, github_url=f"https://github.com/{owner}/{repo}/tree/{REF_NAME}", From dfcfbdeaba3d2c952f84d859c6bcf844be2d23ce Mon Sep 17 00:00:00 2001 From: Eric McGinnis Date: Fri, 7 Aug 2026 14:37:52 -0700 Subject: [PATCH 14/14] clean up artifacts output when data is missing --- bin/attack_data_models.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bin/attack_data_models.py b/bin/attack_data_models.py index 39c2a00f9..b99c66af8 100644 --- a/bin/attack_data_models.py +++ b/bin/attack_data_models.py @@ -244,7 +244,7 @@ def missing_from_cache(self) -> MissingFromCache: """Snapshot missing_from_archive_log as a MissingFromCache model, timestamped with the current time.""" return MissingFromCache(files=dict(self._missing_from_archive_log)) - def write_missing_from_cache(self) -> Optional[Tuple[Path, Path]]: + def write_missing_from_cache(self, test_artifacts_path:Path) -> Optional[Tuple[Path, Path]]: """Write missing_from_cache.yml plus a markdown table into archive_dir, for other tooling to consume. Returns the (yml_path, md_path) written, or None if nothing has missed the cache yet. @@ -253,9 +253,9 @@ def write_missing_from_cache(self) -> Optional[Tuple[Path, Path]]: return None missing = self.missing_from_cache() - self.archive_dir.mkdir(parents=True, exist_ok=True) + test_artifacts_path.mkdir(parents=True, exist_ok=True) - yml_path = self.archive_dir / MISSING_FROM_CACHE_FILE_NAME + yml_path = test_artifacts_path / MISSING_FROM_CACHE_FILE_NAME yml_path.write_text(to_yaml(missing)) md_path = yml_path.with_suffix(".md")