From d3eb934f170bd41ab96f1fce6e899e2b9c284de8 Mon Sep 17 00:00:00 2001
From: Jared Atkinson <jatkinson@specterops.io>
Date: Thu, 10 Sep 2026 13:43:53 -0700
Subject: [PATCH] BED-9729 rebuild lookup database during preprocess

---
 src/openhound_github/main.py |  8 ++++++
 tests/test_preproc.py        | 55 ++++++++++++++++++++++++++++++++++++
 2 files changed, 63 insertions(+)
 create mode 100644 tests/test_preproc.py

diff --git a/src/openhound_github/main.py b/src/openhound_github/main.py
index ab243b5..8ecd030 100644
--- a/src/openhound_github/main.py
+++ b/src/openhound_github/main.py
@@ -1,3 +1,4 @@
+from pathlib import Path
 from typing import Tuple
 
 from dlt.extract.source import DltSource
@@ -11,6 +12,12 @@
 app = OpenHound("github", help="OpenGraph collector for GitHub")
 
 
+def _reset_lookup_database(output_file: Path) -> None:
+    """Remove derived lookup artifacts so each preprocess run starts clean."""
+    output_file.unlink(missing_ok=True)
+    Path(f"{output_file}.wal").unlink(missing_ok=True)
+
+
 @app.collect()
 def collect(ctx: CollectContext) -> DltSource:
     """Register a Typer CLI command that collects GitHub resources and stores them on disk.
@@ -49,6 +56,7 @@ def preproc(ctx: PreProcContext):
     Run before convert:
         openhound preproc github <input_path> lookup.duckdb
     """
+    _reset_lookup_database(ctx.pipeline.output_file)
     return {
         "organizations": "organizations",
         "repositories": "repositories",
diff --git a/tests/test_preproc.py b/tests/test_preproc.py
new file mode 100644
index 0000000..17def2a
--- /dev/null
+++ b/tests/test_preproc.py
@@ -0,0 +1,55 @@
+import gzip
+import json
+from pathlib import Path
+
+import duckdb
+from openhound.core.progress import Progress
+
+from openhound_github.main import preproc
+
+
+def _write_resource_rows(
+    input_path: Path, resource_name: str, rows: list[dict[str, object]]
+) -> None:
+    resource_dir = input_path / resource_name
+    resource_dir.mkdir(exist_ok=True)
+    with gzip.open(resource_dir / "rows.jsonl.gz", "wt", encoding="utf-8") as f:
+        for row in rows:
+            f.write(json.dumps(row))
+            f.write("\n")
+
+
+def test_preproc_rebuilds_lookup_before_optional_table_becomes_populated(
+    tmp_path: Path,
+) -> None:
+    _write_resource_rows(tmp_path, "organizations", [{"id": "ORG_1", "login": "acme"}])
+    lookup_file = tmp_path / "lookup.duckdb"
+
+    preproc(tmp_path, lookup_file, progress=Progress.log)
+
+    with duckdb.connect(str(lookup_file)) as connection:
+        assert connection.execute(
+            "SELECT COUNT(*) FROM github.team_external_groups"
+        ).fetchone() == (0,)
+
+    _write_resource_rows(
+        tmp_path,
+        "team_external_groups",
+        [
+            {
+                "org_login": "acme",
+                "team_database_id": 7,
+                "external_group_id": 100,
+                "external_group_name": "Engineering",
+                "external_group_updated_at": "2026-09-10T00:00:00Z",
+            }
+        ],
+    )
+
+    preproc(tmp_path, lookup_file, progress=Progress.log)
+
+    with duckdb.connect(str(lookup_file)) as connection:
+        assert connection.execute(
+            "SELECT org_login, team_database_id, external_group_name "
+            "FROM github.team_external_groups"
+        ).fetchall() == [("acme", 7, "Engineering")]