fix: Use calibration/validation splits correctly, fix walk-forward bl…

scripts/feature_ablation.py

@@ -28,23 +28,23 @@
 ALERT_RATE = 0.005
 
 
-def evaluate_feature_set(name, features, train, calibration, validation, test, config):
+def evaluate_feature_set(name, features, train, calibration, test, config):
     logger.info(f"Evaluating feature set: '{name}' ({len(features)} features)")
     (
         preprocessor,
         model,
-        X_validation_processed,
-        y_validation,
+        X_calibration_processed,
+        y_calibration,
     ) = fit_model(train, calibration, features, config=config)
     logger.debug(f"Model trained for '{name}'")
 
-    validation_probabilities = model.predict_proba(
-        X_validation_processed
+    calibration_probabilities = model.predict_proba(
+        X_calibration_processed
     )[:, 1]
 
     calibrator = ProbabilityCalibrator().fit(
-        validation_probabilities,
-        y_validation,
+        calibration_probabilities,
+        y_calibration,
     )
 
     test_probabilities = calibrator.predict(
@@ -82,7 +82,7 @@ def main():
     logger.info(f"Loaded {len(df):,} transactions")
 
     logger.info("Performing temporal split...")
-    train, calibration, validation, test = temporal_split(df)
+    train, calibration, _validation, test = temporal_split(df)
 
     config = get_config(fast=args.fast)
     if args.fast:
@@ -95,7 +95,6 @@ def main():
             features,
             train,
             calibration,
-            validation,
             test,
             config,
         )

scripts/generate_report_figures.py

@@ -1,5 +1,6 @@
 """Generate report figures (ROC, PR, feature importance, ablation, typology) from existing artifacts."""
 
+import argparse
 import csv
 import sys
 from pathlib import Path
@@ -25,16 +26,21 @@
 
 
 def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--features", type=Path, default=FEATURE_PATH)
+    parser.add_argument("--artifact", type=Path, default=ARTIFACT_PATH)
+    args = parser.parse_args()
+
     FIGURES_DIR.mkdir(parents=True, exist_ok=True)
 
-    logger.info(f"Loading artifact from {ARTIFACT_PATH}...")
-    artifact = joblib.load(ARTIFACT_PATH)
+    logger.info(f"Loading artifact from {args.artifact}...")
+    artifact = joblib.load(args.artifact)
     preprocessor = artifact["preprocessor"]
     model = artifact["model"]
     calibrator = artifact["calibrator"]
 
-    logger.info(f"Loading features from {FEATURE_PATH}...")
-    df = pd.read_parquet(FEATURE_PATH)
+    logger.info(f"Loading features from {args.features}...")
+    df = pd.read_parquet(args.features)
 
     logger.info("Performing temporal split to recover test set...")
     _, _, _, test = temporal_split(df)

scripts/run_experiments.py

@@ -17,6 +17,11 @@ def main():
     parser.add_argument("--features", type=Path, default=PROJECT_ROOT / "data/processed/transactions_features.parquet")
     parser.add_argument("--artifact", type=Path, default=PROJECT_ROOT / "artifacts/risk_model.joblib")
     parser.add_argument("--fast", action="store_true")
+    parser.add_argument(
+        "--generalization",
+        action="store_true",
+        help="Also run the unseen-entity generalization experiment (research extension, not part of the core workflow).",
+    )
     args = parser.parse_args()
     python = sys.executable
 
@@ -31,6 +36,10 @@ def main():
     run(train)
     run(ablation)
     run(walk_forward)
+    run([python, "scripts/generate_report_figures.py", "--features", str(args.features), "--artifact", str(args.artifact)])
+
+    if args.generalization:
+        run([python, "scripts/unseen_entity_evaluation.py", "--features", str(args.features), "--artifact", str(args.artifact)])
 
 
 if __name__ == "__main__":

scripts/train_model.py

@@ -105,44 +105,50 @@ def main():
     (
         preprocessor,
         model,
-        X_validation_processed,
-        y_validation,
+        X_calibration_processed,
+        y_calibration,
     ) = fit_model(
         train,
         calibration,
         config=config,
     )
 
     # ---------------------------
-    # CALIBRATION
+    # CALIBRATION (fit calibrator on the calibration split)
     # ---------------------------
 
     raw_calibration_prob = (
         model.predict_proba(
-            X_validation_processed
+            X_calibration_processed
         )[:, 1]
     )
 
     calibrator = ProbabilityCalibrator()
 
     calibrator.fit(
         raw_calibration_prob,
-        y_validation,
+        y_calibration,
     )
 
-    calibrated_validation_prob = (
-        calibrator.predict(raw_calibration_prob)
-    )
+    # ---------------------------
+    # VALIDATION (independent set: calibration quality check + threshold selection)
+    # ---------------------------
+
+    X_validation_processed = preprocessor.transform(validation[MODEL_FEATURES])
+    y_validation = validation[TARGET]
+
+    raw_validation_prob = model.predict_proba(X_validation_processed)[:, 1]
+    calibrated_validation_prob = calibrator.predict(raw_validation_prob)
 
     calibration_report = {
         "raw_brier_score": float(
-            evaluate_model(y_validation, raw_calibration_prob)["brier_score"]
+            evaluate_model(y_validation, raw_validation_prob)["brier_score"]
         ),
         "calibrated_brier_score": float(
             evaluate_model(y_validation, calibrated_validation_prob)["brier_score"]
         ),
         "raw_log_loss": float(
-            evaluate_model(y_validation, raw_calibration_prob)["log_loss"]
+            evaluate_model(y_validation, raw_validation_prob)["log_loss"]
         ),
         "calibrated_log_loss": float(
             evaluate_model(y_validation, calibrated_validation_prob)["log_loss"]
@@ -154,7 +160,7 @@ def main():
 
     figure, axis = plt.subplots(figsize=(6, 6))
     for probabilities, label in (
-        (raw_calibration_prob, "Raw XGBoost"),
+        (raw_validation_prob, "Raw XGBoost"),
         (calibrated_validation_prob, "Calibrated XGBoost"),
     ):
         observed, predicted = calibration_curve(
@@ -176,11 +182,11 @@ def main():
     # Use a realistic operational alert capacity
     alert_rate = 0.005
 
-    calibration_alerts = top_k_alert_mask(
+    validation_alerts = top_k_alert_mask(
         calibrated_validation_prob,
         alert_rate,
     )
-    threshold = float(calibrated_validation_prob[calibration_alerts].min())
+    threshold = float(calibrated_validation_prob[validation_alerts].min())
 
     logger.info(
         f"Threshold for {alert_rate:.2%} alert rate: {threshold:.6f}"
@@ -251,12 +257,12 @@ def main():
     logger.info("Training logistic baseline for comparison...")
     baseline = build_logistic_baseline()
     baseline.fit(preprocessor.transform(train[MODEL_FEATURES]), train[TARGET])
-    baseline_validation_prob = baseline.predict_proba(
-        X_validation_processed
+    baseline_calibration_prob = baseline.predict_proba(
+        X_calibration_processed
     )[:, 1]
     baseline_calibrator = ProbabilityCalibrator().fit(
-        baseline_validation_prob,
-        y_validation,
+        baseline_calibration_prob,
+        y_calibration,
     )
     baseline_test_prob = baseline_calibrator.predict(
         baseline.predict_proba(X_test_processed)[:, 1]

scripts/unseen_entity_evaluation.py

@@ -7,6 +7,7 @@
 and evaluates it on three partitions of the same held-out test set.
 """
 
+import argparse
 import json
 import sys
 from pathlib import Path
@@ -19,7 +20,11 @@
 PROJECT_ROOT = Path(__file__).resolve().parents[1]
 sys.path.insert(0, str(PROJECT_ROOT))
 
-from src.evaluation.metrics import evaluate_model, precision_recall_at_alert_rate
+from src.evaluation.metrics import (
+    evaluate_model,
+    metrics_at_threshold,
+    precision_recall_at_alert_rate,
+)
 from src.logging_config import setup_logging
 from src.models.train import MODEL_FEATURES, TARGET, temporal_split
 
@@ -32,6 +37,19 @@
 ALERT_RATE = 0.005
 
 
+def summarize_fixed_threshold(
+    mask: np.ndarray, y_test: np.ndarray, calibrated_prob: np.ndarray, threshold: float, label: str
+) -> dict:
+    """What the artifact's actual production threshold does to this subgroup,
+    as opposed to summarize()'s per-subgroup top-K (best ranking within the
+    subgroup alone)."""
+    subset_y = y_test[mask]
+    subset_prob = calibrated_prob[mask]
+    if len(subset_y) == 0:
+        return {"label": label, "metrics": None}
+    return {"label": label, "metrics": metrics_at_threshold(subset_y, subset_prob, threshold)}
+
+
 def summarize(mask: np.ndarray, y_test: np.ndarray, calibrated_prob: np.ndarray, label: str) -> dict:
     subset_y = y_test[mask]
     subset_prob = calibrated_prob[mask]
@@ -65,14 +83,19 @@ def summarize(mask: np.ndarray, y_test: np.ndarray, calibrated_prob: np.ndarray,
 
 
 def main():
-    logger.info(f"Loading artifact from {ARTIFACT_PATH}...")
-    artifact = joblib.load(ARTIFACT_PATH)
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--features", type=Path, default=FEATURE_PATH)
+    parser.add_argument("--artifact", type=Path, default=ARTIFACT_PATH)
+    args = parser.parse_args()
+
+    logger.info(f"Loading artifact from {args.artifact}...")
+    artifact = joblib.load(args.artifact)
     preprocessor = artifact["preprocessor"]
     model = artifact["model"]
     calibrator = artifact["calibrator"]
 
-    logger.info(f"Loading features from {FEATURE_PATH}...")
-    df = pd.read_parquet(FEATURE_PATH)
+    logger.info(f"Loading features from {args.features}...")
+    df = pd.read_parquet(args.features)
 
     train, _, _, test = temporal_split(df)
 
@@ -99,16 +122,29 @@ def main():
     seen_result = summarize(both_seen, y_test, calibrated_prob, "Both sender and receiver seen during training")
     unseen_result = summarize(unseen_entity, y_test, calibrated_prob, "At least one party unseen during training")
 
+    decision_threshold = artifact["decision_threshold"]
+    fixed_threshold_seen = summarize_fixed_threshold(
+        both_seen, y_test, calibrated_prob, decision_threshold, "Both parties seen"
+    )
+    fixed_threshold_unseen = summarize_fixed_threshold(
+        unseen_entity, y_test, calibrated_prob, decision_threshold, "Unseen entity"
+    )
+
     results = {
         "alert_rate": ALERT_RATE,
         "accounts_seen_in_training": len(seen_accounts),
         "standard_out_of_time": standard,
         "both_parties_seen": seen_result,
         "unseen_entity": unseen_result,
+        "fixed_production_threshold": {
+            "decision_threshold": decision_threshold,
+            "both_parties_seen": fixed_threshold_seen,
+            "unseen_entity": fixed_threshold_unseen,
+        },
     }
 
     logger.info("=" * 70)
-    logger.info("UNSEEN-ENTITY GENERALIZATION RESULTS")
+    logger.info("UNSEEN-ENTITY GENERALIZATION RESULTS (per-subgroup top-K)")
     logger.info("=" * 70)
     for key in ("standard_out_of_time", "both_parties_seen", "unseen_entity"):
         r = results[key]
@@ -119,6 +155,15 @@ def main():
             for metric_key, value in r["metrics"].items():
                 logger.info(f"  {metric_key:30s}: {value:.6f}")
 
+    logger.info("=" * 70)
+    logger.info(f"SAME FIXED PRODUCTION THRESHOLD ({decision_threshold:.6f}) APPLIED TO BOTH SUBGROUPS")
+    logger.info("=" * 70)
+    for r in (fixed_threshold_seen, fixed_threshold_unseen):
+        logger.info(f"\n{r['label']}:")
+        if r["metrics"]:
+            for metric_key, value in r["metrics"].items():
+                logger.info(f"  {metric_key:30s}: {value:.6f}")
+
     OUTPUT_PATH.parent.mkdir(parents=True, exist_ok=True)
     OUTPUT_PATH.write_text(json.dumps(results, indent=2) + "\n")
     logger.info(f"\nResults written to {OUTPUT_PATH.relative_to(PROJECT_ROOT)}")

scripts/walk_forward_backtest.py

@@ -3,6 +3,7 @@
 import sys
 from pathlib import Path
 
+import numpy as np
 import pandas as pd
 
 PROJECT_ROOT = Path(__file__).resolve().parents[1]
@@ -17,31 +18,55 @@
 logger = setup_logging(__name__)
 
 
-def run_backtest(frame, config):
-    logger.info(f"Starting walk-forward backtest on {len(frame):,} transactions")
-    ordered = frame.sort_values("timestamp").reset_index(drop=True)
-    timestamps = ordered["timestamp"].drop_duplicates().sort_values().tolist()
-    block_count = 14
+def window_boundaries(timestamps, block_count=14, window_count=4):
+    """Expanding-window (train/calibration/test) cutoffs spanning the full timestamp range.
+
+    Splits the full range into `block_count` contiguous chronological blocks and
+    walks 4 expanding windows across them, each cutoff being the first timestamp
+    of its block. The last window's test cutoff is None (extends to the end).
+    """
+    timestamps = np.asarray(timestamps)
     if len(timestamps) < block_count:
-        raise ValueError("walk-forward backtesting requires at least 14 unique timestamps")
+        raise ValueError(f"requires at least {block_count} unique timestamps")
 
-    logger.info(f"Found {len(timestamps)} unique timestamps, creating rolling windows...")
-    results = []
-    for window in range(4):
+    blocks = np.array_split(timestamps, block_count)
+    block_starts = [block[0] for block in blocks]
+
+    boundaries = []
+    for window in range(window_count):
         train_end = 4 + window * 2
         calibration_end = train_end + 2
         test_end = calibration_end + 2
-        if test_end > len(timestamps):
+        if test_end > block_count:
             continue
-        train = ordered[ordered["timestamp"] < timestamps[train_end]]
+        train_cutoff = block_starts[train_end]
+        calibration_cutoff = block_starts[calibration_end]
+        test_cutoff = block_starts[test_end] if test_end < block_count else None
+        boundaries.append((train_cutoff, calibration_cutoff, test_cutoff))
+    return boundaries
+
+
+def run_backtest(frame, config):
+    logger.info(f"Starting walk-forward backtest on {len(frame):,} transactions")
+    ordered = frame.sort_values("timestamp").reset_index(drop=True)
+    timestamps = ordered["timestamp"].drop_duplicates().sort_values().to_numpy()
+    boundaries = window_boundaries(timestamps)
+
+    logger.info(f"Found {len(timestamps)} unique timestamps, creating rolling windows...")
+    results = []
+    for window, (train_cutoff, calibration_cutoff, test_cutoff) in enumerate(boundaries):
+        train = ordered[ordered["timestamp"] < train_cutoff]
         calibration = ordered[
-            (ordered["timestamp"] >= timestamps[train_end])
-            & (ordered["timestamp"] < timestamps[calibration_end])
-        ]
-        test = ordered[
-            (ordered["timestamp"] >= timestamps[calibration_end])
-            & (ordered["timestamp"] < timestamps[test_end])
+            (ordered["timestamp"] >= train_cutoff)
+            & (ordered["timestamp"] < calibration_cutoff)
         ]
+        if test_cutoff is not None:
+            test = ordered[
+                (ordered["timestamp"] >= calibration_cutoff)
+                & (ordered["timestamp"] < test_cutoff)
+            ]
+        else:
+            test = ordered[ordered["timestamp"] >= calibration_cutoff]
         if calibration[TARGET].nunique() < 2:
             logger.debug(f"Window {window + 1}: Skipping (insufficient target variance in calibration)")
             continue