From d692b3b8f061e05527c537634d00e5b314ea2fb0 Mon Sep 17 00:00:00 2001 From: PurplePulse Date: Wed, 16 Sep 2026 15:37:10 +0800 Subject: [PATCH] feat(vector-retrieval): add PurplePulse CUDA retrieval engine - implement GPU Exact and IVF-Flat vector search - add correctness validation and benchmark tooling - include benchmark results and technical report --- 09_vector_retrieval/PurplePulse/.clang-format | 4 + .../PurplePulse/.gitattributes | 6 + 09_vector_retrieval/PurplePulse/.gitignore | 63 + .../PurplePulse/CMakeLists.txt | 83 + 09_vector_retrieval/PurplePulse/README.md | 490 ++++ .../exact_fused_k100_batch64_querytile2.conf | 6 + .../exact_fused_k100_batch64_querytile4.conf | 6 + ...used_k10_batch64_blocktile8_blocks128.conf | 8 + ...used_k10_batch64_blocktile8_blocks160.conf | 8 + ...used_k10_batch64_blocktile8_blocks192.conf | 8 + ...fused_k10_batch64_blocktile8_blocks64.conf | 8 + ...fused_k10_batch64_blocktile8_blocks96.conf | 8 + .../exact_fused_k10_batch64_querytile2.conf | 6 + .../exact_fused_k10_batch64_querytile4.conf | 6 + ...used_k10_batch64_querytile4_blocks128.conf | 7 + ...fused_k10_batch64_querytile4_blocks48.conf | 7 + ...fused_k10_batch64_querytile4_blocks64.conf | 7 + ...fused_k10_batch64_querytile4_blocks80.conf | 7 + ...fused_k10_batch64_querytile4_blocks96.conf | 7 + .../exact_fused_k10_batch64_querytile8.conf | 6 + ...used_k10_batch64_querytile8_blocks128.conf | 7 + ...used_k10_batch64_querytile8_blocks160.conf | 7 + ...used_k10_batch64_querytile8_blocks192.conf | 7 + ...fused_k10_batch64_querytile8_blocks96.conf | 7 + .../exact_fused_k50_batch64_querytile2.conf | 6 + .../exact_fused_k50_batch64_querytile4.conf | 6 + .../PurplePulse/configs/exact_l2.conf | 8 + .../PurplePulse/configs/exact_l2_batch1.conf | 5 + .../configs/exact_l2_batch128.conf | 5 + .../PurplePulse/configs/exact_l2_batch16.conf | 5 + .../PurplePulse/configs/exact_l2_batch32.conf | 5 + .../PurplePulse/configs/exact_l2_batch4.conf | 5 + .../PurplePulse/configs/exact_l2_batch64.conf | 5 + .../PurplePulse/configs/exact_l2_block.conf | 8 + .../PurplePulse/configs/exact_l2_k1.conf | 8 + .../PurplePulse/configs/exact_l2_k100.conf | 8 + .../PurplePulse/configs/exact_l2_k50.conf | 8 + .../PurplePulse/configs/exact_l2_simple.conf | 8 + .../configs/exact_million_k100_batch64.conf | 5 + .../exact_million_k100_fused_batch64.conf | 5 + .../exact_million_k10_fused_batch64.conf | 5 + .../configs/exact_million_k1_batch128.conf | 7 + .../configs/exact_million_k50_batch64.conf | 5 + .../exact_million_k50_fused_batch64.conf | 5 + .../PurplePulse/configs/ivf_flat.conf | 7 + .../configs/ivf_flat_100k_balanced.conf | 7 + .../configs/ivf_flat_100k_exact_check.conf | 7 + .../configs/ivf_flat_100k_high_recall.conf | 7 + .../ivf_flat_100k_scalar_baseline.conf | 7 + .../configs/ivf_flat_memory_fused.conf | 20 + .../configs/ivf_flat_memory_rerank.conf | 18 + .../ivf_flat_million_adaptive_k10.conf | 13 + .../ivf_flat_million_adaptive_k100.conf | 13 + .../ivf_flat_million_adaptive_k50.conf | 13 + .../configs/ivf_flat_million_balanced.conf | 7 + .../configs/ivf_flat_million_batch1.conf | 7 + .../configs/ivf_flat_million_batch128.conf | 7 + .../configs/ivf_flat_million_batch16.conf | 7 + .../configs/ivf_flat_million_batch32.conf | 7 + .../configs/ivf_flat_million_batch64.conf | 7 + .../configs/ivf_flat_million_batch8.conf | 7 + .../configs/ivf_flat_million_fast.conf | 7 + .../configs/ivf_flat_million_high_recall.conf | 7 + .../configs/ivf_flat_million_k1.conf | 7 + .../configs/ivf_flat_million_k100.conf | 7 + .../ivf_flat_million_k100_batch128.conf | 7 + .../ivf_flat_million_k100_batch32.conf | 7 + .../ivf_flat_million_k100_batch64.conf | 7 + .../configs/ivf_flat_million_k100_batch8.conf | 7 + .../ivf_flat_million_k100_bucket_major.conf | 7 + .../ivf_flat_million_k100_partitioned.conf | 7 + .../configs/ivf_flat_million_k100_warp1.conf | 7 + .../configs/ivf_flat_million_k100_warp2.conf | 7 + .../configs/ivf_flat_million_k100_warp4.conf | 7 + .../configs/ivf_flat_million_k100_warp8.conf | 7 + .../configs/ivf_flat_million_k50.conf | 7 + .../ivf_flat_million_k50_batch128.conf | 7 + .../configs/ivf_flat_million_k50_batch32.conf | 7 + .../configs/ivf_flat_million_k50_batch64.conf | 7 + .../configs/ivf_flat_million_k50_batch8.conf | 7 + .../ivf_flat_million_k50_bucket_major.conf | 7 + ...ivf_flat_million_k50_partitioned_2x32.conf | 7 + ...ivf_flat_million_k50_partitioned_4x16.conf | 7 + ...ivf_flat_million_k50_partitioned_4x20.conf | 7 + .../ivf_flat_million_k50_partitioned_8x8.conf | 7 + .../configs/ivf_flat_million_k50_warp1.conf | 7 + .../configs/ivf_flat_million_k50_warp2.conf | 7 + .../configs/ivf_flat_million_k50_warp4.conf | 7 + .../configs/ivf_flat_million_k50_warp8.conf | 7 + ...flat_million_k50_warp_compact_batch40.conf | 7 + .../configs/ivf_flat_million_near_exact.conf | 7 + .../PurplePulse/configs/ivf_flat_mnist.conf | 7 + .../PurplePulse/configs/real_exact_k10.conf | 5 + .../PurplePulse/configs/real_exact_k100.conf | 5 + .../PurplePulse/configs/real_exact_k50.conf | 5 + .../configs/real_ivf_nlist1024_k10.conf | 7 + .../configs/real_ivf_nlist1024_k100.conf | 7 + .../configs/real_ivf_nlist1024_k100_safe.conf | 7 + .../configs/real_ivf_nlist1024_k50.conf | 7 + .../configs/real_ivf_nlist1024_k50_safe.conf | 7 + 09_vector_retrieval/PurplePulse/docs/API.md | 127 + .../PurplePulse/docs/BASELINE_RESULTS.md | 856 +++++++ .../PurplePulse/docs/DEVELOPMENT_PLAN.md | 252 ++ .../PurplePulse/docs/FINAL_REPORT.md | 286 +++ .../PurplePulse/docs/REPRODUCIBILITY.md | 514 ++++ .../PurplePulse/docs/STABILITY_MATRIX.md | 57 + .../PurplePulse/docs/SUBMISSION_CHECKLIST.md | 28 + .../evidence/2026-09-04/commands.md | 115 + .../compute_sanitizer_unit_tests.txt | 3 + .../PurplePulse/evidence/2026-09-04/ctest.txt | 20 + .../ctest_after_instrumentation.txt | 20 + .../2026-09-04/data_million_sha256.txt | 2 + .../evidence/2026-09-04/environment.txt | 35 + .../2026-09-04/exact_k100_repeat100.txt | 25 + .../2026-09-04/exact_k10_repeat100.txt | 25 + .../2026-09-04/exact_k50_repeat100.txt | 25 + .../2026-09-04/faiss_cpu_flat_repeat20.txt | 5 + .../2026-09-04/faiss_gpu_repeat100.txt | 17 + .../2026-09-04/faiss_index_sha256.txt | 2 + .../faiss_script_after_instrumentation.txt | 3 + .../gpu_snapshot_after_benchmarks.txt | 1 + .../2026-09-04/index_100k15_sha256.txt | 1 + .../evidence/2026-09-04/ivf_build_100k15.txt | 11 + .../2026-09-04/ivf_k100_repeat100.txt | 101 + .../evidence/2026-09-04/ivf_k10_repeat100.txt | 101 + .../evidence/2026-09-04/ivf_k50_repeat100.txt | 101 + .../ivf_script_after_instrumentation.txt | 28 + .../evidence/2026-09-04/nsys_exact_k10.txt | 100 + .../rebuild_after_instrumentation.txt | 4 + .../source_sha256_before_instrumentation.txt | 7 + .../bucket_major_compute_sanitizer.log | 3 + .../evidence/2026-09-08/commands.md | 176 ++ .../evidence/2026-09-08/compute_sanitizer.txt | 3 + .../2026-09-08/compute_sanitizer_ivf.txt | 3 + .../PurplePulse/evidence/2026-09-08/ctest.txt | 22 + .../evidence/2026-09-08/ctest_final.txt | 22 + .../evidence/2026-09-08/data_sha256.txt | 7 + .../evidence/2026-09-08/environment.txt | 18 + .../evidence/2026-09-08/exact_k100.log | 27 + .../evidence/2026-09-08/faiss_batch40.log | 31 + .../2026-09-08/k100_bucket_major_formal.log | 26 + .../evidence/2026-09-08/k100_heap_formal.log | 38 + .../2026-09-08/k100_selection_negative.log | 46 + .../2026-09-08/k100_warp1_formal_v2.log | 26 + .../2026-09-08/k50_batch40_formal.log | 26 + .../2026-09-08/k50_batch64_formal.log | 26 + .../2026-09-08/k50_bucket_major_formal.log | 26 + .../2026-09-08/k50_warp_compact_formal.log | 26 + .../evidence/2026-09-08/mnist_exact_k10.log | 27 + .../2026-09-08/submit_audit_ctest.log | 22 + .../evidence/2026-09-10/commands.md | 150 ++ .../evidence/2026-09-10/stage7/README.md | 35 + .../stage7/block_tile_source.sha256 | 20 + .../stage7/cosine_norm_ab_source.sha256 | 4 + .../2026-09-10/stage7/current_source.sha256 | 3 + .../evidence/2026-09-10/stage7/gpu.txt | 1 + .../stage7/query_tile_source.sha256 | 10 + .../2026-09-10/stage7/run_exact_profile_ab.sh | 54 + .../evidence/2026-09-10/stage7/source.sha256 | 3 + .../evidence/2026-09-10/stage8/README.md | 10 + .../evidence/2026-09-10/stage8/source.sha256 | 15 + .../evidence/2026-09-11/stage9/README.md | 18 + .../evidence/2026-09-11/stage9/logs/ctest.log | 42 + .../2026-09-11/stage9/logs/exact_verify.log | 1 + .../2026-09-11/stage9/logs/ivf_verify.log | 1 + .../evidence/2026-09-11/stage9/progress.tsv | 24 + .../2026-09-11/stage9/release-validation.json | 20 + .../2026-09-11/stage9/reproduce-plan.json | 125 + .../evidence/2026-09-11/stage9/status | 1 + .../evidence/2026-09-11/stage9/summary.txt | 11 + .../PurplePulse/include/file_io.h | 17 + .../PurplePulse/include/ivf_flat.h | 58 + .../PurplePulse/include/search.h | 46 + .../PurplePulse/include/vector_types.h | 124 + .../pdf/WarpFusion_Technical_Report.pdf | Bin 0 -> 408735 bytes .../results/exact_fused/million_compare.csv | 7 + .../results/figures/architecture.svg | 84 + .../PurplePulse/results/figures/batch-qps.svg | 53 + .../results/figures/recall-latency.svg | 139 ++ .../results/figures/recall-qps.svg | 139 ++ .../results/figures/report-architecture.svg | 47 + .../figures/report-exact-breakdown.svg | 48 + .../results/figures/report-exact-fused.svg | 44 + .../figures/report-k100-partitioned.svg | 51 + .../results/figures/report-k50-scheduling.svg | 55 + .../figures/report-real-comparison.svg | 46 + .../results/figures/report-recall-qps.svg | 91 + .../results/formal_100k15/exact_gpu.csv | 4 + .../results/formal_100k15/faiss_cpu_flat.csv | 4 + .../results/formal_100k15/faiss_gpu.csv | 16 + .../results/formal_100k15/ivf_k10.csv | 5 + .../results/formal_100k15/ivf_k100.csv | 5 + .../results/formal_100k15/ivf_k50.csv | 5 + .../bucket_major_formal.csv | 5 + .../formal_4090d_20260908/build_threads.csv | 6 + .../faiss_gpu_batch40.csv | 2 + .../formal_4090d_20260908/k100_formal.csv | 5 + .../k100_heap_formal.csv | 6 + .../k100_selection_negative.csv | 4 + .../formal_4090d_20260908/k100_warp_ab.csv | 5 + .../formal_4090d_20260908/k50_batch_sweep.csv | 14 + .../k50_bucket_major_sweep.csv | 8 + .../formal_4090d_20260908/k50_warp_sweep.csv | 5 + .../formal_4090d_20260908/mnist_ivf.csv | 7 + .../results/memory_100k/source1_compare.csv | 3 + .../results/million/adaptive_formal_k10.csv | 2 + .../results/million/adaptive_formal_k100.csv | 2 + .../results/million/adaptive_formal_k50.csv | 2 + .../million/adaptive_grouped_formal_k100.csv | 2 + .../results/million/adaptive_k100_t02.csv | 6 + .../results/million/adaptive_k10_t02.csv | 8 + .../results/million/adaptive_k50_t02.csv | 6 + .../million/adaptive_vs_fixed_n152.csv | 7 + .../results/million/faiss_cpu_flat.csv | 4 + .../results/million/faiss_cpu_ivf_rep.csv | 2 + .../results/million/faiss_gpu_formal.csv | 16 + .../results/million/faiss_gpu_formal_v2.csv | 16 + .../results/million/faiss_gpu_formal_v3.csv | 16 + .../results/million/faiss_smoke.csv | 2 + .../results/million/fixed_k100_matched.csv | 6 + .../results/million/fixed_k10_matched.csv | 8 + .../results/million/fixed_k50_matched.csv | 6 + .../million/optimized_v1_batch_sweep.csv | 14 + .../results/million/optimized_v1_k10.csv | 5 + .../results/million/optimized_v1_k100.csv | 5 + .../results/million/optimized_v1_k50.csv | 5 + .../results/million/optimized_v1_k50_b32.csv | 5 + .../million/purplepulse_faiss_compare_k10.csv | 5 + .../purplepulse_faiss_compare_k100.csv | 5 + .../purplepulse_faiss_compare_k100_v2.csv | 5 + .../purplepulse_faiss_compare_k10_v2.csv | 5 + .../million/purplepulse_faiss_compare_k50.csv | 5 + .../purplepulse_faiss_compare_k50_v2.csv | 5 + .../results/million/sweep_smoke.csv | 3 + .../PurplePulse/results/real/sift1m/build.csv | 2 + .../results/real/sift1m/dataset_manifest.json | 58 + .../results/real/sift1m/environment.json | 26 + .../results/real/sift1m/exact_gpu.csv | 4 + .../results/real/sift1m/faiss_gpu.csv | 22 + .../real/sift1m/figures/recall-latency.svg | 163 ++ .../real/sift1m/figures/recall-qps.svg | 163 ++ .../sift1m/groundtruth_score_validation.jsonl | 3 + .../real/sift1m/groundtruth_validation.json | 23 + .../results/real/sift1m/ivf_k10.csv | 7 + .../results/real/sift1m/ivf_k100.csv | 7 + .../real/sift1m/ivf_k100_full_heap.csv | 7 + .../real/sift1m/ivf_k100_partitioned.csv | 7 + .../results/real/sift1m/ivf_k50.csv | 7 + .../results/real/sift1m/ivf_k50_full_heap.csv | 7 + .../real/sift1m/ivf_k50_partitioned.csv | 7 + .../sift1m/partitioned_topk_validation.json | 110 + .../results/real/yahoo-minilm-384/build.csv | 2 + .../yahoo-minilm-384/dataset_manifest.json | 30 + .../real/yahoo-minilm-384/environment.json | 35 + .../real/yahoo-minilm-384/exact_gpu.csv | 4 + .../real/yahoo-minilm-384/faiss_gpu.csv | 22 + .../figures/recall-latency.svg | 163 ++ .../yahoo-minilm-384/figures/recall-qps.svg | 163 ++ .../groundtruth_validation.json | 47 + .../results/real/yahoo-minilm-384/ivf_k10.csv | 7 + .../real/yahoo-minilm-384/ivf_k100.csv | 7 + .../yahoo-minilm-384/ivf_k100_full_heap.csv | 7 + .../yahoo-minilm-384/ivf_k100_partitioned.csv | 7 + .../results/real/yahoo-minilm-384/ivf_k50.csv | 7 + .../yahoo-minilm-384/ivf_k50_full_heap.csv | 7 + .../yahoo-minilm-384/ivf_k50_partitioned.csv | 7 + .../partitioned_topk_validation.json | 110 + .../smoke/synthetic/results.csv | 8 + .../smoke/yahoo/results.csv | 8 + .../synthetic1m_baseline/breakdown.csv | 4 + .../synthetic1m_optimized/breakdown.csv | 4 + .../yahoo_baseline/breakdown.csv | 4 + .../yahoo_optimized/breakdown.csv | 4 + .../synthetic1m/breakdown.csv | 4 + .../yahoo-minilm-384/breakdown.csv | 4 + .../formal/sift_k10/results.csv | 4 + .../formal/sift_k50/results.csv | 4 + .../formal/synthetic_k10/results.csv | 4 + .../formal/synthetic_k50/results.csv | 4 + .../formal/yahoo_k10/results.csv | 4 + .../formal/yahoo_k10_grid/results.csv | 5 + .../formal/yahoo_k50/results.csv | 4 + .../grid_sweep_smoke/sift/results.csv | 7 + .../grid_sweep_smoke/synthetic/results.csv | 7 + .../grid_sweep_smoke/yahoo/results.csv | 7 + .../scaled_grid_smoke/synthetic1m/results.csv | 10 + .../scaled_grid_smoke/yahoo/results.csv | 10 + .../smoke/synthetic1m/results.csv | 10 + .../smoke/yahoo/results.csv | 10 + .../tile8_smoke/synthetic1m/results.csv | 4 + .../tile8_smoke/yahoo/results.csv | 4 + .../results/stage8_stability/coverage.csv | 10 + .../scripts/benchmark_adaptive_nprobe.py | 167 ++ .../PurplePulse/scripts/benchmark_exact.py | 128 + .../scripts/benchmark_exact_breakdown.py | 174 ++ .../scripts/benchmark_exact_fused.py | 148 ++ .../scripts/benchmark_exact_query_tile.py | 128 + .../PurplePulse/scripts/benchmark_faiss.py | 506 ++++ .../scripts/benchmark_ivf_sweep.py | 204 ++ .../scripts/benchmark_memory_modes.py | 150 ++ .../scripts/build_final_report_pdf.py | 739 ++++++ .../scripts/build_report_figures.py | 389 +++ .../scripts/evaluate_ivecs_groundtruth.py | 241 ++ .../PurplePulse/scripts/evaluate_recall.py | 61 + .../PurplePulse/scripts/generate_data.py | 99 + .../scripts/generate_memory_metadata.py | 67 + .../scripts/import_ann_benchmarks.py | 240 ++ .../PurplePulse/scripts/import_mnist.py | 84 + .../PurplePulse/scripts/import_sift1m.py | 231 ++ .../PurplePulse/scripts/plot_results.py | 243 ++ .../scripts/prepare_real_datasets.py | 196 ++ .../PurplePulse/scripts/reproduce.py | 259 ++ .../scripts/run_real_benchmarks.py | 378 +++ .../PurplePulse/scripts/slice_queries.py | 46 + .../PurplePulse/scripts/validate_release.py | 130 + .../PurplePulse/scripts/verify_results.py | 91 + .../PurplePulse/src/build_ivf.cpp | 137 ++ .../PurplePulse/src/cpu_search.cpp | 163 ++ .../PurplePulse/src/file_io.cpp | 533 +++++ .../PurplePulse/src/gpu_ivf_flat.cu | 2098 +++++++++++++++++ .../PurplePulse/src/gpu_search.cu | 1902 +++++++++++++++ .../PurplePulse/src/ivf_flat.cpp | 722 ++++++ 09_vector_retrieval/PurplePulse/src/main.cpp | 234 ++ .../PurplePulse/src/search_ivf.cpp | 391 +++ .../PurplePulse/tests/file_io_tests.cpp | 195 ++ .../PurplePulse/tests/ivf_flat_tests.cpp | 351 +++ .../PurplePulse/tests/stability_tests.cpp | 317 +++ .../tests/test_benchmark_adaptive_nprobe.py | 29 + .../tests/test_benchmark_exact_breakdown.py | 72 + .../tests/test_benchmark_exact_query_tile.py | 57 + .../PurplePulse/tests/test_benchmark_faiss.py | 94 + .../tests/test_benchmark_ivf_sweep.py | 48 + .../tests/test_import_ann_benchmarks.py | 118 + .../PurplePulse/tests/test_import_mnist.py | 49 + .../PurplePulse/tests/test_import_sift1m.py | 122 + .../PurplePulse/tests/test_plot_results.py | 104 + .../PurplePulse/tests/test_stage6_tools.py | 87 + .../PurplePulse/tests/test_stage9_tools.py | 68 + .../PurplePulse/tests/unit_tests.cpp | 365 +++ 339 files changed, 21437 insertions(+) create mode 100644 09_vector_retrieval/PurplePulse/.clang-format create mode 100644 09_vector_retrieval/PurplePulse/.gitattributes create mode 100644 09_vector_retrieval/PurplePulse/.gitignore create mode 100644 09_vector_retrieval/PurplePulse/CMakeLists.txt create mode 100644 09_vector_retrieval/PurplePulse/README.md create mode 100644 09_vector_retrieval/PurplePulse/configs/exact_fused_k100_batch64_querytile2.conf create mode 100644 09_vector_retrieval/PurplePulse/configs/exact_fused_k100_batch64_querytile4.conf create mode 100644 09_vector_retrieval/PurplePulse/configs/exact_fused_k10_batch64_blocktile8_blocks128.conf create mode 100644 09_vector_retrieval/PurplePulse/configs/exact_fused_k10_batch64_blocktile8_blocks160.conf create mode 100644 09_vector_retrieval/PurplePulse/configs/exact_fused_k10_batch64_blocktile8_blocks192.conf create mode 100644 09_vector_retrieval/PurplePulse/configs/exact_fused_k10_batch64_blocktile8_blocks64.conf create mode 100644 09_vector_retrieval/PurplePulse/configs/exact_fused_k10_batch64_blocktile8_blocks96.conf create mode 100644 09_vector_retrieval/PurplePulse/configs/exact_fused_k10_batch64_querytile2.conf create mode 100644 09_vector_retrieval/PurplePulse/configs/exact_fused_k10_batch64_querytile4.conf create mode 100644 09_vector_retrieval/PurplePulse/configs/exact_fused_k10_batch64_querytile4_blocks128.conf create mode 100644 09_vector_retrieval/PurplePulse/configs/exact_fused_k10_batch64_querytile4_blocks48.conf create mode 100644 09_vector_retrieval/PurplePulse/configs/exact_fused_k10_batch64_querytile4_blocks64.conf create mode 100644 09_vector_retrieval/PurplePulse/configs/exact_fused_k10_batch64_querytile4_blocks80.conf create mode 100644 09_vector_retrieval/PurplePulse/configs/exact_fused_k10_batch64_querytile4_blocks96.conf create mode 100644 09_vector_retrieval/PurplePulse/configs/exact_fused_k10_batch64_querytile8.conf create mode 100644 09_vector_retrieval/PurplePulse/configs/exact_fused_k10_batch64_querytile8_blocks128.conf create mode 100644 09_vector_retrieval/PurplePulse/configs/exact_fused_k10_batch64_querytile8_blocks160.conf create mode 100644 09_vector_retrieval/PurplePulse/configs/exact_fused_k10_batch64_querytile8_blocks192.conf create mode 100644 09_vector_retrieval/PurplePulse/configs/exact_fused_k10_batch64_querytile8_blocks96.conf create mode 100644 09_vector_retrieval/PurplePulse/configs/exact_fused_k50_batch64_querytile2.conf create mode 100644 09_vector_retrieval/PurplePulse/configs/exact_fused_k50_batch64_querytile4.conf create mode 100644 09_vector_retrieval/PurplePulse/configs/exact_l2.conf create mode 100644 09_vector_retrieval/PurplePulse/configs/exact_l2_batch1.conf create mode 100644 09_vector_retrieval/PurplePulse/configs/exact_l2_batch128.conf create mode 100644 09_vector_retrieval/PurplePulse/configs/exact_l2_batch16.conf create mode 100644 09_vector_retrieval/PurplePulse/configs/exact_l2_batch32.conf create mode 100644 09_vector_retrieval/PurplePulse/configs/exact_l2_batch4.conf create mode 100644 09_vector_retrieval/PurplePulse/configs/exact_l2_batch64.conf create mode 100644 09_vector_retrieval/PurplePulse/configs/exact_l2_block.conf create mode 100644 09_vector_retrieval/PurplePulse/configs/exact_l2_k1.conf create mode 100644 09_vector_retrieval/PurplePulse/configs/exact_l2_k100.conf create mode 100644 09_vector_retrieval/PurplePulse/configs/exact_l2_k50.conf create mode 100644 09_vector_retrieval/PurplePulse/configs/exact_l2_simple.conf create mode 100644 09_vector_retrieval/PurplePulse/configs/exact_million_k100_batch64.conf create mode 100644 09_vector_retrieval/PurplePulse/configs/exact_million_k100_fused_batch64.conf create mode 100644 09_vector_retrieval/PurplePulse/configs/exact_million_k10_fused_batch64.conf create mode 100644 09_vector_retrieval/PurplePulse/configs/exact_million_k1_batch128.conf create mode 100644 09_vector_retrieval/PurplePulse/configs/exact_million_k50_batch64.conf create mode 100644 09_vector_retrieval/PurplePulse/configs/exact_million_k50_fused_batch64.conf create mode 100644 09_vector_retrieval/PurplePulse/configs/ivf_flat.conf create mode 100644 09_vector_retrieval/PurplePulse/configs/ivf_flat_100k_balanced.conf create mode 100644 09_vector_retrieval/PurplePulse/configs/ivf_flat_100k_exact_check.conf create mode 100644 09_vector_retrieval/PurplePulse/configs/ivf_flat_100k_high_recall.conf create mode 100644 09_vector_retrieval/PurplePulse/configs/ivf_flat_100k_scalar_baseline.conf create mode 100644 09_vector_retrieval/PurplePulse/configs/ivf_flat_memory_fused.conf create mode 100644 09_vector_retrieval/PurplePulse/configs/ivf_flat_memory_rerank.conf create mode 100644 09_vector_retrieval/PurplePulse/configs/ivf_flat_million_adaptive_k10.conf create mode 100644 09_vector_retrieval/PurplePulse/configs/ivf_flat_million_adaptive_k100.conf create mode 100644 09_vector_retrieval/PurplePulse/configs/ivf_flat_million_adaptive_k50.conf create mode 100644 09_vector_retrieval/PurplePulse/configs/ivf_flat_million_balanced.conf create mode 100644 09_vector_retrieval/PurplePulse/configs/ivf_flat_million_batch1.conf create mode 100644 09_vector_retrieval/PurplePulse/configs/ivf_flat_million_batch128.conf create mode 100644 09_vector_retrieval/PurplePulse/configs/ivf_flat_million_batch16.conf create mode 100644 09_vector_retrieval/PurplePulse/configs/ivf_flat_million_batch32.conf create mode 100644 09_vector_retrieval/PurplePulse/configs/ivf_flat_million_batch64.conf create mode 100644 09_vector_retrieval/PurplePulse/configs/ivf_flat_million_batch8.conf create mode 100644 09_vector_retrieval/PurplePulse/configs/ivf_flat_million_fast.conf create mode 100644 09_vector_retrieval/PurplePulse/configs/ivf_flat_million_high_recall.conf create mode 100644 09_vector_retrieval/PurplePulse/configs/ivf_flat_million_k1.conf create mode 100644 09_vector_retrieval/PurplePulse/configs/ivf_flat_million_k100.conf create mode 100644 09_vector_retrieval/PurplePulse/configs/ivf_flat_million_k100_batch128.conf create mode 100644 09_vector_retrieval/PurplePulse/configs/ivf_flat_million_k100_batch32.conf create mode 100644 09_vector_retrieval/PurplePulse/configs/ivf_flat_million_k100_batch64.conf create mode 100644 09_vector_retrieval/PurplePulse/configs/ivf_flat_million_k100_batch8.conf create mode 100644 09_vector_retrieval/PurplePulse/configs/ivf_flat_million_k100_bucket_major.conf create mode 100644 09_vector_retrieval/PurplePulse/configs/ivf_flat_million_k100_partitioned.conf create mode 100644 09_vector_retrieval/PurplePulse/configs/ivf_flat_million_k100_warp1.conf create mode 100644 09_vector_retrieval/PurplePulse/configs/ivf_flat_million_k100_warp2.conf create mode 100644 09_vector_retrieval/PurplePulse/configs/ivf_flat_million_k100_warp4.conf create mode 100644 09_vector_retrieval/PurplePulse/configs/ivf_flat_million_k100_warp8.conf create mode 100644 09_vector_retrieval/PurplePulse/configs/ivf_flat_million_k50.conf create mode 100644 09_vector_retrieval/PurplePulse/configs/ivf_flat_million_k50_batch128.conf create mode 100644 09_vector_retrieval/PurplePulse/configs/ivf_flat_million_k50_batch32.conf create mode 100644 09_vector_retrieval/PurplePulse/configs/ivf_flat_million_k50_batch64.conf create mode 100644 09_vector_retrieval/PurplePulse/configs/ivf_flat_million_k50_batch8.conf create mode 100644 09_vector_retrieval/PurplePulse/configs/ivf_flat_million_k50_bucket_major.conf create mode 100644 09_vector_retrieval/PurplePulse/configs/ivf_flat_million_k50_partitioned_2x32.conf create mode 100644 09_vector_retrieval/PurplePulse/configs/ivf_flat_million_k50_partitioned_4x16.conf create mode 100644 09_vector_retrieval/PurplePulse/configs/ivf_flat_million_k50_partitioned_4x20.conf create mode 100644 09_vector_retrieval/PurplePulse/configs/ivf_flat_million_k50_partitioned_8x8.conf create mode 100644 09_vector_retrieval/PurplePulse/configs/ivf_flat_million_k50_warp1.conf create mode 100644 09_vector_retrieval/PurplePulse/configs/ivf_flat_million_k50_warp2.conf create mode 100644 09_vector_retrieval/PurplePulse/configs/ivf_flat_million_k50_warp4.conf create mode 100644 09_vector_retrieval/PurplePulse/configs/ivf_flat_million_k50_warp8.conf create mode 100644 09_vector_retrieval/PurplePulse/configs/ivf_flat_million_k50_warp_compact_batch40.conf create mode 100644 09_vector_retrieval/PurplePulse/configs/ivf_flat_million_near_exact.conf create mode 100644 09_vector_retrieval/PurplePulse/configs/ivf_flat_mnist.conf create mode 100644 09_vector_retrieval/PurplePulse/configs/real_exact_k10.conf create mode 100644 09_vector_retrieval/PurplePulse/configs/real_exact_k100.conf create mode 100644 09_vector_retrieval/PurplePulse/configs/real_exact_k50.conf create mode 100644 09_vector_retrieval/PurplePulse/configs/real_ivf_nlist1024_k10.conf create mode 100644 09_vector_retrieval/PurplePulse/configs/real_ivf_nlist1024_k100.conf create mode 100644 09_vector_retrieval/PurplePulse/configs/real_ivf_nlist1024_k100_safe.conf create mode 100644 09_vector_retrieval/PurplePulse/configs/real_ivf_nlist1024_k50.conf create mode 100644 09_vector_retrieval/PurplePulse/configs/real_ivf_nlist1024_k50_safe.conf create mode 100644 09_vector_retrieval/PurplePulse/docs/API.md create mode 100644 09_vector_retrieval/PurplePulse/docs/BASELINE_RESULTS.md create mode 100644 09_vector_retrieval/PurplePulse/docs/DEVELOPMENT_PLAN.md create mode 100644 09_vector_retrieval/PurplePulse/docs/FINAL_REPORT.md create mode 100644 09_vector_retrieval/PurplePulse/docs/REPRODUCIBILITY.md create mode 100644 09_vector_retrieval/PurplePulse/docs/STABILITY_MATRIX.md create mode 100644 09_vector_retrieval/PurplePulse/docs/SUBMISSION_CHECKLIST.md create mode 100644 09_vector_retrieval/PurplePulse/evidence/2026-09-04/commands.md create mode 100644 09_vector_retrieval/PurplePulse/evidence/2026-09-04/compute_sanitizer_unit_tests.txt create mode 100644 09_vector_retrieval/PurplePulse/evidence/2026-09-04/ctest.txt create mode 100644 09_vector_retrieval/PurplePulse/evidence/2026-09-04/ctest_after_instrumentation.txt create mode 100644 09_vector_retrieval/PurplePulse/evidence/2026-09-04/data_million_sha256.txt create mode 100644 09_vector_retrieval/PurplePulse/evidence/2026-09-04/environment.txt create mode 100644 09_vector_retrieval/PurplePulse/evidence/2026-09-04/exact_k100_repeat100.txt create mode 100644 09_vector_retrieval/PurplePulse/evidence/2026-09-04/exact_k10_repeat100.txt create mode 100644 09_vector_retrieval/PurplePulse/evidence/2026-09-04/exact_k50_repeat100.txt create mode 100644 09_vector_retrieval/PurplePulse/evidence/2026-09-04/faiss_cpu_flat_repeat20.txt create mode 100644 09_vector_retrieval/PurplePulse/evidence/2026-09-04/faiss_gpu_repeat100.txt create mode 100644 09_vector_retrieval/PurplePulse/evidence/2026-09-04/faiss_index_sha256.txt create mode 100644 09_vector_retrieval/PurplePulse/evidence/2026-09-04/faiss_script_after_instrumentation.txt create mode 100644 09_vector_retrieval/PurplePulse/evidence/2026-09-04/gpu_snapshot_after_benchmarks.txt create mode 100644 09_vector_retrieval/PurplePulse/evidence/2026-09-04/index_100k15_sha256.txt create mode 100644 09_vector_retrieval/PurplePulse/evidence/2026-09-04/ivf_build_100k15.txt create mode 100644 09_vector_retrieval/PurplePulse/evidence/2026-09-04/ivf_k100_repeat100.txt create mode 100644 09_vector_retrieval/PurplePulse/evidence/2026-09-04/ivf_k10_repeat100.txt create mode 100644 09_vector_retrieval/PurplePulse/evidence/2026-09-04/ivf_k50_repeat100.txt create mode 100644 09_vector_retrieval/PurplePulse/evidence/2026-09-04/ivf_script_after_instrumentation.txt create mode 100644 09_vector_retrieval/PurplePulse/evidence/2026-09-04/nsys_exact_k10.txt create mode 100644 09_vector_retrieval/PurplePulse/evidence/2026-09-04/rebuild_after_instrumentation.txt create mode 100644 09_vector_retrieval/PurplePulse/evidence/2026-09-04/source_sha256_before_instrumentation.txt create mode 100644 09_vector_retrieval/PurplePulse/evidence/2026-09-08/bucket_major_compute_sanitizer.log create mode 100644 09_vector_retrieval/PurplePulse/evidence/2026-09-08/commands.md create mode 100644 09_vector_retrieval/PurplePulse/evidence/2026-09-08/compute_sanitizer.txt create mode 100644 09_vector_retrieval/PurplePulse/evidence/2026-09-08/compute_sanitizer_ivf.txt create mode 100644 09_vector_retrieval/PurplePulse/evidence/2026-09-08/ctest.txt create mode 100644 09_vector_retrieval/PurplePulse/evidence/2026-09-08/ctest_final.txt create mode 100644 09_vector_retrieval/PurplePulse/evidence/2026-09-08/data_sha256.txt create mode 100644 09_vector_retrieval/PurplePulse/evidence/2026-09-08/environment.txt create mode 100644 09_vector_retrieval/PurplePulse/evidence/2026-09-08/exact_k100.log create mode 100644 09_vector_retrieval/PurplePulse/evidence/2026-09-08/faiss_batch40.log create mode 100644 09_vector_retrieval/PurplePulse/evidence/2026-09-08/k100_bucket_major_formal.log create mode 100644 09_vector_retrieval/PurplePulse/evidence/2026-09-08/k100_heap_formal.log create mode 100644 09_vector_retrieval/PurplePulse/evidence/2026-09-08/k100_selection_negative.log create mode 100644 09_vector_retrieval/PurplePulse/evidence/2026-09-08/k100_warp1_formal_v2.log create mode 100644 09_vector_retrieval/PurplePulse/evidence/2026-09-08/k50_batch40_formal.log create mode 100644 09_vector_retrieval/PurplePulse/evidence/2026-09-08/k50_batch64_formal.log create mode 100644 09_vector_retrieval/PurplePulse/evidence/2026-09-08/k50_bucket_major_formal.log create mode 100644 09_vector_retrieval/PurplePulse/evidence/2026-09-08/k50_warp_compact_formal.log create mode 100644 09_vector_retrieval/PurplePulse/evidence/2026-09-08/mnist_exact_k10.log create mode 100644 09_vector_retrieval/PurplePulse/evidence/2026-09-08/submit_audit_ctest.log create mode 100644 09_vector_retrieval/PurplePulse/evidence/2026-09-10/commands.md create mode 100644 09_vector_retrieval/PurplePulse/evidence/2026-09-10/stage7/README.md create mode 100644 09_vector_retrieval/PurplePulse/evidence/2026-09-10/stage7/block_tile_source.sha256 create mode 100644 09_vector_retrieval/PurplePulse/evidence/2026-09-10/stage7/cosine_norm_ab_source.sha256 create mode 100644 09_vector_retrieval/PurplePulse/evidence/2026-09-10/stage7/current_source.sha256 create mode 100644 09_vector_retrieval/PurplePulse/evidence/2026-09-10/stage7/gpu.txt create mode 100644 09_vector_retrieval/PurplePulse/evidence/2026-09-10/stage7/query_tile_source.sha256 create mode 100755 09_vector_retrieval/PurplePulse/evidence/2026-09-10/stage7/run_exact_profile_ab.sh create mode 100644 09_vector_retrieval/PurplePulse/evidence/2026-09-10/stage7/source.sha256 create mode 100644 09_vector_retrieval/PurplePulse/evidence/2026-09-10/stage8/README.md create mode 100644 09_vector_retrieval/PurplePulse/evidence/2026-09-10/stage8/source.sha256 create mode 100644 09_vector_retrieval/PurplePulse/evidence/2026-09-11/stage9/README.md create mode 100644 09_vector_retrieval/PurplePulse/evidence/2026-09-11/stage9/logs/ctest.log create mode 100644 09_vector_retrieval/PurplePulse/evidence/2026-09-11/stage9/logs/exact_verify.log create mode 100644 09_vector_retrieval/PurplePulse/evidence/2026-09-11/stage9/logs/ivf_verify.log create mode 100644 09_vector_retrieval/PurplePulse/evidence/2026-09-11/stage9/progress.tsv create mode 100644 09_vector_retrieval/PurplePulse/evidence/2026-09-11/stage9/release-validation.json create mode 100644 09_vector_retrieval/PurplePulse/evidence/2026-09-11/stage9/reproduce-plan.json create mode 100644 09_vector_retrieval/PurplePulse/evidence/2026-09-11/stage9/status create mode 100644 09_vector_retrieval/PurplePulse/evidence/2026-09-11/stage9/summary.txt create mode 100644 09_vector_retrieval/PurplePulse/include/file_io.h create mode 100644 09_vector_retrieval/PurplePulse/include/ivf_flat.h create mode 100644 09_vector_retrieval/PurplePulse/include/search.h create mode 100644 09_vector_retrieval/PurplePulse/include/vector_types.h create mode 100644 09_vector_retrieval/PurplePulse/output/pdf/WarpFusion_Technical_Report.pdf create mode 100644 09_vector_retrieval/PurplePulse/results/exact_fused/million_compare.csv create mode 100644 09_vector_retrieval/PurplePulse/results/figures/architecture.svg create mode 100644 09_vector_retrieval/PurplePulse/results/figures/batch-qps.svg create mode 100644 09_vector_retrieval/PurplePulse/results/figures/recall-latency.svg create mode 100644 09_vector_retrieval/PurplePulse/results/figures/recall-qps.svg create mode 100644 09_vector_retrieval/PurplePulse/results/figures/report-architecture.svg create mode 100644 09_vector_retrieval/PurplePulse/results/figures/report-exact-breakdown.svg create mode 100644 09_vector_retrieval/PurplePulse/results/figures/report-exact-fused.svg create mode 100644 09_vector_retrieval/PurplePulse/results/figures/report-k100-partitioned.svg create mode 100644 09_vector_retrieval/PurplePulse/results/figures/report-k50-scheduling.svg create mode 100644 09_vector_retrieval/PurplePulse/results/figures/report-real-comparison.svg create mode 100644 09_vector_retrieval/PurplePulse/results/figures/report-recall-qps.svg create mode 100644 09_vector_retrieval/PurplePulse/results/formal_100k15/exact_gpu.csv create mode 100644 09_vector_retrieval/PurplePulse/results/formal_100k15/faiss_cpu_flat.csv create mode 100644 09_vector_retrieval/PurplePulse/results/formal_100k15/faiss_gpu.csv create mode 100644 09_vector_retrieval/PurplePulse/results/formal_100k15/ivf_k10.csv create mode 100644 09_vector_retrieval/PurplePulse/results/formal_100k15/ivf_k100.csv create mode 100644 09_vector_retrieval/PurplePulse/results/formal_100k15/ivf_k50.csv create mode 100644 09_vector_retrieval/PurplePulse/results/formal_4090d_20260908/bucket_major_formal.csv create mode 100644 09_vector_retrieval/PurplePulse/results/formal_4090d_20260908/build_threads.csv create mode 100644 09_vector_retrieval/PurplePulse/results/formal_4090d_20260908/faiss_gpu_batch40.csv create mode 100644 09_vector_retrieval/PurplePulse/results/formal_4090d_20260908/k100_formal.csv create mode 100644 09_vector_retrieval/PurplePulse/results/formal_4090d_20260908/k100_heap_formal.csv create mode 100644 09_vector_retrieval/PurplePulse/results/formal_4090d_20260908/k100_selection_negative.csv create mode 100644 09_vector_retrieval/PurplePulse/results/formal_4090d_20260908/k100_warp_ab.csv create mode 100644 09_vector_retrieval/PurplePulse/results/formal_4090d_20260908/k50_batch_sweep.csv create mode 100644 09_vector_retrieval/PurplePulse/results/formal_4090d_20260908/k50_bucket_major_sweep.csv create mode 100644 09_vector_retrieval/PurplePulse/results/formal_4090d_20260908/k50_warp_sweep.csv create mode 100644 09_vector_retrieval/PurplePulse/results/formal_4090d_20260908/mnist_ivf.csv create mode 100644 09_vector_retrieval/PurplePulse/results/memory_100k/source1_compare.csv create mode 100644 09_vector_retrieval/PurplePulse/results/million/adaptive_formal_k10.csv create mode 100644 09_vector_retrieval/PurplePulse/results/million/adaptive_formal_k100.csv create mode 100644 09_vector_retrieval/PurplePulse/results/million/adaptive_formal_k50.csv create mode 100644 09_vector_retrieval/PurplePulse/results/million/adaptive_grouped_formal_k100.csv create mode 100644 09_vector_retrieval/PurplePulse/results/million/adaptive_k100_t02.csv create mode 100644 09_vector_retrieval/PurplePulse/results/million/adaptive_k10_t02.csv create mode 100644 09_vector_retrieval/PurplePulse/results/million/adaptive_k50_t02.csv create mode 100644 09_vector_retrieval/PurplePulse/results/million/adaptive_vs_fixed_n152.csv create mode 100644 09_vector_retrieval/PurplePulse/results/million/faiss_cpu_flat.csv create mode 100644 09_vector_retrieval/PurplePulse/results/million/faiss_cpu_ivf_rep.csv create mode 100644 09_vector_retrieval/PurplePulse/results/million/faiss_gpu_formal.csv create mode 100644 09_vector_retrieval/PurplePulse/results/million/faiss_gpu_formal_v2.csv create mode 100644 09_vector_retrieval/PurplePulse/results/million/faiss_gpu_formal_v3.csv create mode 100644 09_vector_retrieval/PurplePulse/results/million/faiss_smoke.csv create mode 100644 09_vector_retrieval/PurplePulse/results/million/fixed_k100_matched.csv create mode 100644 09_vector_retrieval/PurplePulse/results/million/fixed_k10_matched.csv create mode 100644 09_vector_retrieval/PurplePulse/results/million/fixed_k50_matched.csv create mode 100644 09_vector_retrieval/PurplePulse/results/million/optimized_v1_batch_sweep.csv create mode 100644 09_vector_retrieval/PurplePulse/results/million/optimized_v1_k10.csv create mode 100644 09_vector_retrieval/PurplePulse/results/million/optimized_v1_k100.csv create mode 100644 09_vector_retrieval/PurplePulse/results/million/optimized_v1_k50.csv create mode 100644 09_vector_retrieval/PurplePulse/results/million/optimized_v1_k50_b32.csv create mode 100644 09_vector_retrieval/PurplePulse/results/million/purplepulse_faiss_compare_k10.csv create mode 100644 09_vector_retrieval/PurplePulse/results/million/purplepulse_faiss_compare_k100.csv create mode 100644 09_vector_retrieval/PurplePulse/results/million/purplepulse_faiss_compare_k100_v2.csv create mode 100644 09_vector_retrieval/PurplePulse/results/million/purplepulse_faiss_compare_k10_v2.csv create mode 100644 09_vector_retrieval/PurplePulse/results/million/purplepulse_faiss_compare_k50.csv create mode 100644 09_vector_retrieval/PurplePulse/results/million/purplepulse_faiss_compare_k50_v2.csv create mode 100644 09_vector_retrieval/PurplePulse/results/million/sweep_smoke.csv create mode 100644 09_vector_retrieval/PurplePulse/results/real/sift1m/build.csv create mode 100644 09_vector_retrieval/PurplePulse/results/real/sift1m/dataset_manifest.json create mode 100644 09_vector_retrieval/PurplePulse/results/real/sift1m/environment.json create mode 100644 09_vector_retrieval/PurplePulse/results/real/sift1m/exact_gpu.csv create mode 100644 09_vector_retrieval/PurplePulse/results/real/sift1m/faiss_gpu.csv create mode 100644 09_vector_retrieval/PurplePulse/results/real/sift1m/figures/recall-latency.svg create mode 100644 09_vector_retrieval/PurplePulse/results/real/sift1m/figures/recall-qps.svg create mode 100644 09_vector_retrieval/PurplePulse/results/real/sift1m/groundtruth_score_validation.jsonl create mode 100644 09_vector_retrieval/PurplePulse/results/real/sift1m/groundtruth_validation.json create mode 100644 09_vector_retrieval/PurplePulse/results/real/sift1m/ivf_k10.csv create mode 100644 09_vector_retrieval/PurplePulse/results/real/sift1m/ivf_k100.csv create mode 100644 09_vector_retrieval/PurplePulse/results/real/sift1m/ivf_k100_full_heap.csv create mode 100644 09_vector_retrieval/PurplePulse/results/real/sift1m/ivf_k100_partitioned.csv create mode 100644 09_vector_retrieval/PurplePulse/results/real/sift1m/ivf_k50.csv create mode 100644 09_vector_retrieval/PurplePulse/results/real/sift1m/ivf_k50_full_heap.csv create mode 100644 09_vector_retrieval/PurplePulse/results/real/sift1m/ivf_k50_partitioned.csv create mode 100644 09_vector_retrieval/PurplePulse/results/real/sift1m/partitioned_topk_validation.json create mode 100644 09_vector_retrieval/PurplePulse/results/real/yahoo-minilm-384/build.csv create mode 100644 09_vector_retrieval/PurplePulse/results/real/yahoo-minilm-384/dataset_manifest.json create mode 100644 09_vector_retrieval/PurplePulse/results/real/yahoo-minilm-384/environment.json create mode 100644 09_vector_retrieval/PurplePulse/results/real/yahoo-minilm-384/exact_gpu.csv create mode 100644 09_vector_retrieval/PurplePulse/results/real/yahoo-minilm-384/faiss_gpu.csv create mode 100644 09_vector_retrieval/PurplePulse/results/real/yahoo-minilm-384/figures/recall-latency.svg create mode 100644 09_vector_retrieval/PurplePulse/results/real/yahoo-minilm-384/figures/recall-qps.svg create mode 100644 09_vector_retrieval/PurplePulse/results/real/yahoo-minilm-384/groundtruth_validation.json create mode 100644 09_vector_retrieval/PurplePulse/results/real/yahoo-minilm-384/ivf_k10.csv create mode 100644 09_vector_retrieval/PurplePulse/results/real/yahoo-minilm-384/ivf_k100.csv create mode 100644 09_vector_retrieval/PurplePulse/results/real/yahoo-minilm-384/ivf_k100_full_heap.csv create mode 100644 09_vector_retrieval/PurplePulse/results/real/yahoo-minilm-384/ivf_k100_partitioned.csv create mode 100644 09_vector_retrieval/PurplePulse/results/real/yahoo-minilm-384/ivf_k50.csv create mode 100644 09_vector_retrieval/PurplePulse/results/real/yahoo-minilm-384/ivf_k50_full_heap.csv create mode 100644 09_vector_retrieval/PurplePulse/results/real/yahoo-minilm-384/ivf_k50_partitioned.csv create mode 100644 09_vector_retrieval/PurplePulse/results/real/yahoo-minilm-384/partitioned_topk_validation.json create mode 100644 09_vector_retrieval/PurplePulse/results/stage7_exact_block_tile/smoke/synthetic/results.csv create mode 100644 09_vector_retrieval/PurplePulse/results/stage7_exact_block_tile/smoke/yahoo/results.csv create mode 100644 09_vector_retrieval/PurplePulse/results/stage7_exact_profile/cosine_norm_ab/synthetic1m_baseline/breakdown.csv create mode 100644 09_vector_retrieval/PurplePulse/results/stage7_exact_profile/cosine_norm_ab/synthetic1m_optimized/breakdown.csv create mode 100644 09_vector_retrieval/PurplePulse/results/stage7_exact_profile/cosine_norm_ab/yahoo_baseline/breakdown.csv create mode 100644 09_vector_retrieval/PurplePulse/results/stage7_exact_profile/cosine_norm_ab/yahoo_optimized/breakdown.csv create mode 100644 09_vector_retrieval/PurplePulse/results/stage7_exact_profile/synthetic1m/breakdown.csv create mode 100644 09_vector_retrieval/PurplePulse/results/stage7_exact_profile/yahoo-minilm-384/breakdown.csv create mode 100644 09_vector_retrieval/PurplePulse/results/stage7_exact_query_tile/formal/sift_k10/results.csv create mode 100644 09_vector_retrieval/PurplePulse/results/stage7_exact_query_tile/formal/sift_k50/results.csv create mode 100644 09_vector_retrieval/PurplePulse/results/stage7_exact_query_tile/formal/synthetic_k10/results.csv create mode 100644 09_vector_retrieval/PurplePulse/results/stage7_exact_query_tile/formal/synthetic_k50/results.csv create mode 100644 09_vector_retrieval/PurplePulse/results/stage7_exact_query_tile/formal/yahoo_k10/results.csv create mode 100644 09_vector_retrieval/PurplePulse/results/stage7_exact_query_tile/formal/yahoo_k10_grid/results.csv create mode 100644 09_vector_retrieval/PurplePulse/results/stage7_exact_query_tile/formal/yahoo_k50/results.csv create mode 100644 09_vector_retrieval/PurplePulse/results/stage7_exact_query_tile/grid_sweep_smoke/sift/results.csv create mode 100644 09_vector_retrieval/PurplePulse/results/stage7_exact_query_tile/grid_sweep_smoke/synthetic/results.csv create mode 100644 09_vector_retrieval/PurplePulse/results/stage7_exact_query_tile/grid_sweep_smoke/yahoo/results.csv create mode 100644 09_vector_retrieval/PurplePulse/results/stage7_exact_query_tile/scaled_grid_smoke/synthetic1m/results.csv create mode 100644 09_vector_retrieval/PurplePulse/results/stage7_exact_query_tile/scaled_grid_smoke/yahoo/results.csv create mode 100644 09_vector_retrieval/PurplePulse/results/stage7_exact_query_tile/smoke/synthetic1m/results.csv create mode 100644 09_vector_retrieval/PurplePulse/results/stage7_exact_query_tile/smoke/yahoo/results.csv create mode 100644 09_vector_retrieval/PurplePulse/results/stage7_exact_query_tile/tile8_smoke/synthetic1m/results.csv create mode 100644 09_vector_retrieval/PurplePulse/results/stage7_exact_query_tile/tile8_smoke/yahoo/results.csv create mode 100644 09_vector_retrieval/PurplePulse/results/stage8_stability/coverage.csv create mode 100644 09_vector_retrieval/PurplePulse/scripts/benchmark_adaptive_nprobe.py create mode 100644 09_vector_retrieval/PurplePulse/scripts/benchmark_exact.py create mode 100644 09_vector_retrieval/PurplePulse/scripts/benchmark_exact_breakdown.py create mode 100644 09_vector_retrieval/PurplePulse/scripts/benchmark_exact_fused.py create mode 100644 09_vector_retrieval/PurplePulse/scripts/benchmark_exact_query_tile.py create mode 100644 09_vector_retrieval/PurplePulse/scripts/benchmark_faiss.py create mode 100644 09_vector_retrieval/PurplePulse/scripts/benchmark_ivf_sweep.py create mode 100644 09_vector_retrieval/PurplePulse/scripts/benchmark_memory_modes.py create mode 100644 09_vector_retrieval/PurplePulse/scripts/build_final_report_pdf.py create mode 100644 09_vector_retrieval/PurplePulse/scripts/build_report_figures.py create mode 100644 09_vector_retrieval/PurplePulse/scripts/evaluate_ivecs_groundtruth.py create mode 100644 09_vector_retrieval/PurplePulse/scripts/evaluate_recall.py create mode 100644 09_vector_retrieval/PurplePulse/scripts/generate_data.py create mode 100644 09_vector_retrieval/PurplePulse/scripts/generate_memory_metadata.py create mode 100644 09_vector_retrieval/PurplePulse/scripts/import_ann_benchmarks.py create mode 100644 09_vector_retrieval/PurplePulse/scripts/import_mnist.py create mode 100644 09_vector_retrieval/PurplePulse/scripts/import_sift1m.py create mode 100644 09_vector_retrieval/PurplePulse/scripts/plot_results.py create mode 100644 09_vector_retrieval/PurplePulse/scripts/prepare_real_datasets.py create mode 100644 09_vector_retrieval/PurplePulse/scripts/reproduce.py create mode 100644 09_vector_retrieval/PurplePulse/scripts/run_real_benchmarks.py create mode 100644 09_vector_retrieval/PurplePulse/scripts/slice_queries.py create mode 100644 09_vector_retrieval/PurplePulse/scripts/validate_release.py create mode 100644 09_vector_retrieval/PurplePulse/scripts/verify_results.py create mode 100644 09_vector_retrieval/PurplePulse/src/build_ivf.cpp create mode 100644 09_vector_retrieval/PurplePulse/src/cpu_search.cpp create mode 100644 09_vector_retrieval/PurplePulse/src/file_io.cpp create mode 100644 09_vector_retrieval/PurplePulse/src/gpu_ivf_flat.cu create mode 100644 09_vector_retrieval/PurplePulse/src/gpu_search.cu create mode 100644 09_vector_retrieval/PurplePulse/src/ivf_flat.cpp create mode 100644 09_vector_retrieval/PurplePulse/src/main.cpp create mode 100644 09_vector_retrieval/PurplePulse/src/search_ivf.cpp create mode 100644 09_vector_retrieval/PurplePulse/tests/file_io_tests.cpp create mode 100644 09_vector_retrieval/PurplePulse/tests/ivf_flat_tests.cpp create mode 100644 09_vector_retrieval/PurplePulse/tests/stability_tests.cpp create mode 100644 09_vector_retrieval/PurplePulse/tests/test_benchmark_adaptive_nprobe.py create mode 100644 09_vector_retrieval/PurplePulse/tests/test_benchmark_exact_breakdown.py create mode 100644 09_vector_retrieval/PurplePulse/tests/test_benchmark_exact_query_tile.py create mode 100644 09_vector_retrieval/PurplePulse/tests/test_benchmark_faiss.py create mode 100644 09_vector_retrieval/PurplePulse/tests/test_benchmark_ivf_sweep.py create mode 100644 09_vector_retrieval/PurplePulse/tests/test_import_ann_benchmarks.py create mode 100644 09_vector_retrieval/PurplePulse/tests/test_import_mnist.py create mode 100644 09_vector_retrieval/PurplePulse/tests/test_import_sift1m.py create mode 100644 09_vector_retrieval/PurplePulse/tests/test_plot_results.py create mode 100644 09_vector_retrieval/PurplePulse/tests/test_stage6_tools.py create mode 100644 09_vector_retrieval/PurplePulse/tests/test_stage9_tools.py create mode 100644 09_vector_retrieval/PurplePulse/tests/unit_tests.cpp diff --git a/09_vector_retrieval/PurplePulse/.clang-format b/09_vector_retrieval/PurplePulse/.clang-format new file mode 100644 index 00000000..75295958 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/.clang-format @@ -0,0 +1,4 @@ +BasedOnStyle: Google +ColumnLimit: 80 +IndentWidth: 2 +SortIncludes: CaseSensitive diff --git a/09_vector_retrieval/PurplePulse/.gitattributes b/09_vector_retrieval/PurplePulse/.gitattributes new file mode 100644 index 00000000..99316f64 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/.gitattributes @@ -0,0 +1,6 @@ +*.csv text eol=lf +*.md text eol=lf +*.cpp text eol=lf +*.cu text eol=lf +*.h text eol=lf +*.py text eol=lf diff --git a/09_vector_retrieval/PurplePulse/.gitignore b/09_vector_retrieval/PurplePulse/.gitignore new file mode 100644 index 00000000..e87d3f18 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/.gitignore @@ -0,0 +1,63 @@ +/build/ +/data/ +/tmp/ +/results/* +!/results/figures/ +/results/figures/* +!/results/figures/*.svg +!/results/million/ +/results/million/* +!/results/million/*.csv +!/results/memory_100k/ +/results/memory_100k/* +!/results/memory_100k/source1_compare.csv +!/results/exact_fused/ +/results/exact_fused/* +!/results/exact_fused/million_compare.csv +!/results/formal_100k15/ +/results/formal_100k15/* +!/results/formal_100k15/*.csv +!/results/formal_4090d_20260908/ +/results/formal_4090d_20260908/* +!/results/formal_4090d_20260908/*.csv +!/results/real/ +/results/real/* +!/results/real/*/ +/results/real/*/* +!/results/real/*/*.csv +!/results/real/*/*.json +!/results/real/*/*.jsonl +!/results/real/*/figures/ +/results/real/*/figures/* +!/results/real/*/figures/*.svg +!/results/stage7_exact_profile/ +/results/stage7_exact_profile/* +!/results/stage7_exact_profile/*/ +/results/stage7_exact_profile/*/* +!/results/stage7_exact_profile/*/*.csv +!/results/stage7_exact_profile/cosine_norm_ab/ +/results/stage7_exact_profile/cosine_norm_ab/* +!/results/stage7_exact_profile/cosine_norm_ab/*/ +/results/stage7_exact_profile/cosine_norm_ab/*/* +!/results/stage7_exact_profile/cosine_norm_ab/*/*.csv +!/results/stage7_exact_query_tile/ +/results/stage7_exact_query_tile/* +!/results/stage7_exact_query_tile/*/ +/results/stage7_exact_query_tile/*/* +!/results/stage7_exact_query_tile/*/*/ +/results/stage7_exact_query_tile/*/*/* +!/results/stage7_exact_query_tile/*/*/*.csv +!/results/stage7_exact_block_tile/ +/results/stage7_exact_block_tile/* +!/results/stage7_exact_block_tile/*/ +/results/stage7_exact_block_tile/*/* +!/results/stage7_exact_block_tile/*/*/ +/results/stage7_exact_block_tile/*/*/* +!/results/stage7_exact_block_tile/*/*/*.csv +!/results/stage8_stability/ +/results/stage8_stability/* +!/results/stage8_stability/*.csv +*.bin +*.ncu-rep +*.nsys-rep +__pycache__/ diff --git a/09_vector_retrieval/PurplePulse/CMakeLists.txt b/09_vector_retrieval/PurplePulse/CMakeLists.txt new file mode 100644 index 00000000..6a7cac41 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/CMakeLists.txt @@ -0,0 +1,83 @@ +cmake_minimum_required(VERSION 3.24) + +# 必须在 project() 之前设置，否则 CMake 可能先缓存一个旧架构默认值。 +if(NOT DEFINED CMAKE_CUDA_ARCHITECTURES) + set(CMAKE_CUDA_ARCHITECTURES native CACHE STRING "CUDA GPU architectures") +endif() + +project(purplepulse_vector_retrieval LANGUAGES CXX CUDA) + +set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_STANDARD_REQUIRED ON) +set(CMAKE_CUDA_STANDARD 17) +set(CMAKE_CUDA_STANDARD_REQUIRED ON) + +add_library(vector_engine + src/file_io.cpp + src/cpu_search.cpp + src/ivf_flat.cpp + src/gpu_ivf_flat.cu + src/gpu_search.cu +) +target_include_directories(vector_engine PUBLIC include) +target_compile_options(vector_engine PRIVATE + $<$:-Wall;-Wextra;-Wpedantic;-march=native> + $<$:--extended-lambda> +) + +add_executable(vector_search src/main.cpp) +target_link_libraries(vector_search PRIVATE vector_engine) + +add_executable(ivf_build src/build_ivf.cpp) +target_link_libraries(ivf_build PRIVATE vector_engine) + +add_executable(ivf_search src/search_ivf.cpp) +target_link_libraries(ivf_search PRIVATE vector_engine) + +add_executable(unit_tests tests/unit_tests.cpp) +target_link_libraries(unit_tests PRIVATE vector_engine) + +add_executable(file_io_tests tests/file_io_tests.cpp src/file_io.cpp) +target_include_directories(file_io_tests PRIVATE include) +target_compile_options(file_io_tests PRIVATE -Wall -Wextra -Wpedantic) + +add_executable(ivf_flat_tests tests/ivf_flat_tests.cpp) +target_link_libraries(ivf_flat_tests PRIVATE vector_engine) + +add_executable(stability_tests tests/stability_tests.cpp) +target_link_libraries(stability_tests PRIVATE vector_engine) + +enable_testing() +add_test(NAME unit_tests COMMAND unit_tests) +add_test(NAME file_io_tests COMMAND file_io_tests) +add_test(NAME ivf_flat_tests COMMAND ivf_flat_tests) +add_test(NAME stability_tests COMMAND stability_tests) +add_test(NAME vector_search_help COMMAND vector_search --help) +add_test(NAME ivf_build_help COMMAND ivf_build --help) +add_test(NAME ivf_search_help COMMAND ivf_search --help) + +find_package(Python3 COMPONENTS Interpreter) +if(Python3_Interpreter_FOUND) + add_test(NAME benchmark_faiss_tests + COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tests/test_benchmark_faiss.py) + add_test(NAME benchmark_ivf_sweep_tests + COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tests/test_benchmark_ivf_sweep.py) + add_test(NAME benchmark_adaptive_nprobe_tests + COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tests/test_benchmark_adaptive_nprobe.py) + add_test(NAME import_mnist_tests + COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tests/test_import_mnist.py) + add_test(NAME import_sift1m_tests + COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tests/test_import_sift1m.py) + add_test(NAME import_ann_benchmarks_tests + COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tests/test_import_ann_benchmarks.py) + add_test(NAME plot_results_tests + COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tests/test_plot_results.py) + add_test(NAME stage6_tools_tests + COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tests/test_stage6_tools.py) + add_test(NAME benchmark_exact_breakdown_tests + COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tests/test_benchmark_exact_breakdown.py) + add_test(NAME benchmark_exact_query_tile_tests + COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tests/test_benchmark_exact_query_tile.py) + add_test(NAME stage9_tools_tests + COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tests/test_stage9_tools.py) +endif() diff --git a/09_vector_retrieval/PurplePulse/README.md b/09_vector_retrieval/PurplePulse/README.md new file mode 100644 index 00000000..66723ee6 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/README.md @@ -0,0 +1,490 @@ +# WarpFusion：GPU Exact 与 IVF-Flat 向量检索引擎 + +这是 2026 夏季训练营 CUDA 方向“GPU 向量检索引擎”项目的独立实现。 + +项目从容易理解、结果正确的精确检索 baseline 逐步演进到 GPU Top-K、 +IVF-Flat、多轮性能优化、并行索引构建和真实数据评测。 + +![WarpFusion 系统架构](results/figures/architecture.svg) + +## 当前功能 + +- FP32/FP16 向量库与查询文件读写 +- FP16 在 GPU 上保持 16 位存储，距离计算使用 FP32 累加 +- L2、内积和余弦相似度 +- CPU 精确检索参考实现 +- CUDA 精确距离计算 baseline +- 四种 GPU Top-K：simple、block、two-stage，以及不生成完整距离矩阵的 + fused 路径 +- 精确检索与 IVF 均支持 K = 1、10、50、100（也支持其他 K≤100） +- 批量查询 +- GPU 数据库和工作区常驻显存，区分一次性初始化与稳定查询性能 +- CPU/GPU 结果自动对比测试 +- IVF-Flat CPU baseline：采样 k-means、连续倒排桶、索引保存/加载、固定 + `nprobe` 查询和 recall@K +- L2 使用普通 k-means；内积和余弦使用归一化中心的球面 k-means，避免 + 相似度聚类因中心模长造成严重桶倾斜 +- GPU IVF：中心、桶 offsets 和索引向量常驻显存，GPU 完成中心选择、 + 每桶 block 并行扫描、局部 Top-K 与最终归并；支持 scalar、2/4/8-warp + 和按 K 自动选择并发度的 compact 扫描路径 +- 实验性 `score_mass` 自适应 nprobe：依据每个 query 的中心分数分布选择 + probe 前缀，支持实际 probe 明细导出和 masked/grouped 调度 A/B +- 可选实验扩展：GPU 元数据过滤与融合评分 + +IVF-Flat 的索引、CPU 正确性基线、全 GPU 查询路径、K=1/10/50/100 以及 +100k/百万规模参数扫描、桶扫描优化、FAISS CPU/GPU 对照和首轮自适应 +nprobe 实验已经完成；提交结论与最新数字以 `docs/FINAL_REPORT.md` 为准。 + +## 正式结果与报告 + +完整方法、实验口径、负结果和复现命令见 +[`docs/FINAL_REPORT.md`](docs/FINAL_REPORT.md)；本轮正式环境、训练差异、 +数据/索引校验和、完整命令和延迟定义见 +[`docs/REPRODUCIBILITY.md`](docs/REPRODUCIBILITY.md)。提交前可按 +[`docs/SUBMISSION_CHECKLIST.md`](docs/SUBMISSION_CHECKLIST.md) 逐项核验。 +第八阶段的维度/dtype/metric/batch/K 覆盖与安全回退规则见 +[`docs/STABILITY_MATRIX.md`](docs/STABILITY_MATRIX.md)；CLI 和常驻执行器示例见 +[`docs/API.md`](docs/API.md)。 +图表由原始 CSV 直接生成： + +![Recall–QPS](results/figures/recall-qps.svg) + +![Batch–QPS](results/figures/batch-qps.svg) + +重新生成全部图表： + +```bash +python3 scripts/plot_results.py \ + --results-dir results/formal_100k15 \ + --batch-results-dir results/million \ + --output-dir results/figures +``` + +## 编译 + +```bash +export PATH=/usr/local/cuda/bin:$PATH +cmake -S . -B build -G Ninja -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_CUDA_ARCHITECTURES=native +cmake --build build +ctest --test-dir build --output-on-failure +``` + +`native` 会在配置时检测当前 GPU，因此切换 GPU 型号时不需要修改 +README。如果要生成能在多种 GPU 上运行的二进制，可显式传入用分号分隔的 +多个架构编号。切换设备后建议删除旧 `build/` 或使用新的构建目录， +避免复用 CMake 缓存中的旧架构。 + +## 一条命令复现 + +在已安装 CUDA、CMake、Ninja、Python 依赖和 GPU 版 FAISS 的 Linux 主机上， +以下命令依次执行干净构建、CTest、公开数据准备、WarpFusion/FAISS 正式实验、 +绘图、PDF 报告生成和发布校验。长任务应由 `nohup` 托管；每一步的日志与断点 +状态都会立即写到结果目录： + +```bash +mkdir -p results/real/reproducibility/logs +nohup python3 scripts/reproduce.py --project-root "$PWD" --resume \ + > results/real/reproducibility/nohup.log 2>&1 & +``` + +若数据已准备好，可加 `--skip-download`；若索引已完成，可再加 +`--skip-index-build`。只查看将执行的命令而不改变状态： + +```bash +python3 scripts/reproduce.py --project-root "$PWD" --dry-run +``` + +断点文件为 `results/real/reproducibility/checkpoint.json`。更换源码、配置或数据后 +应使用新的结果目录，避免把旧断点误认为新实验已完成。 + +## 推荐起点 + +这些值是正式单卡实验的起点，不是跨数据和跨 GPU 的通用最优参数： + +| 场景 | 默认/推荐值 | 说明 | +|---|---|---| +| Exact | `warp + fused`, batch 64 | K≤100；未验证 query tile 自动回退 tile1 | +| SIFT1M / Yahoo IVF | nlist 1024, batch 64 | 建库 100k 样本、15 轮、seed 2026 | +| IVF K=10 | `bucket_major` | 扫描 nprobe 后按目标 recall 选点 | +| IVF K=50 | 完整 heap 回退 | 真实数据上分区路径未稳定获益 | +| IVF K=100 | `bucket_major_partitioned` | 4×40 局部候选后最终归并 | +| 基准计时 | warmup 1, repeat 5 | 报告稳定查询 QPS，另列一次性 H2D | + +RTX 4090 D 上公开真实数据的代表点如下，完整曲线和同预算 FAISS 对照以原始 +CSV 与最终报告为准： + +| 数据集 | 查询 | K | nprobe | Recall@K | QPS | +|---|---|---:|---:|---:|---:| +| SIFT1M | Exact fused | 10 | — | 1.0000 | 7,038 | +| SIFT1M | IVF-Flat | 10 | 160 | 0.99987 | 24,579 | +| Yahoo MiniLM 384 | Exact fused | 10 | — | 1.0000 | 3,610 | +| Yahoo MiniLM 384 | IVF-Flat | 10 | 160 | 0.9939 | 15,733 | + +## 已知限制 + +- 单进程、单 GPU、内存内静态索引；无在线增删、压缩、超显存分片或多 GPU。 +- 正式性能只在 RTX 4090 D 上验证；更换维度、分布、K 或 GPU 必须重新扫参。 +- Exact fused 仍落后于 FAISS GPU Flat；当前强项是可解释、可验证的 IVF 路径。 +- IVF recall 依赖训练样本和数据分布；固定 nprobe 不保证逐 query 的最低 recall。 +- 当前交付提供源码级 API，不承诺稳定 ABI，也未提供并发服务层。 + +## 生成测试数据 + +```bash +python3 scripts/generate_data.py \ + --database data/database.bin \ + --queries data/queries.bin \ + --num-vectors 10000 \ + --num-queries 20 \ + --dim 128 \ + --dtype fp32 \ + --metric l2 \ + --seed 2026 +``` + +## 执行检索 + +```bash +./build/vector_search \ + --database data/database.bin \ + --queries data/queries.bin \ + --params configs/exact_l2.conf \ + --backend gpu \ + --output results/exact_l2.txt +``` + +结果文件中每一行依次为 `query_id vector_id score`。 + +### 融合 Exact 路径 + +设置 `distance_mode=warp`、`topk_mode=fused` 后，每个 warp 在计算距离时 +直接维护局部 Top-K，每个 query/chunk 只写回 K 个候选，再由一个小型 kernel +完成最终归并。该路径不再物化 `batch_size × num_vectors` 的完整距离矩阵。 +K=10/50/100 的参考配置分别为 +`configs/exact_million_k{10,50,100}_fused_batch64.conf`。 + +第七阶段增加 `exact_query_tile=2/4/8` 和 `exact_fused_blocks=N` 以复用跨 query +数据库读取。`exact_fused_layout=block_tile` 是保留的负实验入口；默认值始终为 +`warp_query`，不应把 block-tile 用作推荐配置。选择依据与完整 A/B 数字见 +`docs/REPRODUCIBILITY.md`。 + +百万条 FP32、128 维、1000 queries、Cosine、batch=64、预热 1 次并重复 5 次 +的当前测试结果如下；旧 matrix 路径保留用于 A/B： + +| K | matrix QPS | fused QPS | 加速 | matrix / fused 缓冲区 | +|---:|---:|---:|---:|---:| +| 10 | 1775.10 | 5563.72 | 3.13× | 734.34 / 488.44 MiB | +| 50 | 715.87 | 5123.32 | 7.16× | 732.49 / 488.94 MiB | +| 100 | 443.78 | 4700.80 | 10.59× | 732.53 / 489.56 MiB | + +所有成对结果的 Top-K ID 集合一致，记录的最大分数误差为 0。原始数据见 +`results/exact_fused/million_compare.csv`，可重新运行： + +```bash +python3 scripts/benchmark_exact_fused.py \ + --search-binary ./build/vector_search \ + --database data/exact_million/cosine_db.bin \ + --queries data/exact_million/cosine_q.bin \ + --baseline-configs configs/exact_l2_batch64.conf,configs/exact_million_k50_batch64.conf,configs/exact_million_k100_batch64.conf \ + --fused-configs configs/exact_million_k10_fused_batch64.conf,configs/exact_million_k50_fused_batch64.conf,configs/exact_million_k100_fused_batch64.conf \ + --output-dir results/exact_fused/million_outputs \ + --csv results/exact_fused/million_compare.csv \ + --warmup 1 --repeat 5 +``` + +## IVF-Flat baseline + +先构建可持久化索引： + +```bash +./build/ivf_build \ + --database data/database.bin \ + --output data/database.ivf \ + --nlist 1024 \ + --iterations 15 \ + --training-samples 100000 \ + --threads 16 +``` + +再运行 CPU IVF 正确性/质量基线；`--nprobe` 可覆盖配置文件，便于扫描： + +```bash +./build/ivf_search \ + --index data/database.ivf \ + --queries data/queries.bin \ + --params configs/ivf_flat.conf \ + --nprobe 16 \ + --backend gpu \ + --warmup 1 \ + --repeat 5 \ + --output results/ivf_flat.txt + +python3 scripts/evaluate_recall.py \ + --exact results/exact_l2.txt \ + --approximate results/ivf_flat.txt +``` + +输出同时包含 recall、最低单 query recall，以及按名次计算的平均/最大绝对 +分数误差。 + +`configs/ivf_flat_100k_{balanced,high_recall,exact_check}.conf` 是 +100k × 128、K=10 实验的三档参考配置。`nlist` 必须与索引构建参数一致； +迁移到百万级或不同数据分布时，应重新扫描 `nlist/nprobe`，不要直接把这组 +参数当成通用最优值。`ivf_flat_100k_scalar_baseline.conf` 保留旧的 +thread-per-vector 桶扫描，用于和默认 warp 协作版本做 A/B 对照。 +`nprobe` 可取 `1..nlist`；使用 `nprobe=nlist` 可作为结果完整性校验，但 +近似检索应根据目标 recall 和延迟选取更小的值。 + +百万规模 K=10 初测提供四档配置： + +- `ivf_flat_million_fast.conf`：偏吞吐。 +- `ivf_flat_million_balanced.conf`：约 0.87 recall 的平衡档。 +- `ivf_flat_million_high_recall.conf`：约 0.97 recall。 +- `ivf_flat_million_near_exact.conf`：约 0.99 recall。 + +大 K 使用独立模板实例和 warp 共享内存堆；`warp_compact` 在 K≤10、 +11≤K≤50、K>50 时分别使用 8、2、1 个 warp。K=100 的单 warp 路径在 +RTX 4090 D 上比旧两 warp 路径快 9.7%–15.1%。局部 heap 进一步缓存根节点、 +以空穴下沉替代逐层交换，并让单 warp 直接写回有序结果；nprobe=160 的正式 +A/B 提升 2.40%，与 bucket-major 组合后累计提升 3.65%。K=100 的推荐配置 +进一步使用 `bucket_major_partitioned`：4 个 warp 各维护 40 项局部候选，再 +合并为 Top-100。`nprobe=160` 的 100 次 A/B 从 5503.11 提高到 7204.54 QPS， +提升 30.92%；输出逐字一致，recall@100 保持 0.91758，Compute Sanitizer 为 +0 errors。推荐从 +`ivf_flat_million_k50.conf` 或 +`ivf_flat_million_k100.conf` 开始；新平台实测默认 batch 分别为 40 和 64，并提供 +batch=8/32/64/128 以及 warp1/2/4/8 的 A/B 配置。百万级 K10 的 nlist=256 +三档配置默认 batch=64。切换数据、维度或 GPU 后仍应重新扫参。 + +K=50 先启用 `bucket_major`：GPU 将已选 probe 任务按中心桶分组，以 +提高同一 batch 内的 L2 复用。nprobe=160、batch=40 的 100 次 A/B 从 +6009.22 提高到 6964.16 QPS，提升 15.89%，recall@50 保持 0.92482。 +低 nprobe 时收益较小；原始 probe-major 基线可使用 +`ivf_flat_million_k50_warp_compact_batch40.conf`。完整扫描见 +`results/formal_4090d_20260908/k50_bucket_major_sweep.csv`。 +当前 `bucket_major` 和 `bucket_major_partitioned` 只支持固定 nprobe； +score-mass 自适应策略继续使用 +`warp_compact`。 + +K=50 默认配置在 bucket-major 上继续使用 4×16 分区局部 Top-K。固定 +batch=40、nprobe=160、预热 5 次并重复 100 次，完整 heap 为 7165.23 QPS， +4×16 为 7934.10 QPS，提升 10.73%，输出逐字一致，Compute Sanitizer 为 +0 errors。FAISS GPU IVF-Flat 为 8328.11 QPS / 0.92312 recall@50； +WarpFusion 为 7934.10 QPS / 0.92482 recall@50，吞吐差距为 4.97%。原始结果 +见 `results/formal_4090d_20260908/k50_partitioned_tight/` 和 +`results/formal_4090d_20260908/faiss_gpu_batch40.csv`。 + +### MNIST 真实数据验证 + +除百万规模合成数据外，项目提供 MNIST 转换脚本。训练图像作为数据库，测试 +图像作为查询，像素展开为 784 维 FP32 向量并使用 L2： + +```bash +python3 scripts/import_mnist.py \ + --input data/mnist/mnist.npz \ + --database data/mnist/database.bin \ + --queries data/mnist/queries.bin +``` + +RTX 4090 D 上，60k 数据库和 10k queries 的 Exact K=10 为 23.1k QPS； +IVF nprobe=8 达到 0.9784 recall@10 和 197.7k QPS，nprobe=32 达到 +0.9991 recall@10 和 76.6k QPS。原始数据见 +`results/formal_4090d_20260908/mnist_ivf.csv`。 + +这些 recall 只对应固定随机种子生成的测试数据。可用脚本在新数据或新 GPU +上重新生成 CSV，而不是沿用旧机器的结论： + +```bash +python3 scripts/benchmark_ivf_sweep.py \ + --search-binary ./build/ivf_search \ + --index data/million/inner_fp32_nlist256_spherical_100k15.ivf \ + --queries data/million/inner_fp32_q.bin \ + --params configs/ivf_flat_million_batch64.conf \ + --exact results/million/inner_exact_gpu_k10.txt \ + --output-dir results/million/sweep_n256 \ + --csv results/million/sweep_n256.csv \ + --nlist 256 \ + --nprobes 64,128,160,192,224,256 \ + --training-samples 100000 \ + --training-iterations 15 +``` + +### 第六阶段公开真实数据 + +正式外部验证使用两套互补数据：TexMex SIFT1M（1,000,000 × 128，L2）和 +VIBE `yahoo-minilm-384-normalized`（677,305 × 384，文本 embedding， +cosine）。两者都带公开 Top-100 ground truth。准备脚本固定并验证公开文件 +校验和：SIFT archive MD5 为 `b23d1b3b2ee8469d819b61ca900ef0ed`，VIBE +HDF5 SHA-256 为 +`8b0519850249d7ea4aea258eb43ae837dca1bb2f7a85ee0a5145afdb699c4136`。 +VIBE 文件的 `distance=normalized` 会先经过逐块单位范数检查，再明确映射到 +WarpFusion cosine；不会把它静默当作普通 L2 数据。 + +```bash +python3 scripts/prepare_real_datasets.py \ + --dataset all \ + --raw-dir data/real/raw \ + --output-dir data/real +``` + +数据准备需要 NumPy 和 `h5py`。转换后的每套数据都包含 `database.bin`、 +`queries.bin`、`groundtruth.ivecs` 和记录输入/输出 SHA-256 的 +`manifest.json`。完整 K/nprobe 矩阵可用一条命令运行： + +```bash +python3 scripts/run_real_benchmarks.py \ + --dataset sift1m \ + --nlist 1024 \ + --nprobes 32,64,128,160,192,224 \ + --warmup 1 --repeat 5 + +python3 scripts/run_real_benchmarks.py \ + --dataset yahoo-minilm-384 \ + --nlist 1024 \ + --nprobes 32,64,128,160,192,224 \ + --warmup 1 --repeat 5 +``` + +2026-09-10 的 RTX 4090 D 正式结果已经完成。下表固定 `batch=64`、 +`nprobe=160`，同时列出各自独立训练、相同训练预算下的 FAISS GPU IVF-Flat： + +| 数据集 | K | WarpFusion recall / QPS | FAISS recall / QPS | 吞吐比 | +|---|---:|---:|---:|---:| +| SIFT1M | 10 | 0.999870 / 24579 | 0.999120 / 13617 | 1.81× | +| SIFT1M | 50 | 0.999706 / 19237 | 0.999516 / 13443 | 1.43× | +| SIFT1M | 100 | 0.999565 / 17362 | 0.999457 / 13102 | 1.33× | +| Yahoo MiniLM 384 | 10 | 0.993900 / 15733 | 0.995300 / 6216 | 2.53× | +| Yahoo MiniLM 384 | 50 | 0.991520 / 14950 | 0.991280 / 6157 | 2.43× | +| Yahoo MiniLM 384 | 100 | 0.989230 / 14196 | 0.989080 / 6060 | 2.34× | + +K=50 的 4×16 分区路径在 SIFT1M 和 Yahoo 上分别损失 0.000326 和 +0.000980 recall，均超过 `1e-4` 安全阈值，因此正式曲线自动采用完整 heap。 +K=100 的 4×40 路径在两套数据上均为零 recall 损失，`nprobe=160` 下相对 +完整 heap 分别加速 1.38× 和 1.72×。可提交的 CSV、验证 JSON 和 SVG 位于 +`results/real/`；原始 Top-K 输出和日志按数据集保留在实验机 `/data`。 + +该命令依次记录自研 IVF 建库、Exact K=10/50/100、公开 ground truth +交叉验证、自研与 FAISS GPU IVF 扫描、QPS、run/batch P50/P99、显存和图表。 +K=50/100 会同时运行分区局部 Top-K 与完整 heap；若任一 nprobe 的 recall +损失超过 `1e-4`，最终曲线自动回退到完整 heap，同时保留两套原始 CSV 和 +`partitioned_topk_validation.json`。结果写入 `results/real//`。 + +## FAISS 公平对照 + +`scripts/benchmark_faiss.py` 直接读取相同的项目数据文件，支持 +FAISS CPU/GPU、Flat/IVF-Flat、多个 K/nprobe，并把索引构建/加载、GPU +转移、稳定查询 QPS、run/batch P50/P99 及其样本数、recall、平均/最大绝对 +分数误差和 FAISS GPU 内存分类写入 CSV。 +运行它的 Python 环境需要 NumPy 和与当前 CUDA 环境兼容的 FAISS 包。 + +```bash +python3 scripts/benchmark_faiss.py \ + --database data/million/inner_fp32_db.bin \ + --queries data/million/inner_fp32_q.bin \ + --reference-results results/million/inner_exact_gpu_k100.txt \ + --index-dir data/million/faiss \ + --output-dir results/million/faiss_gpu \ + --csv results/million/faiss_gpu.csv \ + --backends gpu \ + --index-types flat,ivf_flat \ + --top-ks 10,50,100 \ + --nlist 256 \ + --nprobes 128,160,192,224 \ + --batch-size 64 \ + --train-samples 100000 \ + --iterations 15 \ + --seed 2026 \ + --warmup 1 \ + --repeat 5 +``` + +脚本默认将 FAISS CPU 线程数固定为 1；可用 `--cpu-threads` 显式改变。 +FAISS Flat 会先与 WarpFusion exact 结果交叉验证，候选 ID 不一致时终止。 +长矩阵每完成一项都会立即更新 CSV，中断时不会丢失已经完成的行。 + +## 实验性自适应 nprobe + +固定策略仍是默认。设置 `nprobe_policy=score_mass` 后，`nprobe` 表示允许的 +最大值；策略将排序后的中心分数转换为相对概率质量，并在 +`adaptive_nprobe_min..nprobe` 内按 `adaptive_nprobe_step` 选择每个 query +的实际 probe 数。`adaptive_temperature` 使用当前 query 的中心分数跨度归一化， +因此不依赖绝对分数尺度。 + +```bash +python3 scripts/benchmark_adaptive_nprobe.py \ + --search-binary ./build/ivf_search \ + --index data/million/inner_fp32_nlist256_spherical.ivf \ + --queries data/million/inner_fp32_q.bin \ + --params configs/ivf_flat_million_adaptive_k10.conf \ + --exact results/million/inner_exact_gpu_k10.txt \ + --output-dir results/million/adaptive_k10 \ + --csv results/million/adaptive_k10.csv \ + --max-nprobe 224 --min-nprobe 64 --step 16 \ + --temperature 0.2 --target-masses 0.80,0.90,0.93,0.97,0.99 +``` + +百万级均匀合成数据上的首轮实验没有超过相同平均 probe 的固定策略：平均 +151.632 probes 的代表点在 K10/K50/K100 上分别慢约 1.0%/2.6%/3.6%， +recall 也略低。因此配置和脚本用于研究与复现，尚不作为推荐生产默认值。 +切换到更异质的真实查询分布后应重新校准并重新判断是否保留。 + +## 可选扩展：Agent 记忆检索 + +本节不属于当前课程项目主报告，只保留已经实现的实验接口。 + +元数据文件按原始向量 ID 保存 `timestamp`、`importance`、`session_id` 和 +`source_type`。`fused` 在桶扫描读取向量前执行过滤，并在局部 Top-K 前融合 +语义分数、重要性和指数时间衰减；`rerank` 先取语义 Top-(K×factor)，再由 +独立 GPU kernel 过滤和重排，作为 A/B baseline。L2 仍保持分数越低越好， +内积和 Cosine 保持越高越好。 + +```bash +python3 scripts/generate_memory_metadata.py \ + --output data/memory.bin --num-vectors 100000 --seed 2026 + +./build/ivf_search \ + --index data/database.ivf --queries data/queries.bin \ + --params configs/ivf_flat_memory_fused.conf \ + --memory-metadata data/memory.bin --memory-mode fused \ + --filter-session-id any --filter-source-type 1 \ + --backend gpu --warmup 2 --repeat 5 \ + --output results/memory_fused.txt +``` + +使用 `scripts/benchmark_memory_modes.py` 可在完全相同的过滤、评分和批量参数 +下运行 fused/rerank 并保存 CSV。`any` 表示不限制对应的离散元数据字段； +若过滤后不足 K 个候选，输出会诚实返回少于 K 条，而不会写入伪造 ID。 + +索引使用版本化二进制格式，保存 FP32 聚类中心、桶 offsets、原始向量 ID +以及按桶连续排列的 FP32/FP16 原始向量。GPU backend 会将这些索引数据 +常驻显存；每批只上传 queries，中心选择、桶扫描和 Top-K 均在 GPU 完成。 + +GPU backend 会先创建常驻检索引擎，数据库只上传一次，然后让 +`--warmup` 和 `--repeat` 的每轮查询复用同一份 GPU 数据和工作区。 +程序分别输出： + +- GPU 一次性初始化和数据库 H2D 时间。 +- 估算冷启动端到端时间。 +- 数据库常驻后的查询时间和 QPS。 +- 引擎显式分配的 GPU 缓冲区总量。 +- query H2D、距离 kernel、Top-K kernel 和结果 D2H 分段时间。 + +`configs/exact_l2_batch*.conf` 保留生成完整距离矩阵的历史 batch 扫描配置。 +新实验推荐以 fused batch=64 配置为起点；融合路径会根据 batch size 调整每个 +query 的 chunk 数，使总并发块数维持在合理范围，同时避免产生过多局部候选。 +切换数据规模、维度或 GPU 后仍应重新扫描 batch size。 + +第七阶段新增 `exact_query_tile=2/4/8`，用于在同一 block 内跨 query 复用数据库 +读取；`exact_fused_blocks=N` 可固定 `grid.x` 做受控 sweep，省略时使用并行度 +启发式。RTX 4090 D、batch=64 的已验证选择为：K=10/128 维使用 tile4， +K=10/384 维使用 tile8/160 blocks，K=50/384 维使用 tile4；128 维 K=50 和 +全部 K=100 继续使用 tile1 baseline。切换维度、metric、batch 或 GPU 时必须 +重新运行 `scripts/benchmark_exact_query_tile.py`，不能直接套用该表。 + +## 项目原则 + +1. 正确性优先，每个 GPU 版本都和 CPU 参考答案比较。 +2. 先实现简单版本，再逐步优化；不直接堆叠难以解释的 CUDA 技巧。 +3. 每次优化都保存测试环境、参数和优化前后的性能数据。 +4. 扩展实验不得替代 Exact、IVF、FAISS 对照和真实数据主线。 diff --git a/09_vector_retrieval/PurplePulse/configs/exact_fused_k100_batch64_querytile2.conf b/09_vector_retrieval/PurplePulse/configs/exact_fused_k100_batch64_querytile2.conf new file mode 100644 index 00000000..4106adca --- /dev/null +++ b/09_vector_retrieval/PurplePulse/configs/exact_fused_k100_batch64_querytile2.conf @@ -0,0 +1,6 @@ +top_k = 100 +search_mode = exact +batch_size = 64 +distance_mode = warp +topk_mode = fused +exact_query_tile = 2 diff --git a/09_vector_retrieval/PurplePulse/configs/exact_fused_k100_batch64_querytile4.conf b/09_vector_retrieval/PurplePulse/configs/exact_fused_k100_batch64_querytile4.conf new file mode 100644 index 00000000..6cb36a75 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/configs/exact_fused_k100_batch64_querytile4.conf @@ -0,0 +1,6 @@ +top_k = 100 +search_mode = exact +batch_size = 64 +distance_mode = warp +topk_mode = fused +exact_query_tile = 4 diff --git a/09_vector_retrieval/PurplePulse/configs/exact_fused_k10_batch64_blocktile8_blocks128.conf b/09_vector_retrieval/PurplePulse/configs/exact_fused_k10_batch64_blocktile8_blocks128.conf new file mode 100644 index 00000000..25c741f5 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/configs/exact_fused_k10_batch64_blocktile8_blocks128.conf @@ -0,0 +1,8 @@ +top_k = 10 +search_mode = exact +batch_size = 64 +distance_mode = warp +topk_mode = fused +exact_query_tile = 8 +exact_fused_layout = block_tile +exact_fused_blocks = 128 diff --git a/09_vector_retrieval/PurplePulse/configs/exact_fused_k10_batch64_blocktile8_blocks160.conf b/09_vector_retrieval/PurplePulse/configs/exact_fused_k10_batch64_blocktile8_blocks160.conf new file mode 100644 index 00000000..828aea18 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/configs/exact_fused_k10_batch64_blocktile8_blocks160.conf @@ -0,0 +1,8 @@ +top_k = 10 +search_mode = exact +batch_size = 64 +distance_mode = warp +topk_mode = fused +exact_query_tile = 8 +exact_fused_layout = block_tile +exact_fused_blocks = 160 diff --git a/09_vector_retrieval/PurplePulse/configs/exact_fused_k10_batch64_blocktile8_blocks192.conf b/09_vector_retrieval/PurplePulse/configs/exact_fused_k10_batch64_blocktile8_blocks192.conf new file mode 100644 index 00000000..7c9f80e7 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/configs/exact_fused_k10_batch64_blocktile8_blocks192.conf @@ -0,0 +1,8 @@ +top_k = 10 +search_mode = exact +batch_size = 64 +distance_mode = warp +topk_mode = fused +exact_query_tile = 8 +exact_fused_layout = block_tile +exact_fused_blocks = 192 diff --git a/09_vector_retrieval/PurplePulse/configs/exact_fused_k10_batch64_blocktile8_blocks64.conf b/09_vector_retrieval/PurplePulse/configs/exact_fused_k10_batch64_blocktile8_blocks64.conf new file mode 100644 index 00000000..fa43f2ac --- /dev/null +++ b/09_vector_retrieval/PurplePulse/configs/exact_fused_k10_batch64_blocktile8_blocks64.conf @@ -0,0 +1,8 @@ +top_k = 10 +search_mode = exact +batch_size = 64 +distance_mode = warp +topk_mode = fused +exact_query_tile = 8 +exact_fused_layout = block_tile +exact_fused_blocks = 64 diff --git a/09_vector_retrieval/PurplePulse/configs/exact_fused_k10_batch64_blocktile8_blocks96.conf b/09_vector_retrieval/PurplePulse/configs/exact_fused_k10_batch64_blocktile8_blocks96.conf new file mode 100644 index 00000000..d005799d --- /dev/null +++ b/09_vector_retrieval/PurplePulse/configs/exact_fused_k10_batch64_blocktile8_blocks96.conf @@ -0,0 +1,8 @@ +top_k = 10 +search_mode = exact +batch_size = 64 +distance_mode = warp +topk_mode = fused +exact_query_tile = 8 +exact_fused_layout = block_tile +exact_fused_blocks = 96 diff --git a/09_vector_retrieval/PurplePulse/configs/exact_fused_k10_batch64_querytile2.conf b/09_vector_retrieval/PurplePulse/configs/exact_fused_k10_batch64_querytile2.conf new file mode 100644 index 00000000..dbbec7c7 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/configs/exact_fused_k10_batch64_querytile2.conf @@ -0,0 +1,6 @@ +top_k = 10 +search_mode = exact +batch_size = 64 +distance_mode = warp +topk_mode = fused +exact_query_tile = 2 diff --git a/09_vector_retrieval/PurplePulse/configs/exact_fused_k10_batch64_querytile4.conf b/09_vector_retrieval/PurplePulse/configs/exact_fused_k10_batch64_querytile4.conf new file mode 100644 index 00000000..ff75ac6f --- /dev/null +++ b/09_vector_retrieval/PurplePulse/configs/exact_fused_k10_batch64_querytile4.conf @@ -0,0 +1,6 @@ +top_k = 10 +search_mode = exact +batch_size = 64 +distance_mode = warp +topk_mode = fused +exact_query_tile = 4 diff --git a/09_vector_retrieval/PurplePulse/configs/exact_fused_k10_batch64_querytile4_blocks128.conf b/09_vector_retrieval/PurplePulse/configs/exact_fused_k10_batch64_querytile4_blocks128.conf new file mode 100644 index 00000000..7c8f5a4d --- /dev/null +++ b/09_vector_retrieval/PurplePulse/configs/exact_fused_k10_batch64_querytile4_blocks128.conf @@ -0,0 +1,7 @@ +top_k = 10 +search_mode = exact +batch_size = 64 +distance_mode = warp +topk_mode = fused +exact_query_tile = 4 +exact_fused_blocks = 128 diff --git a/09_vector_retrieval/PurplePulse/configs/exact_fused_k10_batch64_querytile4_blocks48.conf b/09_vector_retrieval/PurplePulse/configs/exact_fused_k10_batch64_querytile4_blocks48.conf new file mode 100644 index 00000000..7d31b56b --- /dev/null +++ b/09_vector_retrieval/PurplePulse/configs/exact_fused_k10_batch64_querytile4_blocks48.conf @@ -0,0 +1,7 @@ +top_k = 10 +search_mode = exact +batch_size = 64 +distance_mode = warp +topk_mode = fused +exact_query_tile = 4 +exact_fused_blocks = 48 diff --git a/09_vector_retrieval/PurplePulse/configs/exact_fused_k10_batch64_querytile4_blocks64.conf b/09_vector_retrieval/PurplePulse/configs/exact_fused_k10_batch64_querytile4_blocks64.conf new file mode 100644 index 00000000..dead2159 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/configs/exact_fused_k10_batch64_querytile4_blocks64.conf @@ -0,0 +1,7 @@ +top_k = 10 +search_mode = exact +batch_size = 64 +distance_mode = warp +topk_mode = fused +exact_query_tile = 4 +exact_fused_blocks = 64 diff --git a/09_vector_retrieval/PurplePulse/configs/exact_fused_k10_batch64_querytile4_blocks80.conf b/09_vector_retrieval/PurplePulse/configs/exact_fused_k10_batch64_querytile4_blocks80.conf new file mode 100644 index 00000000..0fbeedb9 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/configs/exact_fused_k10_batch64_querytile4_blocks80.conf @@ -0,0 +1,7 @@ +top_k = 10 +search_mode = exact +batch_size = 64 +distance_mode = warp +topk_mode = fused +exact_query_tile = 4 +exact_fused_blocks = 80 diff --git a/09_vector_retrieval/PurplePulse/configs/exact_fused_k10_batch64_querytile4_blocks96.conf b/09_vector_retrieval/PurplePulse/configs/exact_fused_k10_batch64_querytile4_blocks96.conf new file mode 100644 index 00000000..f54b17d1 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/configs/exact_fused_k10_batch64_querytile4_blocks96.conf @@ -0,0 +1,7 @@ +top_k = 10 +search_mode = exact +batch_size = 64 +distance_mode = warp +topk_mode = fused +exact_query_tile = 4 +exact_fused_blocks = 96 diff --git a/09_vector_retrieval/PurplePulse/configs/exact_fused_k10_batch64_querytile8.conf b/09_vector_retrieval/PurplePulse/configs/exact_fused_k10_batch64_querytile8.conf new file mode 100644 index 00000000..a2f7d7b0 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/configs/exact_fused_k10_batch64_querytile8.conf @@ -0,0 +1,6 @@ +top_k = 10 +search_mode = exact +batch_size = 64 +distance_mode = warp +topk_mode = fused +exact_query_tile = 8 diff --git a/09_vector_retrieval/PurplePulse/configs/exact_fused_k10_batch64_querytile8_blocks128.conf b/09_vector_retrieval/PurplePulse/configs/exact_fused_k10_batch64_querytile8_blocks128.conf new file mode 100644 index 00000000..9601480b --- /dev/null +++ b/09_vector_retrieval/PurplePulse/configs/exact_fused_k10_batch64_querytile8_blocks128.conf @@ -0,0 +1,7 @@ +top_k = 10 +search_mode = exact +batch_size = 64 +distance_mode = warp +topk_mode = fused +exact_query_tile = 8 +exact_fused_blocks = 128 diff --git a/09_vector_retrieval/PurplePulse/configs/exact_fused_k10_batch64_querytile8_blocks160.conf b/09_vector_retrieval/PurplePulse/configs/exact_fused_k10_batch64_querytile8_blocks160.conf new file mode 100644 index 00000000..3b6c4f4f --- /dev/null +++ b/09_vector_retrieval/PurplePulse/configs/exact_fused_k10_batch64_querytile8_blocks160.conf @@ -0,0 +1,7 @@ +top_k = 10 +search_mode = exact +batch_size = 64 +distance_mode = warp +topk_mode = fused +exact_query_tile = 8 +exact_fused_blocks = 160 diff --git a/09_vector_retrieval/PurplePulse/configs/exact_fused_k10_batch64_querytile8_blocks192.conf b/09_vector_retrieval/PurplePulse/configs/exact_fused_k10_batch64_querytile8_blocks192.conf new file mode 100644 index 00000000..5efcd3ef --- /dev/null +++ b/09_vector_retrieval/PurplePulse/configs/exact_fused_k10_batch64_querytile8_blocks192.conf @@ -0,0 +1,7 @@ +top_k = 10 +search_mode = exact +batch_size = 64 +distance_mode = warp +topk_mode = fused +exact_query_tile = 8 +exact_fused_blocks = 192 diff --git a/09_vector_retrieval/PurplePulse/configs/exact_fused_k10_batch64_querytile8_blocks96.conf b/09_vector_retrieval/PurplePulse/configs/exact_fused_k10_batch64_querytile8_blocks96.conf new file mode 100644 index 00000000..c2acc8b3 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/configs/exact_fused_k10_batch64_querytile8_blocks96.conf @@ -0,0 +1,7 @@ +top_k = 10 +search_mode = exact +batch_size = 64 +distance_mode = warp +topk_mode = fused +exact_query_tile = 8 +exact_fused_blocks = 96 diff --git a/09_vector_retrieval/PurplePulse/configs/exact_fused_k50_batch64_querytile2.conf b/09_vector_retrieval/PurplePulse/configs/exact_fused_k50_batch64_querytile2.conf new file mode 100644 index 00000000..5c48d95b --- /dev/null +++ b/09_vector_retrieval/PurplePulse/configs/exact_fused_k50_batch64_querytile2.conf @@ -0,0 +1,6 @@ +top_k = 50 +search_mode = exact +batch_size = 64 +distance_mode = warp +topk_mode = fused +exact_query_tile = 2 diff --git a/09_vector_retrieval/PurplePulse/configs/exact_fused_k50_batch64_querytile4.conf b/09_vector_retrieval/PurplePulse/configs/exact_fused_k50_batch64_querytile4.conf new file mode 100644 index 00000000..6ca99247 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/configs/exact_fused_k50_batch64_querytile4.conf @@ -0,0 +1,6 @@ +top_k = 50 +search_mode = exact +batch_size = 64 +distance_mode = warp +topk_mode = fused +exact_query_tile = 4 diff --git a/09_vector_retrieval/PurplePulse/configs/exact_l2.conf b/09_vector_retrieval/PurplePulse/configs/exact_l2.conf new file mode 100644 index 00000000..8fb50ceb --- /dev/null +++ b/09_vector_retrieval/PurplePulse/configs/exact_l2.conf @@ -0,0 +1,8 @@ +top_k = 10 +search_mode = exact +batch_size = 8 +distance_mode = warp +topk_mode = two_stage +nlist = 4096 +nprobe = 16 +pq_m = 16 diff --git a/09_vector_retrieval/PurplePulse/configs/exact_l2_batch1.conf b/09_vector_retrieval/PurplePulse/configs/exact_l2_batch1.conf new file mode 100644 index 00000000..07d72bc7 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/configs/exact_l2_batch1.conf @@ -0,0 +1,5 @@ +top_k = 10 +search_mode = exact +batch_size = 1 +distance_mode = warp +topk_mode = two_stage diff --git a/09_vector_retrieval/PurplePulse/configs/exact_l2_batch128.conf b/09_vector_retrieval/PurplePulse/configs/exact_l2_batch128.conf new file mode 100644 index 00000000..083f5d34 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/configs/exact_l2_batch128.conf @@ -0,0 +1,5 @@ +top_k = 10 +search_mode = exact +batch_size = 128 +distance_mode = warp +topk_mode = two_stage diff --git a/09_vector_retrieval/PurplePulse/configs/exact_l2_batch16.conf b/09_vector_retrieval/PurplePulse/configs/exact_l2_batch16.conf new file mode 100644 index 00000000..04c6fdec --- /dev/null +++ b/09_vector_retrieval/PurplePulse/configs/exact_l2_batch16.conf @@ -0,0 +1,5 @@ +top_k = 10 +search_mode = exact +batch_size = 16 +distance_mode = warp +topk_mode = two_stage diff --git a/09_vector_retrieval/PurplePulse/configs/exact_l2_batch32.conf b/09_vector_retrieval/PurplePulse/configs/exact_l2_batch32.conf new file mode 100644 index 00000000..204df4d7 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/configs/exact_l2_batch32.conf @@ -0,0 +1,5 @@ +top_k = 10 +search_mode = exact +batch_size = 32 +distance_mode = warp +topk_mode = two_stage diff --git a/09_vector_retrieval/PurplePulse/configs/exact_l2_batch4.conf b/09_vector_retrieval/PurplePulse/configs/exact_l2_batch4.conf new file mode 100644 index 00000000..453bd98a --- /dev/null +++ b/09_vector_retrieval/PurplePulse/configs/exact_l2_batch4.conf @@ -0,0 +1,5 @@ +top_k = 10 +search_mode = exact +batch_size = 4 +distance_mode = warp +topk_mode = two_stage diff --git a/09_vector_retrieval/PurplePulse/configs/exact_l2_batch64.conf b/09_vector_retrieval/PurplePulse/configs/exact_l2_batch64.conf new file mode 100644 index 00000000..01b38e5e --- /dev/null +++ b/09_vector_retrieval/PurplePulse/configs/exact_l2_batch64.conf @@ -0,0 +1,5 @@ +top_k = 10 +search_mode = exact +batch_size = 64 +distance_mode = warp +topk_mode = two_stage diff --git a/09_vector_retrieval/PurplePulse/configs/exact_l2_block.conf b/09_vector_retrieval/PurplePulse/configs/exact_l2_block.conf new file mode 100644 index 00000000..514eb6ff --- /dev/null +++ b/09_vector_retrieval/PurplePulse/configs/exact_l2_block.conf @@ -0,0 +1,8 @@ +top_k = 10 +search_mode = exact +batch_size = 8 +distance_mode = simple +topk_mode = block +nlist = 4096 +nprobe = 16 +pq_m = 16 diff --git a/09_vector_retrieval/PurplePulse/configs/exact_l2_k1.conf b/09_vector_retrieval/PurplePulse/configs/exact_l2_k1.conf new file mode 100644 index 00000000..bbb63d56 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/configs/exact_l2_k1.conf @@ -0,0 +1,8 @@ +top_k = 1 +search_mode = exact +batch_size = 8 +distance_mode = warp +topk_mode = two_stage +nlist = 4096 +nprobe = 16 +pq_m = 16 diff --git a/09_vector_retrieval/PurplePulse/configs/exact_l2_k100.conf b/09_vector_retrieval/PurplePulse/configs/exact_l2_k100.conf new file mode 100644 index 00000000..40924ce3 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/configs/exact_l2_k100.conf @@ -0,0 +1,8 @@ +top_k = 100 +search_mode = exact +batch_size = 8 +distance_mode = warp +topk_mode = block +nlist = 4096 +nprobe = 16 +pq_m = 16 diff --git a/09_vector_retrieval/PurplePulse/configs/exact_l2_k50.conf b/09_vector_retrieval/PurplePulse/configs/exact_l2_k50.conf new file mode 100644 index 00000000..f3a67f82 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/configs/exact_l2_k50.conf @@ -0,0 +1,8 @@ +top_k = 50 +search_mode = exact +batch_size = 8 +distance_mode = warp +topk_mode = block +nlist = 4096 +nprobe = 16 +pq_m = 16 diff --git a/09_vector_retrieval/PurplePulse/configs/exact_l2_simple.conf b/09_vector_retrieval/PurplePulse/configs/exact_l2_simple.conf new file mode 100644 index 00000000..3198d354 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/configs/exact_l2_simple.conf @@ -0,0 +1,8 @@ +top_k = 10 +search_mode = exact +batch_size = 8 +distance_mode = simple +topk_mode = simple +nlist = 4096 +nprobe = 16 +pq_m = 16 diff --git a/09_vector_retrieval/PurplePulse/configs/exact_million_k100_batch64.conf b/09_vector_retrieval/PurplePulse/configs/exact_million_k100_batch64.conf new file mode 100644 index 00000000..d9e10c47 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/configs/exact_million_k100_batch64.conf @@ -0,0 +1,5 @@ +top_k = 100 +search_mode = exact +batch_size = 64 +distance_mode = warp +topk_mode = block diff --git a/09_vector_retrieval/PurplePulse/configs/exact_million_k100_fused_batch64.conf b/09_vector_retrieval/PurplePulse/configs/exact_million_k100_fused_batch64.conf new file mode 100644 index 00000000..e1038bf2 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/configs/exact_million_k100_fused_batch64.conf @@ -0,0 +1,5 @@ +top_k = 100 +search_mode = exact +batch_size = 64 +distance_mode = warp +topk_mode = fused diff --git a/09_vector_retrieval/PurplePulse/configs/exact_million_k10_fused_batch64.conf b/09_vector_retrieval/PurplePulse/configs/exact_million_k10_fused_batch64.conf new file mode 100644 index 00000000..a325885d --- /dev/null +++ b/09_vector_retrieval/PurplePulse/configs/exact_million_k10_fused_batch64.conf @@ -0,0 +1,5 @@ +top_k = 10 +search_mode = exact +batch_size = 64 +distance_mode = warp +topk_mode = fused diff --git a/09_vector_retrieval/PurplePulse/configs/exact_million_k1_batch128.conf b/09_vector_retrieval/PurplePulse/configs/exact_million_k1_batch128.conf new file mode 100644 index 00000000..ee839d09 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/configs/exact_million_k1_batch128.conf @@ -0,0 +1,7 @@ +top_k = 1 +search_mode = exact +batch_size = 128 +distance_mode = warp +topk_mode = two_stage +nlist = 1024 +nprobe = 16 diff --git a/09_vector_retrieval/PurplePulse/configs/exact_million_k50_batch64.conf b/09_vector_retrieval/PurplePulse/configs/exact_million_k50_batch64.conf new file mode 100644 index 00000000..435ffa28 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/configs/exact_million_k50_batch64.conf @@ -0,0 +1,5 @@ +top_k = 50 +search_mode = exact +batch_size = 64 +distance_mode = warp +topk_mode = block diff --git a/09_vector_retrieval/PurplePulse/configs/exact_million_k50_fused_batch64.conf b/09_vector_retrieval/PurplePulse/configs/exact_million_k50_fused_batch64.conf new file mode 100644 index 00000000..a7e5abdf --- /dev/null +++ b/09_vector_retrieval/PurplePulse/configs/exact_million_k50_fused_batch64.conf @@ -0,0 +1,5 @@ +top_k = 50 +search_mode = exact +batch_size = 64 +distance_mode = warp +topk_mode = fused diff --git a/09_vector_retrieval/PurplePulse/configs/ivf_flat.conf b/09_vector_retrieval/PurplePulse/configs/ivf_flat.conf new file mode 100644 index 00000000..b583256c --- /dev/null +++ b/09_vector_retrieval/PurplePulse/configs/ivf_flat.conf @@ -0,0 +1,7 @@ +top_k = 10 +search_mode = ivf_flat +batch_size = 8 +distance_mode = warp +topk_mode = two_stage +nlist = 1024 +nprobe = 16 diff --git a/09_vector_retrieval/PurplePulse/configs/ivf_flat_100k_balanced.conf b/09_vector_retrieval/PurplePulse/configs/ivf_flat_100k_balanced.conf new file mode 100644 index 00000000..951e0f09 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/configs/ivf_flat_100k_balanced.conf @@ -0,0 +1,7 @@ +top_k = 10 +search_mode = ivf_flat +batch_size = 8 +distance_mode = warp +topk_mode = two_stage +nlist = 64 +nprobe = 40 diff --git a/09_vector_retrieval/PurplePulse/configs/ivf_flat_100k_exact_check.conf b/09_vector_retrieval/PurplePulse/configs/ivf_flat_100k_exact_check.conf new file mode 100644 index 00000000..d2811b7e --- /dev/null +++ b/09_vector_retrieval/PurplePulse/configs/ivf_flat_100k_exact_check.conf @@ -0,0 +1,7 @@ +top_k = 10 +search_mode = ivf_flat +batch_size = 8 +distance_mode = warp +topk_mode = two_stage +nlist = 64 +nprobe = 64 diff --git a/09_vector_retrieval/PurplePulse/configs/ivf_flat_100k_high_recall.conf b/09_vector_retrieval/PurplePulse/configs/ivf_flat_100k_high_recall.conf new file mode 100644 index 00000000..b31489bf --- /dev/null +++ b/09_vector_retrieval/PurplePulse/configs/ivf_flat_100k_high_recall.conf @@ -0,0 +1,7 @@ +top_k = 10 +search_mode = ivf_flat +batch_size = 8 +distance_mode = warp +topk_mode = two_stage +nlist = 64 +nprobe = 56 diff --git a/09_vector_retrieval/PurplePulse/configs/ivf_flat_100k_scalar_baseline.conf b/09_vector_retrieval/PurplePulse/configs/ivf_flat_100k_scalar_baseline.conf new file mode 100644 index 00000000..518994ca --- /dev/null +++ b/09_vector_retrieval/PurplePulse/configs/ivf_flat_100k_scalar_baseline.conf @@ -0,0 +1,7 @@ +top_k = 10 +search_mode = ivf_flat +batch_size = 8 +distance_mode = simple +topk_mode = two_stage +nlist = 64 +nprobe = 40 diff --git a/09_vector_retrieval/PurplePulse/configs/ivf_flat_memory_fused.conf b/09_vector_retrieval/PurplePulse/configs/ivf_flat_memory_fused.conf new file mode 100644 index 00000000..1a4b6911 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/configs/ivf_flat_memory_fused.conf @@ -0,0 +1,20 @@ +top_k = 10 +search_mode = ivf_flat +batch_size = 64 +distance_mode = warp_compact +nlist = 256 +nprobe = 160 +nprobe_policy = fixed + +# Combined score keeps the native metric direction: lower is better for L2, +# higher is better for inner-product/cosine. +memory_mode = fused +memory_semantic_weight = 1.0 +memory_importance_weight = 0.15 +memory_recency_weight = 0.10 +memory_time_scale = 604800 +memory_now = 1702592000 +filter_min_timestamp = 0 +filter_session_id = any +filter_source_type = any +memory_rerank_factor = 4 diff --git a/09_vector_retrieval/PurplePulse/configs/ivf_flat_memory_rerank.conf b/09_vector_retrieval/PurplePulse/configs/ivf_flat_memory_rerank.conf new file mode 100644 index 00000000..27b4985e --- /dev/null +++ b/09_vector_retrieval/PurplePulse/configs/ivf_flat_memory_rerank.conf @@ -0,0 +1,18 @@ +top_k = 10 +search_mode = ivf_flat +batch_size = 64 +distance_mode = warp_compact +nlist = 256 +nprobe = 160 +nprobe_policy = fixed + +memory_mode = rerank +memory_semantic_weight = 1.0 +memory_importance_weight = 0.15 +memory_recency_weight = 0.10 +memory_time_scale = 604800 +memory_now = 1702592000 +filter_min_timestamp = 0 +filter_session_id = any +filter_source_type = any +memory_rerank_factor = 4 diff --git a/09_vector_retrieval/PurplePulse/configs/ivf_flat_million_adaptive_k10.conf b/09_vector_retrieval/PurplePulse/configs/ivf_flat_million_adaptive_k10.conf new file mode 100644 index 00000000..3e640f0d --- /dev/null +++ b/09_vector_retrieval/PurplePulse/configs/ivf_flat_million_adaptive_k10.conf @@ -0,0 +1,13 @@ +top_k = 10 +search_mode = ivf_flat +batch_size = 64 +distance_mode = warp +topk_mode = two_stage +nlist = 256 +nprobe = 224 +nprobe_policy = score_mass +adaptive_execution = masked +adaptive_nprobe_min = 64 +adaptive_nprobe_step = 16 +adaptive_target_mass = 0.93 +adaptive_temperature = 0.2 diff --git a/09_vector_retrieval/PurplePulse/configs/ivf_flat_million_adaptive_k100.conf b/09_vector_retrieval/PurplePulse/configs/ivf_flat_million_adaptive_k100.conf new file mode 100644 index 00000000..ec05a96f --- /dev/null +++ b/09_vector_retrieval/PurplePulse/configs/ivf_flat_million_adaptive_k100.conf @@ -0,0 +1,13 @@ +top_k = 100 +search_mode = ivf_flat +batch_size = 64 +distance_mode = warp_compact +topk_mode = two_stage +nlist = 256 +nprobe = 224 +nprobe_policy = score_mass +adaptive_execution = masked +adaptive_nprobe_min = 64 +adaptive_nprobe_step = 16 +adaptive_target_mass = 0.93 +adaptive_temperature = 0.2 diff --git a/09_vector_retrieval/PurplePulse/configs/ivf_flat_million_adaptive_k50.conf b/09_vector_retrieval/PurplePulse/configs/ivf_flat_million_adaptive_k50.conf new file mode 100644 index 00000000..5515ed54 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/configs/ivf_flat_million_adaptive_k50.conf @@ -0,0 +1,13 @@ +top_k = 50 +search_mode = ivf_flat +batch_size = 32 +distance_mode = warp_compact +topk_mode = two_stage +nlist = 256 +nprobe = 224 +nprobe_policy = score_mass +adaptive_execution = masked +adaptive_nprobe_min = 64 +adaptive_nprobe_step = 16 +adaptive_target_mass = 0.93 +adaptive_temperature = 0.2 diff --git a/09_vector_retrieval/PurplePulse/configs/ivf_flat_million_balanced.conf b/09_vector_retrieval/PurplePulse/configs/ivf_flat_million_balanced.conf new file mode 100644 index 00000000..6d5be5dd --- /dev/null +++ b/09_vector_retrieval/PurplePulse/configs/ivf_flat_million_balanced.conf @@ -0,0 +1,7 @@ +top_k = 10 +search_mode = ivf_flat +batch_size = 64 +distance_mode = warp +topk_mode = two_stage +nlist = 256 +nprobe = 128 diff --git a/09_vector_retrieval/PurplePulse/configs/ivf_flat_million_batch1.conf b/09_vector_retrieval/PurplePulse/configs/ivf_flat_million_batch1.conf new file mode 100644 index 00000000..7eb1a2d8 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/configs/ivf_flat_million_batch1.conf @@ -0,0 +1,7 @@ +top_k = 10 +search_mode = ivf_flat +batch_size = 1 +distance_mode = warp +topk_mode = two_stage +nlist = 256 +nprobe = 160 diff --git a/09_vector_retrieval/PurplePulse/configs/ivf_flat_million_batch128.conf b/09_vector_retrieval/PurplePulse/configs/ivf_flat_million_batch128.conf new file mode 100644 index 00000000..00a295c0 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/configs/ivf_flat_million_batch128.conf @@ -0,0 +1,7 @@ +top_k = 10 +search_mode = ivf_flat +batch_size = 128 +distance_mode = warp +topk_mode = two_stage +nlist = 256 +nprobe = 160 diff --git a/09_vector_retrieval/PurplePulse/configs/ivf_flat_million_batch16.conf b/09_vector_retrieval/PurplePulse/configs/ivf_flat_million_batch16.conf new file mode 100644 index 00000000..93742a69 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/configs/ivf_flat_million_batch16.conf @@ -0,0 +1,7 @@ +top_k = 10 +search_mode = ivf_flat +batch_size = 16 +distance_mode = warp +topk_mode = two_stage +nlist = 256 +nprobe = 160 diff --git a/09_vector_retrieval/PurplePulse/configs/ivf_flat_million_batch32.conf b/09_vector_retrieval/PurplePulse/configs/ivf_flat_million_batch32.conf new file mode 100644 index 00000000..5915b9a5 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/configs/ivf_flat_million_batch32.conf @@ -0,0 +1,7 @@ +top_k = 10 +search_mode = ivf_flat +batch_size = 32 +distance_mode = warp +topk_mode = two_stage +nlist = 256 +nprobe = 160 diff --git a/09_vector_retrieval/PurplePulse/configs/ivf_flat_million_batch64.conf b/09_vector_retrieval/PurplePulse/configs/ivf_flat_million_batch64.conf new file mode 100644 index 00000000..17bbf0b4 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/configs/ivf_flat_million_batch64.conf @@ -0,0 +1,7 @@ +top_k = 10 +search_mode = ivf_flat +batch_size = 64 +distance_mode = warp +topk_mode = two_stage +nlist = 256 +nprobe = 160 diff --git a/09_vector_retrieval/PurplePulse/configs/ivf_flat_million_batch8.conf b/09_vector_retrieval/PurplePulse/configs/ivf_flat_million_batch8.conf new file mode 100644 index 00000000..e2bbe468 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/configs/ivf_flat_million_batch8.conf @@ -0,0 +1,7 @@ +top_k = 10 +search_mode = ivf_flat +batch_size = 8 +distance_mode = warp +topk_mode = two_stage +nlist = 256 +nprobe = 160 diff --git a/09_vector_retrieval/PurplePulse/configs/ivf_flat_million_fast.conf b/09_vector_retrieval/PurplePulse/configs/ivf_flat_million_fast.conf new file mode 100644 index 00000000..a07d9c3e --- /dev/null +++ b/09_vector_retrieval/PurplePulse/configs/ivf_flat_million_fast.conf @@ -0,0 +1,7 @@ +top_k = 10 +search_mode = ivf_flat +batch_size = 16 +distance_mode = warp +topk_mode = two_stage +nlist = 512 +nprobe = 128 diff --git a/09_vector_retrieval/PurplePulse/configs/ivf_flat_million_high_recall.conf b/09_vector_retrieval/PurplePulse/configs/ivf_flat_million_high_recall.conf new file mode 100644 index 00000000..fb359f37 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/configs/ivf_flat_million_high_recall.conf @@ -0,0 +1,7 @@ +top_k = 10 +search_mode = ivf_flat +batch_size = 64 +distance_mode = warp +topk_mode = two_stage +nlist = 256 +nprobe = 192 diff --git a/09_vector_retrieval/PurplePulse/configs/ivf_flat_million_k1.conf b/09_vector_retrieval/PurplePulse/configs/ivf_flat_million_k1.conf new file mode 100644 index 00000000..1fb218d0 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/configs/ivf_flat_million_k1.conf @@ -0,0 +1,7 @@ +top_k = 1 +search_mode = ivf_flat +batch_size = 16 +distance_mode = warp +topk_mode = two_stage +nlist = 256 +nprobe = 160 diff --git a/09_vector_retrieval/PurplePulse/configs/ivf_flat_million_k100.conf b/09_vector_retrieval/PurplePulse/configs/ivf_flat_million_k100.conf new file mode 100644 index 00000000..80ea98e4 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/configs/ivf_flat_million_k100.conf @@ -0,0 +1,7 @@ +top_k = 100 +search_mode = ivf_flat +batch_size = 64 +distance_mode = bucket_major_partitioned +topk_mode = two_stage +nlist = 256 +nprobe = 160 diff --git a/09_vector_retrieval/PurplePulse/configs/ivf_flat_million_k100_batch128.conf b/09_vector_retrieval/PurplePulse/configs/ivf_flat_million_k100_batch128.conf new file mode 100644 index 00000000..095aa291 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/configs/ivf_flat_million_k100_batch128.conf @@ -0,0 +1,7 @@ +top_k = 100 +search_mode = ivf_flat +batch_size = 128 +distance_mode = warp_compact +topk_mode = two_stage +nlist = 256 +nprobe = 160 diff --git a/09_vector_retrieval/PurplePulse/configs/ivf_flat_million_k100_batch32.conf b/09_vector_retrieval/PurplePulse/configs/ivf_flat_million_k100_batch32.conf new file mode 100644 index 00000000..ad358310 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/configs/ivf_flat_million_k100_batch32.conf @@ -0,0 +1,7 @@ +top_k = 100 +search_mode = ivf_flat +batch_size = 32 +distance_mode = warp_compact +topk_mode = two_stage +nlist = 256 +nprobe = 160 diff --git a/09_vector_retrieval/PurplePulse/configs/ivf_flat_million_k100_batch64.conf b/09_vector_retrieval/PurplePulse/configs/ivf_flat_million_k100_batch64.conf new file mode 100644 index 00000000..60fb1c9e --- /dev/null +++ b/09_vector_retrieval/PurplePulse/configs/ivf_flat_million_k100_batch64.conf @@ -0,0 +1,7 @@ +top_k = 100 +search_mode = ivf_flat +batch_size = 64 +distance_mode = warp_compact +topk_mode = two_stage +nlist = 256 +nprobe = 160 diff --git a/09_vector_retrieval/PurplePulse/configs/ivf_flat_million_k100_batch8.conf b/09_vector_retrieval/PurplePulse/configs/ivf_flat_million_k100_batch8.conf new file mode 100644 index 00000000..67f9b1a7 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/configs/ivf_flat_million_k100_batch8.conf @@ -0,0 +1,7 @@ +top_k = 100 +search_mode = ivf_flat +batch_size = 8 +distance_mode = warp_compact +topk_mode = two_stage +nlist = 256 +nprobe = 160 diff --git a/09_vector_retrieval/PurplePulse/configs/ivf_flat_million_k100_bucket_major.conf b/09_vector_retrieval/PurplePulse/configs/ivf_flat_million_k100_bucket_major.conf new file mode 100644 index 00000000..793978f4 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/configs/ivf_flat_million_k100_bucket_major.conf @@ -0,0 +1,7 @@ +top_k = 100 +search_mode = ivf_flat +batch_size = 64 +distance_mode = bucket_major +topk_mode = two_stage +nlist = 256 +nprobe = 160 diff --git a/09_vector_retrieval/PurplePulse/configs/ivf_flat_million_k100_partitioned.conf b/09_vector_retrieval/PurplePulse/configs/ivf_flat_million_k100_partitioned.conf new file mode 100644 index 00000000..80ea98e4 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/configs/ivf_flat_million_k100_partitioned.conf @@ -0,0 +1,7 @@ +top_k = 100 +search_mode = ivf_flat +batch_size = 64 +distance_mode = bucket_major_partitioned +topk_mode = two_stage +nlist = 256 +nprobe = 160 diff --git a/09_vector_retrieval/PurplePulse/configs/ivf_flat_million_k100_warp1.conf b/09_vector_retrieval/PurplePulse/configs/ivf_flat_million_k100_warp1.conf new file mode 100644 index 00000000..85650101 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/configs/ivf_flat_million_k100_warp1.conf @@ -0,0 +1,7 @@ +top_k = 100 +search_mode = ivf_flat +batch_size = 64 +distance_mode = warp1 +topk_mode = two_stage +nlist = 256 +nprobe = 160 diff --git a/09_vector_retrieval/PurplePulse/configs/ivf_flat_million_k100_warp2.conf b/09_vector_retrieval/PurplePulse/configs/ivf_flat_million_k100_warp2.conf new file mode 100644 index 00000000..8be75cbe --- /dev/null +++ b/09_vector_retrieval/PurplePulse/configs/ivf_flat_million_k100_warp2.conf @@ -0,0 +1,7 @@ +top_k = 100 +search_mode = ivf_flat +batch_size = 64 +distance_mode = warp2 +topk_mode = two_stage +nlist = 256 +nprobe = 160 diff --git a/09_vector_retrieval/PurplePulse/configs/ivf_flat_million_k100_warp4.conf b/09_vector_retrieval/PurplePulse/configs/ivf_flat_million_k100_warp4.conf new file mode 100644 index 00000000..af38607d --- /dev/null +++ b/09_vector_retrieval/PurplePulse/configs/ivf_flat_million_k100_warp4.conf @@ -0,0 +1,7 @@ +top_k = 100 +search_mode = ivf_flat +batch_size = 64 +distance_mode = warp4 +topk_mode = two_stage +nlist = 256 +nprobe = 160 diff --git a/09_vector_retrieval/PurplePulse/configs/ivf_flat_million_k100_warp8.conf b/09_vector_retrieval/PurplePulse/configs/ivf_flat_million_k100_warp8.conf new file mode 100644 index 00000000..411acdec --- /dev/null +++ b/09_vector_retrieval/PurplePulse/configs/ivf_flat_million_k100_warp8.conf @@ -0,0 +1,7 @@ +top_k = 100 +search_mode = ivf_flat +batch_size = 64 +distance_mode = warp8 +topk_mode = two_stage +nlist = 256 +nprobe = 160 diff --git a/09_vector_retrieval/PurplePulse/configs/ivf_flat_million_k50.conf b/09_vector_retrieval/PurplePulse/configs/ivf_flat_million_k50.conf new file mode 100644 index 00000000..ea85ffdf --- /dev/null +++ b/09_vector_retrieval/PurplePulse/configs/ivf_flat_million_k50.conf @@ -0,0 +1,7 @@ +top_k = 50 +search_mode = ivf_flat +batch_size = 40 +distance_mode = bucket_major_partitioned4x16 +topk_mode = two_stage +nlist = 256 +nprobe = 160 diff --git a/09_vector_retrieval/PurplePulse/configs/ivf_flat_million_k50_batch128.conf b/09_vector_retrieval/PurplePulse/configs/ivf_flat_million_k50_batch128.conf new file mode 100644 index 00000000..b044c307 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/configs/ivf_flat_million_k50_batch128.conf @@ -0,0 +1,7 @@ +top_k = 50 +search_mode = ivf_flat +batch_size = 128 +distance_mode = warp_compact +topk_mode = two_stage +nlist = 256 +nprobe = 160 diff --git a/09_vector_retrieval/PurplePulse/configs/ivf_flat_million_k50_batch32.conf b/09_vector_retrieval/PurplePulse/configs/ivf_flat_million_k50_batch32.conf new file mode 100644 index 00000000..903fc1c2 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/configs/ivf_flat_million_k50_batch32.conf @@ -0,0 +1,7 @@ +top_k = 50 +search_mode = ivf_flat +batch_size = 32 +distance_mode = warp_compact +topk_mode = two_stage +nlist = 256 +nprobe = 160 diff --git a/09_vector_retrieval/PurplePulse/configs/ivf_flat_million_k50_batch64.conf b/09_vector_retrieval/PurplePulse/configs/ivf_flat_million_k50_batch64.conf new file mode 100644 index 00000000..7a3526bc --- /dev/null +++ b/09_vector_retrieval/PurplePulse/configs/ivf_flat_million_k50_batch64.conf @@ -0,0 +1,7 @@ +top_k = 50 +search_mode = ivf_flat +batch_size = 64 +distance_mode = warp_compact +topk_mode = two_stage +nlist = 256 +nprobe = 160 diff --git a/09_vector_retrieval/PurplePulse/configs/ivf_flat_million_k50_batch8.conf b/09_vector_retrieval/PurplePulse/configs/ivf_flat_million_k50_batch8.conf new file mode 100644 index 00000000..d29500b6 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/configs/ivf_flat_million_k50_batch8.conf @@ -0,0 +1,7 @@ +top_k = 50 +search_mode = ivf_flat +batch_size = 8 +distance_mode = warp_compact +topk_mode = two_stage +nlist = 256 +nprobe = 160 diff --git a/09_vector_retrieval/PurplePulse/configs/ivf_flat_million_k50_bucket_major.conf b/09_vector_retrieval/PurplePulse/configs/ivf_flat_million_k50_bucket_major.conf new file mode 100644 index 00000000..1814c11b --- /dev/null +++ b/09_vector_retrieval/PurplePulse/configs/ivf_flat_million_k50_bucket_major.conf @@ -0,0 +1,7 @@ +top_k = 50 +search_mode = ivf_flat +batch_size = 40 +distance_mode = bucket_major +topk_mode = two_stage +nlist = 256 +nprobe = 160 diff --git a/09_vector_retrieval/PurplePulse/configs/ivf_flat_million_k50_partitioned_2x32.conf b/09_vector_retrieval/PurplePulse/configs/ivf_flat_million_k50_partitioned_2x32.conf new file mode 100644 index 00000000..43d17367 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/configs/ivf_flat_million_k50_partitioned_2x32.conf @@ -0,0 +1,7 @@ +top_k = 50 +search_mode = ivf_flat +batch_size = 40 +distance_mode = bucket_major_partitioned2 +topk_mode = two_stage +nlist = 256 +nprobe = 160 diff --git a/09_vector_retrieval/PurplePulse/configs/ivf_flat_million_k50_partitioned_4x16.conf b/09_vector_retrieval/PurplePulse/configs/ivf_flat_million_k50_partitioned_4x16.conf new file mode 100644 index 00000000..ea85ffdf --- /dev/null +++ b/09_vector_retrieval/PurplePulse/configs/ivf_flat_million_k50_partitioned_4x16.conf @@ -0,0 +1,7 @@ +top_k = 50 +search_mode = ivf_flat +batch_size = 40 +distance_mode = bucket_major_partitioned4x16 +topk_mode = two_stage +nlist = 256 +nprobe = 160 diff --git a/09_vector_retrieval/PurplePulse/configs/ivf_flat_million_k50_partitioned_4x20.conf b/09_vector_retrieval/PurplePulse/configs/ivf_flat_million_k50_partitioned_4x20.conf new file mode 100644 index 00000000..35266f7b --- /dev/null +++ b/09_vector_retrieval/PurplePulse/configs/ivf_flat_million_k50_partitioned_4x20.conf @@ -0,0 +1,7 @@ +top_k = 50 +search_mode = ivf_flat +batch_size = 40 +distance_mode = bucket_major_partitioned +topk_mode = two_stage +nlist = 256 +nprobe = 160 diff --git a/09_vector_retrieval/PurplePulse/configs/ivf_flat_million_k50_partitioned_8x8.conf b/09_vector_retrieval/PurplePulse/configs/ivf_flat_million_k50_partitioned_8x8.conf new file mode 100644 index 00000000..a93741ea --- /dev/null +++ b/09_vector_retrieval/PurplePulse/configs/ivf_flat_million_k50_partitioned_8x8.conf @@ -0,0 +1,7 @@ +top_k = 50 +search_mode = ivf_flat +batch_size = 40 +distance_mode = bucket_major_partitioned8x8 +topk_mode = two_stage +nlist = 256 +nprobe = 160 diff --git a/09_vector_retrieval/PurplePulse/configs/ivf_flat_million_k50_warp1.conf b/09_vector_retrieval/PurplePulse/configs/ivf_flat_million_k50_warp1.conf new file mode 100644 index 00000000..deeffcd8 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/configs/ivf_flat_million_k50_warp1.conf @@ -0,0 +1,7 @@ +top_k = 50 +search_mode = ivf_flat +batch_size = 64 +distance_mode = warp1 +topk_mode = two_stage +nlist = 256 +nprobe = 160 diff --git a/09_vector_retrieval/PurplePulse/configs/ivf_flat_million_k50_warp2.conf b/09_vector_retrieval/PurplePulse/configs/ivf_flat_million_k50_warp2.conf new file mode 100644 index 00000000..46493326 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/configs/ivf_flat_million_k50_warp2.conf @@ -0,0 +1,7 @@ +top_k = 50 +search_mode = ivf_flat +batch_size = 64 +distance_mode = warp2 +topk_mode = two_stage +nlist = 256 +nprobe = 160 diff --git a/09_vector_retrieval/PurplePulse/configs/ivf_flat_million_k50_warp4.conf b/09_vector_retrieval/PurplePulse/configs/ivf_flat_million_k50_warp4.conf new file mode 100644 index 00000000..7c41fc93 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/configs/ivf_flat_million_k50_warp4.conf @@ -0,0 +1,7 @@ +top_k = 50 +search_mode = ivf_flat +batch_size = 64 +distance_mode = warp4 +topk_mode = two_stage +nlist = 256 +nprobe = 160 diff --git a/09_vector_retrieval/PurplePulse/configs/ivf_flat_million_k50_warp8.conf b/09_vector_retrieval/PurplePulse/configs/ivf_flat_million_k50_warp8.conf new file mode 100644 index 00000000..ae2d3f30 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/configs/ivf_flat_million_k50_warp8.conf @@ -0,0 +1,7 @@ +top_k = 50 +search_mode = ivf_flat +batch_size = 64 +distance_mode = warp8 +topk_mode = two_stage +nlist = 256 +nprobe = 160 diff --git a/09_vector_retrieval/PurplePulse/configs/ivf_flat_million_k50_warp_compact_batch40.conf b/09_vector_retrieval/PurplePulse/configs/ivf_flat_million_k50_warp_compact_batch40.conf new file mode 100644 index 00000000..0fb56e3c --- /dev/null +++ b/09_vector_retrieval/PurplePulse/configs/ivf_flat_million_k50_warp_compact_batch40.conf @@ -0,0 +1,7 @@ +top_k = 50 +search_mode = ivf_flat +batch_size = 40 +distance_mode = warp_compact +topk_mode = two_stage +nlist = 256 +nprobe = 160 diff --git a/09_vector_retrieval/PurplePulse/configs/ivf_flat_million_near_exact.conf b/09_vector_retrieval/PurplePulse/configs/ivf_flat_million_near_exact.conf new file mode 100644 index 00000000..993e0235 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/configs/ivf_flat_million_near_exact.conf @@ -0,0 +1,7 @@ +top_k = 10 +search_mode = ivf_flat +batch_size = 64 +distance_mode = warp +topk_mode = two_stage +nlist = 256 +nprobe = 224 diff --git a/09_vector_retrieval/PurplePulse/configs/ivf_flat_mnist.conf b/09_vector_retrieval/PurplePulse/configs/ivf_flat_mnist.conf new file mode 100644 index 00000000..41f41723 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/configs/ivf_flat_mnist.conf @@ -0,0 +1,7 @@ +top_k = 10 +search_mode = ivf_flat +batch_size = 128 +distance_mode = warp +topk_mode = two_stage +nlist = 256 +nprobe = 32 diff --git a/09_vector_retrieval/PurplePulse/configs/real_exact_k10.conf b/09_vector_retrieval/PurplePulse/configs/real_exact_k10.conf new file mode 100644 index 00000000..a325885d --- /dev/null +++ b/09_vector_retrieval/PurplePulse/configs/real_exact_k10.conf @@ -0,0 +1,5 @@ +top_k = 10 +search_mode = exact +batch_size = 64 +distance_mode = warp +topk_mode = fused diff --git a/09_vector_retrieval/PurplePulse/configs/real_exact_k100.conf b/09_vector_retrieval/PurplePulse/configs/real_exact_k100.conf new file mode 100644 index 00000000..e1038bf2 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/configs/real_exact_k100.conf @@ -0,0 +1,5 @@ +top_k = 100 +search_mode = exact +batch_size = 64 +distance_mode = warp +topk_mode = fused diff --git a/09_vector_retrieval/PurplePulse/configs/real_exact_k50.conf b/09_vector_retrieval/PurplePulse/configs/real_exact_k50.conf new file mode 100644 index 00000000..a7e5abdf --- /dev/null +++ b/09_vector_retrieval/PurplePulse/configs/real_exact_k50.conf @@ -0,0 +1,5 @@ +top_k = 50 +search_mode = exact +batch_size = 64 +distance_mode = warp +topk_mode = fused diff --git a/09_vector_retrieval/PurplePulse/configs/real_ivf_nlist1024_k10.conf b/09_vector_retrieval/PurplePulse/configs/real_ivf_nlist1024_k10.conf new file mode 100644 index 00000000..59b8b5b4 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/configs/real_ivf_nlist1024_k10.conf @@ -0,0 +1,7 @@ +top_k = 10 +search_mode = ivf_flat +batch_size = 64 +distance_mode = bucket_major +topk_mode = two_stage +nlist = 1024 +nprobe = 160 diff --git a/09_vector_retrieval/PurplePulse/configs/real_ivf_nlist1024_k100.conf b/09_vector_retrieval/PurplePulse/configs/real_ivf_nlist1024_k100.conf new file mode 100644 index 00000000..a3d9dff7 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/configs/real_ivf_nlist1024_k100.conf @@ -0,0 +1,7 @@ +top_k = 100 +search_mode = ivf_flat +batch_size = 64 +distance_mode = bucket_major_partitioned +topk_mode = two_stage +nlist = 1024 +nprobe = 160 diff --git a/09_vector_retrieval/PurplePulse/configs/real_ivf_nlist1024_k100_safe.conf b/09_vector_retrieval/PurplePulse/configs/real_ivf_nlist1024_k100_safe.conf new file mode 100644 index 00000000..f825ea00 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/configs/real_ivf_nlist1024_k100_safe.conf @@ -0,0 +1,7 @@ +top_k = 100 +search_mode = ivf_flat +batch_size = 64 +distance_mode = bucket_major +topk_mode = two_stage +nlist = 1024 +nprobe = 160 diff --git a/09_vector_retrieval/PurplePulse/configs/real_ivf_nlist1024_k50.conf b/09_vector_retrieval/PurplePulse/configs/real_ivf_nlist1024_k50.conf new file mode 100644 index 00000000..bbfb2abb --- /dev/null +++ b/09_vector_retrieval/PurplePulse/configs/real_ivf_nlist1024_k50.conf @@ -0,0 +1,7 @@ +top_k = 50 +search_mode = ivf_flat +batch_size = 64 +distance_mode = bucket_major_partitioned4x16 +topk_mode = two_stage +nlist = 1024 +nprobe = 160 diff --git a/09_vector_retrieval/PurplePulse/configs/real_ivf_nlist1024_k50_safe.conf b/09_vector_retrieval/PurplePulse/configs/real_ivf_nlist1024_k50_safe.conf new file mode 100644 index 00000000..6aa8e5b3 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/configs/real_ivf_nlist1024_k50_safe.conf @@ -0,0 +1,7 @@ +top_k = 50 +search_mode = ivf_flat +batch_size = 64 +distance_mode = bucket_major +topk_mode = two_stage +nlist = 1024 +nprobe = 160 diff --git a/09_vector_retrieval/PurplePulse/docs/API.md b/09_vector_retrieval/PurplePulse/docs/API.md new file mode 100644 index 00000000..51a27cd4 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/docs/API.md @@ -0,0 +1,127 @@ +# WarpFusion CLI 与 C++ API + +本文给出面向使用者的最小稳定接口。二进制向量格式、完整参数和正式实验口径见 +[`REPRODUCIBILITY.md`](REPRODUCIBILITY.md)；性能结论见 +[`FINAL_REPORT.md`](FINAL_REPORT.md)。 + +## CLI + +### Exact 查询 + +```bash +./build/vector_search \ + --database data/database.bin \ + --queries data/queries.bin \ + --params configs/real_exact_k10.conf \ + --backend gpu \ + --warmup 1 \ + --repeat 5 \ + --output results/exact.txt +``` + +`--backend` 可取 `cpu` 或 `gpu`。GPU 路径在进程内常驻数据库和工作区， +`--warmup` 不计入正式测量，`--repeat` 的各轮共享同一个执行器。输出每行是 +`query_id vector_id score`；同一 query 内按最好到最差排序，相同分数按较小 +vector ID 排序。 + +### IVF-Flat 建库与查询 + +```bash +./build/ivf_build \ + --database data/database.bin \ + --output data/database.ivf \ + --nlist 1024 \ + --iterations 15 \ + --training-samples 100000 \ + --threads 16 + +./build/ivf_search \ + --index data/database.ivf \ + --queries data/queries.bin \ + --params configs/real_ivf_nlist1024_k10.conf \ + --nprobe 160 \ + --backend gpu \ + --warmup 1 \ + --repeat 5 \ + --output results/ivf.txt +``` + +建库和查询的 `nlist` 必须一致；`nprobe` 的合法范围是 `1..nlist`。CLI 的 +`--nprobe` 会覆盖配置文件，便于扫描质量—吞吐曲线。`nprobe=nlist` 可用于 +完整性检查，不代表推荐的近似检索配置。 + +所有 CLI 在参数缺失、输入格式不合法、维度/dtype/metric 不匹配或配置越界时 +返回非零退出码并向标准错误输出原因。上层服务应检查退出码，不应只判断输出 +文件是否存在。 + +## C++ API + +公开头文件位于 `include/`，命名空间保持为空以兼容现有 CLI。调用者负责保证 +数据库、查询和参数在执行器生命周期内语义一致；执行器不可复制但可移动。 + +### 常驻 Exact 执行器 + +```cpp +#include "file_io.h" +#include "search.h" + +VectorDatabase database = readVectorDatabase("data/database.bin"); +QuerySet initial = readQuerySet("data/queries.bin"); +SearchParams params = readSearchParams("configs/real_exact_k10.conf"); +validateInputs(database, initial, params); + +GpuExactSearchEngine engine(database, initial, params); +SearchStats stats; +SearchResults neighbors = engine.search(initial, &stats); + +double one_time_upload_ms = engine.databaseH2DMilliseconds(); +std::size_t resident_bytes = engine.deviceBytes(); +``` + +构造函数完成数据库上传与工作区分配；后续 `search()` 只上传 queries 并复用 +常驻状态。`SearchStats` 分离 query H2D、距离/局部 Top-K、最终归并、结果 D2H +和 batch 延迟。只需一次查询时也可以调用 `gpuExactSearch()`；CPU 参考为 +`cpuExactSearch()`。 + +### 常驻 IVF-Flat 执行器 + +```cpp +#include "file_io.h" +#include "ivf_flat.h" + +IvfFlatIndex index = readIvfFlatIndex("data/database.ivf"); +QuerySet queries = readQuerySet("data/queries.bin"); +SearchParams params = readSearchParams("configs/real_ivf_nlist1024_k10.conf"); + +GpuIvfFlatSearchEngine engine(index, queries, params); +SearchStats stats; +SearchResults neighbors = engine.search(queries, &stats); + +double one_time_upload_ms = engine.indexH2DMilliseconds(); +std::size_t resident_bytes = engine.deviceBytes(); +``` + +索引也可通过 `buildIvfFlatIndex()` 在进程内构建，并用 +`writeIvfFlatIndex()`/`readIvfFlatIndex()` 持久化。`cpuIvfFlatSearch()` 是 +正确性参考；`recallAtK(exact, approximate)` 计算候选 ID 集合 recall。 + +### 数据与结果对象 + +- `VectorDatabase`/`QuerySet` 支持 FP32 和 FP16 存储；计算使用 FP32 累加。 +- `Metric` 支持 L2、inner product 和 cosine；L2 分数越小越好，另外两种越大 + 越好。 +- `SearchResults` 是按 query 分组的 `Neighbor{id, score}` 列表。 +- `resultsMatch()` 用候选 ID 与分数容差比较两组 Exact 结果。 +- `SearchStats` 中所有时间字段单位为毫秒；一次性数据库/索引上传不计入稳定 + 查询 QPS。 + +## 默认选择与适用范围 + +Exact 默认使用 `distance_mode=warp`。正式大规模配置使用 +`topk_mode=fused`；未经当前维度、K 和 GPU 验证的 query tile 会安全回退到 +`exact_query_tile=1`。IVF 默认使用固定 `nprobe`；`score_mass` 是实验接口， +不是通用默认值。完整的 K/维度选择表与回退规则见 +[`STABILITY_MATRIX.md`](STABILITY_MATRIX.md)。 + +当前 API 是单进程、单 GPU、内存内静态索引接口；不提供 ABI 稳定承诺、并发 +请求调度、增量增删、压缩索引、超显存分片或多 GPU 协调。 diff --git a/09_vector_retrieval/PurplePulse/docs/BASELINE_RESULTS.md b/09_vector_retrieval/PurplePulse/docs/BASELINE_RESULTS.md new file mode 100644 index 00000000..8ff9e65b --- /dev/null +++ b/09_vector_retrieval/PurplePulse/docs/BASELINE_RESULTS.md @@ -0,0 +1,856 @@ +# Baseline 实验记录 + +> 本文记录开发过程中的初步结果，不是最终提交数据。最终实验需要预热、重复运行， +> 并分别统计数据传输、距离 kernel 和 Top-K kernel 时间。 + +## 环境 + +- GPU：NVIDIA GeForce RTX 5090，32607 MiB +- Compute Capability：12.0 +- CUDA Toolkit：13.3 +- CUDA 架构参数：`CMAKE_CUDA_ARCHITECTURES=120` +- C++：GCC 13.3 + +## 2026-08-28：精确检索第一轮 + +参数： + +- 数据库：100,000 个 FP32 向量 +- 维度：128 +- 查询数：100 +- metric：L2（输出平方 L2 距离） +- K：10 +- batch size：8 + +| 实现 | 总时间（ms） | QPS | 说明 | +|---|---:|---:|---| +| CPU reference | 770.509 | 129.784 | `nth_element` 后排序 Top-K | +| GPU 距离 + CPU Top-K | 432.908 | 230.996 | 会把完整距离矩阵复制回 CPU | +| GPU + 单线程 Top-K，sm_120 | 715.431 | 139.776 | 每个 query 只有一个线程选 Top-K | +| GPU + block Top-K，sm_120 | 288.316 | 346.841 | 256 个线程共同归约，重复 K 轮 | + +正确性：100 个 query 的 Top-10 ID 与 CPU reference 全部相同；由于 CPU/GPU +浮点指令顺序不同，观察到的最大分数误差约为 `1.14e-5`。 + +## 初步结论 + +1. 把代码放到 GPU 上不代表更快。单线程 GPU Top-K 成为明显瓶颈。 +2. block 内线程协作使端到端时间相对单线程 Top-K 降低约 59.7%。 +3. 当前 block Top-K 会重复扫描距离数组 K 次。K=50/100 时预计会成为新瓶颈， + 后续需要分块候选和两阶段归并，但应先完成正式的分段计时。 +4. 最初构建缓存错误使用 `sm_75`；RTX 5090 必须显式使用 `sm_120`，否则实验不公平。 + +## 2026-08-28：预热与分段计时 + +在同一 `100,000 × 128` 数据集上，预热 1 次并重复 5 次，block Top-10 的结果为： + +- 平均总时间：约 14.04 ms +- QPS：约 7,123 +- 数据库 H2D：约 2.37 ms +- 距离 kernel：约 2.98 ms +- Top-K kernel：约 7.67 ms + +这说明首次运行中的约 200 ms CUDA Runtime 初始化不能计入稳定吞吐，且 Top-K 已超过 +距离计算，成为主要 kernel 瓶颈。 + +### K 对性能的影响 + +数据规模仍为 100,000 个向量、100 queries，预热后重复测量： + +| K | 平均总时间（ms） | QPS | 距离 kernel（ms） | Top-K kernel（ms） | +|---:|---:|---:|---:|---:| +| 1 | 7.28 | 13,737 | 2.98 | 0.80 | +| 10 | 14.04 | 7,123 | 2.98 | 7.67 | +| 50 | 46.13 | 2,168 | 3.01 | 38.19 | +| 100 | 83.43 | 1,199 | 3.01 | 76.45 | + +重复扫描版本的 Top-K 时间近似随 K 线性增加，因此不能作为最终 Top-K 方案。 + +## 2026-08-28：两阶段 Top-K + +新版本先让 256 个线程各自保留局部 Top-K，再只对 `256 × K` 个候选进行最终归并。 + +| 数据规模 | 旧 block Top-K（ms） | 两阶段 Top-K（ms） | Top-K 降幅 | +|---|---:|---:|---:| +| 100k vectors，100 queries | 7.67 | 5.08 | 33.8% | +| 1M vectors，1000 queries | 826.07 | 424.09 | 48.7% | + +官方规模初测（1M vectors、dim=128、1000 queries、K=10、batch=8）： + +- 平均端到端时间：950.26 ms +- QPS：1,052.34 +- 距离 kernel：427.00 ms +- 两阶段 Top-K：424.09 ms +- batch P50/P99：6.84 / 6.88 ms + +百万级完整 GPU 运行已经通过。另取相同数据库的前 10 个 query 与单线程 CPU reference +逐项比较，Top-10 ID、分数和顺序完全一致。 + +当前 QPS 仍包含每次调用重新分配显存和上传约 489 MiB 数据库的时间；最终报告需要同时给出 +冷启动端到端性能和数据库常驻显存后的查询性能。 + +## 2026-08-28：warp 协作距离计算 + +原始距离 kernel 由一个线程串行计算一个 128 维距离。优化版由一个 warp 共同处理一个向量， +不同 lane 读取相邻维度，并使用 shuffle 完成归约。 + +| 数据规模 | simple 距离 kernel（ms） | warp 距离 kernel（ms） | 距离计算降幅 | +|---|---:|---:|---:| +| 100k vectors，100 queries | 3.00 | 1.09 | 63.6% | +| 1M vectors，1000 queries | 427.00 | 305.65 | 28.4% | + +在官方规模上，平均端到端时间由约 950.26 ms 降到 825.05 ms，QPS 从约 1,052 +提升到 1,212。由于 warp 归约改变了 FP32 加法顺序，10-query 全库验证中的最大距离误差为 +`2.29e-5`，Top-10 ID 集合和顺序不变。 + +当前最大 kernel 瓶颈再次变为 Top-K（约 424 ms），下一阶段应改进局部候选的选择方法， +而不是继续微调距离 kernel。 + +## 2026-08-28：新 GPU 迁移与 FP16 验证 + +新环境： + +- GPU：NVIDIA GeForce RTX 4090 D，24564 MiB +- Compute Capability：8.9 +- Driver：570.124.06 +- CUDA Toolkit：12.8.93 +- 编译架构：`sm_89` + +完整 CUDA 编译和 CTest 通过，`compute-sanitizer --tool memcheck` +报告 0 个错误。测试同时覆盖 FP32/FP16、L2/内积/Cosine 和 +simple/block/two-stage Top-K。 + +### FP16 存储与性能 + +FP16 文件在 CPU 和 GPU 都保持 binary16 存储，距离 kernel 加载后 +转 FP32 累加。在 `1M × 128`、1000 queries、K=10、batch=8、预热 1 次、 +重复 5 次的冒烟测试中： + +| dtype | 总时间（ms） | QPS | 数据库 H2D（ms） | 距离 kernel（ms） | Top-K（ms） | +|---|---:|---:|---:|---:|---:| +| FP32 | 1018.70 | 981.64 | 101.64 | 539.06 | 367.91 | +| FP16 | 692.42 | 1444.21 | 42.48 | 273.15 | 368.06 | + +FP16 的 QPS 相对 FP32 提高约 47.1%，数据库文件和显存存储减半。 +Top-K 时间基本不变，符合它仍使用 FP32 分数矩阵的实现。 + +正确性验证使用同一百万向量库的前 10 个 query 做 CPU/GPU 全库对照： +FP32 最大分数误差约 `2.29e-5`，FP16 约 `3.43e-5`；两者的 +Top-10 ID 集合和顺序均完全一致。K=100 也已对 FP32/FP16 做 CPU/GPU +对照，候选集合和顺序一致，最大分数误差约 `4.58e-5`。因此验证 +脚本默认绝对容差调整为 `5e-5`。 + +> 以上仍是功能与迁移冒烟测试，不是最终成绩。正式对比还需固定频率、 +> 拆分冷启动/常驻数据库路径，并进行更多次重复。 + +## 2026-08-28：数据库常驻 GPU + +新增 `GpuExactSearchEngine`，在初始化时一次性分配 GPU 工作区并上传数据库。 +预热和每轮重复查询只传输 query 和 Top-K 结果，不再重复分配显存或 +上传数据库。原有 `gpuExactSearch` 一次性 API 仍保留。 + +环境和数据与上一节一致：`1M × 128`、1000 queries、K=10、batch=8， +预热 1 次、重复 5 次。 + +| dtype | 初始化（ms） | 一次性 DB H2D（ms） | 常驻查询（ms） | 常驻 QPS | 估算冷启动（ms） | +|---|---:|---:|---:|---:|---:| +| FP32 | 157.62 | 52.57 | 912.23 | 1096.22 | 1069.84 | +| FP16 | 115.39 | 23.45 | 645.90 | 1548.23 | 761.28 | + +与改造前每轮重新创建 GPU 缓冲区和上传数据库的口径相比： + +- FP32：约 1018.70 ms / 981.64 QPS → 912.23 ms / 1096.22 QPS。 +- FP16：约 692.42 ms / 1444.21 QPS → 645.90 ms / 1548.23 QPS。 + +新旧路径的 1000-query Top-K ID 和分数逐值完全一致。CTest 增加同一 +常驻引擎连续查询两次的确定性检查，compute-sanitizer 仍为 0 个错误。 + +当前稳定查询的主要耗时已清晰拆分为距离 kernel 和 Top-K kernel； +下一步可以在不受重复 H2D 干扰的口径下优化 Top-K，并进行 ncu/nsys 分析。 + +## 2026-08-28：nsys 分析与局部 Top-K 优化 + +Nsight Compute 2025.1 可执行，但平台禁止访问 GPU performance counters， +返回 `ERR_NVGPUCTRPERM`。该权限需宿主机管理员开启，容器内无法解决。 +因此改用 Nsight Systems 2024.6.2 获取 CUDA 时间线，并用 +`cuobjdump --dump-resource-usage` 检查 kernel 资源。 + +### 瓶颈证据 + +在 `100k × 128`、100 queries、K=10、batch=8 上，时间线同时覆盖 +预热和测量各 1 次，共 26 个 batch kernel 实例： + +| kernel | 优化前总时间（ms） | 优化前单次（µs） | kernel 时间占比 | +|---|---:|---:|---:| +| 局部候选 | 11.039 | 424.58 | 76.0% | +| warp 距离 | 3.099 | 119.21 | 21.3% | +| 最终 block 归并 | 0.383 | 14.72 | 2.6% | + +瓶颈不在最终归并，而在局部候选。原实现对每个候选从最优位置开始 +扫描 K 个已排序槽位，即使候选无法进入 Top-K，也通常需要 K 次比较。 + +### 优化方法 + +1. 局部 Top-K 改为无序数组，只记录当前最差槽位。 +2. 普通候选只比较一次；真正替换最差项时，才扫描 K 项重算最差槽位。 +3. 最终 block 归并本来就不依赖局部有序，因此不改变结果语义。 +4. 为常用 K=1/10 生成编译期特化 kernel，其他 K 保留通用路径。 + +无序候选版 nsys 复测中，局部候选 kernel 单次平均降至 +`205.95 µs`，相比 `424.58 µs` 降低约 51.5%。最终归并仍为 +约 `14.70 µs`。K=10 模板特化在百万规模上又将 Top-K 从约 +183.85 ms 降到 178.81 ms。K=1 特化 kernel 的 thread stack 从通用路径的 +128 字节降为 0。 + +### 百万规模收益 + +数据、参数和 RTX 4090 D 环境与前一节相同： + +| dtype | 常驻查询优化前（ms） | 优化后（ms） | Top-K 优化前（ms） | 优化后（ms） | QPS 优化前 | 优化后 | +|---|---:|---:|---:|---:|---:|---:| +| FP32 | 912.23 | 722.21 | 368.23 | 178.81 | 1096.22 | 1384.64 | +| FP16 | 645.90 | 456.00 | 368.11 | 178.87 | 1548.23 | 2193.01 | + +Top-K 阶段降低约 51.4%。端到端常驻查询中，FP32 QPS 提高约 +26.3%，FP16 QPS 提高约 41.6%。优化前后 1000-query 的 Top-K ID +和分数逐值完全一致；FP32/FP16 的 CPU/GPU 测试、K=1/10 测试和 +compute-sanitizer 均通过。 + +优化后 FP32 的主要瓶颈转为距离 kernel（约 539 ms）；FP16 中距离 +kernel 约 273 ms，Top-K 约 179 ms。后续 Top-K 继续优化的边际收益已降低， +应先完成 batch size 扫描和距离 kernel 分析，再决定是否继续专项调优。 + +## 2026-08-28：batch size 扫描 + +固定 `1M × 128`、1000 queries、K=10、warp 距离、优化后 two-stage Top-K， +预热 1 次并重复 5 次。扫描 batch=1/4/8/16/32/64/128： + +### FP32 + +| batch | 常驻时间（ms） | QPS | 距离（ms） | Top-K（ms） | batch P50（ms） | batch P99（ms） | +|---:|---:|---:|---:|---:|---:|---:| +| 1 | 1910.51 | 523.42 | 545.33 | 1332.02 | 1.90 | 1.92 | +| 4 | 902.93 | 1107.51 | 539.36 | 354.78 | 3.60 | 3.63 | +| 8 | 722.68 | 1383.75 | 538.94 | 178.86 | 5.77 | 5.79 | +| 16 | 632.31 | 1581.49 | 538.73 | 90.73 | 10.09 | 10.14 | +| 32 | 587.26 | 1702.83 | 538.58 | 47.24 | 18.74 | 18.78 | +| 64 | 563.47 | 1774.73 | 538.49 | 23.71 | 35.98 | 37.91 | +| 128 | 552.09 | 1811.28 | 538.46 | 12.49 | 70.54 | 72.96 | + +### FP16 + +| batch | 常驻时间（ms） | QPS | 距离（ms） | Top-K（ms） | batch P50（ms） | batch P99（ms） | +|---:|---:|---:|---:|---:|---:|---:| +| 1 | 1577.50 | 633.91 | 277.02 | 1269.03 | 1.57 | 1.59 | +| 4 | 637.30 | 1569.11 | 274.05 | 354.86 | 2.54 | 2.56 | +| 8 | 456.53 | 2190.44 | 273.06 | 179.00 | 3.64 | 3.66 | +| 16 | 365.82 | 2733.60 | 272.74 | 90.96 | 5.83 | 5.85 | +| 32 | 320.96 | 3115.70 | 272.49 | 47.24 | 10.22 | 10.25 | +| 64 | 296.98 | 3367.19 | 272.44 | 23.78 | 18.95 | 18.99 | +| 128 | 285.39 | 3503.96 | 272.40 | 12.54 | 36.48 | 36.51 | + +距离 kernel 的总时间从 batch=4 起几乎不变；Top-K 总时间随 batch +增大快速下降，主要是更少的 kernel 启动和更多同时处理的 query。 +batch=128 时，FP32 已达到仅由距离 kernel 时间估算的最大吞吐的约 +97.5%，FP16 约 95.4%；继续增大 batch 的吞吐空间很小，却会继续增加 +工作区显存和单 batch 延迟。 + +建议保留三档： + +- batch=8：低延迟。 +- batch=32：平衡吞吐和延迟。 +- batch=128：离线最大吞吐。 + +batch=1 和 batch=128 的 FP32/FP16 结果逐值完全一致。 + +## 2026-08-31：切换实例后的边界与三度量复验 + +在新的 GPU 实例上从干净构建目录重新配置 `sm_89` Release 版本。CTest +全部通过，`compute-sanitizer --tool memcheck` 报告 0 个错误。测试覆盖： + +- FP32/FP16 与 L2、内积、Cosine 三种度量。 +- simple/warp 两种距离 kernel，以及 simple/block/two-stage 三种 Top-K。 +- Cosine 零向量语义、相同分数按较小 ID 打破平局。 +- `top_k = num_vectors`、查询数不能被 batch size 整除的尾批。 +- `top_k=0`、`batch_size=0`、空输入、元数据长度不一致。 +- 错误 magic/version、截断文件和额外尾部字节。 + +中等规模复验使用 `100,000 × 128` FP32 数据库、101 queries、K=10、 +batch=8；GPU 预热 1 次并重复 5 次： + +| metric | CPU（ms） | GPU 常驻（ms） | GPU QPS | 距离 kernel（ms） | Top-K（ms） | 最大分数误差 | +|---|---:|---:|---:|---:|---:|---:| +| inner product | 1872.35 | 5.480 | 18431.9 | 2.324 | 2.698 | `9.6e-6` | +| cosine | 3461.35 | 5.480 | 18429.5 | 2.341 | 2.707 | `2.68e-7` | + +两种度量的 GPU Top-10 候选集合和排序均与 CPU reference 一致，101 +queries 也实际经过最后一个非满 batch。这里的结果用于迁移和正确性复验， +不替代前面的百万规模吞吐实验。 + +## 2026-08-31：IVF-Flat CPU 正确性基线 + +完成确定性采样 k-means、连续倒排桶布局、版本化索引保存/加载、固定 +`nprobe` 查询与 recall@K 工具。单元测试覆盖 FP32/FP16 与三种 metric； +`nprobe=nlist` 的结果与 exact reference 逐值一致。 + +首轮真实数据使用 100,000 × 128 FP32 inner-product 数据库、101 queries、 +K=10。索引参数为 nlist=64、训练样本 10,000、8 次迭代： + +- 构建时间：1264.19 ms；保存时间：28.05 ms。 +- 索引文件：52,033,320 bytes。 +- 桶大小 min/avg/max：1426 / 1562.5 / 1718；空桶为 0。 + +CPU 查询预热 1 次、重复 3 次： + +| nprobe | 平均查询（ms） | QPS | recall@10 | 最低单 query recall | +|---:|---:|---:|---:|---:| +| 1 | 29.50 | 3423.99 | 0.0782 | 0.0 | +| 2 | 54.74 | 1845.23 | 0.1406 | 0.0 | +| 4 | 108.05 | 934.79 | 0.2327 | 0.0 | +| 8 | 216.44 | 466.65 | 0.3614 | 0.0 | +| 16 | 431.13 | 234.27 | 0.5455 | 0.2 | +| 32 | 862.91 | 117.05 | 0.8119 | 0.4 | +| 64 | 1734.85 | 58.22 | 1.0000 | 1.0 | + +这是 CPU 算法与质量基线，不是最终 IVF 性能。下一步将索引常驻 GPU， +并把中心选择、候选距离和 Top-K 移到 GPU 后重新绘制 recall—QPS 曲线。 + +## 2026-08-31：GPU IVF-Flat 第一版与 block 并行 + +第一版将索引向量和原始 ID 常驻显存，CPU 根据聚类中心选择探测桶并上传 +候选位置，GPU 融合执行候选距离和 Top-K。教学 baseline 每个 query 仅使用 +一个 CUDA 线程；nprobe=16 时耗时 905.79 ms，其中 kernel 888.03 ms， +比 CPU IVF 更慢。 + +优化版改为每个 query 一个 256-thread block。各线程并行扫描候选，使用 +“无序局部 Top-K + 当前最差槽位”保留局部结果，再通过共享内存归约最终 +Top-K。nprobe=16 降到 27.17 ms，相对单线程 GPU 版本加速约 33.3 倍， +相对同 recall 的 CPU IVF（431.13 ms）加速约 15.9 倍。 + +数据、索引和质量口径与上一节相同，GPU 预热 1 次、重复 5 次： + +| nprobe | GPU 查询（ms） | GPU QPS | recall@10 | CPU IVF（ms） | +|---:|---:|---:|---:|---:| +| 1 | 3.79 | 26623.0 | 0.0782 | 29.50 | +| 2 | 5.34 | 18906.2 | 0.1406 | 54.74 | +| 4 | 8.45 | 11950.9 | 0.2327 | 108.05 | +| 8 | 14.69 | 6876.6 | 0.3614 | 216.44 | +| 16 | 27.17 | 3717.8 | 0.5455 | 431.13 | +| 32 | 63.43 | 1592.4 | 0.8119 | 862.91 | +| 64 | 127.17 | 794.2 | 1.0000 | 1734.85 | + +所有 nprobe 的候选集合、顺序和分数均与 CPU IVF 一致；FP32/FP16、 +三种 metric 的自动测试通过，compute-sanitizer 报告 0 个错误。 + +当前版本仍不是最终 GPU IVF：nprobe=64 全探测明显慢于优化后的 exact GPU。 +分段计时显示 nprobe=32 时 CPU 桶选择约 18.91 ms、候选位置 H2D 约 +3.03 ms、GPU 扫描约 34.57 ms。下一步应把中心选择和桶展开迁到 GPU， +并用多个 block/warp 共同处理一个 query，而不是继续优化单 block 内细节。 + +## 2026-08-31：全 GPU 中心选择与每桶 block 扫描 + +第二版把 centers 和 bucket offsets 也常驻显存。查询流水线变为： + +1. GPU 计算 query 与全部中心的分数并选择 `nprobe`。 +2. 每个 `query × probe` 桶启动一个 256-thread block。 +3. block 内并行扫描连续桶数据并产生桶内 Top-K。 +4. 每个 query 再用一个 block 合并 `nprobe × K` 个候选。 + +每批现在只上传 query，不再执行 CPU 中心/桶选择，也不上传候选位置。 +nprobe=16 从上一版单 block 的 27.17 ms 降到 4.19 ms，再加速约 6.5 倍； +相对最初单线程 GPU 版累计加速约 216 倍。 + +同一 100k × 128、101 queries、inner-product、nlist=64 实验： + +| nprobe | 全 GPU 查询（ms） | QPS | recall@10 | 上一版 GPU（ms） | +|---:|---:|---:|---:|---:| +| 1 | 2.65 | 38107.8 | 0.0782 | 3.79 | +| 2 | 2.68 | 37617.6 | 0.1406 | 5.34 | +| 4 | 2.78 | 36327.6 | 0.2327 | 8.45 | +| 8 | 2.94 | 34348.9 | 0.3614 | 14.69 | +| 16 | 4.19 | 24131.4 | 0.5455 | 27.17 | +| 32 | 5.85 | 17267.3 | 0.8119 | 63.43 | +| 64 | 10.29 | 9814.3 | 1.0000 | 127.17 | + +nprobe=16 的平均分段时间为：query H2D 0.10 ms、GPU 中心选择 +0.68 ms、桶扫描与局部 Top-K 2.55 ms、最终归并 0.17 ms、结果 D2H +0.21 ms。所有 nprobe 的 recall 与 CPU 基线一致；nprobe=16 的结果与 +CPU IVF 分数逐值一致。FP32/FP16 与三种 metric 自动测试通过， +compute-sanitizer 报告 0 个错误。 + +在该中等规模上，nprobe=16 的 24.1k QPS 已高于 exact GPU 的约 18.4k +QPS，但 recall 仅为 0.5455；nprobe=32 的 recall 提升至 0.8119，QPS +约 17.3k。下一步应提高聚类质量、扩大 nlist，并进行 nlist × nprobe +质量—性能扫描，而不能只追求低 nprobe 的高吞吐。 + +## 2026-08-31：球面 k-means 与 nlist × nprobe 扫描 + +inner-product 使用普通均值更新中心时，中心模长会影响分桶。nlist=512 的 +桶大小达到 min/avg/max = 2/195.3/5243，最大的桶会破坏并行扫描的 +负载均衡。现在对 inner-product 和 cosine 的初始中心与每轮更新中心做 +L2 归一化，L2 度量仍保持普通 k-means。相同 nlist=512 索引的桶大小变为 +147/195.3/240，且没有空桶。 + +同时把 GPU 中心选择从“每个 query 一个线程维护有序 nprobe”改为 +256-thread block 归约。以 nlist=512、nprobe=128 为例，中心选择从 +30.43 ms 降到 1.24 ms，查询总时间从 33.69 ms 降到 4.53 ms；结果与 +优化前逐值一致。 + +首轮扫描继续使用 100k × 128 FP32 inner-product、101 queries、K=10、 +batch=8、预热 1 次并重复 5 次。下表列出具有代表性的质量—吞吐点： + +| nlist | nprobe | 查询（ms） | QPS | recall@10 | 最低单 query recall | +|---:|---:|---:|---:|---:|---:| +| 256 | 64 | 3.607 | 28001.4 | 0.6257 | 0.1 | +| 64 | 32 | 5.032 | 20073.3 | 0.8267 | 0.6 | +| 64 | 40 | 5.097 | 19816.9 | 0.8931 | 0.7 | +| 64 | 48 | 6.764 | 14931.4 | 0.9465 | 0.7 | +| 64 | 56 | 7.415 | 13620.3 | 0.9822 | 0.8 | +| 64 | 64 | 8.898 | 11350.9 | 1.0000 | 1.0 | + +在这份 100k 数据上，nlist=64 的中高召回点构成更实用的性能前沿： + +- `64/40`：约 19.8k QPS、0.893 recall，作为平衡档。 +- `64/56`：约 13.6k QPS、0.982 recall，作为高召回档。 +- `64/64`：全桶校验档，recall=1，结果与 exact ground truth 一致。 + +这些配置只代表当前数据规模和分布，不直接外推到百万规模。下一轮应在 +百万库上联动扫描 batch size 与 K，并记录显存和 query 延迟。 + +## 2026-09-01：IVF 桶内 warp 协作扫描 + +原桶扫描虽然每个 probe 使用 256-thread block，但每个线程串行计算一个 +完整向量。行主序布局下，相邻线程同一时刻访问相隔 `dim` 的地址，无法形成 +理想的连续读取。新实现让每个 warp 协作计算一个候选：lane 读取相邻维度， +用 shuffle 归约距离，再由 lane 0 维护该 warp 的局部 Top-K。每个 block 的 +8 个 warp 最后在共享内存中归并。 + +`distance_mode=simple` 保留旧 thread-per-vector 内核；默认 `warp` 选择 +新路径。自动测试对 FP32/FP16、L2/inner-product/cosine 同时比较 CPU、 +scalar GPU 与 warp GPU。CTest 3/3 通过，compute-sanitizer memcheck 为 +0 errors。 + +在同一 100k × 128、101-query、inner-product、nlist=64 实验中： + +| nprobe | 实现 | 总查询（ms） | 桶扫描（ms） | QPS | recall@10 | +|---:|---|---:|---:|---:|---:| +| 40 | scalar | 5.121 | 3.711 | 19723.8 | 0.8931 | +| 40 | warp | 4.094 | 2.566 | 24671.0 | 0.8931 | +| 56 | scalar | 7.438 | 5.917 | 13578.2 | 0.9822 | +| 56 | warp | 4.632 | 3.158 | 21807.2 | 0.9822 | + +nprobe=40 的总查询加速约 1.25 倍；nprobe=56 约 1.61 倍。两种内核输出的 +Top-K ID 和顺序完全一致，最大分数误差为 `9.6e-6`，来自 FP32 warp 归约 +顺序变化。全桶 nprobe=64 的 warp 查询为 5.342 ms、18.9k QPS，recall=1， +并与 exact ground truth 候选一致。 + +资源用量也更健康：FP32/FP16 warp kernel 均为 40 registers、0-byte stack、 +1060-byte shared memory；scalar kernel 为 40 registers、128-byte stack、 +3072-byte shared memory。 + +warp 优化后的代表性质量—吞吐前沿为： + +| nlist/nprobe | QPS | recall@10 | 说明 | +|---:|---:|---:|---| +| 256/64 | 32186.3 | 0.6257 | 最大吞吐参考点 | +| 128/64 | 26146.1 | 0.8327 | 吞吐—质量档 | +| 64/40 | 24671.0 | 0.8931 | 平衡档 | +| 64/48 | 21952.2 | 0.9465 | 高召回档 | +| 64/56 | 21807.2 | 0.9822 | 近精确档 | +| 64/64 | 18907.4 | 1.0000 | 全桶校验档 | + +下一阶段不再继续微调 100k 参数，而是在百万规模检查桶大小、GPU 显存、 +batch size 和 K 对这一布局的影响。 + +## 2026-09-01：百万规模 IVF 联合参数扫描 + +数据固定为 1,000,000 × 128 FP32 inner-product 数据库、1000 queries， +随机种子 2026。K=10 exact GPU ground truth 使用 batch=128、预热 1 次、 +重复 5 次：551.85 ms、1812.1 QPS，显式 GPU 缓冲区 980.39 MiB。 + +球面 k-means 使用 50,000 个训练样本和 8 次迭代： + +| nlist | 构建（ms） | 桶 min/avg/max | 空桶 | 索引 bytes | +|---:|---:|---:|---:|---:| +| 256 | 39401 | 3599/3906.25/4177 | 0 | 520133160 | +| 512 | 78075 | 1809/1953.12/2147 | 0 | 520266280 | +| 1024 | 155191 | 858/976.56/1088 | 0 | 520532520 | + +构建时间近似随 nlist 线性增加，桶保持均衡。当前 CPU 训练和全库分配已经 +成为离线阶段主要成本，后续可考虑 GPU k-means/assignment，但不影响查询 +性能结论。 + +### nlist × nprobe 前沿 + +下表统一使用 batch=16、K=10、预热 1 次、重复 5 次： + +| nlist/nprobe | 查询（ms） | QPS | recall@10 | 最低 query recall | +|---:|---:|---:|---:|---:| +| 1024/64 | 40.77 | 24526.0 | 0.3611 | 0.0 | +| 512/64 | 59.50 | 16808.1 | 0.4900 | 0.0 | +| 1024/128 | 67.03 | 14919.4 | 0.5293 | 0.1 | +| 256/64 | 92.32 | 10832.5 | 0.6613 | 0.2 | +| 512/128 | 96.45 | 10367.7 | 0.6897 | 0.2 | +| 1024/256 | 128.83 | 7762.4 | 0.7287 | 0.3 | +| 256/128 | 160.59 | 6227.0 | 0.8716 | 0.4 | +| 256/160 | 198.70 | 5032.8 | 0.9332 | 0.5 | +| 256/192 | 245.34 | 4076.0 | 0.9708 | 0.7 | +| 256/224 | 287.90 | 3473.5 | 0.9929 | 0.8 | +| 256/256 | 323.78 | 3088.5 | 1.0000 | 1.0 | + +更大的 nlist 在低召回区间提供更高吞吐，但中心选择和更多小桶 block 的 +开销使其在高召回区间被 nlist=256 支配。代表性预设因此选择： + +- fast：512/128，约 10.4k QPS、0.690 recall。 +- balanced：256/128，约 6.23k QPS、0.872 recall。 +- high recall：256/192，约 4.08k QPS、0.971 recall。 +- near exact：256/224，约 3.47k QPS、0.993 recall。 + +256/256 的 recall=1，候选与 exact ground truth 一致；其 3089 QPS 仍为 +当前 exact 路径的约 1.70 倍，因为分桶布局直接产生每桶 Top-K，不需要写出 +完整 `batch × database` 分数矩阵。 + +### batch size + +固定 256/160、K=10： + +| batch | 查询（ms） | QPS | batch P50/P99（ms） | +|---:|---:|---:|---:| +| 1 | 657.69 | 1520.5 | 0.643/0.659 | +| 8 | 207.40 | 4821.7 | 1.606/1.678 | +| 16 | 198.11 | 5047.7 | 3.076/3.222 | +| 32 | 205.95 | 4855.6 | 6.407/6.766 | +| 64 | 223.71 | 4470.1 | 14.016/15.705 | +| 128 | 250.74 | 3988.2 | 31.813/32.537 | + +batch=16 是当前吞吐峰值。batch 太小时启动和中心选择开销占比高；太大时 +单次提交的 `query × probe` blocks 过多，桶扫描总时间反而增加。这与 exact +search 在 batch=128 达到最高吞吐的结论不同。 + +### K 与显存 + +固定 256/160、batch=16：K=1 为 176.16 ms、5676.7 QPS、recall@1=0.943； +K=10 为约 198.70 ms、5032.8 QPS、recall@10=0.933。下一步需要用模板化 +局部候选和分层归并扩展 K=50/100，避免扩大运行时数组拖慢 K=10。 + +显存现在由程序直接统计显式 CUDA 缓冲区：exact batch128/K10 为 +980.39 MiB，IVF 256/160/b16/K10 为 496.37 MiB，降低约 49.4%。IVF 不保留 +完整距离矩阵是主要原因。CTest 3/3、compute-sanitizer 0 errors；新增 +`benchmark_ivf_sweep.py` 已实测可自动输出性能、显存、recall 和分段时间 CSV。 + +## 2026-09-01：IVF K=50/100 与 warp heap + +为避免 K100 拖慢 K10，桶扫描分别实例化 MaximumK=10/50/100。第一版仍用 +无序局部数组跟踪最差槽位，并在每个 probe 输出阶段重复扫描数组；虽然结果 +正确，但输出有序 K 项产生 O(K²) 工作。nlist=256、nprobe=160、batch=16 +时，K50/K100 分别为 502.36/1392.80 ms。 + +优化版让每个 warp 在共享内存维护“最差项在根”的 heap：候选替换从 O(K) +降为 O(log K)，扫描完成后原地 heap sort 一次，再对 8 个 warp 的有序列表 +做 k-way merge。跨 probe 的最终归并也改为维护每个 probe 的当前 rank， +每轮只归约各列表头，不再给每个线程分配大型局部 Top-K。 + +资源用量证明 K10 未被大 K 拖累： + +| kernel | K | registers | stack | static shared | +|---|---:|---:|---:|---:| +| warp bucket scan | 10 | 40 | 0 | 1072 B | +| warp bucket scan | 50 | 40 | 0 | 4912 B | +| warp bucket scan | 100 | 40 | 0 | 9712 B | +| probe k-way merge | runtime | 18 | 0 | 4096 B | + +merge 另使用 `nprobe × 4` bytes 动态共享内存保存游标。GPU IVF 将 nprobe +限制为 8192，因此静态与动态共享内存保持在普通单 block 上限内。 + +### 优化收益 + +固定 1M×128、1000 queries、inner-product、256/160、batch=16： + +| K | heap 前（ms） | heap 后（ms） | 加速 | heap 后 QPS | recall@K | +|---:|---:|---:|---:|---:|---:| +| 10 | 198.70 | 188.31 | 1.06× | 5310.6 | 0.9332 | +| 50 | 502.36 | 268.38 | 1.87× | 3726.1 | 0.9224 | +| 100 | 1392.80 | 338.42 | 4.12× | 2954.9 | 0.9163 | + +优化前后 K10/50/100 的候选、顺序和分数逐值相同。256/256 全桶查询也与 +exact K50/K100 ground truth 逐项一致，最大分数误差为 0。 + +最终 probe merge 只占很小比例：上述 K10/50/100 分别约 +0.67/2.45/4.98 ms；主要成本仍是每个桶内的候选距离和局部 heap。 + +### 大 K batch 与高召回点 + +固定 256/160 时，大 K 的最佳 batch 高于 K10： + +| K | batch | 查询（ms） | QPS | batch P50/P99（ms） | +|---:|---:|---:|---:|---:| +| 50 | 8 | 278.40 | 3591.9 | 2.182/2.211 | +| 50 | 32 | 264.15 | 3785.7 | 8.273/8.713 | +| 50 | 64 | 257.25 | 3887.2 | 16.123/16.845 | +| 50 | 128 | 264.16 | 3785.6 | 33.161/35.268 | +| 100 | 8 | 364.51 | 2743.4 | 2.861/2.891 | +| 100 | 32 | 326.04 | 3067.1 | 10.234/10.489 | +| 100 | 64 | 319.40 | 3130.9 | 20.035/20.548 | +| 100 | 128 | 316.88 | 3155.8 | 39.689/40.778 | + +K50 的吞吐峰值为 batch64。K100 的 batch128 只比 64 高约 0.8%，延迟却 +接近翻倍，因此默认仍选择 batch64。 + +使用 batch64 的质量—吞吐点： + +| K | nprobe | QPS | recall@K | 最低 query recall | +|---:|---:|---:|---:|---:| +| 50 | 160 | 3887.2 | 0.9224 | 0.80 | +| 50 | 192 | 3185.5 | 0.9636 | 0.86 | +| 50 | 224 | 2629.3 | 0.9904 | 0.92 | +| 100 | 160 | 3130.9 | 0.9163 | 0.83 | +| 100 | 192 | 2576.5 | 0.9611 | 0.90 | +| 100 | 224 | 2201.2 | 0.9890 | 0.94 | + +旧 exact block ground truth 的 K50/K100 吞吐分别约 254/138 QPS；近似 IVF +在 0.92 左右 recall 下分别约快 15.3×/22.7×。自动测试覆盖 +FP32/FP16 × L2/IP/cosine × K50/K100；CTest 3/3，compute-sanitizer +0 errors。 + +## 2026-09-01：FAISS CPU/GPU 公平对照 + +本轮固定使用同一份 1,000,000 × 128 FP32 inner-product 数据库、1000 +queries、batch=64、随机种子 2026。FAISS 版本为 1.14.1（CUDA 12 wheel）， +运行环境为 RTX 4090 D、driver 570.124.06、CUDA 12.8。GPU 查询均完整预热 +1 轮并重复 5 轮；FAISS CPU 使用单线程。FAISS IVF 使用 nlist=256、 +100,000 个训练样本、15 次聚类迭代，并和自研 IVF 扫描完全相同的 nprobe。 + +FAISS Flat K100 和 WarpFusion exact K100 的全部候选 ID 完全一致， +recall@100=1、最低单 query recall=1。FAISS IVF 使用自己训练的聚类中心， +因此不是复用自研索引；比较口径是相同算法参数与实际达到的 recall，而不是 +强行要求两个索引访问相同桶。 + +### Exact / Flat + +| K | WarpFusion GPU QPS | FAISS GPU QPS | FAISS 加速 | FAISS CPU QPS | WarpFusion/FAISS batch P99（ms） | +|---:|---:|---:|---:|---:|---:| +| 10 | 1774.9 | 20583.6 | 11.60× | 75.92 | 36.046 / 3.232 | +| 50 | 715.9 | 20611.3 | 28.79× | 75.71 | 88.177 / 3.150 | +| 100 | 444.6 | 19906.4 | 44.77× | 73.06 | 141.487 / 3.754 | + +这里的 WarpFusion Exact 也重新统一为 batch64、预热 1 次、重复 5 次。 +差距主要来自当前 exact 路径仍写回完整距离矩阵，且 block Top-K 随 K 增大 +迅速变慢。FAISS Flat 的 K10/50/100 吞吐接近，表明其距离扫描与选择融合程度 +明显更高。自研 IVF 仍是项目性能主线，Exact 主要保留作正确性 ground truth。 + +### IVF-Flat 质量—吞吐 + +下表给出平衡档 nprobe=160；两个实现的 recall 非常接近： + +| K | WarpFusion QPS / recall | FAISS QPS / recall | WarpFusion/FAISS QPS | batch P99 WarpFusion/FAISS（ms） | +|---:|---:|---:|---:|---:| +| 10 | 4771.1 / 0.9332 | 4756.1 / 0.9369 | 1.003× | 14.495 / 19.408 | +| 50 | 3879.4 / 0.9224 | 8721.0 / 0.9231 | 0.445× | 17.032 / 11.133 | +| 100 | 3111.3 / 0.9163 | 7896.0 / 0.9163 | 0.394× | 20.663 / 8.296 | + +近精确档 nprobe=224： + +| K | WarpFusion QPS / recall | FAISS QPS / recall | WarpFusion/FAISS QPS | batch P99 WarpFusion/FAISS（ms） | +|---:|---:|---:|---:|---:| +| 10 | 3314.1 / 0.9929 | 3049.7 / 0.9930 | 1.087× | 19.961 / 23.000 | +| 50 | 2647.5 / 0.9904 | 6553.5 / 0.9899 | 0.404× | 24.621 / 9.946 | +| 100 | 2192.1 / 0.9890 | 5620.9 / 0.9888 | 0.390× | 30.712 / 11.475 | + +K10 在 nprobe=160 基本持平，在 192/224 分别比 FAISS 快约 5.4%/8.7%； +说明中心选择、warp 协作距离计算和 K10 路径已经具备竞争力。K50/K100 则 +稳定落后约 2.25--2.56 倍，且 P99 同样较高。结合 kernel 分段时间，最终 +probe merge 只占约 0.7--1.4 ms，下一优化目标应是桶内大 K 候选选择，而 +不是中心选择或最终归并。 + +FAISS GPU IVF 的 K10 路径在这组环境中稳定慢于 K50；完整 query-set 预热 +后复测仍然如此，因此保留实测结果，不用推测值修正。CPU 代表点 +IVF K10/nprobe128 为 43.57 QPS、recall=0.8795，反而慢于单线程 Flat 的 +75.92 QPS，故没有继续浪费时间扫描 CPU IVF 的所有 K/nprobe 组合。 + +### 构建、加载与显存口径 + +- FAISS IVF 首次 CPU 建索引约 7.02 s；持久化索引加载约 286.6 ms，GPU + 转移约 414.6 ms。后续基准复用保存的 CPU 索引。 +- FAISS Flat/IVF 的 GPU 索引数据分别约 492.10/497.98 MiB； + `StandardGpuResources` 默认另保留 1536 MiB 临时工作区。 +- WarpFusion IVF 显式缓冲区随 K/nprobe 约 497--513 MiB。这个数不包含 + CUDA context 或第三方库保留区，因此不能直接和 FAISS 的“索引 + 临时 + 工作区”相减后宣称节省比例。 +- 原始数据保存在 `results/million/faiss_gpu_formal_v3.csv`、 + `faiss_cpu_flat.csv`、`faiss_cpu_ivf_rep.csv` 和 + `purplepulse_faiss_compare_k{10,50,100}_v2.csv`。两个 sweep 脚本现在都会 + 保存 run/batch P50/P99、初始化、H2D、QPS、recall 和内存字段，并在每个 + FAISS 配置完成后立即落盘。 + +本阶段结论不是“全面超过 FAISS”：K10 IVF 在高召回区已经可竞争，但大 K +选择以及 Exact 路径仍有明显工程差距。这个结果为下一轮优化给出了清晰、 +可量化的目标。 + +## 2026-09-01：IVF 桶扫描第二轮优化 + +本轮只改变 GPU 桶扫描实现，不改变索引、访问的桶或排序规则。优化包含： + +1. 将 L2、inner-product、cosine 作为模板参数实例化，去掉候选内层循环的 + 运行时 metric 分支。 +2. 每个候选只执行当前 metric 所需的 warp 归约；例如 inner-product 不再 + 对恒为 0 的 L2 累加器执行 5 次 shuffle。 +3. 维度数据不超过 8 KiB 时，将 query 缓存在 block 动态共享内存；拷贝与 + heap 初始化共用原有 barrier。 +4. 新增 2/4/8-warp A/B 路径。`warp_compact` 对 K50/K100 使用 2-warp + block，降低每个 probe block 的 heap 共享内存并允许更多 block 驻留； + K10 保持 8 warps。 + +曾测试在距离维循环加入 `#pragma unroll 4`，K10/K50/K100 分别回退到约 +5684/5732/5049 QPS，因此撤销该改动。保留失败实验可以避免后续将“更多 +展开”误当成必然优化。 + +### 逐步 A/B + +固定 1M×128、1000 queries、inner-product、nlist/nprobe=256/160、 +batch=64。下表使用同一批量口径隔离 kernel 改动： + +| 实现阶段 | K10 QPS | K50 QPS | K100 QPS | +|---|---:|---:|---:| +| 上轮正式基线 | 4771.1 | 3879.4 | 3111.3 | +| compact heap + 删除无效归约 | 5776.0 | 5275.6 | 4637.4 | +| metric 编译期特化 | 6309.8 | 5812.1 | 5101.3 | +| query 共享内存缓存 | 7000.1 | 6441.3 | 5390.5 | + +核心路径相对上轮基线分别提升约 46.7%、66.0%、73.2%。资源检查中 +inner-product/cosine 使用 38/40 个寄存器，stack=0；compact K50/K100 +静态共享内存为 1232/2432 B。旧 8-warp K50/K100 路径需要 4928/9728 B。 + +### 最终质量—吞吐 + +优化后重新扫描 nprobe；K10/K100 使用 batch64，K50 根据新 batch sweep +使用 batch32： + +| K | nprobe | 查询（ms） | QPS | recall@K | 最低 query recall | batch P99（ms） | +|---:|---:|---:|---:|---:|---:|---:| +| 10 | 128 | 114.49 | 8734.3 | 0.8716 | 0.40 | 7.278 | +| 10 | 160 | 142.81 | 7002.2 | 0.9332 | 0.50 | 9.368 | +| 10 | 192 | 172.77 | 5788.1 | 0.9708 | 0.70 | 11.411 | +| 10 | 224 | 199.94 | 5001.5 | 0.9929 | 0.80 | 13.255 | +| 50 | 128 | 124.92 | 8005.3 | 0.8576 | 0.68 | 4.028 | +| 50 | 160 | 149.80 | 6675.4 | 0.9224 | 0.80 | 4.853 | +| 50 | 192 | 192.87 | 5184.9 | 0.9636 | 0.86 | 6.260 | +| 50 | 224 | 215.97 | 4630.2 | 0.9904 | 0.92 | 7.037 | +| 100 | 128 | 145.74 | 6861.7 | 0.8496 | 0.73 | 9.266 | +| 100 | 160 | 186.16 | 5371.8 | 0.9163 | 0.83 | 12.128 | +| 100 | 192 | 226.51 | 4414.9 | 0.9611 | 0.90 | 14.712 | +| 100 | 224 | 266.67 | 3749.9 | 0.9890 | 0.94 | 17.482 | + +推荐配置在 nprobe=160 相对上轮正式基线提升：K10 46.8%、K50 72.1%、 +K100 72.7%。与同口径 FAISS GPU IVF 相比，WarpFusion K10 快约 1.47×； +FAISS K50/K100 分别仍快约 1.31×/1.47×，但差距已从 2.25--2.54× 明显缩小。 + +### 优化后的 batch 选择 + +固定 nprobe=160，吞吐峰值分别是 K10/batch64、K50/batch32、 +K100/batch64： + +| K | 推荐 batch | QPS | batch P50/P99（ms） | +|---:|---:|---:|---:| +| 10 | 64 | 6973.0 | 8.897/9.395 | +| 50 | 32 | 6635.6 | 4.670/4.877 | +| 100 | 64 | 5359.2 | 11.548/12.163 | + +最终 K10/K50/K100 与优化前结果文件逐字节一致。自动测试还覆盖 +FP32/FP16 × L2/IP/cosine × K50/K100 × warp2/4/8/compact，CTest 5/5； +`compute-sanitizer --tool memcheck` 报告 0 errors。原始扫描结果保存在 +`optimized_v1_k10.csv`、`optimized_v1_k50_b32.csv`、 +`optimized_v1_k100.csv` 和 `optimized_v1_batch_sweep.csv`。 + +## 2026-09-02：score-mass 自适应 nprobe + +### 实现 + +GPU 中心选择现在可额外保留排序后的前 `max_nprobe` 个分数。每个 query 将 +中心分数按 top-1 到最后一个候选中心的跨度归一化，再用 softmax 风格质量 +选择达到 `target_mass` 的最短 probe 前缀。策略支持 min/max/step 和无量纲 +temperature；L2 使用负距离作为 utility，IP/cosine 使用原相似度。 + +桶扫描和最终归并都读取每个 query 的实际 nprobe。CLI 会报告 +min/avg/max，并可用 `--probe-counts-output` 保存逐 query 明细；新增 +`benchmark_adaptive_nprobe.py` 可扫描 target mass，逐项落盘结果和 CSV。 + +实现了两种执行方式： + +- `masked`：保持一个最大网格，超过该 query 实际 nprobe 的 block 立即退出。 +- `grouped`：取回每批 probe 计数、按 tier 压紧 query，再为每个 tier 启动 + 精确大小的扫描网格。 + +两条路径的候选 ID、顺序和分数逐值一致。K100、平均 151.632 probes 的正式 +A/B 中，masked 为 183.852 ms / 5439.2 QPS，grouped 为 204.680 ms / +4885.7 QPS；grouped 虽消除了空 block，但多个较小网格降低并行度并增加 +launch 开销，最终慢约 11.3%，因此配置默认使用 masked。 + +### target mass 前沿 + +固定 max=224、min=64、step=16、temperature=0.2。三种 K 得到相同的实际 +probe 分布；下表列出 K10 的完整前沿： + +| target mass | 实际平均 nprobe | QPS | recall@10 | 最低 query recall | +|---:|---:|---:|---:|---:| +| 0.80 | 92.720 | 11748.6 | 0.7753 | 0.30 | +| 0.85 | 110.544 | 9879.3 | 0.8275 | 0.30 | +| 0.90 | 133.648 | 8239.5 | 0.8837 | 0.40 | +| 0.93 | 151.632 | 7233.8 | 0.9201 | 0.50 | +| 0.95 | 166.736 | 6641.8 | 0.9416 | 0.50 | +| 0.97 | 186.096 | 5923.8 | 0.9654 | 0.70 | +| 0.99 | 210.128 | 5306.2 | 0.9860 | 0.80 | + +### 同计算预算公平对照 + +代表点 target_mass=0.93 的平均值为 151.632，与固定 nprobe=152 对照。 +两者使用相同数据、batch、预热 1 次、重复 5 次： + +| K | 策略 | 实际平均 nprobe | QPS | recall@K | batch P99（ms） | +|---:|---|---:|---:|---:|---:| +| 10 | fixed | 152.000 | 7307.7 | 0.92120 | 8.693 | +| 10 | score-mass | 151.632 | 7233.8 | 0.92010 | 8.947 | +| 50 | fixed | 152.000 | 6954.4 | 0.90874 | 4.573 | +| 50 | score-mass | 151.632 | 6773.0 | 0.90742 | 4.775 | +| 100 | fixed | 152.000 | 5642.5 | 0.90179 | 11.583 | +| 100 | score-mass | 151.632 | 5439.2 | 0.89993 | 11.955 | + +按实际 probe 分组分析表明，策略分配更多 probes 的 query 在固定预算下确实 +更难，说明难度信号不是随机的；但相关性太弱，无法提高总体 recall，而且 +变长工作量使 GPU 批处理略慢。结论是：在当前均匀合成查询上不应替换固定 +nprobe。配置、代码和原始结果保留，等待更异质的真实 Agent 查询分布复验。 + +固定路径与此前输出逐字节一致，masked/grouped 自适应输出也逐字节一致； +CTest 6/6、compute-sanitizer memcheck 0 errors。原始数据保存在 +`adaptive_k{10,50,100}_t02.csv`、`adaptive_formal_k{10,50,100}.csv`、 +`adaptive_grouped_formal_k100.csv`、`fixed_k{10,50,100}_matched.csv` 和 +`adaptive_vs_fixed_n152.csv`。 + +## 2026-09-03：GPU 元数据过滤与记忆评分 + +### 实现与正确性 + +新增版本化 `PPMETA01` 元数据文件，按原始向量 ID 保存时间戳、重要性、 +会话 ID 和来源类型。fused 路径在桶扫描中先检查时间、会话和来源过滤条件， +通过后才读取向量，并在局部 Top-K 前加入重要性与指数时间衰减。独立 rerank +路径先执行语义 Top-(K×factor)，再用第二个 GPU kernel 过滤和重排。 + +CPU baseline 与 GPU fused/rerank 已覆盖 FP32/FP16、L2/IP/Cosine；新 GPU +环境使用 CUDA 12.8、计算能力 8.9，CTest 6/6。20k 端到端文件链路中，CPU +与 GPU 两条模式的候选集合相同，最大分数误差为 7.6e-6。正式结果中的 +source_type 过滤违规数为 0；`compute-sanitizer --tool memcheck` 报告 0 errors。 + +### fused 与独立重排 A/B + +固定 100k×128、1000 queries、Cosine、nlist/nprobe=64/64、K=10、 +batch=64、source_type=1、预热 2 次、重复 5 次。两种模式使用相同评分权重； +rerank 从语义 Top-100 开始，fused 直接在完整扫描候选中融合。 + +| 模式 | 查询（ms） | QPS | batch P99（ms） | 扫描（ms） | 重排（ms） | GPU 缓冲区（MiB） | recall@10 vs fused | +|---|---:|---:|---:|---:|---:|---:|---:| +| fused | 20.8149 | 48042.6 | 1.29066 | 18.6603 | 0 | 52.0687 | 1.000 | +| rerank Top-100 | 46.3539 | 21573.2 | 2.96419 | 42.4594 | 0.8408 | 56.3534 | 0.507 | + +fused 吞吐是独立重排的约 2.23×，显式缓冲区少约 4.28 MiB。独立重排的 +最低单 query recall 只有 0.1：语义 Top-100 在过滤和记忆加权之前已经截断， +无法恢复被丢弃的高重要性或近期记忆。因此 fused 同时改善了性能和该评分定义 +下的结果质量。原始结果与汇总位于 `results/memory_100k/`。 diff --git a/09_vector_retrieval/PurplePulse/docs/DEVELOPMENT_PLAN.md b/09_vector_retrieval/PurplePulse/docs/DEVELOPMENT_PLAN.md new file mode 100644 index 00000000..3c236df0 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/docs/DEVELOPMENT_PLAN.md @@ -0,0 +1,252 @@ +# 开发计划 + +## 第一阶段：正确性地基 + +- [x] 定义带版本号的二进制文件格式 +- [x] 实现数据生成、读取和检查 +- [x] 实现 CPU 精确检索参考答案 +- [x] 实现容易理解的 CUDA 距离计算 baseline +- [x] 建立自动正确性与边界测试 +- [x] 在中等规模上复验 L2、内积和 Cosine 的 CPU/GPU 一致性 + +## 第二阶段：精确检索性能 + +- [x] 将 Top-K 从 CPU 移到 GPU +- [x] 实现分块与两阶段 Top-K +- [x] 将局部 Top-K 从有序插入改为无序候选 + 最差项跟踪 +- [x] 为 K=1/10 增加编译期模板特化 +- [x] 实现 FP16 输入、GPU 16 位存储和 FP32 累加 +- [x] 扫描 batch=1/4/8/16/32/64/128 的吞吐与延迟 +- [x] 融合 Exact 距离计算与局部 Top-K，删除完整距离矩阵写回 +- [x] 用 nsys 记录 Top-K 优化前后的 kernel 时间线 +- [x] 完成 ncu 可用性诊断；确认赛事平台宿主权限限制并在报告中记录，不虚构计数器数据 + +## 第三阶段：IVF-Flat + +- [x] 用确定性采样 k-means 训练聚类中心并构建倒排表 +- [x] 使用连续桶布局保存和加载版本化索引 +- [x] 实现 CPU 固定 nprobe 正确性基线 +- [x] 对比 exact search 的 recall@K +- [x] 实现第一版 GPU 常驻索引、候选扫描和 Top-K +- [x] 将中心选择迁到 GPU，并消除候选位置 H2D +- [x] 使用每个 probe 一个 block 提高单 query 候选扫描并行度 +- [x] 对相似度度量使用球面 k-means，修复大 nlist 下的桶倾斜 +- [x] 使用 block 归约并行选择中心，替代单线程有序插入 +- [x] 完成 100k × 128、K=10 的首轮 nlist × nprobe 扫描 +- [x] 使用 warp 协作改善桶内向量读取与距离计算，并保留 scalar A/B 基线 +- [x] 百万规模扫描 nlist=256/512/1024 与对应 nprobe +- [x] 在百万规模扫描 IVF batch=1/8/16/32/64/128 +- [x] 对 K=1/10 完成首轮质量与吞吐对比 +- [x] CLI 报告 exact/IVF 显式 GPU 缓冲区，并提供自动 CSV sweep 脚本 +- [x] 使用 K=10/50/100 模板化 warp heap 扩展 IVF 局部 Top-K +- [x] 使用 probe 有序列表 k-way merge 高效归并 K=50/100 +- [x] 百万规模完成 K50/K100 的 batch 与 nprobe 首轮扫描 +- [x] 将 metric 编译期特化，并删除每候选无效的 warp shuffle 归约 +- [x] 将小于等于 8 KiB 的 query 缓存到 block 共享内存 +- [x] 为大 K 使用 compact 2-warp block，保留 warp2/4/8 A/B 路径 +- [x] 优化后重新扫描 K10/K50/K100 的 batch 与 nprobe +- [x] 实现 GPU bucket-major 任务分组，验证高 nprobe 下的跨 query L2 复用 +- [x] 缓存单 warp heap 根节点，以空穴下沉减少插入与排序的共享内存写入 +- [x] 为单 warp probe 删除多 warp 合并同步并直接写回有序 Top-K +- [x] 评估分布式寄存器 Top-K、8 候选缓冲和 warp bitonic sort；依据 + +0.43%、-22.68% 和 -18.72% 的 A/B 结果拒绝替换稳定实现 + +## 第四阶段：Agent 场景创新 + +- [x] 根据归一化中心分数质量为每个 query 自适应选择 nprobe +- [x] 实现 masked/grouped 两种变长 GPU 调度并完成 A/B +- [x] 与相同平均 probe 的固定 nprobe 完成百万级公平实验 +- [x] 根据负结果保留实验接口但不替换固定策略默认值 +- [x] 在 GPU 候选扫描中融合元数据过滤和记忆评分 +- [x] 与分离重排序方案进行公平实验 + +## 第五阶段：正式实验与报告 + +- [x] 百万向量、128 维、1000 queries 固定数据集 +- [x] 记录 QPS、run/batch P50/P99、显存和 recall@K +- [x] 完成 FAISS CPU/GPU Flat/IVF-Flat 与自研引擎首轮公平对照 +- [x] 补齐 K=50、batch=40 的 FAISS GPU 同批次 100 次正式对照 +- [x] 保存 FAISS 与自研 nprobe 扫描原始 CSV +- [x] 输出 recall-latency、recall-QPS 和 batch-QPS 曲线图 +- [x] 记录赛事平台无法开放 ncu 计数器的原因、影响范围和替代证据 +- [x] 完成最终总结报告与复现说明 + +Synthetic-1M 最终结果中，WarpFusion K10 在 nprobe=160 下比同口径 FAISS +GPU IVF 快 1.20 倍。K50 经 batch=40、bucket-major 和 4×16 分区局部 Top-K +优化后达到 7934 QPS，与 FAISS 的差距缩小到 4.97%；K100 使用 4×40 分区 +局部 Top-K 达到 7205 QPS，差距缩小到 9.64%。Exact fused 在百万规模 +K10/K50/K100 上相对矩阵路径分别加速约 3.13×/7.16×/10.59×，显式 GPU +缓冲区减少约 33%，但仍比 FAISS GPU Flat 慢 3.18--3.71 倍。上述结果已经 +通过正式 A/B、输出一致性检查、CTest 和 Compute Sanitizer 验证。 + +## 第六阶段：真实公开数据验证（已完成） + +目标是验证 WarpFusion 的优化能否从 Synthetic-1M 推广到真实向量分布。 +继续使用当前 RTX 4090，不以更换硬件作为达标条件。 + +- [x] 接入 SIFT1M，并保留公开数据校验和、转换命令和 ground truth。 +- [x] 接入至少一套 384 或 768 维公开文本 embedding 数据。 +- [x] 在两套真实数据上完整评测 K=10/50/100 与 + nprobe=32/64/128/160/192/224。 +- [x] 同时报告 WarpFusion、FAISS GPU IVF-Flat 和 Exact ground truth 的 + QPS、P50/P99、recall、显存与建库时间。 +- [x] 验证分区局部 Top-K 的外部有效性；若产生不可接受的 recall 损失，提供 + 自动回退到完整 heap 的安全路径。 + +验收门槛：真实数据上输出正确，实验可以一条命令复现，并形成至少两张完整的 +recall-QPS 曲线。WarpFusion 在相近 recall 下保持接近 FAISS 的竞争力，而非 +只在 Synthetic-1M 上成立。 + +验收结果（2026-09-10）：通过。SIFT1M 与 384 维 Yahoo MiniLM 的完整矩阵、 +公开 ground truth 交叉验证、FAISS GPU 对照和四张曲线均已落盘。在 +`nprobe=160` 下，WarpFusion 在六组 K/数据集组合中达到 FAISS GPU IVF-Flat +的 1.33--2.53 倍吞吐。K=50 分区路径在两套真实数据上触发 recall 安全回退， +K=100 分区路径保持零 recall 损失并分别加速 1.38 倍和 1.72 倍，证明自动回退 +是必要的，而不是只保留未验证的快速路径。 + +## 第七阶段：Exact fused 性能冲刺（已完成） + +- [x] 分离距离计算、局部选择和最终归并耗时，确定 K=10/50/100 的真实瓶颈。 + 2026-09-10 的 differential profile 显示：Synthetic-1M 的距离阶段占 + 80.72%--98.26%，Yahoo MiniLM 384 占 89.58%--98.73%；最终归并最多 + 1.03%，不是当前主瓶颈。局部 Top-K 使用 `fused - distance-only` 估算， + 不是硬件计数器的独立时间线。 +- [x] 比较当前 heap、分层 Warp/Block Top-K、候选缓冲后归并以及块级 + radix/select 方案。当前 fused 本身采用 8 个 warp 局部 heap + block 归并； + 已实测的 matrix K=10 候选缓冲后归并、K=50/100 block 反复选择路径慢 + 3.13--10.59 倍且多用约 33% buffer。 + 全量 radix/select 若保持精确语义，只能物化 score 或重复距离扫描，违反本阶段 + fused 约束，故在设计审查后不实现；不将其写成 GPU 实测结果。 +- [x] 对不同 K 独立选择实现，不要求一套 kernel 覆盖所有场景。首轮 + query-tiling 结果选择 K=10/128 维 tile4、K=10/384 维 tile8、K=50/384 维 + tile4；K=50/128 维和 K=100 回退原 fused heap。 +- [x] 保持融合路径不写回完整距离矩阵，且候选 ID 与精确参考逐项一致。 + query tile 的全部 smoke 与正式 A/B 均通过逐 query、逐 rank ID/score 校验， + Compute Sanitizer 报告 0 errors。 + +验收门槛：在 Synthetic-1M 和至少一套真实数据上，WarpFusion Exact 相对 +FAISS GPU Flat 的吞吐差距缩小到 2 倍以内；若未达到，必须用可复现 profile +明确剩余瓶颈，并记录被否决方案和原因。 + +阶段进展（2026-09-10）：跨 query 复用数据库读取的 tiled fused kernel 已完成 +实现和三数据集正式验证。Synthetic-1M/SIFT1M K=10 分别提升 37.3%/33.0%； +Yahoo K=10 使用 tile8/160 blocks 提升 88.7%，K=50 使用 tile4 提升 21.1%。 +128 维 K=50 和全部 K=100 路径退化，已记录并回退。优化后相对 FAISS GPU +Flat 的 K=10 差距仍为 Synthetic 2.30 倍、SIFT1M 2.24 倍、Yahoo 3.15 倍， +尚未达到本阶段 2 倍门槛。后续 block 级数据库 tile 在 Synthetic/Yahoo 的 +最佳值分别为 9050/4794 QPS，均低于已接纳的 warp-query 9265/6989 QPS， +因此记录为负实验且未扩展到正式长跑。K 专用选择结构的最终裁决见上方清单与 +复现说明：保留当前分层 heap，不引入需要完整 score 物化或重复扫描的方案。 + +验收结论（2026-09-10）：按计划的失败分支完成。吞吐差距未缩小到 2 倍以内， +但已有可复现 differential profile 明确显示距离/数据库读取占 80.72%--98.73%， +且 cosine 范数预计算与 block-tile 两条候选均保存负结果和原因。选择结构比较 +确认完整候选物化、重复距离扫描和全量 radix/select 不能在保持 fused 约束下 +解决主瓶颈；已接纳的 query-tile 逐 K 回退，默认行为不扩大到退化组合。 + +## 第八阶段：稳定性与适用范围（已完成） + +- [x] 覆盖维度 64/128/384/768，batch 1/8/32/40/64/128，以及 + K=1/10/50/100。 +- [x] 覆盖 FP32/FP16、L2/inner-product/cosine、空桶、倾斜桶、重复向量、 + 相同距离和非整 warp 维度。 +- [x] 所有默认模式通过 CTest、Python 回归测试和 Compute Sanitizer。 +- [x] 给出按 K、维度和 GPU 资源选择实现的规则，并为非推荐组合提供安全回退。 + +验收结果（2026-09-10）：通过。新增独立 `stability_tests`，完成 576 组 Exact +交叉矩阵、6 组满 batch + 尾 batch、6 组 65 维 dtype/metric 组合、3 组已接纳 +query-tile 和 2 组 fail-closed 回退。另构造 8 桶中 4 个空桶、最大桶占 +125/128 的病理 IVF 索引，完成 48 组 warp/bucket-major CPU/GPU 对照。 +远端 CTest 14/14 通过，稳定性矩阵 Compute Sanitizer 报告 0 errors；选择规则 +和边界记录在 `docs/STABILITY_MATRIX.md`，原始摘要位于 +`results/stage8_stability/coverage.csv`。 + +## 第九阶段：工程化与报告定稿（已完成） + +- [x] 提供稳定的构建、建库、查询和 benchmark CLI，减少与本机目录绑定的配置。 +- [x] 提供一条命令完成数据准备、测试、FAISS 对照、绘图和报告数据校验。 +- [x] 将真实数据结果、Exact 新实现、负实验和适用边界写入技术报告。 +- [x] 补充 API 示例、默认配置表、架构图、性能表和已知限制。 +- [x] 发布前在一套干净 Linux + CUDA 环境完成从零构建和最小查询验证。 + +验收门槛：四个技术阶段的代码、实验、原始 CSV、图表、报告和复现说明相互 +对应；课程提交版本与后续开源候选共用同一套可追溯证据。 + +验收结果（2026-09-11）：通过。新增项目根目录无关的 `scripts/reproduce.py`， +串联构建、18 项 CTest、两套公开数据准备、WarpFusion/FAISS 实验、绘图、PDF +和发布校验，并以原子 checkpoint 支持 `--resume`。`scripts/validate_release.py` +校验正式 CSV、Stage 8 覆盖、SVG、核心文档和 PDF；`docs/API.md` 与 README +补齐 CLI/C++ 示例、默认配置、代表性能和限制。RTX 4090 D 的 CUDA 12.8 干净 +构建通过 18/18 CTest，257×65、13 queries 的 Exact/IVF CPU/GPU 最小查询均 +通过候选与分数校验；摘要证据位于 `evidence/2026-09-11/stage9/`。 + +## 第十阶段：开源发布冲刺（条件阶段） + +只有第六至第九阶段全部通过验收后，才启动本阶段。目标是把完成验证的工程 +整理为可公开发布、可独立复现的 GPU 向量检索研究原型，而不是先发布再补证据。 + +- [ ] 完成面向外部使用者的 README、快速开始、贡献指南和变更记录。 +- [ ] 清理提交历史中的凭据、机器地址、临时文件和不可公开数据。 +- [ ] 选择并加入开源许可证，列清第三方依赖及其许可证。 +- [ ] 发布版本必须绑定源码提交、配置、数据校验和和原始 CSV，确保报告数字可追溯。 +- [ ] 建立公开 issue 模板、版本标签和最小持续集成检查。 + +完成第六至第十阶段后，才将仓库标记为 WarpFusion 的首个公开版本候选。 +建议版本号为 `v0.1.0`，定位为“单卡、内存内、Exact 与 IVF-Flat GPU 向量 +检索研究原型”，不宣称生产级服务能力。 + +## 第十一阶段：Agent 应用方向（暂停，不纳入当前提交） + +> 本节以下内容属于 Agent 应用方向，不改变项目主体。本项目仍是一套 +> 可独立构建、运行和评测的 C++/CUDA GPU 向量检索引擎；Agent 是用于验证 +> 元数据融合检索价值的真实工作负载，Pi 只是首个轻量客户端。 + +### 11.1 范围与边界 + +- GPU 检索核心继续负责 Exact、IVF-Flat、GPU Top-K、常驻索引、批量 + 查询、元数据前置过滤以及语义/时间/重要性融合评分。 +- Agent 侧只负责提取记忆、生成 query、调用检索接口和组装上下文，不在 + 检索引擎内嵌 LLM，也不让 Agent 框架接管索引与排序逻辑。 +- 第一版采用 session 结束后的批量摄取和下一 session 的在线检索；暂不实现 + 实时增量索引、分布式服务、多 Agent 编排或 Web UI。 +- 接口保持框架无关，Pi adapter 放在 `clients/pi/`，后续可以增加其他 Agent + 客户端而无需修改 CUDA 核心。 + +### 11.2 最小可交付链路 + +- [ ] 定义 Agent Memory JSONL 格式，包含 content、memory_type、project_id、 + session_id、timestamp、importance 和 source_type。 +- [ ] 实现 session ingestion，将事实、经历、成功步骤和失败教训映射到文本、 + embedding 与现有项目 metadata 文件。 +- [ ] 增加 embedding provider 抽象；演示实现可替换，不把具体模型写死在 + 检索引擎中。 +- [ ] 保存 `vector_id -> content/provenance` 映射，使 GPU Top-K ID 能恢复成 + 可注入 Agent 上下文的记忆文本。 +- [ ] 提供最小常驻查询接口，复用已加载的 IVF 索引和 GPU 工作区，避免每个 + Agent tool call 都重新初始化 GPU。 +- [ ] 实现框架无关的 `memory_search` 与 `memory_record` 客户端协议。 +- [ ] 实现 Pi adapter：`memory_search` 工具、session 结束记忆记录和带来源的 + context builder。 +- [ ] 建立双 session CUDA 开发演示：后一 session 在空上下文下召回前一 + session 的成功方案或失败教训。 + +### 11.3 评测与验收 + +- [ ] 对比 No Memory、纯语义检索和本项目 fused memory 三组方案。 +- [ ] 记录记忆 Recall@K/MRR、错误重复率、任务成功率和注入 token 数。 +- [ ] 记录检索 QPS、P50/P99、GPU kernel 时间、显存以及错误 project/session + 的过滤泄漏率。 +- [ ] 验证 Agent adapter 关闭后，原有 CTest、百万规模 benchmark 和 CLI + 行为完全不变。 +- [ ] 在 README 中先展示 GPU 引擎架构与性能，再将 Agent Memory 作为应用 + 案例；Agent 内容不覆盖或替代 Exact/IVF/FAISS 主线。 + +### 11.4 本阶段明确不做 + +- MiniDeepSeek 或其他小模型训练。 +- MoE、GRPO、DPO 或 learned retrieval policy。 +- Base/Delta 在线索引、后台 compaction 和分布式多租户调度。 +- 完整 RAG、复杂网页、多个 Agent 框架同时适配。 + +完成上述最小链路后再根据时间评估在线增量索引；任何扩展不得降低 +GPU 检索核心测试、性能复现和代码可解释性。 diff --git a/09_vector_retrieval/PurplePulse/docs/FINAL_REPORT.md b/09_vector_retrieval/PurplePulse/docs/FINAL_REPORT.md new file mode 100644 index 00000000..d3adc140 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/docs/FINAL_REPORT.md @@ -0,0 +1,286 @@ +# WarpFusion：面向大规模向量检索的 CUDA 并行设计与性能优化 + +**作者：花生　　学员 ID：PurplePulse** + +## 摘要 + +面向百万级高维向量的近邻检索，系统需要同时承担距离计算、候选筛选、索引访问和批量调度等开销。仅优化单个距离计算核函数，通常难以显著改善完整检索流程的吞吐和延迟。本文设计并实现了单 GPU 向量检索系统 WarpFusion。系统提供精确检索（Exact Search）和基于倒排文件的 IVF-Flat 近似检索两条路径，支持 FP32 与 FP16 数据，以及 L2 距离、内积和余弦相似度三种度量方式。 + +在精确检索中，本文将距离计算与局部 Top-K 选择合并执行，避免生成规模为查询数乘以数据库向量数的完整距离矩阵。在 IVF-Flat 路径中，针对不同 K 值下的候选维护开销，设计了按桶任务调度和分区局部 Top-K 两项优化，并以结果一致性和召回率阈值决定是否启用优化。系统还配套实现了索引持久化、CPU 参考计算、异常输入检查、稳定性测试和实验复现流程，用于验证功能正确性和性能结果。 + +在 NVIDIA GeForce RTX 4090 D 上，Exact 融合路径相较距离矩阵基线的加速比为 3.13× 至 10.59×，并将显式 GPU 缓冲区减少约三分之一。在 SIFT1M 与 Yahoo MiniLM 两个公开数据集上，当训练样本数、聚类轮数、批大小和 `nprobe` 保持一致时，WarpFusion IVF-Flat 在六组 K 值实验中的吞吐为 FAISS GPU IVF-Flat 的 1.33× 至 2.53×。除 Yahoo MiniLM 的 K=10 配置外，其余五组实验的召回率均不低于 FAISS；在该例外配置中，WarpFusion 的 Recall@10 低 0.14 个百分点。实验还表明，分区候选结构在 K=50 和 K=100 时具有不同的适用性，因此候选容量需要结合真实数据分布验证，不能仅依据合成数据确定。 + +**关键词：** GPU 向量检索；CUDA；IVF-Flat；Top-K 选择；并行调度；近似最近邻 + +## 1　研究背景与目标 + +向量检索广泛应用于图像匹配、语义搜索、推荐系统和检索增强生成等任务。给定查询向量，系统需要从数据库中返回距离最近或相似度最高的 K 个向量。随着数据库规模和向量维度增加，检索过程不仅包含大规模距离计算，还需要持续维护候选集合并完成跨线程结果归并。对于百万规模数据，系统瓶颈往往由显存访问、Top-K 更新和任务调度共同决定。 + +精确检索需要比较查询与数据库中的全部向量。若查询数为 Q、数据库规模为 N、向量维度为 D，其计算复杂度约为 O(QND)。朴素 GPU 实现通常先生成大小为 Q×N 的距离矩阵，再执行 Top-K 选择。该方法结构清晰，适合作为正确性与性能基线，但会产生大量中间结果读写。IVF-Flat 则通过聚类将数据库划分为多个倒排桶，查询时只扫描与查询最相关的部分桶，从而以一定的召回率损失换取更高吞吐。FAISS 等成熟向量检索系统也采用了这类 GPU 并行检索方法[1]。 + +本文的目标不是构建完整的在线向量数据库，而是研究单机单卡、数据常驻显存条件下的 CUDA 向量检索实现与优化。具体目标包括： + +1. 实现从数据读取、索引构建、GPU 查询到结果输出的完整检索流程； +2. 支持 Exact 与 IVF-Flat 两种检索方式、FP32 与 FP16 两种数据类型、三种距离度量，并支持 K 分别取 1、10、50 和 100； +3. 对距离计算、候选筛选和任务调度进行协同优化，而非仅比较孤立核函数； +4. 通过 CPU 参考计算、公开数据的标准近邻集合和 FAISS 对照验证正确性与性能； +5. 保留原始实验数据、未采用方案和复现脚本，明确结论的适用范围。 + +本文的主要工作可概括为三点。第一，构建了可执行、可测试的单 GPU 检索系统，并统一了不同计算路径的排序规则和错误处理方式。第二，针对完整距离矩阵和大 K 候选维护的开销，分别实现融合 Exact、按桶调度和分区局部 Top-K。第三，在合成数据与三个真实数据集上进行端到端评估，并结合召回率、延迟、显存占用和稳定性分析性能收益。 + +## 2　系统架构与检索流程 + +![WarpFusion 系统架构](../results/figures/report-architecture.svg) + +**图 1.** WarpFusion 的系统组成与数据流。CPU 端负责输入校验、索引构建与参考计算，GPU 端执行主要检索流程，最终输出查询结果及性能、正确性记录。 + +### 2.1　数据组织与接口语义 + +向量数据库和查询文件采用带版本号的二进制格式。文件头记录数据形状、数值类型和距离度量；加载阶段检查文件标识、版本、维度及数据长度，避免程序在未报错的情况下接受截断文件或类型不匹配的数据。索引文件保存聚类中心、桶边界、原始向量 ID 以及按桶连续排列的向量。训练和分桶完成后，系统可以直接加载索引，无需在每次查询前重新聚类。 + +系统支持 L2 距离、内积和余弦相似度。L2 距离采用普通 k-means 训练中心；内积和余弦相似度采用中心归一化的球面 k-means，以减轻中心模长差异造成的桶规模失衡。不同计算路径均遵循统一排序规则：首先比较距离或相似度分数，分数相同时优先返回较小的原始向量 ID。该规则用于减少并行执行顺序不同造成的结果歧义。 + +### 2.2　Exact 检索路径 + +距离矩阵基线包含两个主要阶段：首先计算全部 Q×N 个距离或相似度分数并写入全局显存；随后逐查询读取矩阵并选择 Top-K。虽然该实现便于验证，但中间矩阵的写回与再次读取会增加显存流量，其空间开销也随 Q×N 线性增长。 + +WarpFusion 的融合 Exact 路径将距离计算与局部候选筛选合并。一个 CUDA 线程束（warp）负责扫描数据库中的连续区域，距离计算完成后立即更新线程束内部的局部 Top-K 缓冲区，仅将少量候选写入全局显存。随后由第二个规模较小的 GPU 核函数归并各局部结果。该方法没有改变精确检索的计算范围，但将中间数据量从完整距离矩阵缩减为与局部候选数量相关的缓冲区，因而主要收益来自减少全局显存读写。 + +### 2.3　IVF-Flat 检索路径 + +IVF-Flat 的建库阶段将数据库划分为 `nlist` 个聚类桶。查询阶段首先计算查询与全部聚类中心的分数，再选择最相关的 `nprobe` 个桶，仅扫描这些桶中的原始向量。与乘积量化方法[2]不同，IVF-Flat 不压缩桶内向量，因此已扫描候选的距离仍按原始精度计算；它的近似性来自只扫描部分倒排桶，而非扫描整个数据库。 + +单批查询的 GPU 流程包括以下五个阶段： + +1. 计算查询向量与聚类中心的距离或相似度； +2. 选择每个查询对应的 `nprobe` 个倒排桶； +3. 将“查询-桶”组合转换为可并行执行的扫描任务； +4. 扫描桶内向量并维护局部 Top-K 候选； +5. 合并同一查询的局部候选，输出最终 ID 与分数。 + +数据库向量、聚类中心和主要工作区在查询期间常驻显存。每批查询只传输查询向量与最终结果，因此稳定查询性能不包含数据库重复上传成本。索引加载和数据库上传作为一次性开销单独记录。 + +### 2.4　评价指标 + +本文使用以下指标评价正确性、性能与资源占用。 + +| 指标 | 定义与用途 | +|---|---| +| QPS | 每秒完成的查询数量，用于评价系统吞吐 | +| Recall@K | 近似 Top-K 与精确 Top-K 的平均重合比例，用于评价检索精度 | +| P50 / P99 | 延迟分布的第 50 / 99 百分位数；本文分别对整轮延迟和批次延迟进行统计 | +| 显式 GPU 缓冲区 | 程序可直接统计的索引与工作区内存，不包含驱动和运行时内部保留 | +| nlist | 建库阶段的聚类中心数量，决定倒排索引粒度 | +| nprobe | 单次查询扫描的桶数量，通常越大召回率越高、计算量也越大 | + +## 3　CUDA 并行设计与优化 + +### 3.1　面向不同 K 值的候选维护 + +候选维护开销与 K 密切相关。K 较小时，局部 Top-K 缓冲区较小，可以为同一桶分配更多线程束以提高扫描并行度；K 增大后，复制多份候选缓冲区会增加共享内存占用、候选更新和最终归并成本。WarpFusion 因此按照 K 的范围调整每个桶的线程束数量，并对 K=50 与 K=100 分别评估更细的候选结构。 + +| K 范围 | 基础并行方式 | 设计依据 | +|---:|---|---| +| 1-10 | 每个桶 8 个局部候选缓冲区 | 候选集合较小，可通过增加扫描并行度提高吞吐 | +| 11-50 | 每个桶 2 个局部候选缓冲区 | 在扫描并行度与归并开销之间折中 | +| 51-100 | 每个桶 1 个完整候选缓冲区 | 限制大候选集合的复制与维护成本 | +| K=50/100 优化 | 多个容量受限的分区候选缓冲区 | 降低单个大候选结构的更新成本 | + +该策略的核心不是预设某种数据结构一定更优，而是针对不同 K 值控制局部状态规模和并行任务数量。所有候选结构均以端到端查询时间和最终输出正确性为验收依据。 + +### 3.2　分区局部 Top-K + +当 K 较大时，单个完整候选缓冲区的更新与排序开销逐渐增加。分区局部 Top-K 将同一扫描任务分配给多个线程束，每个线程束只保留一定数量的局部候选，最后再归并全部局部结果。下文以“4×40”表示 4 个线程束各保留 40 项候选；例如 K=100 时，系统最终从 160 个中间候选中选出 100 项。类似地，“4×16”表示 4 个线程束各保留 16 项候选。 + +该方法以较小的局部状态换取更高并行度，但局部容量不足可能在归并前丢弃真实 Top-K 元素。因此本文不将吞吐提升作为唯一启用条件，而是同时运行完整候选路径进行对照。若任一 `nprobe` 配置的召回率损失超过 1 × 10^(-4)，该数据集的正式结果自动回退到完整候选路径。这一机制使优化策略与正确性约束保持一致。 + +### 3.3　按桶任务调度 + +常规任务顺序以查询为主：完成一个查询的多个桶任务后，再处理下一个查询。该顺序实现简单，但来自不同查询、访问同一倒排桶的任务在时间上可能相距较远。WarpFusion 增加按桶调度方式，对一个批次内的“查询-桶”任务重新排序，使访问相同或相邻桶的任务更集中地提交给 GPU。 + +任务重排不减少距离计算次数，也不改变候选集合及最终输出。其潜在收益来自相邻任务对相同桶向量的重复访问，从而提高缓存数据被再次使用的机会。由于当前实验环境无法获取 Nsight Compute 的缓存命中率等硬件计数器，本文将缓存复用作为基于访问模式和计时结果的机制解释，而不将其表述为已经直接测量的硬件结论。 + +### 3.4　多线程索引构建 + +IVF 建库主要包含训练样本到中心的分配，以及全部数据库向量到倒排桶的分配。两部分均采用固定连续区间的 CPU 多线程并行方式：各线程维护局部统计，完成后再按固定次序合并。与共享计数器上的细粒度同步相比，该设计减少了锁竞争，同时保持建库结果和桶内布局的确定性。 + +## 4　实验设计 + +### 4.1　实验平台与数据集 + +正式 GPU 实验运行在 NVIDIA GeForce RTX 4090 D 上，显存容量为 24,564 MiB，CUDA 计算能力为 8.9，CUDA 工具链版本为 12.8.61。CPU 平台为双路 Intel Xeon Platinum 8358P，共 64 个物理核心。程序采用 Release 模式，并针对当前 GPU 架构编译。 + +实验使用一组合成数据和三个真实数据集，以覆盖参数分析、公开基准、文本向量和非均匀数据分布。 + +| 数据集 | 数据规模与维度 | 查询数 | 距离度量 | 主要用途 | +|---|---:|---:|---|---| +| Synthetic-1M | 1,000,000 × 128 | 1,000 | 内积 | 参数扫描与优化消融 | +| MNIST | 60,000 × 784 | 10,000 | L2 | 非均匀真实分布检查 | +| SIFT1M | 1,000,000 × 128 | 10,000 | L2 | 公开标准近邻集合与 FAISS 对照 | +| Yahoo MiniLM | 677,305 × 384 | 1,000 | 余弦 | 文本嵌入数据与 FAISS 对照 | + +SIFT1M 和 Yahoo MiniLM 均提供预先计算的 Top-100 标准近邻集合（ground truth）[3]。数据准备阶段检查公开文件的校验和，并记录格式转换后文件的 SHA-256，以降低数据版本差异对结果的影响。 + +### 4.2　对照设置与计时方法 + +WarpFusion 与 FAISS 使用相同的数据集、K、批大小、`nlist`、`nprobe`、训练样本数和训练轮数。两套系统分别训练聚类中心，因此本文比较的是相同训练与搜索预算下的端到端系统表现，而不是共享同一索引后单个 CUDA 核函数的速度。 + +不同实验采用与其目的相匹配的重复设置。图 2 的距离矩阵与融合路径成对 A/B 实验预热 1 次并重复 5 次；Synthetic-1M 后续正式参数对照预热 5 次并重复 100 次；公开数据实验预热 1 次并重复 5 次。QPS 根据查询总数和整轮平均时间计算；整轮延迟和批次延迟均分别统计 P50 与 P99。索引加载和数据库上传不计入稳定查询 QPS，但相关耗时和显存占用仍保留在原始结果中。进行优化前后对照时，除被考察因素外，其余输入和运行参数均保持一致。 + +### 4.3　正确性与优化验收标准 + +Exact 路径逐查询比较 Top-K 向量 ID 和分数。IVF 路径报告 Recall@K、最低单查询召回率和分数误差。对于距离完全相同但 ID 不同的候选，系统重新计算原始分数，以区分数值或排序错误与等价近邻的不同选择。 + +任何优化在出现以下情况之一时均不进入默认路径：Exact 输出集合不一致；分数误差超过预设容差；IVF 召回率损失超过设定阈值；或 Compute Sanitizer 检测到越界访问。该验收规则用于避免以正确性退化换取表面性能提升。 + +## 5　实验结果与分析 + +### 5.1　正确性与稳定性 + +发布验证在全新构建目录中完成 CTest 18/18。稳定性测试覆盖 FP32/FP16、L2/内积/余弦、64/128/384/768 维、批大小 1/8/32/40/64/128，以及 K=1/10/50/100。边界场景包括 65 维非对齐输入、查询数不足一个完整批次、空桶、倾斜桶、重复向量和全零查询。 + +完整测试矩阵包含 576 组 Exact CPU/GPU 对照和 48 组 IVF 极端桶分布对照。`compute-sanitizer --tool memcheck` 对稳定性测试报告 0 个错误。在发布前的最小 GPU 查询测试中，Exact 与全桶 IVF 均与 CPU 候选集合一致，平均绝对分数误差为 3.04 × 10^(-6)，最大分数误差为 9.6 × 10^(-6)。这些结果表明，后续性能比较所采用的实现已经通过边界条件和内存安全检查。 + +### 5.2　Exact 融合路径 + +![Exact 融合路径吞吐](../results/figures/report-exact-fused.svg) + +**图 2.** Synthetic-1M 上距离矩阵基线与融合 Exact 路径的吞吐对比。随着 K 增大，距离矩阵路径的二次读取与 Top-K 选择开销更为显著。 + +| K | 距离矩阵 QPS | 融合路径 QPS | 加速比 | 显式 GPU 缓冲区（基线 / 融合） | +|---:|---:|---:|---:|---:| +| 10 | 1,775 | 5,564 | 3.13× | 734 / 488 MiB | +| 50 | 716 | 5,123 | 7.16× | 732 / 489 MiB | +| 100 | 444 | 4,701 | 10.59× | 733 / 490 MiB | + +两条路径返回的 Top-K 候选集合完全一致，记录的最大分数误差为 0。融合路径的吞吐优势随 K 增大而扩大：K=10 时加速比为 3.13×，K=100 时达到 10.59×。这一趋势说明，基线的主要额外成本不仅是距离矩阵写回，还包括随后针对更大 K 执行的矩阵读取和候选选择。融合实现同时将显式 GPU 缓冲区从约 733 MiB 降至约 490 MiB，减少约三分之一。 + +为验证 GPU 实现相对 CPU 精确检索的实际收益，本文在相同的 Synthetic-1M、FP32、内积和批大小 64 条件下，将 WarpFusion Exact 与单线程 FAISS CPU Flat 进行对照。GPU 结果预热 5 次并重复 100 次，CPU 结果重复 20 次；两者均执行完整精确检索，不涉及召回率损失。 + +| K | FAISS CPU Flat QPS | WarpFusion Exact QPS | GPU / CPU 加速比 | +|---:|---:|---:|---:| +| 10 | 76.45 | 6,678 | 87.35× | +| 50 | 76.45 | 5,974 | 78.15× | +| 100 | 73.55 | 5,406 | 73.51× | + +该对照表明，WarpFusion Exact 在百万规模数据上相对单线程 CPU 基线取得 73.51× 至 87.35× 的吞吐提升，满足主要计算由 GPU 加速的项目目标。表中数据来自独立的正式重复实验，因此不与图 2 中专门用于比较两条 GPU 路径的成对 A/B 数据混用。 + +上述结果证明融合设计优于项目内部基线，但不能据此得出 Exact 已超过成熟库的结论。在 SIFT1M 上，WarpFusion Exact 仍比 FAISS GPU Flat 慢 2.94 至 3.51 倍；在 384 维 Yahoo MiniLM 上慢 5.88 至 6.25 倍。为进一步定位内部耗时，本文拆分了融合路径的 GPU 执行阶段。 + +![Exact 耗时构成](../results/figures/report-exact-breakdown.svg) + +**图 3.** Exact 融合路径的 GPU 执行时间分解。距离计算及数据库读取占总核函数时间的 80.7% 至 98.7%，最终候选归并占比较低。 + +在 128 维、K=100 的配置中，局部候选维护约占 18.3%，说明大 K 已使 Top-K 更新成为不可忽略的成本；在 384 维数据上，距离计算和数据库读取仍占绝对主导。由此可见，仅继续缩短最终归并阶段难以显著提高整体吞吐。更有价值的方向包括跨查询复用数据库读取，以及为 K=50/100 设计更低开销的候选选择结构。 + +Nsight Systems 2024.6.2 的代表性采样进一步验证了这一判断。在 Synthetic-1M、K=10、批大小 64 的运行中，融合距离计算与局部 Top-K 核函数占 GPU 核函数总时间约 99.9%，最终归并核函数约占 0.1%。程序内部阶段计时得到的结果与之相符：融合核函数平均耗时 144.49 ms，最终归并平均耗时 0.21 ms。由此可以确认，当前 Exact 路径的主要优化对象是数据库读取、距离计算和局部候选维护，而不是最终归并。该分析使用 Nsight Systems 的时间线和核函数统计；由于平台未开放性能计数器，本文不据此推断缓存命中率或实际显存带宽。 + +### 5.3　K=50 条件下的按桶调度 + +![K=50 按桶调度](../results/figures/report-k50-scheduling.svg) + +**图 4.** Synthetic-1M、批大小 40、K=50 时，按查询顺序与按桶顺序调度的吞吐对比。随着 `nprobe` 增加，按桶调度的相对收益逐步扩大。 + +在 `nprobe=160` 的 100 次正式对照中，原始任务顺序达到 6,009 QPS，按桶调度达到 6,964 QPS，提升 15.89%；整轮 P99 从 169.04 ms 降至 145.53 ms，且输出逐项一致。该结果说明，在计算量和候选集合不变的条件下，改变任务访问顺序即可改善吞吐与尾部延迟。 + +在按桶调度基础上采用 4×16 分区候选后，吞吐进一步提高至 7,934 QPS，相较完整候选路径提高 10.73%。同为批大小 40、`nprobe=160` 时，FAISS GPU IVF-Flat 达到 8,328 QPS、Recall@50 为 0.92312；WarpFusion 达到 7,934 QPS、Recall@50 为 0.92482。以 WarpFusion 为基准，FAISS 的吞吐高 4.97%。 + +但 4×16 分区候选在两套公开数据上分别产生 0.000326 和 0.000980 的召回率损失，即 0.0326 和 0.0980 个百分点，均超过 1 × 10^(-4) 的验收阈值。因此，公开数据的 K=50 正式结果使用完整候选路径。该结果表明，合成数据上的输出一致不能替代真实数据上的召回率验证，局部候选容量需要随数据分布重新评估。 + +### 5.4　K=100 条件下的分区候选结构 + +![K=100 分区候选](../results/figures/report-k100-partitioned.svg) + +**图 5.** SIFT1M 上 K=100 的完整候选路径与 4×40 分区候选路径。两条路径在全部测试 `nprobe` 点上取得相同 Recall@100。 + +与 K=50 不同，K=100 的 4×40 分区候选在两套公开数据上均通过召回率检查。在 `nprobe=160` 时，SIFT1M 的吞吐从 12,591 QPS 提高到 17,362 QPS，加速比为 1.38×；Yahoo MiniLM 从 8,261 QPS 提高到 14,196 QPS，加速比为 1.72×。两套数据的全部六个 `nprobe` 点均未观测到召回率下降，因此 K=100 的正式路径保留分区候选优化。 + +K=50 与 K=100 使用相似的分区思路，却在公开数据上表现出不同的可靠性。这说明分区数量和局部容量之间不存在仅由 K 决定的通用配置；候选分布、桶规模和数据相似度结构都会影响局部截断风险。因此，每个数据集和参数点都需要用完整候选路径复核，不能将合成数据上的最优配置直接视为通用配置。 + +### 5.5　公开数据集上的 FAISS 对照 + +![公开数据 Recall-QPS 权衡](../results/figures/report-recall-qps.svg) + +**图 6.** SIFT1M 与 Yahoo MiniLM 在 K=10 时的 Recall-QPS 曲线。每条曲线包含 `nprobe=32/64/128/160/192/224` 六个实测点，展示增加搜索桶数量后召回率提高、吞吐下降的完整变化过程。 + +在 SIFT1M 的六个 `nprobe` 点上，WarpFusion 的 Recall@10 均略高于 FAISS，吞吐比从 1.05× 增加到 1.95×。在 Yahoo MiniLM 上，`nprobe=32` 时两者召回率几乎相同，WarpFusion 的吞吐比为 1.21×；随着 `nprobe` 增加到 224，吞吐比提高到 2.70×，但 Recall@10 低 0.05 个百分点。由于两套系统分别训练聚类中心，曲线不能视为共享索引条件下的核函数对照，但它表明下表所选的 `nprobe=160` 并非孤立的有利参数点。 + +![公开数据对比](../results/figures/report-real-comparison.svg) + +**图 7.** SIFT1M 与 Yahoo MiniLM 在 `nprobe=160`、批大小 64 条件下的端到端吞吐。柱顶给出 WarpFusion 相对 FAISS 的吞吐比，柱下给出两者 Recall@K。 + +| 数据集 | K | WarpFusion Recall / QPS | FAISS GPU Recall / QPS | 吞吐比 | +|---|---:|---:|---:|---:| +| SIFT1M | 10 | 0.999870 / 24,579 | 0.999120 / 13,617 | 1.81× | +| SIFT1M | 50 | 0.999706 / 19,237 | 0.999516 / 13,443 | 1.43× | +| SIFT1M | 100 | 0.999565 / 17,362 | 0.999457 / 13,102 | 1.33× | +| Yahoo MiniLM | 10 | 0.993900 / 15,733 | 0.995300 / 6,216 | 2.53× | +| Yahoo MiniLM | 50 | 0.991520 / 14,950 | 0.991280 / 6,157 | 2.43× | +| Yahoo MiniLM | 100 | 0.989230 / 14,196 | 0.989080 / 6,060 | 2.34× | + +在 SIFT1M 上，WarpFusion 的三组 Recall@K 均略高于 FAISS，对应吞吐比为 1.33× 至 1.81×。在 Yahoo MiniLM 上，K=50 和 K=100 的召回率与 FAISS 接近，吞吐比分别为 2.43× 和 2.34×；K=10 的吞吐比为 2.53×，但 Recall@10 低 0.0014，即低 0.14 个百分点。因而 Yahoo K=10 的结果应理解为较高吞吐下的轻微精度差异，而不能只报告速度优势。 + +两套系统分别训练聚类中心，因此倒排桶及候选集合并不完全相同。这组对照反映的是在相同训练参数和搜索预算下，完整 IVF-Flat 流程的系统表现，而不是单个 GPU 核函数之间的直接速度比较。公开标准近邻集合与 WarpFusion Exact 结果之间存在极少数分数相同但 ID 不同的情况；重新计算原始分数后，两套数据按分数判断的等价查询比例均为 1.0。SIFT1M 的名次和分数误差均为 0，Yahoo 的最大分数误差为 2.98 × 10^(-7)，说明召回率结果没有因 Exact 参考结果错误而被系统性抬高。 + +### 5.6　延迟、显存与索引构建 + +在 Synthetic-1M 的 IVF-Flat 推荐配置上，K=10/50/100 分别达到 5,697、7,934 和 7,205 QPS，整轮 P99 分别为 176.8、127.8 和 140.6 ms，显式 GPU 缓冲区约为 497 至 508 MiB。K=50 的吞吐高于 K=10 并不表示更大的 K 天然更快，而是因为不同 K 使用了不同批大小、并行配置和候选路径，结果应在各自推荐配置内解释。 + +公开数据 `nprobe=160` 时，WarpFusion IVF 的显式 GPU 缓冲区在 SIFT1M 上为 498 至 509 MiB，在 Yahoo MiniLM 上为 1,000 至 1,011 MiB。FAISS 分别报告 504 MiB 和 1,022 MiB 的索引显存，并额外预留 1,536 MiB 临时显存。由于两套系统的内存统计口径并非完全相同，本文仅将其作为资源规模参考，不据此宣称确定的显存优势。 + +Synthetic-1M 的索引构建时间从单线程 67.53 秒降至 16 线程 6.48 秒，加速 10.43 倍。继续增加到 32 或 64 线程后性能反而下降，表明该双路 CPU 平台已受到同步、归并和 NUMA 访问开销影响。因此默认线程上限设为 16，而不是简单使用全部逻辑核心。 + +### 5.7　MNIST 补充实验 + +MNIST 的规模小于两套百万级数据，但其分布与均匀随机向量存在明显差异。Exact K=10 达到约 23,100 QPS；IVF 在 `nprobe=8` 时取得 Recall@10=0.9784 和约 197,700 QPS，在 `nprobe=32` 时取得 Recall@10=0.9991 和约 76,600 QPS。该实验一方面说明实现并非只在均匀合成数据上有效，另一方面展示了 IVF 的基本精度-性能权衡：增加搜索桶数量可以提高召回率，但会增加扫描工作量并降低吞吐。 + +## 6　未采用方案及原因分析 + +为了避免仅保留正向结果，本文同时记录未进入默认实现的优化方案。 + +1. **预计算余弦范数。** 该方案在 Yahoo K=100 上提高约 2.5%，但在其他配置中下降 1% 至 2.6%。收益缺乏一致性，因此未作为默认路径。 +2. **线程块级数据库分块。** 该实现优于项目早期基线，但仍比最终采用的跨查询线程束方案慢 2.3% 至 31.4%。其额外同步与数据搬运未能抵消局部复用收益，故仅保留为实验入口。 +3. **自适应 `nprobe`。** 在当前 Synthetic-1M 上，该方案相较相同平均搜索桶数的固定策略慢 1.0% 至 3.6%，召回率也略低。由于合成查询的难度差异有限，自适应决策成本没有转化为有效收益。 + +这些负向结果进一步明确了潜在优化方向。Exact 路径的首要问题仍是数据库读取与高维距离计算；大 K 场景需要更高效且具有安全容量约束的候选结构；自适应搜索则更适合在查询难度差异显著的数据集上重新评估。 + +## 7　适用范围、局限与复现 + +### 7.1　结论适用范围 + +本文实验结论受以下条件约束： + +1. 正式 GPU 数据来自单张 RTX 4090 D，不能直接外推至 T4、A100、H100、Blackwell 或其他架构； +2. 系统面向单进程、单 GPU、数据常驻显存的静态 IVF-Flat，不包含 PQ/SQ 压缩、动态增删、超显存分块和多 GPU 执行； +3. Exact 融合路径虽然显著优于内部距离矩阵基线，但仍明显落后 FAISS GPU Flat；当前性能优势主要出现在本文测试的 IVF-Flat 配置中； +4. IVF 参数与分区候选容量依赖数据分布、维度和硬件，更换平台或数据后需要重新扫描并执行召回率回退检查。 + +### 7.2　硬件分析的限制 + +当前实验环境无法读取 Nsight Compute 的硬件计数器，因此本文未直接报告 GPU 占用率（occupancy）、实际显存带宽、缓存命中率和线程停顿原因。关于按桶调度改善缓存复用、Exact 主要受数据库读取限制等判断，来自对访问模式、阶段计时和性能趋势的综合分析，尚未得到硬件计数器的直接验证。该限制不影响吞吐和正确性结果本身，但使本文暂时无法对底层原因作进一步的定量归因。 + +### 7.3　参数迁移要求 + +IVF 的最优 `nlist`、`nprobe`、批大小和候选分区容量与数据分布、向量维度及 GPU 架构相关。本文给出的推荐配置仅代表 RTX 4090 D 和当前数据集上的测量结果。迁移至其他数据或硬件时，应重新执行吞吐-召回率扫描，并以完整候选路径复核分区候选结构，避免直接沿用参数导致精度或性能退化。 + +### 7.4　实验复现与材料组织 + +`scripts/reproduce.py` 将全新构建、CTest、公开数据准备、WarpFusion 与 FAISS 基准测试、绘图、PDF 生成和发布校验组织为连续流程。各阶段分别保存日志，并通过检查点（checkpoint）支持中断后继续执行。发布校验会检查公开数据 CSV 字段、稳定性矩阵、SVG 图表和最终 PDF。 + +仓库保留原始 CSV、运行环境快照、数据校验和、标准近邻集合检查结果以及 Compute Sanitizer 日志。公开数据的完整实验命令见 `docs/REPRODUCIBILITY.md`，命令行参数和错误处理规则见 `docs/API.md`。这些材料用于支持结果核验和实验复现，是保证实验结论可信的重要依据。 + +## 8　结论与可扩展方向 + +本文完成了 WarpFusion 单 GPU 向量检索系统的设计、实现与评估。系统覆盖数据格式、索引构建、CPU 参考计算、GPU Exact、GPU IVF-Flat、结果验证和性能记录，重点研究距离计算、候选维护和任务调度之间的协同优化。 + +实验结果表明，融合距离计算与局部 Top-K 可以避免完整距离矩阵的中间读写，使 Exact 路径相较内部基线的加速比达到 3.13× 至 10.59×，相对单线程 FAISS CPU Flat 的加速比达到 73.51× 至 87.35×。K=50 时，按桶任务调度使吞吐提高 15.89%；分区候选在合成数据上又提高 10.73%，但未通过公开数据的召回率门槛。K=100 时，4×40 分区候选在两套公开数据上均未降低召回率，内部加速比为 1.38× 至 1.72×。在相同训练参数和搜索预算下，WarpFusion IVF-Flat 在 SIFT1M 和 Yahoo MiniLM 六组实验中的吞吐为 FAISS GPU IVF-Flat 的 1.33× 至 2.53×。 + +上述结果说明，GPU 向量检索性能不能仅依靠距离计算核函数优化，还取决于中间结果规模、候选结构和任务访问顺序。实验也表明，合成数据上的最优策略未必能够安全应用于真实数据，因此性能优化必须与正确性和召回率检查共同进行。 + +在现有工作基础上，系统仍可从三个方向进一步扩展：利用 Nsight Compute 对显存访问与缓存行为进行定量分析；研究面向 K=50/100 的低开销、可证明容量安全的候选选择结构；在不同 GPU 架构和更大规模数据上重新评估参数与性能，以判断当前优化的硬件适用范围。上述方向超出本文的实验范围，不影响当前系统实现、正确性验证与性能结论的完整性。 + +## 参考文献 + +1. J. Johnson, M. Douze, and H. Jégou. Billion-scale similarity search with GPUs. IEEE Transactions on Big Data, 2019. +2. H. Jégou, M. Douze, and C. Schmid. Product quantization for nearest neighbor search. IEEE Transactions on Pattern Analysis and Machine Intelligence, 2011. +3. M. Aumüller, E. Bernhardsson, and A. Faithfull. ANN-Benchmarks: A benchmarking tool for approximate nearest neighbor algorithms. Information Systems, 2020. diff --git a/09_vector_retrieval/PurplePulse/docs/REPRODUCIBILITY.md b/09_vector_retrieval/PurplePulse/docs/REPRODUCIBILITY.md new file mode 100644 index 00000000..e1a0f3af --- /dev/null +++ b/09_vector_retrieval/PurplePulse/docs/REPRODUCIBILITY.md @@ -0,0 +1,514 @@ +# WarpFusion 正式实验复现记录 + +## 2026-09-11 第九阶段工程化入口 + +统一入口为： + +```bash +mkdir -p results/real/reproducibility/logs +nohup python3 scripts/reproduce.py --project-root "$PWD" --resume \ + > results/real/reproducibility/nohup.log 2>&1 & +``` + +默认流水线包含 CMake/Ninja Release 构建、完整 CTest、SIFT1M 与 Yahoo MiniLM +384 维数据准备、WarpFusion/FAISS Exact 与 IVF-Flat 对照、两套图表、技术报告 +PDF 和发布校验。`--dataset` 可只选择一套数据，`--skip-download` 与 +`--skip-index-build` 用于复用已核验产物，`--dry-run` 输出完整绝对路径计划而不 +执行。每个阶段写独立日志，完成一步后原子更新 +`results/real/reproducibility/checkpoint.json`；中断后 `--resume` 不重复已经成功 +的步骤。 + +发布校验器： + +```bash +python3 scripts/validate_release.py \ + --project-root "$PWD" \ + --report output/pdf/WarpFusion_Technical_Report.pdf \ + --json-output evidence/2026-09-11/stage9/release-validation.json +``` + +它检查两套公开数据的 10 个正式 CSV、Stage 8 覆盖矩阵、架构/性能 SVG、核心 +文档标记和 PDF 完整性。它不会启动长实验，因此适合作为复现流水线的最后一道 +fail-closed 检查。 + +干净 Linux + CUDA 验证使用 CUDA 12.8、GCC 13.3、RTX 4090 D 和全新构建目录， +并以 257 vectors、65 维、13 queries 的 FP32/L2 数据运行 Exact CPU/GPU 与 +IVF CPU/GPU。IVF 使用 nlist=16、nprobe=16，确保最小查询同时校验索引读写和 +全 probe CPU/GPU 等价。CTest 18/18 通过；Exact 与 IVF 各输出 130 行，候选 +集合相同，平均绝对分数误差 3.039×10⁻⁶，最大误差 9.6×10⁻⁶，无近似并列换位。 + +远端 `nohup` 任务结果位于 +`/data/purplepulse-work/PurplePulse/results/stage9_clean_20260911_v2/`，阶段状态、 +单项日志、摘要和输出校验和均分阶段落盘。仓库仅保留小型证据： +`evidence/2026-09-11/stage9/`。首次配置因非登录 shell 的 `PATH` 不含 +`/usr/local/cuda/bin` 而 fail-closed；修正 PATH 后使用新的 `-v2` 目录重跑， +没有删除或复用失败构建，也没有重复已经完成的 SIFT1M 或 Yahoo MiniLM 长实验。 + +## 2026-09-10 第八阶段稳定性矩阵 + +新增 `stability_tests`，使用包含重复向量与全零 query 的确定性、FP16 可精确 +表示数据。Exact 主矩阵覆盖 64/128/384/768 维、batch +1/8/32/40/64/128、K=1/10/50/100、FP32/FP16 和 L2/IP/cosine，共 576 组； +另用 65 维覆盖非整 warp 和每种 `batch+1` query 的满 batch + 尾 batch。 + +IVF 测试人工构造 8 个桶：4 个空桶，最大桶包含 125/128 个向量。在 65 维、 +K=1/10/50/100、FP32/FP16、三种 metric 下分别运行 warp 与 bucket-major， +共 48 组。所有 GPU 结果均按 query/rank 比较 ID 与 score；相同分数验证较小 +ID 优先，而不是只比较 recall 或结果数量。 + +远端任务与结果： + +- `stage8-stability-build-20260910`：nohup 构建和完整 CTest，状态 0，14/14。 +- `stage8-stability-sanitizer-20260910`：先落盘覆盖摘要，再运行 + `compute-sanitizer --tool memcheck`，状态 0、0 errors。 +- 结果目录：`/data/purplepulse-work/PurplePulse/results/stage8_stability/`；仓库 + 保存 `../results/stage8_stability/coverage.csv`，大日志继续只留在 `/data`。 + +保守的 Exact/IVF 实现选择、GPU 适用范围和 fail-closed 回退规则见 +`STABILITY_MATRIX.md`。FP16、非 batch64、64/768 维或其他 GPU 虽已验证正确性， +但没有被错误标记为已有正式性能调优；这些组合默认回退到完整 warp-query heap。 + +## 2026-09-10 第七阶段 Exact fused 剖析 + +新增 `--profile-breakdown 1 --profile-repeat N` 和 +`scripts/benchmark_exact_breakdown.py`。剖析器使用与 fused 路径相同的网格、 +query cache 和距离归约运行一个 distance-only kernel；该 kernel 每个 warp +仅写一个 checksum，以阻止编译器删除计算。最终归并由 CUDA event 直接计时， +局部 Top-K 则定义为 `fused kernel - distance-only kernel`。因此它是可复现的 +differential estimate，不应解释为三个硬件计数器区间的严格可加时间线。 + +正式运行使用 RTX 4090 D、batch=64、预热 5 次、查询重复 20 次、剖析重复 +20 次。正式基线先完成候选集合与分数校验；随后同一基线在 A/B 复测中使用加严 +后的驱动，按 query、rank、候选 ID 和分数与既有 Exact 参考逐项比较。结果如下。 + +| 数据集 | K | QPS | distance-only | 局部 Top-K 估算 | 最终归并 | 占比（距离 / 局部 / 归并） | +|---|---:|---:|---:|---:|---:|---:| +| Synthetic-1M | 10 | 6743 | 144.94 ms | 2.38 ms | 0.19 ms | 98.26% / 1.61% / 0.13% | +| Synthetic-1M | 50 | 6083 | 145.53 ms | 17.13 ms | 0.83 ms | 89.02% / 10.48% / 0.51% | +| Synthetic-1M | 100 | 5484 | 146.29 ms | 33.08 ms | 1.86 ms | 80.72% / 18.25% / 1.03% | +| Yahoo MiniLM 384 | 10 | 3525 | 279.28 ms | 3.40 ms | 0.19 ms | 98.73% / 1.20% / 0.07% | +| Yahoo MiniLM 384 | 50 | 3366 | 280.78 ms | 13.61 ms | 0.88 ms | 95.09% / 4.61% / 0.30% | +| Yahoo MiniLM 384 | 100 | 3172 | 281.07 ms | 30.70 ms | 2.00 ms | 89.58% / 9.79% / 0.64% | + +完整 CSV 位于 `../results/stage7_exact_profile/`。这组结果把后续优先级指向 +数据库读取与跨 query 数据复用；只优化最终 merge 不可能达到阶段目标。 + +### 已否决：cosine 范数预计算 + +第一项 A/B 尝试在建库时预计算数据库范数，并在每批预计算 query 范数，避免 +cosine fused kernel 重复归约范数。基线和候选版本在同一任务中顺序运行，均 +预热 5 次、重复 20 次，并通过逐 query、逐 rank 的 ID/score 校验。 + +| 数据集 | K | 基线 QPS | 候选 QPS | 候选 / 基线 | +|---|---:|---:|---:|---:| +| Synthetic-1M | 10 | 6741 | 6628 | 0.983× | +| Synthetic-1M | 50 | 6067 | 5993 | 0.988× | +| Synthetic-1M | 100 | 5478 | 5425 | 0.990× | +| Yahoo MiniLM 384 | 10 | 3497 | 3407 | 0.974× | +| Yahoo MiniLM 384 | 50 | 3341 | 3330 | 0.997× | +| Yahoo MiniLM 384 | 100 | 3160 | 3240 | 1.025× | + +该方案只在 Yahoo K=100 提升 2.5%，Yahoo K=10 下降 2.6%，Synthetic 全部 +下降 1%--2%，因此未进入默认代码。候选版本源码已撤回，基线与候选 CSV、 +二进制/源码校验和、恢复后的当前源码校验和及可重放脚本保存在 +`../results/stage7_exact_profile/cosine_norm_ab/` 与 +`../evidence/2026-09-10/stage7/`。撤回后在远端重新构建，CTest 12/12 通过。 + +### 已保留：跨 query 数据库读取复用 + +`exact_query_tile=2/4/8` 让一个 fused block 对同一向量 chunk 同时计算多个 +query，每个数据库分量只从全局显存读取一次，再用于 tile 内所有 query。 +`exact_fused_blocks=N` 可固定 `grid.x`；默认值 0 使用并行度启发式。初版直接 +减少 `grid.y` 导致并行度下降，修订版按 query tile 放大 `grid.x`，使总 block +数维持在约 1024。两版 smoke 均保留在结果目录，避免只报告成功版本。 + +正式项统一预热 5 次、重复 20 次，驱动在每个 variant 后 checkpoint CSV，并 +按 query、rank、ID 和 score 与第六阶段 Exact 参考逐项校验。 + +| 数据集 | K | 基线 QPS | 最佳配置 | 最佳 QPS | 加速 | +|---|---:|---:|---|---:|---:| +| Synthetic-1M | 10 | 6739 | tile4 / 64 blocks | 9250 | 1.373× | +| SIFT1M | 10 | 6940 | tile4 / 64 blocks | 9228 | 1.330× | +| Yahoo MiniLM 384 | 10 | 3577 | tile8 / 160 blocks | 6747 | 1.887× | +| Yahoo MiniLM 384 | 50 | 3363 | tile4 / 64 blocks | 4074 | 1.211× | + +Synthetic/SIFT K=50 的最佳策略仍是 baseline；tile2 分别为 0.986×/0.962×， +tile4 为 0.917×/0.890×。K=100 smoke 中 tile2/tile4 在 Synthetic 仅为 +0.684×/0.567×，Yahoo 为 0.712×/0.698×，因此没有继续正式长跑。K=10 的 +最终归并仍低于 0.3 ms；Yahoo 最佳配置相对 baseline 只增加约 1.06 MiB +显式 buffer。完整 CSV 位于 `../results/stage7_exact_query_tile/`，源码与配置 +校验和位于 `../evidence/2026-09-10/stage7/query_tile_source.sha256`。 + +优化后的 K=10 相对 FAISS GPU Flat 差距为 Synthetic 2.30 倍、SIFT1M +2.24 倍、Yahoo 3.15 倍，尚未达到 2 倍阶段门槛。后续应改用 block 级数据库 +tile 保持 query 并行度，或针对 K=50/100 比较专用选择结构，而不是继续扩大 +同一 warp 内的 query tile。 + +### 已否决：block 级数据库 tile + +`exact_fused_layout=block_tile` 让一个 block 的 8 个 warp 各自负责一个 query， +并通过共享内存复用 8 个数据库向量，目的是避免 query-tile 中单 warp 串行维护 +多个 query heap。筛选统一使用 batch=64、K=10、预热 1 次、重复 3 次，扫描 +`grid.x=64/96/128/160/192`；每个 variant 均与既有 Exact 参考逐 query、逐 rank +比较 ID 和 score。CSV 在每个 variant 完成后落盘。 + +| 数据集 | baseline QPS | warp-query 最优 | block-tile 最优 | block / warp | +|---|---:|---:|---:|---:| +| Synthetic-1M | 6929 | 9265（tile4/64） | 9050（tile8/128） | 0.977× | +| Yahoo MiniLM 384 | 3620 | 6989（tile8/160） | 4794（tile8/128） | 0.686× | + +block-tile 虽分别比未 tiled baseline 快 30.6% 和 32.5%，但没有超过已经接纳的 +warp-query；Yahoo 还退化 31.4%。同步与共享内存中转成本抵消了保留 query +并行度的收益，因此不再运行 SIFT 或 20 次正式长跑。该路径保留为显式 opt-in +负实验，默认 `exact_fused_layout=warp_query` 不变；原始筛选 CSV 位于 +`../results/stage7_exact_block_tile/smoke/`。 + +### 选择结构裁决与阶段验收 + +当前 Exact fused 并非单线程 heap：每个 block 的 8 个 warp 分别维护局部 heap， +块内先合并成 `K` 项，再由第二个 kernel 归并各 chunk，已经是分层 +Warp/Block Top-K。实际运行过的候选物化对照是旧 matrix 路径：它先写回 +`batch × N` score；K=10 使用每线程候选缓冲后归并，K=50/100 使用 block +反复选择。Synthetic-1M K=10/50/100 +分别只有 1775/716/444 QPS，而对应 fused 为 5564/5123/4701 QPS，慢 +3.13--10.59 倍且显式 buffer 多约 33%。 + +对全量 block radix/select 的裁决属于设计边界，不冒充实测：若只保留少量局部 +候选，它仍需先使用 heap/selection 得到这些候选；若要对全部精确 score 做 +radix/select，则必须物化 `batch × N` score，或为每个 radix pass 重算距离。 +前者回到已否决 matrix 路径，后者破坏单遍扫描并放大已经占 80.72%--98.73% +的主瓶颈，因此没有为“列齐算法名称”而实现不可接纳路径。第七阶段最终未达到 +2 倍硬目标，但满足计划规定的失败分支：保存可复现 profile、严格正确性结果、 +被否决方案、默认回退和原因。 + +## 2026-09-10 第六阶段公开数据 + +第六阶段使用 SIFT1M 与 VIBE `yahoo-minilm-384-normalized`。数据准备、校验、 +转换和完整基准入口分别为 `scripts/prepare_real_datasets.py` 与 +`scripts/run_real_benchmarks.py`。公开源与固定校验和如下： + +| 数据集 | 公开源 | 规模 | 校验和 | +|---|---|---:|---| +| SIFT1M | `ftp://ftp.irisa.fr/local/texmex/corpus/sift.tar.gz` | 1M × 128 | MD5 `b23d1b3b2ee8469d819b61ca900ef0ed` | +| yahoo-minilm-384-normalized | `https://huggingface.co/datasets/vector-index-bench/vibe` | 677,305 × 384 | SHA-256 `8b0519850249d7ea4aea258eb43ae837dca1bb2f7a85ee0a5145afdb699c4136` | + +正式矩阵统一使用 nlist=1024、训练样本 100,000、15 次迭代、seed 2026、 +batch=64，以及 K=10/50/100、nprobe=32/64/128/160/192/224。完整命令见 +README 的“第六阶段公开真实数据”。每套结果目录已包含 `build.csv`、 +`exact_gpu.csv`、三份有效 IVF CSV、FAISS GPU CSV、公开 ground truth 验证、 +分区 Top-K 回退判定、环境快照、原始输出和 recall-QPS/latency 图。 + +### 正式结果摘要 + +正式运行使用 RTX 4090 D、CUDA 12.8.61、`batch=64`、预热 1 次和重复 5 次。 +下表固定 `nprobe=160`；两套系统使用相同的 nlist、训练样本、迭代次数、K、 +nprobe 和 batch，但各自训练聚类中心。 + +| 数据集 | K | WarpFusion recall / QPS | FAISS GPU recall / QPS | WF / FAISS | +|---|---:|---:|---:|---:| +| SIFT1M | 10 | 0.999870 / 24579.1 | 0.999120 / 13616.6 | 1.81× | +| SIFT1M | 50 | 0.999706 / 19236.8 | 0.999516 / 13442.7 | 1.43× | +| SIFT1M | 100 | 0.999565 / 17362.2 | 0.999457 / 13102.4 | 1.33× | +| Yahoo MiniLM 384 | 10 | 0.993900 / 15733.2 | 0.995300 / 6216.2 | 2.53× | +| Yahoo MiniLM 384 | 50 | 0.991520 / 14949.7 | 0.991280 / 6157.2 | 2.43× | +| Yahoo MiniLM 384 | 100 | 0.989230 / 14196.2 | 0.989080 / 6059.8 | 2.34× | + +WarpFusion/FAISS IVF 建库时间分别为 SIFT1M 25.47/31.69 秒、Yahoo +87.54/77.89 秒。WarpFusion Exact K=10/50/100 吞吐为 SIFT1M +7038/6337/5622 QPS、Yahoo 3610/3471/3286 QPS。Exact 与公开 ground truth +存在极少数同分 ID 选择差异，但两套数据的 score-equivalent query fraction +均为 1.0；SIFT 的 rank/report score error 为 0，Yahoo 的最大报告分数误差为 +`2.98e-7`。 + +K=50 的分区路径在 SIFT1M 和 Yahoo 上分别损失 0.000326 和 0.000980 recall， +超过预设 `1e-4` 阈值，`ivf_k50.csv` 因此自动复制完整 heap 结果。K=100 在 +两套数据的全部六个 nprobe 点上均保持零 recall 损失，正式结果保留分区路径； +`nprobe=160` 下相对完整 heap 分别快 1.38 倍和 1.72 倍。提交仓库保存 +`results/real/` 下的 CSV、JSON 和 SVG；原始 Top-K 输出与日志保留在实验机 +`/data/purplepulse-work/PurplePulse/results/real/`。 + +## 2026-09-08 增量实验 + +新 GPU 实例仍为 RTX 4090 D / CUDA 12.8，但从空工作目录重新同步源码、配置 +并构建。CTest 8/8 通过，Compute Sanitizer 报告 0 errors。环境、测试与数据 +校验和位于 `../evidence/2026-09-08/`。 + +本轮增加六组结果：K=50 batch 调优、缓存感知桶任务调度、K=100 单 warp +正式 A/B、K=100 分区局部 Top-K、IVF 多线程建库和 MNIST 真实数据。原始 CSV 位于 +`../results/formal_4090d_20260908/`。百万数据并行建库： + +```bash +./build/ivf_build \ + --database data/million/inner_fp32_db.bin \ + --output data/million/inner_fp32_nlist256.ivf \ + --nlist 256 --iterations 15 --training-samples 100000 --threads 16 +``` + +K=50 在 nprobe=160 下先以 10 次重复扫描 batch=8/32/36/40/44/48/56/64/ +80/96/128，再对 batch=40 和 batch=64 各预热 1 次、正式重复 100 次： + +```bash +./build/ivf_search \ + --index data/million/inner_fp32_nlist256.ivf \ + --queries data/million/inner_fp32_q.bin \ + --params configs/ivf_flat_million_k50_warp_compact_batch40.conf \ + --output results/k50_batch40.txt \ + --backend gpu --warmup 1 --repeat 100 + +./build/ivf_search \ + --index data/million/inner_fp32_nlist256.ivf \ + --queries data/million/inner_fp32_q.bin \ + --params configs/ivf_flat_million_k50_batch64.conf \ + --output results/k50_batch64.txt \ + --backend gpu --warmup 1 --repeat 100 + +python3 scripts/evaluate_recall.py \ + --exact results/new_gpu/exact_k50.txt \ + --approximate results/k50_batch40.txt +``` + +batch=40 的正式结果为 6019.91 QPS、run P50/P99=166.303/168.097 ms、 +recall@50=0.92482；同实例 batch=64 对照为 5737.20 QPS，严格 A/B 提升 +4.93%。筛选表、warp 对照和原始日志分别位于 +`../results/formal_4090d_20260908/` 与 `../evidence/2026-09-08/`。 + +缓存感知调度使用相同索引、K、batch、nprobe 和输出语义，仅改变 probe block +的执行顺序。正式 A/B 命令为： + +```bash +./build/ivf_search \ + --index data/million/inner_fp32_nlist256.ivf \ + --queries data/million/inner_fp32_q.bin \ + --params configs/ivf_flat_million_k50_warp_compact_batch40.conf \ + --output results/k50_probe_major.txt \ + --backend gpu --warmup 5 --repeat 100 + +./build/ivf_search \ + --index data/million/inner_fp32_nlist256.ivf \ + --queries data/million/inner_fp32_q.bin \ + --params configs/ivf_flat_million_k50_bucket_major.conf \ + --output results/k50_bucket_major.txt \ + --backend gpu --warmup 5 --repeat 100 + +cmp results/k50_probe_major.txt results/k50_bucket_major.txt +``` + +K=50 的正式结果为 6009.22 与 6964.16 QPS，提升 15.89%；两份输出完全 +一致。nprobe=32/64/96/128/160/192/224 的扫描结果位于 +`k50_bucket_major_sweep.csv`。K=100 单独启用 bucket-major 提升 2.00%；加入 +heap 根节点寄存器缓存、空穴下沉和单 warp 直写后，同轮 100 次 A/B 从 +5298.37 提高到 5425.61 QPS，再与 bucket-major 组合达到 5491.76 QPS，累计 +提升 3.65%。三条路径的输出逐字一致，该组合构成后续分区实验的完整 heap +基线。 + +K=50 的分区候选筛选包含 2×32、4×20、4×16 与 8×8。未接纳的 2×32、4×20 +和 8×8 分别取得 7.37%、9.51% 与 2.88% 的增益；4×16 是唯一超过 10% 接纳线的 +配置。正式命令为： + +```bash +./build/ivf_search \ + --index data/million/inner_fp32_nlist256.ivf \ + --queries data/million/inner_fp32_q.bin \ + --params configs/ivf_flat_million_k50_partitioned_4x16.conf \ + --output results/k50_partitioned_4x16.txt \ + --backend gpu --warmup 5 --repeat 100 +``` + +同一二进制的完整 heap 基线为 7165.23 QPS，4×16 为 7934.10 QPS，提升 +10.73%；输出逐字一致，recall@50 均为 0.92482。七个 nprobe 点的增益范围为 +6.04% 至 22.86%，均保持输出一致。原始 CSV、输出和 memcheck 日志位于 +`../results/formal_4090d_20260908/k50_partitioned_tight/`；2×32 与 4×20 记录 +位于 `../results/formal_4090d_20260908/k50_partitioned/`。 + +K=100 的分区局部 Top-K 使用 4 个 warp、每个 warp 40 项局部候选。正式复测 +保持同一二进制、索引、`batch=64` 和 `nprobe=160`： + +```bash +./build/ivf_search \ + --index data/million/inner_fp32_nlist256.ivf \ + --queries data/million/inner_fp32_q.bin \ + --params configs/ivf_flat_million_k100_partitioned.conf \ + --output results/k100_partitioned.txt \ + --backend gpu --warmup 5 --repeat 100 +``` + +完整 heap 基线为 5503.11 QPS，分区路径为 7204.54 QPS，提升 30.92%；两份 +输出逐字一致，recall@100 均为 0.91758。`nprobe=128/160/192/224` 的 10 次 +扫描分别提升 26.49%/31.28%/34.32%/36.80%。原始 CSV、结果文件与 memcheck +日志位于 `../results/formal_4090d_20260908/k100_partitioned/`。 + +三项未采用的 K=100 选择或排序实验记录在 +`results/formal_4090d_20260908/k100_selection_negative.csv`。分布式寄存器 +Top-K 重复 100 次，仅提升 0.43%；buffer-8 快速重复 10 次后退化 22.68%， +因此按停止规则未进入 100 次阶段。原始摘要、资源用量和回退校验见 +`evidence/2026-09-08/k100_selection_negative.log`。 +warp bitonic sort 也在 10 次快速 A/B 中退化 18.72%，输出逐字一致，因而 +同样按停止规则终止。 + +K=50 的 FAISS 同批次补测固定 batch=40、nprobe=160、预热 5 次、重复 100 次。 +隔离环境使用 Python 3.12.3、FAISS GPU 1.14.1 和 NumPy 2.2.6，并复用主机 +CUDA 12.8 的 runtime/cuBLAS。FAISS Flat 与 WarpFusion Exact 的候选 ID +完全一致。FAISS IVF-Flat 为 8328.11 QPS、0.92312 recall@50；WarpFusion +最终 4×16 路径为 7934.10 QPS、0.92482 recall@50，FAISS 吞吐高 4.97%。完整命令见 +`../evidence/2026-09-08/commands.md`，原始 CSV 为 +`../results/formal_4090d_20260908/faiss_gpu_batch40.csv`。 + +MNIST 归档来自 Keras 公共数据地址 +`https://storage.googleapis.com/tensorflow/tf-keras-datasets/mnist.npz`。转换、 +建库与扫描命令为： + +```bash +python3 scripts/import_mnist.py \ + --input data/mnist/mnist.npz \ + --database data/mnist/database.bin \ + --queries data/mnist/queries.bin + +./build/ivf_build \ + --database data/mnist/database.bin --output data/mnist/nlist256.ivf \ + --nlist 256 --iterations 15 --training-samples 60000 --threads 16 + +python3 scripts/benchmark_ivf_sweep.py \ + --search-binary ./build/ivf_search \ + --index data/mnist/nlist256.ivf --queries data/mnist/queries.bin \ + --params configs/ivf_flat_mnist.conf \ + --exact results/mnist_exact_k10.txt \ + --output-dir results/mnist_ivf --csv results/mnist_ivf.csv \ + --nlist 256 --nprobes 8,16,32,64,96,128 \ + --training-samples 60000 --training-iterations 15 \ + --warmup 2 --repeat 10 +``` + +下文保留 2026-09-04 的主 FAISS 对照口径。 + +本文固定本轮正式结果的数据、训练、延迟和环境口径。公开结果文件不包含原始 +数据与索引；二者通过生成命令和 SHA-256 校验和审计。 + +## 1. 环境 + +- GPU：NVIDIA GeForce RTX 4090 D，24,564 MiB，compute capability 8.9。 +- Driver：570.124.06。 +- OS：Ubuntu 24.04.1 LTS。 +- CUDA Toolkit / NVCC：12.8 / 12.8.61。 +- CMake / Ninja / GCC：3.31.4 / 1.11.1 / 13.3.0。 +- FAISS 专用 venv：FAISS 1.14.1 GPU，NumPy 2.2.6，Python 3.12.3；CPU + 对照固定 1 线程。系统 Python 不含 FAISS，不用于 FAISS 正式实验。 +- 构建类型：Release，`CMAKE_CUDA_ARCHITECTURES=native`。 + +GPU 机上的部署目录不是 Git checkout。正式实验前，核心源码和本地基准提交 +`fea41f6` 的 SHA-256 逐文件一致；随后新增的评测字段不改变检索 kernel。 +完整环境输出、测试与 profiler 摘要保存在 `../evidence/2026-09-04/`。 + +## 2. 数据与索引 + +```bash +python3 scripts/generate_data.py \ + --database data/million/inner_fp32_db.bin \ + --queries data/million/inner_fp32_q.bin \ + --num-vectors 1000000 \ + --num-queries 1000 \ + --dim 128 \ + --dtype fp32 \ + --metric inner_product \ + --seed 2026 +``` + +| 文件 | SHA-256 | +|---|---| +| `inner_fp32_db.bin` | `d4dde1418ea7626e5226e594e592f9ae5ffbe4c412c326395d96388f95df9800` | +| `inner_fp32_q.bin` | `a58ebe43fce865b71a2a651791286fa0bd1567348e07d6c0601b69beaebceb45` | +| `inner_fp32_nlist256_spherical_100k15.ivf` | `f5bd87fcc72d37164fa931436e5bfd293dfb7f953d09cb28169a77c7cdadc415` | +| FAISS Flat index | `760e3a94f24c5becfa21e89a57f67395013d5ad5375c20eb3f5d5cfb4c3720a1` | +| FAISS IVF-Flat index（100k/15/seed 2026） | `a66a63bb28ed9e70ae83c042f57a8ca79de3ee01a6414377452a7610738431b1` | + +WarpFusion 正式索引的构建命令为： + +```bash +./build/ivf_build \ + --database data/million/inner_fp32_db.bin \ + --output data/million/inner_fp32_nlist256_spherical_100k15.ivf \ + --nlist 256 \ + --iterations 15 \ + --training-samples 100000 +``` + +构建耗时 67,678 ms，桶大小 min/avg/max 为 +3734/3906.25/4091，无空桶，索引大小 520,133,160 bytes。 + +两种实现都使用 100,000 个训练向量和 15 次迭代，但不强制共享中心： + +- WarpFusion 从全库等间距、确定性抽取训练样本，初始中心也等间距选择； + 当前实现没有单独的训练随机种子。 +- FAISS 用 NumPy `default_rng(2026)` 无放回抽样，并把聚类 seed 设为 2026。 + +所以这是相同训练预算和检索参数下的端到端系统比较，不是相同桶扫描比较。 + +## 3. 正式查询命令 + +Exact 的 K=10/50/100 均使用 batch=64、预热 5 次、正式重复 100 次： + +```bash +for k in 10 50 100; do + ./build/vector_search \ + --database data/million/inner_fp32_db.bin \ + --queries data/million/inner_fp32_q.bin \ + --params configs/exact_million_k${k}_fused_batch64.conf \ + --backend gpu \ + --output results/formal_100k15/exact_k${k}.txt \ + --warmup 5 --repeat 100 +done +``` + +IVF 对 nprobe=128/160/192/224 扫描。K=10 的示例为： + +```bash +python3 scripts/benchmark_ivf_sweep.py \ + --search-binary ./build/ivf_search \ + --index data/million/inner_fp32_nlist256_spherical_100k15.ivf \ + --queries data/million/inner_fp32_q.bin \ + --params configs/ivf_flat_million_batch64.conf \ + --exact results/formal_100k15/exact_k10.txt \ + --output-dir results/formal_100k15/ivf_k10 \ + --csv results/formal_100k15/ivf_k10.csv \ + --nlist 256 --nprobes 128,160,192,224 \ + --training-samples 100000 --training-iterations 15 \ + --warmup 5 --repeat 100 +``` + +K=50/100 分别替换为对应的 batch64 配置和 Exact 文件。FAISS 使用同一数据、 +K、batch、nlist、nprobe、训练预算、预热和重复次数；完整命令保存在 +`../evidence/2026-09-04/commands.md`。 + +## 4. 延迟与质量定义 + +- `run P50/P99`：一次处理全部 1000 queries 的端到端稳定查询时间。 +- `batch P50/P99`：每个 batch 的 GPU 时间，从 query H2D 开始，到 Top-K + ID/score D2H 完成为止；包含 H2D、所有检索 kernel 和 D2H。 +- 稳定查询不包含数据库/索引的一次性加载、一次性 H2D 和引擎初始化；这些 + 时间另列，不能混入常驻 QPS。 +- GPU 正式项每个 run 有 16 个 batch；100 次重复共产生 1600 个 batch 延迟 + 样本。单线程 CPU Flat 重复 20 次，共 320 个 batch 延迟样本。 +- `recall@K` 是返回 ID 集合与 Exact Top-K 的交集比例。 +- `mean/max absolute score error` 按名次比较近似 Top-K 与 Exact Top-K 的 + 分数绝对差；它衡量漏召回造成的分数质量损失，不等同于同 ID 数值误差。 + Exact kernel A/B 的分数误差则按相同候选 ID 比较。 + +## 5. 自动验证 + +```bash +ctest --test-dir build --output-on-failure +compute-sanitizer --tool memcheck --error-exitcode 99 ./build/unit_tests +``` + +本轮 CTest 8/8 通过，`compute-sanitizer` 报告 0 errors。Nsight Systems 的 +Exact K10 代表性采样显示 fused 距离+局部 Top-K kernel 占 GPU kernel 时间 +约 99.9%；文本摘要已入库，原始 `.nsys-rep` 因体积和机器相关性不提交。 + +新增评测字段和两处样本数日志上传后，GPU 端已再次完成增量编译，CTest 仍为 +8/8。新版 IVF 和 FAISS 脚本分别完成百万数据 smoke test，确认训练元数据、 +run/batch 样本数与平均/最大分数误差均能写入 CSV；对应日志文件名包含 +`after_instrumentation`。 + +为排除构建目录、数据文件和历史结果造成的隐式依赖，另将源码排除 +`build/`、`data/`、`results/`、`evidence/` 和 `.git/` 后解压到新目录，从零 +执行 CMake、编译和 CTest，8/8 通过。输出见 +`../evidence/2026-09-08/submit_audit_ctest.log`。 diff --git a/09_vector_retrieval/PurplePulse/docs/STABILITY_MATRIX.md b/09_vector_retrieval/PurplePulse/docs/STABILITY_MATRIX.md new file mode 100644 index 00000000..135ae6f2 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/docs/STABILITY_MATRIX.md @@ -0,0 +1,57 @@ +# 稳定性矩阵与实现选择规则 + +## 第八阶段覆盖 + +`stability_tests` 使用可被 FP16 精确表示的确定性数据，并为数据库加入重复向量、 +为查询加入全零向量，使相同分数必须按较小 ID 排序。GPU 结果按候选 ID、名次和 +score 与 CPU Exact/IVF 参考比较，而不是只检查结果数量。 + +| 类别 | 覆盖 | +|---|---| +| Exact 主矩阵 | 64/128/384/768 维 × batch 1/8/32/40/64/128 × K 1/10/50/100 × FP32/FP16 × L2/IP/cosine，共 576 组 | +| batch 边界 | 65 维，每个 batch 使用 `batch+1` 个 query，覆盖一个满 batch 和尾 batch | +| 非整 warp 维度 | 65 维 × FP32/FP16 × L2/IP/cosine，K=100 | +| Exact 推荐路径 | 128/K10/tile4/64 blocks、384/K10/tile8/160 blocks、384/K50/tile4/64 blocks | +| Exact 安全回退 | 不支持的 FP32 768 维 block-tile 与 K50/tile8 均 fail closed；改用 warp-query/tile1 后通过 | +| IVF 病理桶 | 65 维，8 桶中 4 个空桶、最大桶 125/128；K 1/10/50/100 × FP32/FP16 × 三 metric × warp/bucket-major，共 48 组 | + +原始摘要位于 `../results/stage8_stability/coverage.csv`。远端完整 CTest 14/14 +通过；`compute-sanitizer --tool memcheck` 对整个 `stability_tests` 报告 +`ERROR SUMMARY: 0 errors`。 + +## Exact 选择规则 + +下表中的优化项只适用于已有正式 A/B 的 RTX 4090 D、FP32、batch=64 组合。 +规则刻意保守：正确性测试通过不等于在未测组合上已有性能证据。 + +| K | 维度/metric | 推荐 layout | query tile / blocks | 其他情况 | +|---:|---|---|---|---| +| 10 | 128，L2 或 inner product | warp-query | 4 / 64 | 回退 1 / 0 | +| 10 | 384，cosine | warp-query | 8 / 160 | 回退 1 / 0 | +| 50 | 384，cosine | warp-query | 4 / 64 | 回退 1 / 0 | +| 1、100 | 任意已测维度 | warp-query | 1 / 0 | 同左 | +| 任意 | 64、768、FP16、非 batch64 或其他 GPU | warp-query | 1 / 0 | 同左 | + +其中 `blocks=0` 表示使用稳定的并行度启发式。`block_tile` 在 Synthetic/Yahoo +均未超过 warp-query，不进入任何推荐组合。显式请求不支持的实验组合时，参数 +验证会拒绝启动 kernel；调用方必须改回 +`exact_fused_layout=warp_query, exact_query_tile=1, exact_fused_blocks=0`。 +这是一种 fail-closed 回退，不会静默运行未经验证的近似路径。 + +## IVF 选择与回退 + +- K=10、固定 nprobe：使用完整候选的 `bucket_major`。 +- K=50/100：未做数据集级 recall 验证时使用完整 heap 的 `bucket_major`。 +- 分区 K=50/100 只作为候选运行。`scripts/run_real_benchmarks.py` 会将其与完整 + heap 的全部 nprobe 点比较；任一点 recall 损失超过 `1e-4`，最终 CSV 自动 + 回退到 `*_safe.conf` 的完整 heap 结果。 +- 自适应 nprobe 或 memory scoring 不使用分区模式；参数验证会拒绝不兼容组合。 +- 空桶、严重倾斜桶、重复向量、相同分数和 65 维均已验证完整 heap 路径；这些 + 场景不会自动切换到分区候选。 + +## 适用边界 + +本阶段证明的是单卡、内存内、K≤100、维度≤768 的已列组合正确且无 memcheck +错误，不代表其他 GPU 上的最优性能选择。切换 GPU 架构、batch、dtype 或数据 +分布时，先使用安全 baseline，再按第六/七阶段的严格 ID/score 或 recall 门槛 +重新接纳优化配置。 diff --git a/09_vector_retrieval/PurplePulse/docs/SUBMISSION_CHECKLIST.md b/09_vector_retrieval/PurplePulse/docs/SUBMISSION_CHECKLIST.md new file mode 100644 index 00000000..528fcf98 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/docs/SUBMISSION_CHECKLIST.md @@ -0,0 +1,28 @@ +# GPU 向量检索项目提交核对表 + +本表只用于提交前核验。项目方法、结果和局限以 `FINAL_REPORT.md` 为准。 + +| 课程要求 | 实现与证据 | 状态 | +|---|---|---:| +| 二进制向量库、查询与参数输入 | `src/file_io.cpp`、`src/main.cpp`、`src/search_ivf.cpp` | 通过 | +| 有序 Top-K 输出 | Exact 与 IVF CLI 均输出 ID 和分数；相同分数按较小 ID 排序 | 通过 | +| CPU 正确性参考 | `src/cpu_search.cpp`、`tests/unit_tests.cpp` | 通过 | +| GPU 精确检索 | matrix 与 fused 两条路径，候选集合一致 | 通过 | +| 至少一种近似检索 | GPU IVF-Flat、索引保存与加载、recall@K 评测 | 通过 | +| K=1/10/50/100 | Exact 与 IVF 自动测试覆盖 | 通过 | +| 百万规模、128 维、1000 queries | Synthetic-1M 正式实验与原始 CSV | 通过 | +| 性能超过 CPU 同级实现 | Exact 相对单线程 FAISS CPU Flat 加速 73.5–87.3 倍 | 通过 | +| 查询性能与质量报告 | QPS、run/batch P50/P99、显存、recall 和误差 | 通过 | +| 平均距离/分数误差 | nprobe 扫描 CSV 与报告正确性章节均给出 rank 对齐 MAE | 通过 | +| FAISS 对照 | 同数据、K、batch、nlist、nprobe 和训练预算 | 通过 | +| 真实分布实验 | MNIST 60k 数据库、10k queries | 通过 | +| 自动测试与内存安全 | 新平台 CTest 18/18；Compute Sanitizer 0 errors | 通过 | +| 独立源码构建 | CUDA 12.8 全新目录从零构建，CTest 18/18，Exact/IVF 最小 GPU 查询通过 | 通过 | +| 可复现材料 | 固定随机种子、环境、命令、校验和、CSV 和图表 | 通过 | + +提交目录应保留源码、头文件、测试、配置、脚本、报告、CSV 和 SVG；不提交 +`build/`、原始数据、二进制索引、检索输出、profiler 二进制文件或远端凭据。 +Agent 相关内容属于可选扩展，不纳入本次报告结论。 + +提交前仍需人工确认仓库路径满足课程规定的 `/<选题>/<学员 ID>/`。当前目录名 +为 `PurplePulse`；该字符串仅表示学员 ID，在未获得目标 PR 路径前不做自动重命名。 diff --git a/09_vector_retrieval/PurplePulse/evidence/2026-09-04/commands.md b/09_vector_retrieval/PurplePulse/evidence/2026-09-04/commands.md new file mode 100644 index 00000000..2866e458 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/evidence/2026-09-04/commands.md @@ -0,0 +1,115 @@ +# 2026-09-04 正式验收命令 + +所有命令均在项目根目录执行。登录信息、主机地址和密码不属于实验口径，未写入 +证据文件。 + +## 构建与测试 + +```bash +export PATH=/usr/local/cuda/bin:$PATH +cmake -S . -B build -G Ninja -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_CUDA_ARCHITECTURES=native +cmake --build build +ctest --test-dir build --output-on-failure +compute-sanitizer --tool memcheck --error-exitcode 99 ./build/unit_tests +``` + +## 数据与自研索引 + +```bash +python3 scripts/generate_data.py \ + --database data/million/inner_fp32_db.bin \ + --queries data/million/inner_fp32_q.bin \ + --num-vectors 1000000 --num-queries 1000 --dim 128 \ + --dtype fp32 --metric inner_product --seed 2026 + +sha256sum data/million/inner_fp32_db.bin data/million/inner_fp32_q.bin + +./build/ivf_build \ + --database data/million/inner_fp32_db.bin \ + --output data/million/inner_fp32_nlist256_spherical_100k15.ivf \ + --nlist 256 --iterations 15 --training-samples 100000 + +sha256sum data/million/inner_fp32_nlist256_spherical_100k15.ivf +``` + +## PurplePulse Exact + +```bash +for k in 10 50 100; do + ./build/vector_search \ + --database data/million/inner_fp32_db.bin \ + --queries data/million/inner_fp32_q.bin \ + --params configs/exact_million_k${k}_fused_batch64.conf \ + --backend gpu \ + --output results/formal_100k15/exact_k${k}.txt \ + --warmup 5 --repeat 100 +done +``` + +## PurplePulse IVF + +K=10/50/100 分别使用 +`ivf_flat_million_batch64.conf`、`ivf_flat_million_k50_batch64.conf` 和 +`ivf_flat_million_k100_batch64.conf`。以下是 K=10 命令，其余两项只替换 +配置、Exact 文件、输出目录和 CSV 名称。 + +```bash +python3 scripts/benchmark_ivf_sweep.py \ + --search-binary ./build/ivf_search \ + --index data/million/inner_fp32_nlist256_spherical_100k15.ivf \ + --queries data/million/inner_fp32_q.bin \ + --params configs/ivf_flat_million_batch64.conf \ + --exact results/formal_100k15/exact_k10.txt \ + --output-dir results/formal_100k15/ivf_k10 \ + --csv results/formal_100k15/ivf_k10.csv \ + --nlist 256 --nprobes 128,160,192,224 \ + --training-samples 100000 --training-iterations 15 \ + --warmup 5 --repeat 100 +``` + +## FAISS GPU + +```bash +OMP_NUM_THREADS=1 python3 scripts/benchmark_faiss.py \ + --database data/million/inner_fp32_db.bin \ + --queries data/million/inner_fp32_q.bin \ + --reference-results results/formal_100k15/exact_k100.txt \ + --index-dir data/million/faiss_formal_100k15 \ + --output-dir results/formal_100k15/faiss_gpu \ + --csv results/formal_100k15/faiss_gpu.csv \ + --backends gpu --index-types flat,ivf_flat \ + --top-ks 10,50,100 --nlist 256 --nprobes 128,160,192,224 \ + --batch-size 64 --train-samples 100000 --iterations 15 --seed 2026 \ + --warmup 5 --repeat 100 --cpu-threads 1 --rebuild +``` + +## FAISS CPU Exact + +```bash +OMP_NUM_THREADS=1 python3 scripts/benchmark_faiss.py \ + --database data/million/inner_fp32_db.bin \ + --queries data/million/inner_fp32_q.bin \ + --reference-results results/formal_100k15/exact_k100.txt \ + --index-dir data/million/faiss_formal_100k15 \ + --output-dir results/formal_100k15/faiss_cpu \ + --csv results/formal_100k15/faiss_cpu_flat_repeat20.csv \ + --backends cpu --index-types flat --top-ks 10,50,100 \ + --nlist 256 --nprobes 160 --batch-size 64 \ + --train-samples 100000 --iterations 15 --seed 2026 \ + --warmup 1 --repeat 20 --cpu-threads 1 +``` + +## Nsight Systems 代表性采样 + +```bash +nsys profile --force-overwrite true --stats=true \ + -o results/evidence/exact_k10_nsys \ + ./build/vector_search \ + --database data/million/inner_fp32_db.bin \ + --queries data/million/inner_fp32_q.bin \ + --params configs/exact_million_k10_fused_batch64.conf \ + --backend gpu \ + --output results/formal_100k15/exact_k10_nsys.txt \ + --warmup 1 --repeat 5 +``` diff --git a/09_vector_retrieval/PurplePulse/evidence/2026-09-04/compute_sanitizer_unit_tests.txt b/09_vector_retrieval/PurplePulse/evidence/2026-09-04/compute_sanitizer_unit_tests.txt new file mode 100644 index 00000000..55289ea6 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/evidence/2026-09-04/compute_sanitizer_unit_tests.txt @@ -0,0 +1,3 @@ +========= COMPUTE-SANITIZER +FP32/FP16 的 CPU/GPU 正确性与边界测试通过 +========= ERROR SUMMARY: 0 errors diff --git a/09_vector_retrieval/PurplePulse/evidence/2026-09-04/ctest.txt b/09_vector_retrieval/PurplePulse/evidence/2026-09-04/ctest.txt new file mode 100644 index 00000000..6b0187f9 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/evidence/2026-09-04/ctest.txt @@ -0,0 +1,20 @@ +Internal ctest changing into directory: /root/PurplePulse/build +Test project /root/PurplePulse/build + Start 1: unit_tests +1/7 Test #1: unit_tests ........................ Passed 0.24 sec + Start 2: file_io_tests +2/7 Test #2: file_io_tests ..................... Passed 0.01 sec + Start 3: ivf_flat_tests +3/7 Test #3: ivf_flat_tests .................... Passed 0.23 sec + Start 4: benchmark_faiss_tests +4/7 Test #4: benchmark_faiss_tests ............. Passed 0.93 sec + Start 5: benchmark_ivf_sweep_tests +5/7 Test #5: benchmark_ivf_sweep_tests ......... Passed 0.07 sec + Start 6: benchmark_adaptive_nprobe_tests +6/7 Test #6: benchmark_adaptive_nprobe_tests ... Passed 0.07 sec + Start 7: plot_results_smoke +7/7 Test #7: plot_results_smoke ................ Passed 0.06 sec + +100% tests passed, 0 tests failed out of 7 + +Total Test time (real) = 1.60 sec diff --git a/09_vector_retrieval/PurplePulse/evidence/2026-09-04/ctest_after_instrumentation.txt b/09_vector_retrieval/PurplePulse/evidence/2026-09-04/ctest_after_instrumentation.txt new file mode 100644 index 00000000..7e2f4457 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/evidence/2026-09-04/ctest_after_instrumentation.txt @@ -0,0 +1,20 @@ +Internal ctest changing into directory: /root/PurplePulse/build +Test project /root/PurplePulse/build + Start 1: unit_tests +1/7 Test #1: unit_tests ........................ Passed 0.24 sec + Start 2: file_io_tests +2/7 Test #2: file_io_tests ..................... Passed 0.01 sec + Start 3: ivf_flat_tests +3/7 Test #3: ivf_flat_tests .................... Passed 0.22 sec + Start 4: benchmark_faiss_tests +4/7 Test #4: benchmark_faiss_tests ............. Passed 1.46 sec + Start 5: benchmark_ivf_sweep_tests +5/7 Test #5: benchmark_ivf_sweep_tests ......... Passed 0.07 sec + Start 6: benchmark_adaptive_nprobe_tests +6/7 Test #6: benchmark_adaptive_nprobe_tests ... Passed 0.07 sec + Start 7: plot_results_smoke +7/7 Test #7: plot_results_smoke ................ Passed 0.06 sec + +100% tests passed, 0 tests failed out of 7 + +Total Test time (real) = 2.14 sec diff --git a/09_vector_retrieval/PurplePulse/evidence/2026-09-04/data_million_sha256.txt b/09_vector_retrieval/PurplePulse/evidence/2026-09-04/data_million_sha256.txt new file mode 100644 index 00000000..c3ec1f0e --- /dev/null +++ b/09_vector_retrieval/PurplePulse/evidence/2026-09-04/data_million_sha256.txt @@ -0,0 +1,2 @@ +d4dde1418ea7626e5226e594e592f9ae5ffbe4c412c326395d96388f95df9800 data/million/inner_fp32_db.bin +a58ebe43fce865b71a2a651791286fa0bd1567348e07d6c0601b69beaebceb45 data/million/inner_fp32_q.bin diff --git a/09_vector_retrieval/PurplePulse/evidence/2026-09-04/environment.txt b/09_vector_retrieval/PurplePulse/evidence/2026-09-04/environment.txt new file mode 100644 index 00000000..200635b8 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/evidence/2026-09-04/environment.txt @@ -0,0 +1,35 @@ +2026-09-04T07:42:26,935679805+00:00 +Linux 2a615599b24c 5.4.0-196-generic #216-Ubuntu SMP Thu Aug 29 13:26:53 UTC 2024 x86_64 x86_64 x86_64 GNU/Linux +PRETTY_NAME="Ubuntu 24.04.1 LTS" +NAME="Ubuntu" +VERSION_ID="24.04" +VERSION="24.04.1 LTS (Noble Numbat)" +VERSION_CODENAME=noble +ID=ubuntu +ID_LIKE=debian +HOME_URL="https://www.ubuntu.com/" +SUPPORT_URL="https://help.ubuntu.com/" +BUG_REPORT_URL="https://bugs.launchpad.net/ubuntu/" +PRIVACY_POLICY_URL="https://www.ubuntu.com/legal/terms-and-policies/privacy-policy" +UBUNTU_CODENAME=noble +LOGO=ubuntu-logo +NVIDIA GeForce RTX 4090 D, 570.124.06, 24564 MiB, 8.9 +nvcc: NVIDIA (R) Cuda compiler driver +Copyright (c) 2005-2025 NVIDIA Corporation +Built on Wed_Jan_15_19:20:09_PST_2025 +Cuda compilation tools, release 12.8, V12.8.61 +Build cuda_12.8.r12.8/compiler.35404655_0 +cmake version 3.31.4 + +CMake suite maintained and supported by Kitware (kitware.com/cmake). +1.11.1.git.kitware.jobserver-1 +gcc (Ubuntu 13.3.0-6ubuntu2~24.04) 13.3.0 +Python 3.12.3 +Traceback (most recent call last): + File "", line 1, in +ModuleNotFoundError: No module named 'faiss' +numpy 1.26.4 +faiss-python 3.12.3 +faiss-numpy 2.2.6 +faiss 1.14.1 +faiss-gpus 1 diff --git a/09_vector_retrieval/PurplePulse/evidence/2026-09-04/exact_k100_repeat100.txt b/09_vector_retrieval/PurplePulse/evidence/2026-09-04/exact_k100_repeat100.txt new file mode 100644 index 00000000..b032c0c7 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/evidence/2026-09-04/exact_k100_repeat100.txt @@ -0,0 +1,25 @@ +向量库: 1000000 x 128 +查询数: 1000 +dtype: fp32 +metric: inner_product +top_k: 100 +batch_size: 64 +distance_mode: warp +topk_mode: fused +warmup/repeat: 5/100 +backend: gpu +GPU 一次性初始化总时间: 172.278 ms +数据库一次性 H2D: 48.5765 ms +GPU 缓冲区: 489.558 MiB +估算冷启动端到端: 357.244 ms +平均常驻查询时间: 184.966 ms +run P50: 185.035 ms +run P99: 187.947 ms +常驻 QPS: 5406.4 +结果: "results/formal_100k15/exact_k100.txt" +平均查询 H2D: 0.232269 ms +平均融合距离+局部 Top-K kernel: 181.896 ms +平均最终 Top-K kernel: 1.88726 ms +平均结果 D2H: 0.492971 ms +batch P50: 11.7263 ms +batch P99: 12.1334 ms diff --git a/09_vector_retrieval/PurplePulse/evidence/2026-09-04/exact_k10_repeat100.txt b/09_vector_retrieval/PurplePulse/evidence/2026-09-04/exact_k10_repeat100.txt new file mode 100644 index 00000000..b6ee64f3 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/evidence/2026-09-04/exact_k10_repeat100.txt @@ -0,0 +1,25 @@ +向量库: 1000000 x 128 +查询数: 1000 +dtype: fp32 +metric: inner_product +top_k: 10 +batch_size: 64 +distance_mode: warp +topk_mode: fused +warmup/repeat: 5/100 +backend: gpu +GPU 一次性初始化总时间: 200.701 ms +数据库一次性 H2D: 48.602 ms +GPU 缓冲区: 488.437 MiB +估算冷启动端到端: 350.456 ms +平均常驻查询时间: 149.755 ms +run P50: 150.025 ms +run P99: 151.143 ms +常驻 QPS: 6677.57 +结果: "results/formal_100k15/exact_k10.txt" +平均查询 H2D: 0.22932 ms +平均融合距离+局部 Top-K kernel: 148.772 ms +平均最终 Top-K kernel: 0.187956 ms +平均结果 D2H: 0.363535 ms +batch P50: 9.53296 ms +batch P99: 9.64477 ms diff --git a/09_vector_retrieval/PurplePulse/evidence/2026-09-04/exact_k50_repeat100.txt b/09_vector_retrieval/PurplePulse/evidence/2026-09-04/exact_k50_repeat100.txt new file mode 100644 index 00000000..61251dcc --- /dev/null +++ b/09_vector_retrieval/PurplePulse/evidence/2026-09-04/exact_k50_repeat100.txt @@ -0,0 +1,25 @@ +向量库: 1000000 x 128 +查询数: 1000 +dtype: fp32 +metric: inner_product +top_k: 50 +batch_size: 64 +distance_mode: warp +topk_mode: fused +warmup/repeat: 5/100 +backend: gpu +GPU 一次性初始化总时间: 173.011 ms +数据库一次性 H2D: 49.114 ms +GPU 缓冲区: 488.935 MiB +估算冷启动端到端: 340.398 ms +平均常驻查询时间: 167.387 ms +run P50: 167.671 ms +run P99: 169.027 ms +常驻 QPS: 5974.19 +结果: "results/formal_100k15/exact_k50.txt" +平均查询 H2D: 0.232645 ms +平均融合距离+局部 Top-K kernel: 165.594 ms +平均最终 Top-K kernel: 0.844275 ms +平均结果 D2H: 0.411974 ms +batch P50: 10.6347 ms +batch P99: 10.8679 ms diff --git a/09_vector_retrieval/PurplePulse/evidence/2026-09-04/faiss_cpu_flat_repeat20.txt b/09_vector_retrieval/PurplePulse/evidence/2026-09-04/faiss_cpu_flat_repeat20.txt new file mode 100644 index 00000000..8782ef73 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/evidence/2026-09-04/faiss_cpu_flat_repeat20.txt @@ -0,0 +1,5 @@ +FAISS Flat vs PurplePulse exact recall@100: 1.000000000, min=1.000000000 +{"implementation": "faiss-1.14.1", "backend": "cpu", "index_type": "flat", "metric": "inner_product", "num_vectors": 1000000, "num_queries": 1000, "dim": 128, "dtype": "fp32", "cpu_threads": 1, "top_k": 10, "batch_size": 64, "nlist": 0, "nprobe": 0, "train_samples": 0, "build_ms": 0.0, "load_ms": 325.0962197780609, "gpu_transfer_ms": 0.0, "gpu_index_mib": 0.0, "gpu_temp_mib": 0.0, "average_query_ms": 13079.959641397, "qps": 76.45283528513971, "batch_p50_ms": 834.8349779844284, "batch_p99_ms": 846.3362380862236, "recall_at_k": 1.0, "min_query_recall": 1.0} +{"implementation": "faiss-1.14.1", "backend": "cpu", "index_type": "flat", "metric": "inner_product", "num_vectors": 1000000, "num_queries": 1000, "dim": 128, "dtype": "fp32", "cpu_threads": 1, "top_k": 50, "batch_size": 64, "nlist": 0, "nprobe": 0, "train_samples": 0, "build_ms": 0.0, "load_ms": 325.0962197780609, "gpu_transfer_ms": 0.0, "gpu_index_mib": 0.0, "gpu_temp_mib": 0.0, "average_query_ms": 13080.880285799503, "qps": 76.44745446417637, "batch_p50_ms": 835.1001441478729, "batch_p99_ms": 847.1395075321198, "recall_at_k": 1.0, "min_query_recall": 1.0} +{"implementation": "faiss-1.14.1", "backend": "cpu", "index_type": "flat", "metric": "inner_product", "num_vectors": 1000000, "num_queries": 1000, "dim": 128, "dtype": "fp32", "cpu_threads": 1, "top_k": 100, "batch_size": 64, "nlist": 0, "nprobe": 0, "train_samples": 0, "build_ms": 0.0, "load_ms": 325.0962197780609, "gpu_transfer_ms": 0.0, "gpu_index_mib": 0.0, "gpu_temp_mib": 0.0, "average_query_ms": 13596.499814093113, "qps": 73.54834065186945, "batch_p50_ms": 867.8400814533234, "batch_p99_ms": 881.017416715622, "recall_at_k": 1.0, "min_query_recall": 1.0} +CSV: results/formal_100k15/faiss_cpu_flat_repeat20_raw.csv diff --git a/09_vector_retrieval/PurplePulse/evidence/2026-09-04/faiss_gpu_repeat100.txt b/09_vector_retrieval/PurplePulse/evidence/2026-09-04/faiss_gpu_repeat100.txt new file mode 100644 index 00000000..0e0d5cd7 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/evidence/2026-09-04/faiss_gpu_repeat100.txt @@ -0,0 +1,17 @@ +FAISS Flat vs PurplePulse exact recall@100: 1.000000000, min=1.000000000 +{"implementation": "faiss-1.14.1", "backend": "gpu", "index_type": "flat", "metric": "inner_product", "num_vectors": 1000000, "num_queries": 1000, "dim": 128, "dtype": "fp32", "cpu_threads": 1, "top_k": 10, "batch_size": 64, "nlist": 0, "nprobe": 0, "train_samples": 0, "build_ms": 261.927954852581, "load_ms": 0.0, "gpu_transfer_ms": 338.9444574713707, "gpu_index_mib": 492.095947265625, "gpu_temp_mib": 1536.0, "average_query_ms": 47.11136355996132, "qps": 21226.301351419028, "batch_p50_ms": 2.9594004154205322, "batch_p99_ms": 2.972230315208435, "recall_at_k": 1.0, "min_query_recall": 1.0} +{"implementation": "faiss-1.14.1", "backend": "gpu", "index_type": "flat", "metric": "inner_product", "num_vectors": 1000000, "num_queries": 1000, "dim": 128, "dtype": "fp32", "cpu_threads": 1, "top_k": 50, "batch_size": 64, "nlist": 0, "nprobe": 0, "train_samples": 0, "build_ms": 261.927954852581, "load_ms": 0.0, "gpu_transfer_ms": 338.9444574713707, "gpu_index_mib": 492.095947265625, "gpu_temp_mib": 1536.0, "average_query_ms": 48.20252262055874, "qps": 20745.802203586176, "batch_p50_ms": 3.023482859134674, "batch_p99_ms": 3.048732876777649, "recall_at_k": 1.0, "min_query_recall": 1.0} +{"implementation": "faiss-1.14.1", "backend": "gpu", "index_type": "flat", "metric": "inner_product", "num_vectors": 1000000, "num_queries": 1000, "dim": 128, "dtype": "fp32", "cpu_threads": 1, "top_k": 100, "batch_size": 64, "nlist": 0, "nprobe": 0, "train_samples": 0, "build_ms": 261.927954852581, "load_ms": 0.0, "gpu_transfer_ms": 338.9444574713707, "gpu_index_mib": 492.095947265625, "gpu_temp_mib": 1536.0, "average_query_ms": 49.90370690822601, "qps": 20038.591558719705, "batch_p50_ms": 3.1260475516319275, "batch_p99_ms": 3.1578615307807922, "recall_at_k": 1.0, "min_query_recall": 1.0} +{"implementation": "faiss-1.14.1", "backend": "gpu", "index_type": "ivf_flat", "metric": "inner_product", "num_vectors": 1000000, "num_queries": 1000, "dim": 128, "dtype": "fp32", "cpu_threads": 1, "top_k": 10, "batch_size": 64, "nlist": 256, "nprobe": 128, "train_samples": 100000, "build_ms": 6923.940047621727, "load_ms": 0.0, "gpu_transfer_ms": 784.832239151001, "gpu_index_mib": 497.984375, "gpu_temp_mib": 1536.0, "average_query_ms": 145.7934482395649, "qps": 6859.01878359321, "batch_p50_ms": 9.360350668430328, "batch_p99_ms": 10.512225329875946, "recall_at_k": 0.8795000000000001, "min_query_recall": 0.5} +{"implementation": "faiss-1.14.1", "backend": "gpu", "index_type": "ivf_flat", "metric": "inner_product", "num_vectors": 1000000, "num_queries": 1000, "dim": 128, "dtype": "fp32", "cpu_threads": 1, "top_k": 50, "batch_size": 64, "nlist": 256, "nprobe": 128, "train_samples": 100000, "build_ms": 6923.940047621727, "load_ms": 0.0, "gpu_transfer_ms": 784.832239151001, "gpu_index_mib": 497.984375, "gpu_temp_mib": 1536.0, "average_query_ms": 90.35561479628086, "qps": 11067.380840190588, "batch_p50_ms": 5.7607293128967285, "batch_p99_ms": 5.909278988838196, "recall_at_k": 0.85972, "min_query_recall": 0.68} +{"implementation": "faiss-1.14.1", "backend": "gpu", "index_type": "ivf_flat", "metric": "inner_product", "num_vectors": 1000000, "num_queries": 1000, "dim": 128, "dtype": "fp32", "cpu_threads": 1, "top_k": 100, "batch_size": 64, "nlist": 256, "nprobe": 128, "train_samples": 100000, "build_ms": 6923.940047621727, "load_ms": 0.0, "gpu_transfer_ms": 784.832239151001, "gpu_index_mib": 497.984375, "gpu_temp_mib": 1536.0, "average_query_ms": 100.42996548116207, "qps": 9957.187530723317, "batch_p50_ms": 6.3972026109695435, "batch_p99_ms": 6.478957831859589, "recall_at_k": 0.8490999999999999, "min_query_recall": 0.7} +{"implementation": "faiss-1.14.1", "backend": "gpu", "index_type": "ivf_flat", "metric": "inner_product", "num_vectors": 1000000, "num_queries": 1000, "dim": 128, "dtype": "fp32", "cpu_threads": 1, "top_k": 10, "batch_size": 64, "nlist": 256, "nprobe": 160, "train_samples": 100000, "build_ms": 6923.940047621727, "load_ms": 0.0, "gpu_transfer_ms": 784.832239151001, "gpu_index_mib": 497.984375, "gpu_temp_mib": 1536.0, "average_query_ms": 209.93292428553104, "qps": 4763.42623913481, "batch_p50_ms": 13.52139562368393, "batch_p99_ms": 14.626868069171906, "recall_at_k": 0.9369000000000001, "min_query_recall": 0.7} +{"implementation": "faiss-1.14.1", "backend": "gpu", "index_type": "ivf_flat", "metric": "inner_product", "num_vectors": 1000000, "num_queries": 1000, "dim": 128, "dtype": "fp32", "cpu_threads": 1, "top_k": 50, "batch_size": 64, "nlist": 256, "nprobe": 160, "train_samples": 100000, "build_ms": 6923.940047621727, "load_ms": 0.0, "gpu_transfer_ms": 784.832239151001, "gpu_index_mib": 497.984375, "gpu_temp_mib": 1536.0, "average_query_ms": 113.9417802542448, "qps": 8776.411933959984, "batch_p50_ms": 7.0451050996780396, "batch_p99_ms": 11.37205958366394, "recall_at_k": 0.92312, "min_query_recall": 0.8} +{"implementation": "faiss-1.14.1", "backend": "gpu", "index_type": "ivf_flat", "metric": "inner_product", "num_vectors": 1000000, "num_queries": 1000, "dim": 128, "dtype": "fp32", "cpu_threads": 1, "top_k": 100, "batch_size": 64, "nlist": 256, "nprobe": 160, "train_samples": 100000, "build_ms": 6923.940047621727, "load_ms": 0.0, "gpu_transfer_ms": 784.832239151001, "gpu_index_mib": 497.984375, "gpu_temp_mib": 1536.0, "average_query_ms": 126.59424863755703, "qps": 7899.253013168305, "batch_p50_ms": 8.060820400714874, "batch_p99_ms": 8.196607232093811, "recall_at_k": 0.9163399999999999, "min_query_recall": 0.8} +{"implementation": "faiss-1.14.1", "backend": "gpu", "index_type": "ivf_flat", "metric": "inner_product", "num_vectors": 1000000, "num_queries": 1000, "dim": 128, "dtype": "fp32", "cpu_threads": 1, "top_k": 10, "batch_size": 64, "nlist": 256, "nprobe": 192, "train_samples": 100000, "build_ms": 6923.940047621727, "load_ms": 0.0, "gpu_transfer_ms": 784.832239151001, "gpu_index_mib": 497.984375, "gpu_temp_mib": 1536.0, "average_query_ms": 268.18819120526314, "qps": 3728.724950587515, "batch_p50_ms": 17.25468784570694, "batch_p99_ms": 19.591324031352997, "recall_at_k": 0.9722000000000001, "min_query_recall": 0.7} +{"implementation": "faiss-1.14.1", "backend": "gpu", "index_type": "ivf_flat", "metric": "inner_product", "num_vectors": 1000000, "num_queries": 1000, "dim": 128, "dtype": "fp32", "cpu_threads": 1, "top_k": 50, "batch_size": 64, "nlist": 256, "nprobe": 192, "train_samples": 100000, "build_ms": 6923.940047621727, "load_ms": 0.0, "gpu_transfer_ms": 784.832239151001, "gpu_index_mib": 497.984375, "gpu_temp_mib": 1536.0, "average_query_ms": 134.80234675109386, "qps": 7418.268480492054, "batch_p50_ms": 8.60326737165451, "batch_p99_ms": 8.92588496208191, "recall_at_k": 0.96408, "min_query_recall": 0.84} +{"implementation": "faiss-1.14.1", "backend": "gpu", "index_type": "ivf_flat", "metric": "inner_product", "num_vectors": 1000000, "num_queries": 1000, "dim": 128, "dtype": "fp32", "cpu_threads": 1, "top_k": 100, "batch_size": 64, "nlist": 256, "nprobe": 192, "train_samples": 100000, "build_ms": 6923.940047621727, "load_ms": 0.0, "gpu_transfer_ms": 784.832239151001, "gpu_index_mib": 497.984375, "gpu_temp_mib": 1536.0, "average_query_ms": 152.11867079138756, "qps": 6573.815001127506, "batch_p50_ms": 9.70180332660675, "batch_p99_ms": 9.853780269622803, "recall_at_k": 0.96127, "min_query_recall": 0.88} +{"implementation": "faiss-1.14.1", "backend": "gpu", "index_type": "ivf_flat", "metric": "inner_product", "num_vectors": 1000000, "num_queries": 1000, "dim": 128, "dtype": "fp32", "cpu_threads": 1, "top_k": 10, "batch_size": 64, "nlist": 256, "nprobe": 224, "train_samples": 100000, "build_ms": 6923.940047621727, "load_ms": 0.0, "gpu_transfer_ms": 784.832239151001, "gpu_index_mib": 497.984375, "gpu_temp_mib": 1536.0, "average_query_ms": 327.7342140674591, "qps": 3051.252988173414, "batch_p50_ms": 21.058037877082825, "batch_p99_ms": 22.79273420572281, "recall_at_k": 0.993, "min_query_recall": 0.8} +{"implementation": "faiss-1.14.1", "backend": "gpu", "index_type": "ivf_flat", "metric": "inner_product", "num_vectors": 1000000, "num_queries": 1000, "dim": 128, "dtype": "fp32", "cpu_threads": 1, "top_k": 50, "batch_size": 64, "nlist": 256, "nprobe": 224, "train_samples": 100000, "build_ms": 6923.940047621727, "load_ms": 0.0, "gpu_transfer_ms": 784.832239151001, "gpu_index_mib": 497.984375, "gpu_temp_mib": 1536.0, "average_query_ms": 152.58803740143776, "qps": 6553.593696006064, "batch_p50_ms": 9.723648428916931, "batch_p99_ms": 9.950794279575348, "recall_at_k": 0.9898800000000001, "min_query_recall": 0.92} +{"implementation": "faiss-1.14.1", "backend": "gpu", "index_type": "ivf_flat", "metric": "inner_product", "num_vectors": 1000000, "num_queries": 1000, "dim": 128, "dtype": "fp32", "cpu_threads": 1, "top_k": 100, "batch_size": 64, "nlist": 256, "nprobe": 224, "train_samples": 100000, "build_ms": 6923.940047621727, "load_ms": 0.0, "gpu_transfer_ms": 784.832239151001, "gpu_index_mib": 497.984375, "gpu_temp_mib": 1536.0, "average_query_ms": 177.92695373296738, "qps": 5620.283936860961, "batch_p50_ms": 11.344268918037415, "batch_p99_ms": 11.505231261253357, "recall_at_k": 0.9887900000000001, "min_query_recall": 0.95} +CSV: results/formal_100k15/faiss_gpu_raw.csv diff --git a/09_vector_retrieval/PurplePulse/evidence/2026-09-04/faiss_index_sha256.txt b/09_vector_retrieval/PurplePulse/evidence/2026-09-04/faiss_index_sha256.txt new file mode 100644 index 00000000..c2efd7e6 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/evidence/2026-09-04/faiss_index_sha256.txt @@ -0,0 +1,2 @@ +760e3a94f24c5becfa21e89a57f67395013d5ad5375c20eb3f5d5cfb4c3720a1 data/million/faiss_formal_100k15/faiss_flat_ip_1000000x128_nlist256.index +a66a63bb28ed9e70ae83c042f57a8ca79de3ee01a6414377452a7610738431b1 data/million/faiss_formal_100k15/faiss_ivf_flat_ip_1000000x128_nlist256.index diff --git a/09_vector_retrieval/PurplePulse/evidence/2026-09-04/faiss_script_after_instrumentation.txt b/09_vector_retrieval/PurplePulse/evidence/2026-09-04/faiss_script_after_instrumentation.txt new file mode 100644 index 00000000..f3271a0a --- /dev/null +++ b/09_vector_retrieval/PurplePulse/evidence/2026-09-04/faiss_script_after_instrumentation.txt @@ -0,0 +1,3 @@ +FAISS Flat vs PurplePulse exact recall@10: 1.000000000, min=1.000000000, mean_score_error=2.1645546e-06, max_score_error=1.14440918e-05 +{"implementation": "faiss-1.14.1", "backend": "gpu", "index_type": "flat", "metric": "inner_product", "num_vectors": 1000000, "num_queries": 1000, "dim": 128, "dtype": "fp32", "cpu_threads": 1, "top_k": 10, "batch_size": 64, "nlist": 0, "nprobe": 0, "train_samples": 0, "training_iterations": 0, "training_seed": 0, "build_ms": 0.0, "load_ms": 296.24930024147034, "gpu_transfer_ms": 668.2284846901894, "gpu_index_mib": 492.095947265625, "gpu_temp_mib": 1536.0, "average_query_ms": 48.66527020931244, "qps": 20548.534831902423, "run_p50_ms": 48.66527020931244, "run_p99_ms": 48.66527020931244, "run_samples": 1, "batch_p50_ms": 3.057025372982025, "batch_p99_ms": 3.069080412387848, "batch_latency_samples": 16, "recall_at_k": 1.0, "min_query_recall": 1.0, "mean_absolute_score_error": 2.152252197265625e-06, "max_absolute_score_error": 1.1444091796875e-05} +CSV: results/validation_after_instrumentation/faiss_gpu.csv diff --git a/09_vector_retrieval/PurplePulse/evidence/2026-09-04/gpu_snapshot_after_benchmarks.txt b/09_vector_retrieval/PurplePulse/evidence/2026-09-04/gpu_snapshot_after_benchmarks.txt new file mode 100644 index 00000000..55c78f31 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/evidence/2026-09-04/gpu_snapshot_after_benchmarks.txt @@ -0,0 +1 @@ +2026/09/04 08:05:44.331, NVIDIA GeForce RTX 4090 D, 32, 22.39 W, 210 MHz, 405 MHz diff --git a/09_vector_retrieval/PurplePulse/evidence/2026-09-04/index_100k15_sha256.txt b/09_vector_retrieval/PurplePulse/evidence/2026-09-04/index_100k15_sha256.txt new file mode 100644 index 00000000..3971a8e3 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/evidence/2026-09-04/index_100k15_sha256.txt @@ -0,0 +1 @@ +f5bd87fcc72d37164fa931436e5bfd293dfb7f953d09cb28169a77c7cdadc415 data/million/inner_fp32_nlist256_spherical_100k15.ivf diff --git a/09_vector_retrieval/PurplePulse/evidence/2026-09-04/ivf_build_100k15.txt b/09_vector_retrieval/PurplePulse/evidence/2026-09-04/ivf_build_100k15.txt new file mode 100644 index 00000000..98fa2c56 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/evidence/2026-09-04/ivf_build_100k15.txt @@ -0,0 +1,11 @@ +IVF-Flat 索引构建完成 +向量库: 1000000 x 128 +dtype/metric: fp32/inner_product +nlist: 256 +迭代/训练样本: 15/100000 +构建时间: 67678 ms +保存时间: 201.959 ms +桶大小 min/avg/max: 3734/3906.25/4091 +空桶: 0 +索引大小: 520133160 bytes +输出: "data/million/inner_fp32_nlist256_spherical_100k15.ivf" diff --git a/09_vector_retrieval/PurplePulse/evidence/2026-09-04/ivf_k100_repeat100.txt b/09_vector_retrieval/PurplePulse/evidence/2026-09-04/ivf_k100_repeat100.txt new file mode 100644 index 00000000..68f8926e --- /dev/null +++ b/09_vector_retrieval/PurplePulse/evidence/2026-09-04/ivf_k100_repeat100.txt @@ -0,0 +1,101 @@ +IVF-Flat gpu 查询完成 +索引: 1000000 x 128 +dtype/metric: fp32/inner_product +nlist/nprobe/top_k: 256/128/100 +nprobe policy: fixed +memory mode: disabled +索引加载: 353.542 ms +GPU 初始化: 222.296557 ms +索引 H2D: 51.038208 ms +GPU 缓冲区: 505.610847 MiB +平均查询: 164.37 ms +run P50/P99: 164.701/166.278 ms +QPS: 6083.83 +输出: "results/formal_100k15/ivf_k100_raw/nlist256_nprobe128.txt" +平均 CPU 中心/桶选择: 0 ms +平均 query H2D: 0.230585 ms +平均 GPU 中心选择: 1.42848 ms +平均自适应分层开销: 0 ms +平均 GPU 桶扫描+局部 Top-K: 156.251 ms +平均 GPU 最终 Top-K 归并: 1.33227 ms +平均记忆独立重排: 0 ms +平均结果 D2H: 0.526344 ms +batch P50/P99: 10.2194/10.4293 ms +实际 nprobe min/avg/max: 128/128/128 +recall@100=0.851910, min_query_recall=0.700000 +IVF-Flat gpu 查询完成 +索引: 1000000 x 128 +dtype/metric: fp32/inner_product +nlist/nprobe/top_k: 256/160/100 +nprobe policy: fixed +memory mode: disabled +索引加载: 348.273 ms +GPU 初始化: 177.593285 ms +索引 H2D: 49.840225 ms +GPU 缓冲区: 507.962410 MiB +平均查询: 211.3 ms +run P50/P99: 211.196/213.17 ms +QPS: 4732.6 +输出: "results/formal_100k15/ivf_k100_raw/nlist256_nprobe160.txt" +平均 CPU 中心/桶选择: 0 ms +平均 query H2D: 0.230195 ms +平均 GPU 中心选择: 1.71171 ms +平均自适应分层开销: 0 ms +平均 GPU 桶扫描+局部 Top-K: 202.791 ms +平均 GPU 最终 Top-K 归并: 1.36999 ms +平均记忆独立重排: 0 ms +平均结果 D2H: 0.538248 ms +batch P50/P99: 13.2468/13.5739 ms +实际 nprobe min/avg/max: 160/160/160 +recall@100=0.917580, min_query_recall=0.830000 +IVF-Flat gpu 查询完成 +索引: 1000000 x 128 +dtype/metric: fp32/inner_product +nlist/nprobe/top_k: 256/192/100 +nprobe policy: fixed +memory mode: disabled +索引加载: 349.271 ms +GPU 初始化: 184.404243 ms +索引 H2D: 50.775040 ms +GPU 缓冲区: 510.313972 MiB +平均查询: 262.99 ms +run P50/P99: 263.133/264.838 ms +QPS: 3802.43 +输出: "results/formal_100k15/ivf_k100_raw/nlist256_nprobe192.txt" +平均 CPU 中心/桶选择: 0 ms +平均 query H2D: 0.234722 ms +平均 GPU 中心选择: 1.99681 ms +平均自适应分层开销: 0 ms +平均 GPU 桶扫描+局部 Top-K: 254.169 ms +平均 GPU 最终 Top-K 归并: 1.38832 ms +平均记忆独立重排: 0 ms +平均结果 D2H: 0.53602 ms +batch P50/P99: 16.5511/16.997 ms +实际 nprobe min/avg/max: 192/192/192 +recall@100=0.962280, min_query_recall=0.900000 +IVF-Flat gpu 查询完成 +索引: 1000000 x 128 +dtype/metric: fp32/inner_product +nlist/nprobe/top_k: 256/224/100 +nprobe policy: fixed +memory mode: disabled +索引加载: 345.818 ms +GPU 初始化: 177.716485 ms +索引 H2D: 49.881088 ms +GPU 缓冲区: 512.665535 MiB +平均查询: 315.231 ms +run P50/P99: 315.293/317.578 ms +QPS: 3172.27 +输出: "results/formal_100k15/ivf_k100_raw/nlist256_nprobe224.txt" +平均 CPU 中心/桶选择: 0 ms +平均 query H2D: 0.234026 ms +平均 GPU 中心选择: 2.27418 ms +平均自适应分层开销: 0 ms +平均 GPU 桶扫描+局部 Top-K: 306.111 ms +平均 GPU 最终 Top-K 归并: 1.42648 ms +平均记忆独立重排: 0 ms +平均结果 D2H: 0.514689 ms +batch P50/P99: 19.943/20.4702 ms +实际 nprobe min/avg/max: 224/224/224 +recall@100=0.989200, min_query_recall=0.930000 +CSV: results/formal_100k15/ivf_k100_raw.csv diff --git a/09_vector_retrieval/PurplePulse/evidence/2026-09-04/ivf_k10_repeat100.txt b/09_vector_retrieval/PurplePulse/evidence/2026-09-04/ivf_k10_repeat100.txt new file mode 100644 index 00000000..5598c9f3 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/evidence/2026-09-04/ivf_k10_repeat100.txt @@ -0,0 +1,101 @@ +IVF-Flat gpu 查询完成 +索引: 1000000 x 128 +dtype/metric: fp32/inner_product +nlist/nprobe/top_k: 256/128/10 +nprobe policy: fixed +memory mode: disabled +索引加载: 362.111 ms +GPU 初始化: 249.238430 ms +索引 H2D: 50.082817 ms +GPU 缓冲区: 497.107430 MiB +平均查询: 142.972 ms +run P50/P99: 143.046/144.389 ms +QPS: 6994.35 +输出: "results/formal_100k15/ivf_k10_raw/nlist256_nprobe128.txt" +平均 CPU 中心/桶选择: 0 ms +平均 query H2D: 0.226118 ms +平均 GPU 中心选择: 1.4063 ms +平均自适应分层开销: 0 ms +平均 GPU 桶扫描+局部 Top-K: 136.9 ms +平均 GPU 最终 Top-K 归并: 0.177871 ms +平均记忆独立重排: 0 ms +平均结果 D2H: 0.35093 ms +batch P50/P99: 8.91104/8.99946 ms +实际 nprobe min/avg/max: 128/128/128 +recall@10=0.887600, min_query_recall=0.400000 +IVF-Flat gpu 查询完成 +索引: 1000000 x 128 +dtype/metric: fp32/inner_product +nlist/nprobe/top_k: 256/160/10 +nprobe policy: fixed +memory mode: disabled +索引加载: 349.508 ms +GPU 初始化: 187.561331 ms +索引 H2D: 52.624416 ms +GPU 缓冲区: 497.349617 MiB +平均查询: 175.542 ms +run P50/P99: 175.499/176.839 ms +QPS: 5696.66 +输出: "results/formal_100k15/ivf_k10_raw/nlist256_nprobe160.txt" +平均 CPU 中心/桶选择: 0 ms +平均 query H2D: 0.2292 ms +平均 GPU 中心选择: 1.71734 ms +平均自适应分层开销: 0 ms +平均 GPU 桶扫描+局部 Top-K: 169.143 ms +平均 GPU 最终 Top-K 归并: 0.183409 ms +平均记忆独立重排: 0 ms +平均结果 D2H: 0.35601 ms +batch P50/P99: 10.9579/11.1168 ms +实际 nprobe min/avg/max: 160/160/160 +recall@10=0.940900, min_query_recall=0.600000 +IVF-Flat gpu 查询完成 +索引: 1000000 x 128 +dtype/metric: fp32/inner_product +nlist/nprobe/top_k: 256/192/10 +nprobe policy: fixed +memory mode: disabled +索引加载: 338.971 ms +GPU 初始化: 178.590357 ms +索引 H2D: 50.026463 ms +GPU 缓冲区: 497.591805 MiB +平均查询: 213.302 ms +run P50/P99: 213.26/214.714 ms +QPS: 4688.19 +输出: "results/formal_100k15/ivf_k10_raw/nlist256_nprobe192.txt" +平均 CPU 中心/桶选择: 0 ms +平均 query H2D: 0.227254 ms +平均 GPU 中心选择: 2.00364 ms +平均自适应分层开销: 0 ms +平均 GPU 桶扫描+局部 Top-K: 206.648 ms +平均 GPU 最终 Top-K 归并: 0.187812 ms +平均记忆独立重排: 0 ms +平均结果 D2H: 0.311597 ms +batch P50/P99: 13.3933/13.5356 ms +实际 nprobe min/avg/max: 192/192/192 +recall@10=0.973400, min_query_recall=0.700000 +IVF-Flat gpu 查询完成 +索引: 1000000 x 128 +dtype/metric: fp32/inner_product +nlist/nprobe/top_k: 256/224/10 +nprobe policy: fixed +memory mode: disabled +索引加载: 348.109 ms +GPU 初始化: 179.173165 ms +索引 H2D: 50.736130 ms +GPU 缓冲区: 497.833992 MiB +平均查询: 250.392 ms +run P50/P99: 250.245/252.798 ms +QPS: 3993.74 +输出: "results/formal_100k15/ivf_k10_raw/nlist256_nprobe224.txt" +平均 CPU 中心/桶选择: 0 ms +平均 query H2D: 0.234291 ms +平均 GPU 中心选择: 2.28967 ms +平均自适应分层开销: 0 ms +平均 GPU 桶扫描+局部 Top-K: 243.249 ms +平均 GPU 最终 Top-K 归并: 0.190981 ms +平均记忆独立重排: 0 ms +平均结果 D2H: 0.33933 ms +batch P50/P99: 15.7794/15.9556 ms +实际 nprobe min/avg/max: 224/224/224 +recall@10=0.993100, min_query_recall=0.800000 +CSV: results/formal_100k15/ivf_k10_raw.csv diff --git a/09_vector_retrieval/PurplePulse/evidence/2026-09-04/ivf_k50_repeat100.txt b/09_vector_retrieval/PurplePulse/evidence/2026-09-04/ivf_k50_repeat100.txt new file mode 100644 index 00000000..292b24eb --- /dev/null +++ b/09_vector_retrieval/PurplePulse/evidence/2026-09-04/ivf_k50_repeat100.txt @@ -0,0 +1,101 @@ +IVF-Flat gpu 查询完成 +索引: 1000000 x 128 +dtype/metric: fp32/inner_product +nlist/nprobe/top_k: 256/128/50 +nprobe policy: fixed +memory mode: disabled +索引加载: 349.347 ms +GPU 初始化: 219.744959 ms +索引 H2D: 51.002369 ms +GPU 缓冲区: 500.886726 MiB +平均查询: 137.089 ms +run P50/P99: 137.215/139.576 ms +QPS: 7294.52 +输出: "results/formal_100k15/ivf_k50_raw/nlist256_nprobe128.txt" +平均 CPU 中心/桶选择: 0 ms +平均 query H2D: 0.23926 ms +平均 GPU 中心选择: 1.43383 ms +平均自适应分层开销: 0 ms +平均 GPU 桶扫描+局部 Top-K: 129.996 ms +平均 GPU 最终 Top-K 归并: 0.649493 ms +平均记忆独立重排: 0 ms +平均结果 D2H: 0.471183 ms +batch P50/P99: 8.49632/8.70291 ms +实际 nprobe min/avg/max: 128/128/128 +recall@50=0.862480, min_query_recall=0.700000 +IVF-Flat gpu 查询完成 +索引: 1000000 x 128 +dtype/metric: fp32/inner_product +nlist/nprobe/top_k: 256/160/50 +nprobe policy: fixed +memory mode: disabled +索引加载: 328.392 ms +GPU 初始化: 175.771099 ms +索引 H2D: 48.182274 ms +GPU 缓冲区: 502.066414 MiB +平均查询: 178.399 ms +run P50/P99: 178.397/180.654 ms +QPS: 5605.41 +输出: "results/formal_100k15/ivf_k50_raw/nlist256_nprobe160.txt" +平均 CPU 中心/桶选择: 0 ms +平均 query H2D: 0.213315 ms +平均 GPU 中心选择: 1.73579 ms +平均自适应分层开销: 0 ms +平均 GPU 桶扫描+局部 Top-K: 171.117 ms +平均 GPU 最终 Top-K 归并: 0.716433 ms +平均记忆独立重排: 0 ms +平均结果 D2H: 0.416466 ms +batch P50/P99: 11.18/11.4261 ms +实际 nprobe min/avg/max: 160/160/160 +recall@50=0.924820, min_query_recall=0.800000 +IVF-Flat gpu 查询完成 +索引: 1000000 x 128 +dtype/metric: fp32/inner_product +nlist/nprobe/top_k: 256/192/50 +nprobe policy: fixed +memory mode: disabled +索引加载: 310.692 ms +GPU 初始化: 176.079546 ms +索引 H2D: 48.196609 ms +GPU 缓冲区: 503.246101 MiB +平均查询: 223.565 ms +run P50/P99: 223.664/226.713 ms +QPS: 4472.96 +输出: "results/formal_100k15/ivf_k50_raw/nlist256_nprobe192.txt" +平均 CPU 中心/桶选择: 0 ms +平均 query H2D: 0.213812 ms +平均 GPU 中心选择: 2.00779 ms +平均自适应分层开销: 0 ms +平均 GPU 桶扫描+局部 Top-K: 215.991 ms +平均 GPU 最终 Top-K 归并: 0.727207 ms +平均记忆独立重排: 0 ms +平均结果 D2H: 0.404718 ms +batch P50/P99: 14.0604/14.5124 ms +实际 nprobe min/avg/max: 192/192/192 +recall@50=0.965800, min_query_recall=0.820000 +IVF-Flat gpu 查询完成 +索引: 1000000 x 128 +dtype/metric: fp32/inner_product +nlist/nprobe/top_k: 256/224/50 +nprobe policy: fixed +memory mode: disabled +索引加载: 313.004 ms +GPU 初始化: 177.229013 ms +索引 H2D: 49.465343 ms +GPU 缓冲区: 504.425789 MiB +平均查询: 273.458 ms +run P50/P99: 273.591/276.441 ms +QPS: 3656.87 +输出: "results/formal_100k15/ivf_k50_raw/nlist256_nprobe224.txt" +平均 CPU 中心/桶选择: 0 ms +平均 query H2D: 0.217707 ms +平均 GPU 中心选择: 2.27946 ms +平均自适应分层开销: 0 ms +平均 GPU 桶扫描+局部 Top-K: 265.604 ms +平均 GPU 最终 Top-K 归并: 0.743718 ms +平均记忆独立重排: 0 ms +平均结果 D2H: 0.379669 ms +batch P50/P99: 17.2885/17.7533 ms +实际 nprobe min/avg/max: 224/224/224 +recall@50=0.990540, min_query_recall=0.920000 +CSV: results/formal_100k15/ivf_k50_raw.csv diff --git a/09_vector_retrieval/PurplePulse/evidence/2026-09-04/ivf_script_after_instrumentation.txt b/09_vector_retrieval/PurplePulse/evidence/2026-09-04/ivf_script_after_instrumentation.txt new file mode 100644 index 00000000..4dcea24f --- /dev/null +++ b/09_vector_retrieval/PurplePulse/evidence/2026-09-04/ivf_script_after_instrumentation.txt @@ -0,0 +1,28 @@ +IVF-Flat gpu 查询完成 +索引: 1000000 x 128 +dtype/metric: fp32/inner_product +nlist/nprobe/top_k: 256/160/10 +nprobe policy: fixed +memory mode: disabled +索引加载: 335.638 ms +GPU 初始化: 299.387560 ms +索引 H2D: 48.644096 ms +GPU 缓冲区: 497.349617 MiB +平均查询: 171.168 ms +run P50/P99: 171.168/171.168 ms +run samples: 1 +QPS: 5842.22 +输出: "results/validation_after_instrumentation/ivf_k10/nlist256_nprobe160.txt" +平均 CPU 中心/桶选择: 0 ms +平均 query H2D: 0.266208 ms +平均 GPU 中心选择: 1.62509 ms +平均自适应分层开销: 0 ms +平均 GPU 桶扫描+局部 Top-K: 164.851 ms +平均 GPU 最终 Top-K 归并: 0.178112 ms +平均记忆独立重排: 0 ms +平均结果 D2H: 0.30512 ms +batch P50/P99: 10.6724/10.7368 ms +batch samples: 16 +实际 nprobe min/avg/max: 160/160/160 +recall@10=0.940900, min_query_recall=0.600000, mean_absolute_score_error=0.042425712, max_absolute_score_error=3.1734657 +CSV: results/validation_after_instrumentation/ivf_k10.csv diff --git a/09_vector_retrieval/PurplePulse/evidence/2026-09-04/nsys_exact_k10.txt b/09_vector_retrieval/PurplePulse/evidence/2026-09-04/nsys_exact_k10.txt new file mode 100644 index 00000000..41759a4d --- /dev/null +++ b/09_vector_retrieval/PurplePulse/evidence/2026-09-04/nsys_exact_k10.txt @@ -0,0 +1,100 @@ +向量库: 1000000 x 128 +查询数: 1000 +dtype: fp32 +metric: inner_product +top_k: 10 +batch_size: 64 +distance_mode: warp +topk_mode: fused +warmup/repeat: 1/5 +backend: gpu +GPU 一次性初始化总时间: 387.492 ms +数据库一次性 H2D: 49.1047 ms +GPU 缓冲区: 488.437 MiB +估算冷启动端到端: 533.198 ms +平均常驻查询时间: 145.706 ms +run P50: 145.789 ms +run P99: 146.261 ms +常驻 QPS: 6863.12 +结果: "results/formal_100k15/exact_k10_nsys.txt" +平均查询 H2D: 0.33273 ms +平均融合距离+局部 Top-K kernel: 144.491 ms +平均最终 Top-K kernel: 0.209363 ms +平均结果 D2H: 0.399002 ms +batch P50: 9.25331 ms +batch P99: 9.6127 ms +Collecting data... +Generating '/tmp/nsys-report-450f.qdstrm' +[1/8] [0% ] exact_k10_nsys.nsys-rep[1/8] [0% ] exact_k10_nsys.nsys-rep[1/8] [==================76% ] exact_k10_nsys.nsys-rep[1/8] [========================100%] exact_k10_nsys.nsys-rep[1/8] [========================100%] exact_k10_nsys.nsys-rep +[2/8] [0% ] exact_k10_nsys.sqlite[2/8] [1% ] exact_k10_nsys.sqlite[2/8] [2% ] exact_k10_nsys.sqlite[2/8] [3% ] exact_k10_nsys.sqlite[2/8] [4% ] exact_k10_nsys.sqlite[2/8] [5% ] exact_k10_nsys.sqlite[2/8] [6% ] exact_k10_nsys.sqlite[2/8] [7% ] exact_k10_nsys.sqlite[2/8] [8% ] exact_k10_nsys.sqlite[2/8] [9% ] exact_k10_nsys.sqlite[2/8] [10% ] exact_k10_nsys.sqlite[2/8] [11% ] exact_k10_nsys.sqlite[2/8] [12% ] exact_k10_nsys.sqlite[2/8] [13% ] exact_k10_nsys.sqlite[2/8] [14% ] exact_k10_nsys.sqlite[2/8] [=15% ] exact_k10_nsys.sqlite[2/8] [=16% ] exact_k10_nsys.sqlite[2/8] [=17% ] exact_k10_nsys.sqlite[2/8] [==18% ] exact_k10_nsys.sqlite[2/8] [==19% ] exact_k10_nsys.sqlite[2/8] [==20% ] exact_k10_nsys.sqlite[2/8] [==21% ] exact_k10_nsys.sqlite[2/8] [===22% ] exact_k10_nsys.sqlite[2/8] [===23% ] exact_k10_nsys.sqlite[2/8] [===24% ] exact_k10_nsys.sqlite[2/8] [====25% ] exact_k10_nsys.sqlite[2/8] [====26% ] exact_k10_nsys.sqlite[2/8] [====27% ] exact_k10_nsys.sqlite[2/8] [====28% ] exact_k10_nsys.sqlite[2/8] [=====29% ] exact_k10_nsys.sqlite[2/8] [=====30% ] exact_k10_nsys.sqlite[2/8] [=====31% ] exact_k10_nsys.sqlite[2/8] [=====32% ] exact_k10_nsys.sqlite[2/8] [======33% ] exact_k10_nsys.sqlite[2/8] [======34% ] exact_k10_nsys.sqlite[2/8] [======35% ] exact_k10_nsys.sqlite[2/8] [=======36% ] exact_k10_nsys.sqlite[2/8] [=======37% ] exact_k10_nsys.sqlite[2/8] [=======38% ] exact_k10_nsys.sqlite[2/8] [=======39% ] exact_k10_nsys.sqlite[2/8] [========40% ] exact_k10_nsys.sqlite[2/8] [========41% ] exact_k10_nsys.sqlite[2/8] [========42% ] exact_k10_nsys.sqlite[2/8] [=========43% ] exact_k10_nsys.sqlite[2/8] [=========44% ] exact_k10_nsys.sqlite[2/8] [=========45% ] exact_k10_nsys.sqlite[2/8] [=========46% ] exact_k10_nsys.sqlite[2/8] [==========47% ] exact_k10_nsys.sqlite[2/8] [==========48% ] exact_k10_nsys.sqlite[2/8] [==========49% ] exact_k10_nsys.sqlite[2/8] [===========50% ] exact_k10_nsys.sqlite[2/8] [===========51% ] exact_k10_nsys.sqlite[2/8] [===========52% ] exact_k10_nsys.sqlite[2/8] [===========53% ] exact_k10_nsys.sqlite[2/8] [============54% ] exact_k10_nsys.sqlite[2/8] [============55% ] exact_k10_nsys.sqlite[2/8] [============56% ] exact_k10_nsys.sqlite[2/8] [============57% ] exact_k10_nsys.sqlite[2/8] [=============58% ] exact_k10_nsys.sqlite[2/8] [=============59% ] exact_k10_nsys.sqlite[2/8] [=============60% ] exact_k10_nsys.sqlite[2/8] [==============61% ] exact_k10_nsys.sqlite[2/8] [==============62% ] exact_k10_nsys.sqlite[2/8] [==============63% ] exact_k10_nsys.sqlite[2/8] [==============64% ] exact_k10_nsys.sqlite[2/8] [===============65% ] exact_k10_nsys.sqlite[2/8] [===============66% ] exact_k10_nsys.sqlite[2/8] [===============67% ] exact_k10_nsys.sqlite[2/8] [================68% ] exact_k10_nsys.sqlite[2/8] [================69% ] exact_k10_nsys.sqlite[2/8] [================70% ] exact_k10_nsys.sqlite[2/8] [================71% ] exact_k10_nsys.sqlite[2/8] [=================72% ] exact_k10_nsys.sqlite[2/8] [=================73% ] exact_k10_nsys.sqlite[2/8] [=================74% ] exact_k10_nsys.sqlite[2/8] [==================75% ] exact_k10_nsys.sqlite[2/8] [==================76% ] exact_k10_nsys.sqlite[2/8] [==================77% ] exact_k10_nsys.sqlite[2/8] [==================78% ] exact_k10_nsys.sqlite[2/8] [===================79% ] exact_k10_nsys.sqlite[2/8] [===================80% ] exact_k10_nsys.sqlite[2/8] [===================81% ] exact_k10_nsys.sqlite[2/8] [===================82% ] exact_k10_nsys.sqlite[2/8] [====================83% ] exact_k10_nsys.sqlite[2/8] [====================84% ] exact_k10_nsys.sqlite[2/8] [====================85% ] exact_k10_nsys.sqlite[2/8] [=====================86% ] exact_k10_nsys.sqlite[2/8] [=====================87% ] exact_k10_nsys.sqlite[2/8] [=====================88% ] exact_k10_nsys.sqlite[2/8] [=====================89% ] exact_k10_nsys.sqlite[2/8] [======================90% ] exact_k10_nsys.sqlite[2/8] [======================91% ] exact_k10_nsys.sqlite[2/8] [======================92% ] exact_k10_nsys.sqlite[2/8] [=======================93% ] exact_k10_nsys.sqlite[2/8] [=======================94% ] exact_k10_nsys.sqlite[2/8] [=======================95% ] exact_k10_nsys.sqlite[2/8] [=======================96% ] exact_k10_nsys.sqlite[2/8] [========================97% ] exact_k10_nsys.sqlite[2/8] [========================98% ] exact_k10_nsys.sqlite[2/8] [========================99% ] exact_k10_nsys.sqlite[2/8] [========================100%] exact_k10_nsys.sqlite[2/8] [========================100%] exact_k10_nsys.sqlite +SKIPPED: /root/PurplePulse/results/evidence/exact_k10_nsys.sqlite does not contain NV Tools Extension (NVTX) data. +[3/8] Executing 'nvtx_sum' stats report +[4/8] Executing 'osrt_sum' stats report + + Time (%) Total Time (ns) Num Calls Avg (ns) Med (ns) Min (ns) Max (ns) StdDev (ns) Name + -------- --------------- --------- ----------- ----------- --------- --------- ----------- ---------------------- + 50.5 1229257719 21 58536081.9 41666582.0 4009 227645472 60313247.0 poll + 41.1 1000173936 2 500086968.0 500086968.0 500085280 500088656 2387.2 pthread_cond_timedwait + 4.5 108441993 19 5707473.3 2347.0 1028 108319891 24848738.0 read + 3.8 92281614 552 167176.8 15264.5 1034 16349452 834161.2 ioctl + 0.1 1445352 25 57814.1 4230.0 2799 1135032 225138.3 mmap64 + 0.0 458294 9 50921.6 47450.0 44364 69199 8896.6 sem_timedwait + 0.0 308424 75 4112.3 1614.0 1244 60791 7551.7 fopen + 0.0 199870 43 4648.1 3864.0 1716 10003 1941.4 open64 + 0.0 141161 1 141161.0 141161.0 141161 141161 0.0 pthread_cond_wait + 0.0 122397 26 4707.6 3610.0 3177 15967 3203.3 writev + 0.0 110603 3 36867.7 35896.0 27886 46821 9504.8 pthread_create + 0.0 88462 13 6804.8 2546.0 1350 36082 9437.1 mmap + 0.0 46148 11 4195.3 3895.0 1685 7795 1879.7 write + 0.0 41415 11 3765.0 1962.0 1010 10948 3549.7 fclose + 0.0 39814 1 39814.0 39814.0 39814 39814 0.0 fgets + 0.0 25760 3 8586.7 4555.0 1766 19439 9501.3 pipe2 + 0.0 25583 6 4263.8 4722.0 1795 6360 1915.0 open + 0.0 20692 6 3448.7 3280.5 1891 5558 1403.1 munmap + 0.0 20296 3 6765.3 4667.0 4282 11347 3972.5 pthread_cond_broadcast + 0.0 15064 3 5021.3 2905.0 2787 9372 3768.2 stat + 0.0 13074 2 6537.0 6537.0 4600 8474 2739.3 socket + 0.0 8873 1 8873.0 8873.0 8873 8873 0.0 connect + 0.0 6874 3 2291.3 2418.0 1040 3416 1193.1 fwrite + 0.0 2770 1 2770.0 2770.0 2770 2770 0.0 bind + 0.0 1234 1 1234.0 1234.0 1234 1234 0.0 listen + 0.0 1156 1 1156.0 1156.0 1156 1156 0.0 fcntl + +[5/8] Executing 'cuda_api_sum' stats report + + Time (%) Total Time (ns) Num Calls Avg (ns) Med (ns) Min (ns) Max (ns) StdDev (ns) Name + -------- --------------- --------- ---------- -------- -------- -------- ----------- ---------------------- + 89.9 919006727 289 3179954.1 13477.0 9090 49083340 5059558.9 cudaMemcpy + 9.7 99655891 6 16609315.2 3164.5 1521 99579244 40646806.2 cudaMalloc + 0.2 1653566 482 3430.6 3266.5 2894 23561 1083.3 cudaEventRecord + 0.2 1584689 192 8253.6 3812.0 3165 815332 58578.6 cudaLaunchKernel + 0.1 608876 6 101479.3 23387.0 2171 376971 151258.0 cudaFree + 0.0 253967 97 2618.2 2630.0 1364 3568 259.8 cudaEventSynchronize + 0.0 31958 32 998.7 520.0 403 7999 1462.2 cudaEventCreate + 0.0 16409 32 512.8 355.5 288 1888 375.6 cudaEventDestroy + 0.0 1284 1 1284.0 1284.0 1284 1284 0.0 cuModuleGetLoadingMode + +[6/8] Executing 'cuda_gpu_kern_sum' stats report + + Time (%) Total Time (ns) Instances Avg (ns) Med (ns) Min (ns) Max (ns) StdDev (ns) Name + -------- --------------- --------- --------- --------- -------- -------- ----------- ---------------------------------------------------------------------------------------------------- + 99.9 866184910 96 9022759.5 9188328.0 6364230 9554682 668168.8 void exactFusedWarpTopKKernel(const T1 *, const T1 *, float *, … + 0.1 877669 96 9142.4 9120.0 8992 9696 104.0 blockTopKKernel(float *, const unsigned long *, float *, unsigned long *, unsigned long, unsigned i… + +[7/8] Executing 'cuda_gpu_mem_time_sum' stats report + + Time (%) Total Time (ns) Count Avg (ns) Med (ns) Min (ns) Max (ns) StdDev (ns) Operation + -------- --------------- ----- -------- -------- -------- -------- ----------- ---------------------------- + 99.5 49358062 97 508846.0 3936.0 2688 48983659 4973141.1 [CUDA memcpy Host-to-Device] + 0.5 241888 192 1259.8 1280.0 1184 1728 73.5 [CUDA memcpy Device-to-Host] + +[8/8] Executing 'cuda_gpu_mem_size_sum' stats report + + Total (MB) Count Avg (MB) Med (MB) Min (MB) Max (MB) StdDev (MB) Operation + ---------- ----- -------- -------- -------- -------- ----------- ---------------------------- + 515.072 97 5.310 0.033 0.020 512.000 51.982 [CUDA memcpy Host-to-Device] + 0.720 192 0.004 0.003 0.002 0.005 0.001 [CUDA memcpy Device-to-Host] + +Generated: + /root/PurplePulse/results/evidence/exact_k10_nsys.nsys-rep + /root/PurplePulse/results/evidence/exact_k10_nsys.sqlite diff --git a/09_vector_retrieval/PurplePulse/evidence/2026-09-04/rebuild_after_instrumentation.txt b/09_vector_retrieval/PurplePulse/evidence/2026-09-04/rebuild_after_instrumentation.txt new file mode 100644 index 00000000..6e5fda3c --- /dev/null +++ b/09_vector_retrieval/PurplePulse/evidence/2026-09-04/rebuild_after_instrumentation.txt @@ -0,0 +1,4 @@ +[1/4] Building CXX object CMakeFiles/vector_search.dir/src/main.cpp.o +[2/4] Linking CXX executable vector_search +[3/4] Building CXX object CMakeFiles/ivf_search.dir/src/search_ivf.cpp.o +[4/4] Linking CXX executable ivf_search diff --git a/09_vector_retrieval/PurplePulse/evidence/2026-09-04/source_sha256_before_instrumentation.txt b/09_vector_retrieval/PurplePulse/evidence/2026-09-04/source_sha256_before_instrumentation.txt new file mode 100644 index 00000000..83cd8927 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/evidence/2026-09-04/source_sha256_before_instrumentation.txt @@ -0,0 +1,7 @@ +eeeffc41d9f0ab84e28aaa6f880724fff4c8d5705865802cbc0b33d289743630 CMakeLists.txt +32ed6b1be004536f50253595156d16546337c7da1919ed913e268e5a70af78b4 README.md +2e0b17c978b82dbe7ce893054cd0bea7ce69b8c30d7a460c63307161d56c8802 docs/FINAL_REPORT.md +6407c1299670abfc45b8dc6b3236d3c3ceeafac0f8b66dc6a525d00db6a499db docs/BASELINE_RESULTS.md +52fc4696718ca42a34210c5c708d3356c3d2cb4b43bc0deadccef4aac0b49bfe src/gpu_search.cu +fb547bdb955d692ef5b727741e56af0e10bba2078213f427a703159a6fc05d33 src/gpu_ivf_flat.cu +f9015d3923c57610548234da66e80781989a88da05309320d8aedc0339cfe915 scripts/evaluate_recall.py diff --git a/09_vector_retrieval/PurplePulse/evidence/2026-09-08/bucket_major_compute_sanitizer.log b/09_vector_retrieval/PurplePulse/evidence/2026-09-08/bucket_major_compute_sanitizer.log new file mode 100644 index 00000000..317a6b81 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/evidence/2026-09-08/bucket_major_compute_sanitizer.log @@ -0,0 +1,3 @@ +========= COMPUTE-SANITIZER +IVF-Flat 建库、持久化、查询与 recall 测试通过 +========= ERROR SUMMARY: 0 errors diff --git a/09_vector_retrieval/PurplePulse/evidence/2026-09-08/commands.md b/09_vector_retrieval/PurplePulse/evidence/2026-09-08/commands.md new file mode 100644 index 00000000..bd6a1b12 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/evidence/2026-09-08/commands.md @@ -0,0 +1,176 @@ +# 2026-09-08 validation commands + +The remote host address and credentials are intentionally not recorded. + +```bash +export PATH=/usr/local/cuda/bin:$PATH +cmake -S . -B build -G Ninja -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_CUDA_ARCHITECTURES=native +cmake --build build -j 8 +ctest --test-dir build --output-on-failure +compute-sanitizer --tool memcheck --error-exitcode 99 ./build/unit_tests +compute-sanitizer --tool memcheck --error-exitcode 99 ./build/ivf_flat_tests +``` + +```bash +python3 scripts/generate_data.py \ + --database data/million/inner_fp32_db.bin \ + --queries data/million/inner_fp32_q.bin \ + --num-vectors 1000000 --num-queries 1000 --dim 128 \ + --dtype fp32 --metric inner_product --seed 2026 + +./build/ivf_build \ + --database data/million/inner_fp32_db.bin \ + --output data/million/inner_fp32_nlist256.ivf \ + --nlist 256 --iterations 15 --training-samples 100000 --threads 16 + +python3 scripts/benchmark_ivf_sweep.py \ + --search-binary ./build/ivf_search \ + --index data/million/inner_fp32_nlist256.ivf \ + --queries data/million/inner_fp32_q.bin \ + --params configs/ivf_flat_million_k100_batch64.conf \ + --exact results/exact_k100.txt \ + --output-dir results/k100_formal --csv results/k100_formal.csv \ + --nlist 256 --nprobes 128,160,192,224 \ + --training-samples 100000 --training-iterations 15 \ + --warmup 5 --repeat 100 +``` + +```bash +./build/ivf_search \ + --index data/million/inner_fp32_nlist256.ivf \ + --queries data/million/inner_fp32_q.bin \ + --params configs/ivf_flat_million_k50_warp_compact_batch40.conf \ + --output results/k50_batch40.txt \ + --backend gpu --warmup 1 --repeat 100 + +./build/ivf_search \ + --index data/million/inner_fp32_nlist256.ivf \ + --queries data/million/inner_fp32_q.bin \ + --params configs/ivf_flat_million_k50_batch64.conf \ + --output results/k50_batch64.txt \ + --backend gpu --warmup 1 --repeat 100 + +python3 scripts/evaluate_recall.py \ + --exact results/new_gpu/exact_k50.txt \ + --approximate results/k50_batch40.txt +``` + +```bash +./build/ivf_search \ + --index data/million/inner_fp32_nlist256.ivf \ + --queries data/million/inner_fp32_q.bin \ + --params configs/ivf_flat_million_k50_warp_compact_batch40.conf \ + --output results/k50_probe_major.txt \ + --backend gpu --warmup 5 --repeat 100 + +./build/ivf_search \ + --index data/million/inner_fp32_nlist256.ivf \ + --queries data/million/inner_fp32_q.bin \ + --params configs/ivf_flat_million_k50_bucket_major.conf \ + --output results/k50_bucket_major.txt \ + --backend gpu --warmup 5 --repeat 100 + +cmp results/k50_probe_major.txt results/k50_bucket_major.txt +compute-sanitizer --tool memcheck --error-exitcode 99 \ + ./build/ivf_flat_tests +``` + +K=100 heap and combined bucket-major A/B used the retained pre-optimization +binary and the rebuilt binary on the same host: + +```bash +./build/ivf_search_pre_k100_heap \ + --index data/million/inner_fp32_nlist256.ivf \ + --queries data/million/inner_fp32_q.bin \ + --params configs/ivf_flat_million_k100_warp1.conf \ + --output results/k100_pre_heap.txt \ + --backend gpu --warmup 5 --repeat 100 + +./build/ivf_search \ + --index data/million/inner_fp32_nlist256.ivf \ + --queries data/million/inner_fp32_q.bin \ + --params configs/ivf_flat_million_k100_bucket_major.conf \ + --output results/k100_heap_bucket_major.txt \ + --backend gpu --warmup 5 --repeat 100 + +cmp results/k100_pre_heap.txt results/k100_heap_bucket_major.txt +``` + +Rejected K=100 selection prototypes were tested against the retained stable +binary on the same host. The buffer-8 prototype was stopped after 10 repeats +because its QPS regression exceeded 20%. Kernel resources were inspected from +the compiled object before restoring the retained source. + +```bash +/usr/local/cuda/bin/cuobjdump --dump-resource-usage \ + build/CMakeFiles/vector_engine.dir/src/gpu_ivf_flat.cu.o + +sha256sum src/gpu_ivf_flat.cu +cmake --build build -j 8 +ctest --test-dir build --output-on-failure +``` + +The same-batch FAISS comparison reused the host CUDA 12.8 libraries. The venv +contained FAISS 1.14.1 and NumPy 2.2.6; disabling wheel preloading avoids +installing a second copy of CUDA runtime and cuBLAS. + +```bash +export _FAISS_WHEEL_DISABLE_CUDA_PRELOAD=1 +export LD_LIBRARY_PATH=/usr/local/cuda/lib64 + +/data/purplepulse-faiss-system-venv/bin/python scripts/benchmark_faiss.py \ + --database data/million/inner_fp32_db.bin \ + --queries data/million/inner_fp32_q.bin \ + --output-dir results/faiss_batch40 \ + --csv results/faiss_batch40.csv \ + --index-dir /data/purplepulse-faiss-indexes \ + --reference-results results/new_gpu/exact_k50.txt \ + --backends gpu --index-types ivf_flat \ + --top-ks 50 --nprobes 160 --nlist 256 --batch-size 40 \ + --train-samples 100000 --iterations 15 --seed 2026 \ + --warmup 5 --repeat 100 --cpu-threads 1 +``` + +```bash +python3 scripts/import_mnist.py \ + --input data/mnist/mnist.npz \ + --database data/mnist/database.bin \ + --queries data/mnist/queries.bin + +./build/ivf_build \ + --database data/mnist/database.bin --output data/mnist/nlist256.ivf \ + --nlist 256 --iterations 15 --training-samples 60000 --threads 16 + +./build/vector_search \ + --database data/mnist/database.bin --queries data/mnist/queries.bin \ + --params configs/exact_million_k10_fused_batch64.conf --backend gpu \ + --output results/mnist_exact_k10.txt --warmup 2 --repeat 10 + +python3 scripts/benchmark_ivf_sweep.py \ + --search-binary ./build/ivf_search \ + --index data/mnist/nlist256.ivf --queries data/mnist/queries.bin \ + --params configs/ivf_flat_mnist.conf \ + --exact results/mnist_exact_k10.txt \ + --output-dir results/mnist_ivf --csv results/mnist_ivf.csv \ + --nlist 256 --nprobes 8,16,32,64,96,128 \ + --training-samples 60000 --training-iterations 15 \ + --warmup 2 --repeat 10 +``` + +The final source-only audit excluded generated artifacts before configuring a new +directory: + +```bash +tar --exclude='./build' --exclude='./data' --exclude='./results' \ + --exclude='./evidence' --exclude='./.git' \ + -cf /data/purplepulse-submit-audit.tar . +mkdir /data/purplepulse-submit-audit +tar -xf /data/purplepulse-submit-audit.tar \ + -C /data/purplepulse-submit-audit +cmake -S /data/purplepulse-submit-audit \ + -B /data/purplepulse-submit-audit/build -G Ninja \ + -DCMAKE_BUILD_TYPE=Release +cmake --build /data/purplepulse-submit-audit/build -j 16 +ctest --test-dir /data/purplepulse-submit-audit/build --output-on-failure +``` diff --git a/09_vector_retrieval/PurplePulse/evidence/2026-09-08/compute_sanitizer.txt b/09_vector_retrieval/PurplePulse/evidence/2026-09-08/compute_sanitizer.txt new file mode 100644 index 00000000..55289ea6 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/evidence/2026-09-08/compute_sanitizer.txt @@ -0,0 +1,3 @@ +========= COMPUTE-SANITIZER +FP32/FP16 的 CPU/GPU 正确性与边界测试通过 +========= ERROR SUMMARY: 0 errors diff --git a/09_vector_retrieval/PurplePulse/evidence/2026-09-08/compute_sanitizer_ivf.txt b/09_vector_retrieval/PurplePulse/evidence/2026-09-08/compute_sanitizer_ivf.txt new file mode 100644 index 00000000..317a6b81 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/evidence/2026-09-08/compute_sanitizer_ivf.txt @@ -0,0 +1,3 @@ +========= COMPUTE-SANITIZER +IVF-Flat 建库、持久化、查询与 recall 测试通过 +========= ERROR SUMMARY: 0 errors diff --git a/09_vector_retrieval/PurplePulse/evidence/2026-09-08/ctest.txt b/09_vector_retrieval/PurplePulse/evidence/2026-09-08/ctest.txt new file mode 100644 index 00000000..af1690c9 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/evidence/2026-09-08/ctest.txt @@ -0,0 +1,22 @@ +Internal ctest changing into directory: /data/purplepulse-work/PurplePulse/build +Test project /data/purplepulse-work/PurplePulse/build + Start 1: unit_tests +1/8 Test #1: unit_tests ........................ Passed 0.26 sec + Start 2: file_io_tests +2/8 Test #2: file_io_tests ..................... Passed 0.01 sec + Start 3: ivf_flat_tests +3/8 Test #3: ivf_flat_tests .................... Passed 0.24 sec + Start 4: benchmark_faiss_tests +4/8 Test #4: benchmark_faiss_tests ............. Passed 0.87 sec + Start 5: benchmark_ivf_sweep_tests +5/8 Test #5: benchmark_ivf_sweep_tests ......... Passed 0.07 sec + Start 6: benchmark_adaptive_nprobe_tests +6/8 Test #6: benchmark_adaptive_nprobe_tests ... Passed 0.07 sec + Start 7: import_mnist_tests +7/8 Test #7: import_mnist_tests ................ Passed 0.28 sec + Start 8: plot_results_tests +8/8 Test #8: plot_results_tests ................ Passed 0.06 sec + +100% tests passed, 0 tests failed out of 8 + +Total Test time (real) = 1.86 sec diff --git a/09_vector_retrieval/PurplePulse/evidence/2026-09-08/ctest_final.txt b/09_vector_retrieval/PurplePulse/evidence/2026-09-08/ctest_final.txt new file mode 100644 index 00000000..704f4b73 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/evidence/2026-09-08/ctest_final.txt @@ -0,0 +1,22 @@ +Internal ctest changing into directory: /data/purplepulse-work/PurplePulse/build +Test project /data/purplepulse-work/PurplePulse/build + Start 1: unit_tests +1/8 Test #1: unit_tests ........................ Passed 0.22 sec + Start 2: file_io_tests +2/8 Test #2: file_io_tests ..................... Passed 0.01 sec + Start 3: ivf_flat_tests +3/8 Test #3: ivf_flat_tests .................... Passed 0.24 sec + Start 4: benchmark_faiss_tests +4/8 Test #4: benchmark_faiss_tests ............. Passed 0.27 sec + Start 5: benchmark_ivf_sweep_tests +5/8 Test #5: benchmark_ivf_sweep_tests ......... Passed 0.07 sec + Start 6: benchmark_adaptive_nprobe_tests +6/8 Test #6: benchmark_adaptive_nprobe_tests ... Passed 0.07 sec + Start 7: import_mnist_tests +7/8 Test #7: import_mnist_tests ................ Passed 0.25 sec + Start 8: plot_results_tests +8/8 Test #8: plot_results_tests ................ Passed 0.06 sec + +100% tests passed, 0 tests failed out of 8 + +Total Test time (real) = 1.18 sec diff --git a/09_vector_retrieval/PurplePulse/evidence/2026-09-08/data_sha256.txt b/09_vector_retrieval/PurplePulse/evidence/2026-09-08/data_sha256.txt new file mode 100644 index 00000000..fb3e430b --- /dev/null +++ b/09_vector_retrieval/PurplePulse/evidence/2026-09-08/data_sha256.txt @@ -0,0 +1,7 @@ +d4dde1418ea7626e5226e594e592f9ae5ffbe4c412c326395d96388f95df9800 data/million/inner_fp32_db.bin +a58ebe43fce865b71a2a651791286fa0bd1567348e07d6c0601b69beaebceb45 data/million/inner_fp32_q.bin +f5bd87fcc72d37164fa931436e5bfd293dfb7f953d09cb28169a77c7cdadc415 data/million/inner_fp32_nlist256.ivf +731c5ac602752760c8e48fbffcf8c3b850d9dc2a2aedcf2cc48468fc17b673d1 data/mnist/mnist.npz +12e47ef4a46843d3bda71db30dd2163eb86b33c7404fa5c7f04e52e4f7c77007 data/mnist/database.bin +12725522c337401372be0a80ba95204578ec693b133472cd7251848e365ab3be data/mnist/queries.bin +b5290c789c8f28a6f2bc2f691c2b1a9c008ccda6b7d98600d022d05f08e8d479 data/mnist/nlist256.ivf diff --git a/09_vector_retrieval/PurplePulse/evidence/2026-09-08/environment.txt b/09_vector_retrieval/PurplePulse/evidence/2026-09-08/environment.txt new file mode 100644 index 00000000..72c65f99 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/evidence/2026-09-08/environment.txt @@ -0,0 +1,18 @@ +2026-09-08T08:02:37+00:00 +NVIDIA GeForce RTX 4090 D, 24564 MiB, 8.9, 570.124.06 +nvcc: NVIDIA (R) Cuda compiler driver +Copyright (c) 2005-2025 NVIDIA Corporation +Built on Wed_Jan_15_19:20:09_PST_2025 +Cuda compilation tools, release 12.8, V12.8.61 +Build cuda_12.8.r12.8/compiler.35404655_0 +cmake version 3.31.4 +1.11.1.git.kitware.jobserver-1 +g++ (Ubuntu 13.3.0-6ubuntu2~24.04) 13.3.0 +CPU(s): 128 +Model name: Intel(R) Xeon(R) Platinum 8358P CPU @ 2.60GHz +Thread(s) per core: 2 +Core(s) per socket: 32 +Socket(s): 2 +NUMA node(s): 2 +NUMA node0 CPU(s): 0-31,64-95 +NUMA node1 CPU(s): 32-63,96-127 diff --git a/09_vector_retrieval/PurplePulse/evidence/2026-09-08/exact_k100.log b/09_vector_retrieval/PurplePulse/evidence/2026-09-08/exact_k100.log new file mode 100644 index 00000000..bc3fe74e --- /dev/null +++ b/09_vector_retrieval/PurplePulse/evidence/2026-09-08/exact_k100.log @@ -0,0 +1,27 @@ +向量库: 1000000 x 128 +查询数: 1000 +dtype: fp32 +metric: inner_product +top_k: 100 +batch_size: 64 +distance_mode: warp +topk_mode: fused +warmup/repeat: 2/10 +backend: gpu +GPU 一次性初始化总时间: 171.032 ms +数据库一次性 H2D: 48.3343 ms +GPU 缓冲区: 489.558 MiB +估算冷启动端到端: 351.168 ms +平均常驻查询时间: 180.136 ms +run P50: 178.79 ms +run P99: 183.509 ms +run samples: 10 +常驻 QPS: 5551.36 +结果: "results/new_gpu/exact_k100.txt" +平均查询 H2D: 0.255754 ms +平均融合距离+局部 Top-K kernel: 176.98 ms +平均最终 Top-K kernel: 1.8362 ms +平均结果 D2H: 0.583744 ms +batch P50: 11.3988 ms +batch P99: 11.7988 ms +batch samples: 160 diff --git a/09_vector_retrieval/PurplePulse/evidence/2026-09-08/faiss_batch40.log b/09_vector_retrieval/PurplePulse/evidence/2026-09-08/faiss_batch40.log new file mode 100644 index 00000000..dfc01463 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/evidence/2026-09-08/faiss_batch40.log @@ -0,0 +1,31 @@ +FAISS same-batch comparison, RTX 4090 D +FAISS 1.14.1 GPU; NumPy 2.2.6; Python 3.12.3 +K=50, nlist=256, nprobe=160, batch=40 +Training samples=100000, iterations=15, seed=2026 +Warmup: 5 full query sets; repeat: 100 + +FAISS Flat vs PurplePulse Exact: +recall@50=1.000000000 +minimum_query_recall=1.000000000 +mean_score_error=1.9556427e-06 +max_score_error=1.33514404e-05 + +FAISS GPU IVF-Flat: +average_query_ms=120.07527381181717 +qps=8328.109262255002 +run_p50_ms=119.6761205792427 +run_p99_ms=126.95340067148209 +batch_p50_ms=4.623956978321075 +batch_p99_ms=5.8260709047317505 +run_samples=100 +batch_latency_samples=2500 +recall_at_50=0.92312 +minimum_query_recall=0.8 + +PurplePulse bucket-major, same K/nprobe/batch/repeat: +average_query_ms=144.077 +qps=6940.71 +run_p99_ms=146.137 +recall_at_50=0.92482 + +FAISS/PurplePulse QPS ratio=1.1999x. diff --git a/09_vector_retrieval/PurplePulse/evidence/2026-09-08/k100_bucket_major_formal.log b/09_vector_retrieval/PurplePulse/evidence/2026-09-08/k100_bucket_major_formal.log new file mode 100644 index 00000000..ff8bc4c3 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/evidence/2026-09-08/k100_bucket_major_formal.log @@ -0,0 +1,26 @@ +IVF-Flat gpu 查询完成 +索引: 1000000 x 128 +dtype/metric: fp32/inner_product +nlist/nprobe/top_k: 256/160/100 +nprobe policy: fixed +memory mode: disabled +索引加载: 297.36 ms +GPU 初始化: 205.505429 ms +索引 H2D: 49.871902 ms +GPU 缓冲区: 508.042488 MiB +平均查询: 185.624 ms +run P50/P99: 185.513/187.381 ms +run samples: 100 +QPS: 5387.23 +输出: "results/new_gpu/bucket_major/k100_bucket_major_formal.txt" +平均 CPU 中心/桶选择: 0 ms +平均 query H2D: 0.233426 ms +平均 GPU 中心选择: 1.62975 ms +平均自适应分层开销: 0 ms +平均 GPU 桶扫描+局部 Top-K: 177.212 ms +平均 GPU 最终 Top-K 归并: 1.30388 ms +平均记忆独立重排: 0 ms +平均结果 D2H: 0.537924 ms +batch P50/P99: 11.4996/11.5487 ms +batch samples: 1600 +实际 nprobe min/avg/max: 160/160/160 diff --git a/09_vector_retrieval/PurplePulse/evidence/2026-09-08/k100_heap_formal.log b/09_vector_retrieval/PurplePulse/evidence/2026-09-08/k100_heap_formal.log new file mode 100644 index 00000000..8af8fcc7 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/evidence/2026-09-08/k100_heap_formal.log @@ -0,0 +1,38 @@ +K=100 heap optimization, RTX 4090 D, nprobe=160, batch=64 +Warmup: 5 full query sets; repeat: 100; queries per run: 1000 + +warp1_pre_heap +average_query_ms=188.737 +qps=5298.37 +run_p50_ms=188.738 +run_p99_ms=190.409 +batch_p50_ms=11.6936 +batch_p99_ms=11.8875 +scan_ms=180.438 +merge_ms=1.30079 + +warp1_heap_hole +average_query_ms=184.311 +qps=5425.61 +run_p50_ms=184.179 +run_p99_ms=185.741 +batch_p50_ms=11.3594 +batch_p99_ms=11.6603 +scan_ms=175.945 +merge_ms=1.30443 +speedup_vs_pre_heap=2.4015% + +bucket_major_heap_hole +average_query_ms=182.091 +qps=5491.76 +run_p50_ms=181.988 +run_p99_ms=184.321 +batch_p50_ms=11.2720 +batch_p99_ms=11.3244 +scan_ms=173.743 +merge_ms=1.30183 +speedup_vs_pre_heap=3.6500% + +All three result files were compared byte for byte with cmp: equal. +CTest: 8/8 passed. +Compute Sanitizer ivf_flat_tests: ERROR SUMMARY: 0 errors. diff --git a/09_vector_retrieval/PurplePulse/evidence/2026-09-08/k100_selection_negative.log b/09_vector_retrieval/PurplePulse/evidence/2026-09-08/k100_selection_negative.log new file mode 100644 index 00000000..ae2bd569 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/evidence/2026-09-08/k100_selection_negative.log @@ -0,0 +1,46 @@ +K=100 selection experiments rejected after same-host A/B +Platform: NVIDIA GeForce RTX 4090 D +Dataset: Synthetic-1M, 128-dimensional FP32, inner product +Index: IVF-Flat, nlist=256, nprobe=160 +Batch size: 64 + +distributed_register_topk + baseline: 5483.49 QPS, average 182.366 ms, P99 183.618 ms, scan 174.026 ms + variant: 5507.23 QPS, average 181.579 ms, P99 183.087 ms, scan 173.294 ms + change: +0.4329% + repeat: 100 after warm-up + output: byte-identical + decision: reject; below the predeclared 1% acceptance threshold + +candidate_buffer_8 + baseline: 5508.19 QPS, average 181.548 ms, P99 182.597 ms, scan 173.114 ms + variant: 4258.85 QPS, average 234.805 ms, P99 239.191 ms, scan 225.342 ms + change: -22.6815% + repeat: 10 after warm-up; stopped before the 100-run formal phase + output: byte-identical + CTest: 8/8 passed + Compute Sanitizer: 0 errors + resource diagnostic for K=100, one-warp, inner-product, bucket-major kernel: + stable: REG 48, STACK 0 B, SHARED 1200 B, LOCAL 0 B + buffer8: REG 48, STACK 96 B, SHARED 1200 B, LOCAL 0 B + decision: reject; per-thread candidate arrays spill to a 96-byte stack and the + best-first flush adds extra selection work + +warp_bitonic_sort_128 + baseline: 5496.78 QPS, average 181.925 ms, P99 182.873 ms, scan 173.398 ms + variant: 4467.83 QPS, average 223.822 ms, P99 229.924 ms, scan 214.114 ms + change: -18.7191% + repeat: 10 after warm-up; stopped before the 100-run formal phase + output: byte-identical + CTest: 8/8 passed + Compute Sanitizer: 0 errors + resource diagnostic for K=100, one-warp, inner-product, bucket-major kernel: + stable: REG 48, STACK 0 B, SHARED 1200 B, LOCAL 0 B + bitonic: REG 56, STACK 0 B, SHARED 1536 B, LOCAL 0 B + decision: reject; the fixed 28-stage sorting network, synchronization, and + extra comparisons cost more than parallel lane participation saves + +After each experiment, local and remote source were restored to the retained +stable baseline: + sha256 0a4dd5ddaaf7471893f3a2471908d2c9417f8f014c961e16153584f7ca6e45ac +Remote CTest after restoration: 8/8 passed. diff --git a/09_vector_retrieval/PurplePulse/evidence/2026-09-08/k100_warp1_formal_v2.log b/09_vector_retrieval/PurplePulse/evidence/2026-09-08/k100_warp1_formal_v2.log new file mode 100644 index 00000000..e76d0d55 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/evidence/2026-09-08/k100_warp1_formal_v2.log @@ -0,0 +1,26 @@ +IVF-Flat gpu 查询完成 +索引: 1000000 x 128 +dtype/metric: fp32/inner_product +nlist/nprobe/top_k: 256/160/100 +nprobe policy: fixed +memory mode: disabled +索引加载: 300.863 ms +GPU 初始化: 208.406499 ms +索引 H2D: 49.795135 ms +GPU 缓冲区: 507.962410 MiB +平均查询: 189.329 ms +run P50/P99: 189.296/190.881 ms +run samples: 100 +QPS: 5281.82 +输出: "results/new_gpu/bucket_major/k100_warp1_formal.txt" +平均 CPU 中心/桶选择: 0 ms +平均 query H2D: 0.229897 ms +平均 GPU 中心选择: 1.63722 ms +平均自适应分层开销: 0 ms +平均 GPU 桶扫描+局部 Top-K: 180.957 ms +平均 GPU 最终 Top-K 归并: 1.30519 ms +平均记忆独立重排: 0 ms +平均结果 D2H: 0.491604 ms +batch P50/P99: 11.7282/11.9142 ms +batch samples: 1600 +实际 nprobe min/avg/max: 160/160/160 diff --git a/09_vector_retrieval/PurplePulse/evidence/2026-09-08/k50_batch40_formal.log b/09_vector_retrieval/PurplePulse/evidence/2026-09-08/k50_batch40_formal.log new file mode 100644 index 00000000..fcc2efd9 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/evidence/2026-09-08/k50_batch40_formal.log @@ -0,0 +1,26 @@ +IVF-Flat gpu 查询完成 +索引: 1000000 x 128 +dtype/metric: fp32/inner_product +nlist/nprobe/top_k: 256/160/50 +nprobe policy: fixed +memory mode: disabled +索引加载: 318.318 ms +GPU 初始化: 227.746493 ms +索引 H2D: 63.280128 ms +GPU 缓冲区: 499.805611 MiB +平均查询: 166.116 ms +run P50/P99: 166.303/168.097 ms +run samples: 100 +QPS: 6019.91 +输出: "results/new_gpu/k50_tuning/batch40_formal.txt" +平均 CPU 中心/桶选择: 0 ms +平均 query H2D: 0.296211 ms +平均 GPU 中心选择: 2.65625 ms +平均自适应分层开销: 0 ms +平均 GPU 桶扫描+局部 Top-K: 157.225 ms +平均 GPU 最终 Top-K 归并: 1.02493 ms +平均记忆独立重排: 0 ms +平均结果 D2H: 0.641596 ms +batch P50/P99: 6.49126/6.62845 ms +batch samples: 2500 +实际 nprobe min/avg/max: 160/160/160 diff --git a/09_vector_retrieval/PurplePulse/evidence/2026-09-08/k50_batch64_formal.log b/09_vector_retrieval/PurplePulse/evidence/2026-09-08/k50_batch64_formal.log new file mode 100644 index 00000000..5e95fe41 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/evidence/2026-09-08/k50_batch64_formal.log @@ -0,0 +1,26 @@ +IVF-Flat gpu 查询完成 +索引: 1000000 x 128 +dtype/metric: fp32/inner_product +nlist/nprobe/top_k: 256/160/50 +nprobe policy: fixed +memory mode: disabled +索引加载: 294.715 ms +GPU 初始化: 201.381904 ms +索引 H2D: 48.189442 ms +GPU 缓冲区: 502.066414 MiB +平均查询: 174.301 ms +run P50/P99: 174.55/176.348 ms +run samples: 100 +QPS: 5737.2 +输出: "results/new_gpu/k50_tuning/batch64_formal.txt" +平均 CPU 中心/桶选择: 0 ms +平均 query H2D: 0.229836 ms +平均 GPU 中心选择: 1.69307 ms +平均自适应分层开销: 0 ms +平均 GPU 桶扫描+局部 Top-K: 166.943 ms +平均 GPU 最终 Top-K 归并: 0.699045 ms +平均记忆独立重排: 0 ms +平均结果 D2H: 0.419959 ms +batch P50/P99: 10.9121/11.2116 ms +batch samples: 1600 +实际 nprobe min/avg/max: 160/160/160 diff --git a/09_vector_retrieval/PurplePulse/evidence/2026-09-08/k50_bucket_major_formal.log b/09_vector_retrieval/PurplePulse/evidence/2026-09-08/k50_bucket_major_formal.log new file mode 100644 index 00000000..4411f280 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/evidence/2026-09-08/k50_bucket_major_formal.log @@ -0,0 +1,26 @@ +IVF-Flat gpu 查询完成 +索引: 1000000 x 128 +dtype/metric: fp32/inner_product +nlist/nprobe/top_k: 256/160/50 +nprobe policy: fixed +memory mode: disabled +索引加载: 287.3 ms +GPU 初始化: 200.789089 ms +索引 H2D: 50.339840 ms +GPU 缓冲区: 499.856392 MiB +平均查询: 143.592 ms +run P50/P99: 143.5/145.534 ms +run samples: 100 +QPS: 6964.16 +输出: "results/new_gpu/bucket_major/v2_bucket_major_formal.txt" +平均 CPU 中心/桶选择: 0 ms +平均 query H2D: 0.293489 ms +平均 GPU 中心选择: 2.54214 ms +平均自适应分层开销: 0 ms +平均 GPU 桶扫描+局部 Top-K: 134.757 ms +平均 GPU 最终 Top-K 归并: 1.04728 ms +平均记忆独立重排: 0 ms +平均结果 D2H: 0.60983 ms +batch P50/P99: 5.56659/5.62608 ms +batch samples: 2500 +实际 nprobe min/avg/max: 160/160/160 diff --git a/09_vector_retrieval/PurplePulse/evidence/2026-09-08/k50_warp_compact_formal.log b/09_vector_retrieval/PurplePulse/evidence/2026-09-08/k50_warp_compact_formal.log new file mode 100644 index 00000000..e6d41de1 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/evidence/2026-09-08/k50_warp_compact_formal.log @@ -0,0 +1,26 @@ +IVF-Flat gpu 查询完成 +索引: 1000000 x 128 +dtype/metric: fp32/inner_product +nlist/nprobe/top_k: 256/160/50 +nprobe policy: fixed +memory mode: disabled +索引加载: 302.472 ms +GPU 初始化: 223.422613 ms +索引 H2D: 64.267265 ms +GPU 缓冲区: 499.805611 MiB +平均查询: 166.411 ms +run P50/P99: 166.384/169.04 ms +run samples: 100 +QPS: 6009.22 +输出: "results/new_gpu/bucket_major/v2_compact_formal.txt" +平均 CPU 中心/桶选择: 0 ms +平均 query H2D: 0.307637 ms +平均 GPU 中心选择: 2.65958 ms +平均自适应分层开销: 0 ms +平均 GPU 桶扫描+局部 Top-K: 157.329 ms +平均 GPU 最终 Top-K 归并: 1.02553 ms +平均记忆独立重排: 0 ms +平均结果 D2H: 0.66075 ms +batch P50/P99: 6.4944/6.62154 ms +batch samples: 2500 +实际 nprobe min/avg/max: 160/160/160 diff --git a/09_vector_retrieval/PurplePulse/evidence/2026-09-08/mnist_exact_k10.log b/09_vector_retrieval/PurplePulse/evidence/2026-09-08/mnist_exact_k10.log new file mode 100644 index 00000000..cdc2251e --- /dev/null +++ b/09_vector_retrieval/PurplePulse/evidence/2026-09-08/mnist_exact_k10.log @@ -0,0 +1,27 @@ +向量库: 60000 x 784 +查询数: 10000 +dtype: fp32 +metric: l2 +top_k: 10 +batch_size: 64 +distance_mode: warp +topk_mode: fused +warmup/repeat: 2/10 +backend: gpu +GPU 一次性初始化总时间: 188.657 ms +数据库一次性 H2D: 23.2746 ms +GPU 缓冲区: 179.759 MiB +估算冷启动端到端: 621.072 ms +平均常驻查询时间: 432.415 ms +run P50: 430.865 ms +run P99: 437.408 ms +run samples: 10 +常驻 QPS: 23125.9 +结果: "results/new_gpu/mnist_exact_k10.txt" +平均查询 H2D: 6.91412 ms +平均融合距离+局部 Top-K kernel: 418.37 ms +平均最终 Top-K kernel: 1.67326 ms +平均结果 D2H: 3.2634 ms +batch P50: 2.74214 ms +batch P99: 2.81181 ms +batch samples: 1570 diff --git a/09_vector_retrieval/PurplePulse/evidence/2026-09-08/submit_audit_ctest.log b/09_vector_retrieval/PurplePulse/evidence/2026-09-08/submit_audit_ctest.log new file mode 100644 index 00000000..3089980a --- /dev/null +++ b/09_vector_retrieval/PurplePulse/evidence/2026-09-08/submit_audit_ctest.log @@ -0,0 +1,22 @@ +Internal ctest changing into directory: /data/purplepulse-submit-audit/build +Test project /data/purplepulse-submit-audit/build + Start 1: unit_tests +1/8 Test #1: unit_tests ........................ Passed 0.24 sec + Start 2: file_io_tests +2/8 Test #2: file_io_tests ..................... Passed 0.01 sec + Start 3: ivf_flat_tests +3/8 Test #3: ivf_flat_tests .................... Passed 0.24 sec + Start 4: benchmark_faiss_tests +4/8 Test #4: benchmark_faiss_tests ............. Passed 2.55 sec + Start 5: benchmark_ivf_sweep_tests +5/8 Test #5: benchmark_ivf_sweep_tests ......... Passed 0.07 sec + Start 6: benchmark_adaptive_nprobe_tests +6/8 Test #6: benchmark_adaptive_nprobe_tests ... Passed 0.07 sec + Start 7: import_mnist_tests +7/8 Test #7: import_mnist_tests ................ Passed 0.26 sec + Start 8: plot_results_tests +8/8 Test #8: plot_results_tests ................ Passed 0.07 sec + +100% tests passed, 0 tests failed out of 8 + +Total Test time (real) = 3.50 sec diff --git a/09_vector_retrieval/PurplePulse/evidence/2026-09-10/commands.md b/09_vector_retrieval/PurplePulse/evidence/2026-09-10/commands.md new file mode 100644 index 00000000..47704d8e --- /dev/null +++ b/09_vector_retrieval/PurplePulse/evidence/2026-09-10/commands.md @@ -0,0 +1,150 @@ +# 2026-09-10 第六阶段公开数据命令 + +所有长任务在实验机项目目录执行并用 `nohup` 脱离 SSH。登录地址和凭据不记录 +在仓库中。原始数据、索引、Top-K 输出和日志保存在 `/data`；仓库只同步 CSV、 +JSON 和 SVG 摘要。 + +## 现有产物检查 + +启动 Yahoo 前先检查进程和两套结果目录。SIFT1M 已包含 `build.csv`、 +`exact_gpu.csv`、K=10/50/100 的完整 IVF 矩阵、21 行 FAISS 数据、ground truth +验证和图表，因此没有重复运行。Yahoo 数据已转换但结果目录为空。 + +```bash +pgrep -af 'prepare_real_datasets|run_real_benchmarks|ivf_build|ivf_search|vector_search|benchmark_faiss' +find results/real/sift1m -maxdepth 2 -type f -print +find results/real/yahoo-minilm-384 -maxdepth 2 -type f -print +sha256sum data/real/yahoo-minilm-384-normalized/manifest.json +``` + +Yahoo manifest 记录原始 HDF5 SHA-256 为 +`8b0519850249d7ea4aea258eb43ae837dca1bb2f7a85ee0a5145afdb699c4136`，转换结果为 +677,305 × 384、1,000 queries、cosine、Top-100 ground truth。 + +## Yahoo MiniLM 384 正式运行 + +任务信息写入 `/data/purplepulse-work/jobs/stage6-yahoo-minilm-384-20260910/`，结果 +写入 `/data/purplepulse-work/PurplePulse/results/real/yahoo-minilm-384/`。 + +```bash +cd /data/purplepulse-work/PurplePulse +job_dir=/data/purplepulse-work/jobs/stage6-yahoo-minilm-384-20260910 +mkdir -p "$job_dir" + +nohup env \ + PYTHONUNBUFFERED=1 \ + PATH=/usr/local/cuda/bin:$PATH \ + LD_LIBRARY_PATH=/usr/local/cuda/lib64 \ + _FAISS_WHEEL_DISABLE_CUDA_PRELOAD=1 \ + /data/purplepulse-faiss-system-venv/bin/python \ + scripts/run_real_benchmarks.py \ + --dataset yahoo-minilm-384 \ + --project-root /data/purplepulse-work/PurplePulse \ + --data-root data/real \ + --results-root results/real \ + --build-dir build \ + --nlist 1024 \ + --nprobes 32,64,128,160,192,224 \ + --training-samples 100000 \ + --iterations 15 \ + --threads 16 \ + --batch-size 64 \ + --warmup 1 \ + --repeat 5 \ + > "$job_dir/driver.log" 2>&1 < /dev/null & + +printf '%s\n' "$!" > "$job_dir/driver.pid" +``` + +驱动于 2026-09-10 11:55 UTC 完成。阶段结果由各 benchmark 脚本逐行更新 CSV， +即使 SSH 断开也不依赖终端缓冲。完整结果摘要与回退结论见 +`docs/REPRODUCIBILITY.md`，可提交证据位于 `results/real/`。 + +## 第七阶段 Exact fused 剖析 + +长任务仍通过 `nohup` 运行，状态、PID 和日志均位于 `/data`。正式剖析任务为 +`/data/purplepulse-work/jobs/stage7-exact-profile-formal-20260910/`，结果逐 K +checkpoint 到 `results/stage7_exact_profile/`。命令模板为： + +```bash +cd /data/purplepulse-work/PurplePulse +job_dir=/data/purplepulse-work/jobs/stage7-exact-profile-formal-20260910 +mkdir -p "$job_dir" +nohup bash -lc ' + cd /data/purplepulse-work/PurplePulse + python3 scripts/benchmark_exact_breakdown.py \ + --search-binary ./build/vector_search \ + --database data/million/inner_fp32_db.bin \ + --queries data/million/inner_fp32_q.bin \ + --configs 10:configs/exact_million_k10_fused_batch64.conf,50:configs/exact_million_k50_fused_batch64.conf,100:configs/exact_million_k100_fused_batch64.conf \ + --reference-results 10:results/new_gpu/exact_k10.txt,50:results/new_gpu/exact_k50.txt,100:results/new_gpu/exact_k100.txt \ + --output-dir results/stage7_exact_profile/synthetic1m/outputs \ + --csv results/stage7_exact_profile/synthetic1m/breakdown.csv \ + --warmup 5 --repeat 20 --profile-repeat 20 +' > "$job_dir/driver.log" 2>&1 < /dev/null & +printf '%s\n' "$!" > "$job_dir/driver.pid" +``` + +Yahoo 使用 `data/real/yahoo-minilm-384-normalized/{database,queries}.bin`、 +`configs/real_exact_k{10,50,100}.conf` 和第六阶段 Exact 输出作为逐 rank 参考。 +首项 cosine 范数预计算 A/B 的完整可执行脚本为 +`stage7/run_exact_profile_ab.sh`；对应 nohup 任务目录是 +`/data/purplepulse-work/jobs/stage7-cosine-norm-ab-20260910/`。实验否决并恢复 +baseline 后，`stage7-restore-profiler-baseline-20260910` 状态为 0，CTest +12/12 通过。 + +## 第七阶段 query tiling + +query-tile A/B 使用 `scripts/benchmark_exact_query_tile.py`。驱动接受多个 +`--variant label=K:config`，每完成一个 variant 就重写 checkpoint CSV，并用 +`--reference-results` 做逐 query、逐 rank 的 ID/score 校验。远端长任务全部 +通过 `nohup` 运行，主要目录如下： + +- `stage7-query-tile-formal-20260910`：Synthetic/Yahoo K=10、K=50 正式 A/B。 +- `stage7-query-tile-sift-formal-20260910`：只复测既有 SIFT1M 的 K=10/K=50。 +- `stage7-query-tile-grid-sweep-smoke-20260910`：固定 `grid.x` 的短筛选。 +- `stage7-query-tile-yahoo-grid-formal-20260910`：Yahoo K=10 的 20 次正式 + 96/160/192-block 对照。 +- `stage7-query-tile-sanitizer-20260910`：`compute-sanitizer --tool memcheck` + 执行 `unit_tests`，状态 0、0 errors。 + +所有 CSV 写入 +`/data/purplepulse-work/PurplePulse/results/stage7_exact_query_tile/`；仓库只同步 +CSV，不同步大型 Top-K 输出和日志。 + +## 第七阶段 block-tile 负实验 + +源码上传与构建任务分别位于 `stage7-block-tile-upload-20260910` 和 +`stage7-block-tile-build-20260910`；上传前保存了 query-tile 最佳二进制， +构建状态为 0，CTest 13/13。筛选任务 `stage7-block-tile-smoke-20260910` +通过 `nohup` 运行 Synthetic/Yahoo K=10 的 baseline、warp-query 最优项和 +block-tile 64/96/128/160/192 blocks。结果逐项 checkpoint 到 +`results/stage7_exact_block_tile/smoke/{synthetic,yahoo}/results.csv`，全部通过 +逐 query、逐 rank 的 ID/score 校验。候选未超过 warp-query，因此没有重复 +SIFT 已完成工作，也没有启动 20 次正式长跑。Compute Sanitizer 任务目录为 +`stage7-block-tile-sanitizer-20260910`。 + +## 第八阶段稳定性矩阵 + +上传任务目录为 `stage8-stability-upload-20260910`，其中保存上传前 CMake、测试 +文件（若存在）和 `vector_search` 二进制。构建与完整回归通过以下 nohup 任务 +执行，日志和状态均位于 `/data`： + +```bash +cd /data/purplepulse-work/PurplePulse +job_dir=/data/purplepulse-work/jobs/stage8-stability-build-20260910 +nohup bash -lc ' + cd /data/purplepulse-work/PurplePulse + export PATH=/usr/local/cuda/bin:$PATH + cmake -S . -B build -G Ninja -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_CUDA_ARCHITECTURES=native && + cmake --build build && + ctest --test-dir build --output-on-failure +' > "$job_dir/driver.log" 2>&1 < /dev/null & +``` + +CTest 14/14 通过。`stage8-stability-sanitizer-20260910` 先执行 +`build/stability_tests` 并把摘要写入 `results/stage8_stability/coverage.log`， +再以 `compute-sanitizer --tool memcheck --error-exitcode 99` 运行同一矩阵；两段 +状态均为 0，错误汇总为 0。大日志只保留在远端 `/data`，仓库保存小型 +`coverage.csv` 和校验和。 diff --git a/09_vector_retrieval/PurplePulse/evidence/2026-09-10/stage7/README.md b/09_vector_retrieval/PurplePulse/evidence/2026-09-10/stage7/README.md new file mode 100644 index 00000000..f729185e --- /dev/null +++ b/09_vector_retrieval/PurplePulse/evidence/2026-09-10/stage7/README.md @@ -0,0 +1,35 @@ +# 第七阶段证据索引 + +- `gpu.txt`：正式实验 GPU 型号、驱动和显存。 +- `source.sha256`：首次正式分段剖析时的二进制、CUDA 源码和驱动脚本校验和。 +- `current_source.sha256`：加严逐 rank 校验并撤回负优化后的远端二进制、CUDA + 源码和驱动脚本校验和；恢复任务状态为 0，CTest 12/12 通过。 +- `cosine_norm_ab_source.sha256`：范数预计算 A/B 的基线/候选二进制、候选源码 + 与重放脚本校验和。 +- `run_exact_profile_ab.sh`：在远端 `/data` 项目目录重放四组严格 A/B 的脚本。 +- `query_tile_source.sha256`：query-tiled fused 最终二进制、源码、驱动和保留 + 配置的校验和，以及 CTest/Sanitizer 状态。 +- `block_tile_source.sha256`：block-tile 二进制、源码、配置、两份 smoke CSV + 的校验和，以及 CTest/Sanitizer 状态。 +- `../../../results/stage7_exact_profile/`：Synthetic-1M 与 Yahoo MiniLM 384 + 分段基线 CSV，以及范数预计算的四份 A/B CSV。 +- `../../../results/stage7_exact_query_tile/`：两轮 smoke、grid.x sweep，以及 + Synthetic/SIFT/Yahoo 的 20 次正式 A/B CSV。 +- `../../../results/stage7_exact_block_tile/smoke/`：Synthetic/Yahoo 的 + block-tile grid.x 筛选 CSV；该候选未超过 warp-query，未扩展正式长跑。 + +分段方法是 differential estimate：distance-only kernel 保留 fused 路径的 +网格、query cache 和距离归约，局部选择估算为两者差值；final merge 独立计时。 +首次正式基线做候选集合与分数校验；随后 A/B 复测加严为按 query、rank、ID +和分数比较。长任务使用 `nohup`，原始输出和日志仅保存在远端 +`/data/purplepulse-work/`。 + +query-tile 长任务目录以 `stage7-query-tile-` 开头。最终构建 CTest 13/13 +通过，`stage7-query-tile-sanitizer-20260910` 状态为 0，Compute Sanitizer +报告 0 errors。大日志和 Top-K 文本输出只保存在远端 `/data`。 + +block-tile 使用 `stage7-block-tile-build-20260910` 构建并通过 CTest 13/13； +筛选任务为 `stage7-block-tile-smoke-20260910`。默认 layout 仍是 +`warp_query`，block-tile 仅在配置显式请求时启用。 +`stage7-block-tile-sanitizer-20260910` 状态为 0，Compute Sanitizer 报告 +0 errors。 diff --git a/09_vector_retrieval/PurplePulse/evidence/2026-09-10/stage7/block_tile_source.sha256 b/09_vector_retrieval/PurplePulse/evidence/2026-09-10/stage7/block_tile_source.sha256 new file mode 100644 index 00000000..30444823 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/evidence/2026-09-10/stage7/block_tile_source.sha256 @@ -0,0 +1,20 @@ +bd61902328a23c815393ecdc324a66911c7248764460ef17d649bfc6ada30bda build/vector_search +012a421cae285d297207c8627c55fa69d41f804ad812195f522d80770cf13f4e src/gpu_search.cu +357cf1ed201e511508ba85985080b39bd66d2aed25ec6eb9b578b163be3637af src/file_io.cpp +04e0e9479adaf3f334246e9daeab2c4fcaabca469f30677d96315849dec4ef0b src/main.cpp +38d0a5a99bc6aae1875874c41d6c183482ea094e2c3ba9f67ff3b283ba80db3b include/vector_types.h +ff29350fcda815214c7a4b730ea5a9ea07f1ac335131f8813b343b23ed97aae3 scripts/benchmark_exact_query_tile.py +7ad6329dbb33264fe49ad23113aa26cef0f9d8beb41880231882c1d26879c645 tests/unit_tests.cpp +da20d6e976f16478ca0e61598944a31df2d009fbf53167ad2f4afd08b0f204a5 tests/file_io_tests.cpp +306093be152d295e2d9584a15cc6ef9c38ad333433b9c49d57180f86011417e1 tests/test_benchmark_exact_query_tile.py +1357bf04fa9c275e14983187fa1837cc5688923361846f536d398c74cefaed8a configs/exact_fused_k10_batch64_blocktile8_blocks64.conf +636780a393d0c59e8f604639b9ce367e138f1c8b3a56ef2608933761bd268ff5 configs/exact_fused_k10_batch64_blocktile8_blocks96.conf +1d0bb80086965acb08157dd3ff58ff738a050ff64fb47045c6ac65ad081d1301 configs/exact_fused_k10_batch64_blocktile8_blocks128.conf +eb2a045d50e2c16aacefc30efa12805ebd995123ebb8a33442a6b6fd0d0911f3 configs/exact_fused_k10_batch64_blocktile8_blocks160.conf +0c1e6d2d2243db09753b06146f4427d9b65046111ae46491b64222f5746fe419 configs/exact_fused_k10_batch64_blocktile8_blocks192.conf +3ef6ebf060a3efb4ba0e887cde3f2b51138e7579a48cee3caa62e42d01470c3b results/stage7_exact_block_tile/smoke/synthetic/results.csv +70dda44634311e029730f38ba894c28c223d8c98da6e3f6551a548d625521bff results/stage7_exact_block_tile/smoke/yahoo/results.csv +build_status=0 +ctest=13/13 +compute_sanitizer_status=0 +compute_sanitizer_error_summary=0 diff --git a/09_vector_retrieval/PurplePulse/evidence/2026-09-10/stage7/cosine_norm_ab_source.sha256 b/09_vector_retrieval/PurplePulse/evidence/2026-09-10/stage7/cosine_norm_ab_source.sha256 new file mode 100644 index 00000000..47bcbe8c --- /dev/null +++ b/09_vector_retrieval/PurplePulse/evidence/2026-09-10/stage7/cosine_norm_ab_source.sha256 @@ -0,0 +1,4 @@ +6fab50001ddb98f11dd3227b06cf5488cb284196fe0c95acd18a9ed29470830b /data/purplepulse-work/jobs/stage7-cosine-norm-build-20260910/vector_search_baseline +58eb60f859bfc3a8287030ec77db1b75972e3215d2090b20e635ffe1dacaeb46 /data/purplepulse-work/PurplePulse/build/vector_search +4fc6144f72878808f558b9519f94497e8d37a2404095897d37afe84f69e79036 /data/purplepulse-work/PurplePulse/src/gpu_search.cu +b5127ce72cb5b57a70188558b3be33fc6148366224e01766960879d26577f445 /data/purplepulse-work/jobs/stage7-cosine-norm-ab-20260910/run.sh diff --git a/09_vector_retrieval/PurplePulse/evidence/2026-09-10/stage7/current_source.sha256 b/09_vector_retrieval/PurplePulse/evidence/2026-09-10/stage7/current_source.sha256 new file mode 100644 index 00000000..baa6b5d1 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/evidence/2026-09-10/stage7/current_source.sha256 @@ -0,0 +1,3 @@ +f976c061e4cbd695635cb942e9a36b57a09033171b83ae6eed88b594c9d76cd9 build/vector_search +4ce613866f23cce5e9fc1aca0daaddba4cb1a628ed25bdfcc03d60722b1e28bc src/gpu_search.cu +4a05cac327565e0a901111e1780d164d163b6955dfc98669f8b3c8cdf588badc scripts/benchmark_exact_breakdown.py diff --git a/09_vector_retrieval/PurplePulse/evidence/2026-09-10/stage7/gpu.txt b/09_vector_retrieval/PurplePulse/evidence/2026-09-10/stage7/gpu.txt new file mode 100644 index 00000000..d0955f10 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/evidence/2026-09-10/stage7/gpu.txt @@ -0,0 +1 @@ +NVIDIA GeForce RTX 4090 D, 570.124.06, 24564 MiB diff --git a/09_vector_retrieval/PurplePulse/evidence/2026-09-10/stage7/query_tile_source.sha256 b/09_vector_retrieval/PurplePulse/evidence/2026-09-10/stage7/query_tile_source.sha256 new file mode 100644 index 00000000..b7b55a69 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/evidence/2026-09-10/stage7/query_tile_source.sha256 @@ -0,0 +1,10 @@ +dce7c95fcb9bfe2610a765caba2e52a6100d468d8241e2aed7a2dca0199ffaca build/vector_search +892eccea7ce042c4323513bb94a919387d2e129d231fced56924cae82a532aec src/gpu_search.cu +d5dd5d78cb3169798654a267c8b94716716492b22cd555492657e365eadd6532 src/file_io.cpp +f713ac71d61f75dda851e22dc94d12125a1e336430b421c44ab2f10542fec7c1 include/vector_types.h +285dca96efd4597e183d7cd37931466d2ce8081a5a9804ab1f67ddf5a877938e scripts/benchmark_exact_query_tile.py +6f5008eeb95452f311300fc416d1cac2106bef6d2f7249166c95b5e8b93a6a89 configs/exact_fused_k10_batch64_querytile4.conf +7f26d004d22bafc5d24f0d2ef903f1778687bb8fc604e1f5bee95dd1c702e18d configs/exact_fused_k10_batch64_querytile8_blocks160.conf +e28a2874f8db9942038dd8834d222de3c8d3c4ffd084ab2d1c4f6e9a229228ec configs/exact_fused_k50_batch64_querytile4.conf +ctest_status=0 +compute_sanitizer_status=0 diff --git a/09_vector_retrieval/PurplePulse/evidence/2026-09-10/stage7/run_exact_profile_ab.sh b/09_vector_retrieval/PurplePulse/evidence/2026-09-10/stage7/run_exact_profile_ab.sh new file mode 100755 index 00000000..13bfa972 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/evidence/2026-09-10/stage7/run_exact_profile_ab.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env bash +set -euo pipefail + +project_root=${1:-/data/purplepulse-work/PurplePulse} +baseline_binary=${2:-/data/purplepulse-work/jobs/stage7-cosine-norm-build-20260910/vector_search_baseline} +job_dir=${3:-/data/purplepulse-work/jobs/stage7-cosine-norm-ab-20260910} +results_root="$project_root/results/stage7_exact_profile/cosine_norm_ab" + +cd "$project_root" +mkdir -p \ + "$results_root/synthetic1m_baseline/outputs" \ + "$results_root/synthetic1m_optimized/outputs" \ + "$results_root/yahoo_baseline/outputs" \ + "$results_root/yahoo_optimized/outputs" + +synthetic_configs=10:configs/exact_million_k10_fused_batch64.conf,50:configs/exact_million_k50_fused_batch64.conf,100:configs/exact_million_k100_fused_batch64.conf +synthetic_references=10:results/new_gpu/exact_k10.txt,50:results/new_gpu/exact_k50.txt,100:results/new_gpu/exact_k100.txt +yahoo_configs=10:configs/real_exact_k10.conf,50:configs/real_exact_k50.conf,100:configs/real_exact_k100.conf +yahoo_references=10:results/real/yahoo-minilm-384/exact_outputs/exact_k10.txt,50:results/real/yahoo-minilm-384/exact_outputs/exact_k50.txt,100:results/real/yahoo-minilm-384/exact_outputs/exact_k100.txt + +run_profile() { + local binary=$1 + local database=$2 + local queries=$3 + local configs=$4 + local references=$5 + local output=$6 + python3 scripts/benchmark_exact_breakdown.py \ + --search-binary "$binary" \ + --database "$database" \ + --queries "$queries" \ + --configs "$configs" \ + --reference-results "$references" \ + --output-dir "$results_root/$output/outputs" \ + --csv "$results_root/$output/breakdown.csv" \ + --warmup 5 --repeat 20 --profile-repeat 20 +} + +run_profile "$baseline_binary" \ + data/million/inner_fp32_db.bin data/million/inner_fp32_q.bin \ + "$synthetic_configs" "$synthetic_references" synthetic1m_baseline +run_profile ./build/vector_search \ + data/million/inner_fp32_db.bin data/million/inner_fp32_q.bin \ + "$synthetic_configs" "$synthetic_references" synthetic1m_optimized +run_profile "$baseline_binary" \ + data/real/yahoo-minilm-384-normalized/database.bin \ + data/real/yahoo-minilm-384-normalized/queries.bin \ + "$yahoo_configs" "$yahoo_references" yahoo_baseline +run_profile ./build/vector_search \ + data/real/yahoo-minilm-384-normalized/database.bin \ + data/real/yahoo-minilm-384-normalized/queries.bin \ + "$yahoo_configs" "$yahoo_references" yahoo_optimized + +touch "$job_dir/complete" diff --git a/09_vector_retrieval/PurplePulse/evidence/2026-09-10/stage7/source.sha256 b/09_vector_retrieval/PurplePulse/evidence/2026-09-10/stage7/source.sha256 new file mode 100644 index 00000000..00670f5c --- /dev/null +++ b/09_vector_retrieval/PurplePulse/evidence/2026-09-10/stage7/source.sha256 @@ -0,0 +1,3 @@ +6fab50001ddb98f11dd3227b06cf5488cb284196fe0c95acd18a9ed29470830b build/vector_search +4ce613866f23cce5e9fc1aca0daaddba4cb1a628ed25bdfcc03d60722b1e28bc src/gpu_search.cu +5e231c55a3df94e83cb13f09b0dd7050d6da3ade6fa009cab7cf9c9fb57d0fc7 scripts/benchmark_exact_breakdown.py diff --git a/09_vector_retrieval/PurplePulse/evidence/2026-09-10/stage8/README.md b/09_vector_retrieval/PurplePulse/evidence/2026-09-10/stage8/README.md new file mode 100644 index 00000000..f322192c --- /dev/null +++ b/09_vector_retrieval/PurplePulse/evidence/2026-09-10/stage8/README.md @@ -0,0 +1,10 @@ +# 第八阶段证据索引 + +- `source.sha256`：CMake、稳定性测试、远端测试二进制、远端覆盖摘要和本地 + `coverage.csv` 的校验和，以及构建/CTest/Sanitizer 状态。 +- `../../../results/stage8_stability/coverage.csv`：覆盖类别、组合数和结果摘要。 +- `../../../docs/STABILITY_MATRIX.md`：Exact/IVF 实现选择、适用边界和安全回退。 + +远端长任务全部使用 nohup：`stage8-stability-build-20260910` 和 +`stage8-stability-sanitizer-20260910`。日志与逐任务状态保存在 +`/data/purplepulse-work/`；仓库不复制 sanitizer 大日志。 diff --git a/09_vector_retrieval/PurplePulse/evidence/2026-09-10/stage8/source.sha256 b/09_vector_retrieval/PurplePulse/evidence/2026-09-10/stage8/source.sha256 new file mode 100644 index 00000000..cc2d7391 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/evidence/2026-09-10/stage8/source.sha256 @@ -0,0 +1,15 @@ +0f969ab1acc2240ab6b52b02023282e3cf44a2b2120221e0ba247465a9e554c6 CMakeLists.txt +8db44443bf335387ff74c4fbd30dde9f0f94913fbae1f919fa0d1010913c34ab tests/stability_tests.cpp +fb2bc60f2d6cdb547b962260e830c2e07dd7b6dc50b0af53461ac0e3c07b24dd build/stability_tests +bd61902328a23c815393ecdc324a66911c7248764460ef17d649bfc6ada30bda build/vector_search +a8626c69c04eaad003f7bc0e3ec7e421f002da6f7acbd98341e3fa75f66e457d results/stage8_stability/coverage.log +9a271f2a916b0b6ee6cecb2426f0b3206ef074578be55d9bc94f6f3fe3ab86aa results/stage8_stability/coverage.status +9a271f2a916b0b6ee6cecb2426f0b3206ef074578be55d9bc94f6f3fe3ab86aa results/stage8_stability/sanitizer.status +2d1b4b722ea111df21243f4d81d02cf64393424f2ccc6e5d7c85e9440f56b0c8 results/stage8_stability/coverage.csv +build_status=0 +ctest=14/14 +compute_sanitizer_status=0 +compute_sanitizer_error_summary=0 +gpu=NVIDIA GeForce RTX 4090 D +compute_capability=8.9 +gpu_memory_mib=24564 diff --git a/09_vector_retrieval/PurplePulse/evidence/2026-09-11/stage9/README.md b/09_vector_retrieval/PurplePulse/evidence/2026-09-11/stage9/README.md new file mode 100644 index 00000000..9f8703fc --- /dev/null +++ b/09_vector_retrieval/PurplePulse/evidence/2026-09-11/stage9/README.md @@ -0,0 +1,18 @@ +# Stage 9 clean-build evidence + +Remote environment: RTX 4090 D, CUDA 12.8, GCC 13.3. The successful job was +started with `nohup` and wrote phased state to +`/data/purplepulse-work/PurplePulse/results/stage9_clean_20260911_v2/`. + +- `progress.tsv` and `status` show the phase sequence and terminal state. +- `logs/ctest.log` contains the 18/18 CTest summary. +- `logs/exact_verify.log` and `logs/ivf_verify.log` contain the CPU/GPU checks. +- `summary.txt` records row counts and SHA-256 hashes for the remote outputs. +- `reproduce-plan.json` is the machine-resolved Yahoo MiniLM dry-run plan. +- `release-validation.json` is the local release-artifact validator result. + +The first clean configure attempt wrote its failure state under the sibling remote +directory `stage9_clean_20260911`: the non-login shell did not include +`/usr/local/cuda/bin` in `PATH`, so CMake could not find `nvcc` and stopped before +compilation. The successful run used a new `-v2` build/result directory with that +PATH made explicit; no failed build directory was reused or deleted. diff --git a/09_vector_retrieval/PurplePulse/evidence/2026-09-11/stage9/logs/ctest.log b/09_vector_retrieval/PurplePulse/evidence/2026-09-11/stage9/logs/ctest.log new file mode 100644 index 00000000..0a6723cf --- /dev/null +++ b/09_vector_retrieval/PurplePulse/evidence/2026-09-11/stage9/logs/ctest.log @@ -0,0 +1,42 @@ +Internal ctest changing into directory: /data/purplepulse-work/PurplePulse/build-stage9-clean-20260911-v2 +Test project /data/purplepulse-work/PurplePulse/build-stage9-clean-20260911-v2 + Start 1: unit_tests + 1/18 Test #1: unit_tests ......................... Passed 0.24 sec + Start 2: file_io_tests + 2/18 Test #2: file_io_tests ...................... Passed 0.01 sec + Start 3: ivf_flat_tests + 3/18 Test #3: ivf_flat_tests ..................... Passed 0.26 sec + Start 4: stability_tests + 4/18 Test #4: stability_tests .................... Passed 1.66 sec + Start 5: vector_search_help + 5/18 Test #5: vector_search_help ................. Passed 0.00 sec + Start 6: ivf_build_help + 6/18 Test #6: ivf_build_help ..................... Passed 0.00 sec + Start 7: ivf_search_help + 7/18 Test #7: ivf_search_help .................... Passed 0.00 sec + Start 8: benchmark_faiss_tests + 8/18 Test #8: benchmark_faiss_tests .............. Passed 1.66 sec + Start 9: benchmark_ivf_sweep_tests + 9/18 Test #9: benchmark_ivf_sweep_tests .......... Passed 0.07 sec + Start 10: benchmark_adaptive_nprobe_tests +10/18 Test #10: benchmark_adaptive_nprobe_tests .... Passed 0.07 sec + Start 11: import_mnist_tests +11/18 Test #11: import_mnist_tests ................. Passed 0.31 sec + Start 12: import_sift1m_tests +12/18 Test #12: import_sift1m_tests ................ Passed 0.31 sec + Start 13: import_ann_benchmarks_tests +13/18 Test #13: import_ann_benchmarks_tests ........ Passed 0.25 sec + Start 14: plot_results_tests +14/18 Test #14: plot_results_tests ................. Passed 0.07 sec + Start 15: stage6_tools_tests +15/18 Test #15: stage6_tools_tests ................. Passed 0.23 sec + Start 16: benchmark_exact_breakdown_tests +16/18 Test #16: benchmark_exact_breakdown_tests .... Passed 0.07 sec + Start 17: benchmark_exact_query_tile_tests +17/18 Test #17: benchmark_exact_query_tile_tests ... Passed 0.07 sec + Start 18: stage9_tools_tests +18/18 Test #18: stage9_tools_tests ................. Passed 0.08 sec + +100% tests passed, 0 tests failed out of 18 + +Total Test time (real) = 5.38 sec diff --git a/09_vector_retrieval/PurplePulse/evidence/2026-09-11/stage9/logs/exact_verify.log b/09_vector_retrieval/PurplePulse/evidence/2026-09-11/stage9/logs/exact_verify.log new file mode 100644 index 00000000..5d062456 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/evidence/2026-09-11/stage9/logs/exact_verify.log @@ -0,0 +1 @@ +通过：Top-K 候选集合相同，平均绝对分数误差=3.039230769056408e-06，最大分数误差=9.599999998499698e-06，近似并列换位数=0 diff --git a/09_vector_retrieval/PurplePulse/evidence/2026-09-11/stage9/logs/ivf_verify.log b/09_vector_retrieval/PurplePulse/evidence/2026-09-11/stage9/logs/ivf_verify.log new file mode 100644 index 00000000..5d062456 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/evidence/2026-09-11/stage9/logs/ivf_verify.log @@ -0,0 +1 @@ +通过：Top-K 候选集合相同，平均绝对分数误差=3.039230769056408e-06，最大分数误差=9.599999998499698e-06，近似并列换位数=0 diff --git a/09_vector_retrieval/PurplePulse/evidence/2026-09-11/stage9/progress.tsv b/09_vector_retrieval/PurplePulse/evidence/2026-09-11/stage9/progress.tsv new file mode 100644 index 00000000..c7581803 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/evidence/2026-09-11/stage9/progress.tsv @@ -0,0 +1,24 @@ +2026-09-11T03:11:04Z running:preflight +2026-09-11T03:11:04Z running:configure +2026-09-11T03:11:12Z passed:configure +2026-09-11T03:11:12Z running:build +2026-09-11T03:11:44Z passed:build +2026-09-11T03:11:44Z running:ctest +2026-09-11T03:11:50Z passed:ctest +2026-09-11T03:11:50Z running:generate +2026-09-11T03:11:50Z passed:generate +2026-09-11T03:11:50Z running:exact_cpu +2026-09-11T03:11:50Z passed:exact_cpu +2026-09-11T03:11:50Z running:exact_gpu +2026-09-11T03:11:50Z passed:exact_gpu +2026-09-11T03:11:50Z running:ivf_build +2026-09-11T03:11:50Z passed:ivf_build +2026-09-11T03:11:50Z running:ivf_cpu +2026-09-11T03:11:50Z passed:ivf_cpu +2026-09-11T03:11:50Z running:ivf_gpu +2026-09-11T03:11:51Z passed:ivf_gpu +2026-09-11T03:11:51Z running:reproduce_dry_run +2026-09-11T03:11:51Z passed:reproduce_dry_run +2026-09-11T03:11:51Z running:summarize +2026-09-11T03:11:51Z passed:summarize +2026-09-11T03:11:51Z complete diff --git a/09_vector_retrieval/PurplePulse/evidence/2026-09-11/stage9/release-validation.json b/09_vector_retrieval/PurplePulse/evidence/2026-09-11/stage9/release-validation.json new file mode 100644 index 00000000..b145975d --- /dev/null +++ b/09_vector_retrieval/PurplePulse/evidence/2026-09-11/stage9/release-validation.json @@ -0,0 +1,20 @@ +{ + "status": "passed", + "project_root": "PurplePulse", + "validated_csv_rows": { + "results/real/sift1m/exact_gpu.csv": 3, + "results/real/sift1m/faiss_gpu.csv": 21, + "results/real/sift1m/ivf_k10.csv": 6, + "results/real/sift1m/ivf_k50.csv": 6, + "results/real/sift1m/ivf_k100.csv": 6, + "results/real/yahoo-minilm-384/exact_gpu.csv": 3, + "results/real/yahoo-minilm-384/faiss_gpu.csv": 21, + "results/real/yahoo-minilm-384/ivf_k10.csv": 6, + "results/real/yahoo-minilm-384/ivf_k50.csv": 6, + "results/real/yahoo-minilm-384/ivf_k100.csv": 6, + "results/formal_100k15/exact_gpu.csv": 3, + "results/formal_100k15/faiss_cpu_flat.csv": 3, + "results/stage8_stability/coverage.csv": 9 + }, + "report_bytes": 408735 +} diff --git a/09_vector_retrieval/PurplePulse/evidence/2026-09-11/stage9/reproduce-plan.json b/09_vector_retrieval/PurplePulse/evidence/2026-09-11/stage9/reproduce-plan.json new file mode 100644 index 00000000..5fdacb61 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/evidence/2026-09-11/stage9/reproduce-plan.json @@ -0,0 +1,125 @@ +{ + "project_root": "/data/purplepulse-work/PurplePulse", + "steps": [ + { + "name": "configure", + "command": [ + "cmake", + "-S", + "/data/purplepulse-work/PurplePulse", + "-B", + "/data/purplepulse-work/PurplePulse/build-reproduce", + "-G", + "Ninja", + "-DCMAKE_BUILD_TYPE=Release", + "-DCMAKE_CUDA_ARCHITECTURES=native" + ], + "log": "/data/purplepulse-work/PurplePulse/results/real/reproducibility/logs/configure.log" + }, + { + "name": "build", + "command": [ + "cmake", + "--build", + "/data/purplepulse-work/PurplePulse/build-reproduce" + ], + "log": "/data/purplepulse-work/PurplePulse/results/real/reproducibility/logs/build.log" + }, + { + "name": "test", + "command": [ + "ctest", + "--test-dir", + "/data/purplepulse-work/PurplePulse/build-reproduce", + "--output-on-failure" + ], + "log": "/data/purplepulse-work/PurplePulse/results/real/reproducibility/logs/ctest.log" + }, + { + "name": "prepare", + "command": [ + "/usr/bin/python3", + "/data/purplepulse-work/PurplePulse/scripts/prepare_real_datasets.py", + "--dataset", + "yahoo-minilm-384-normalized", + "--raw-dir", + "/data/purplepulse-work/PurplePulse/data/real/raw", + "--output-dir", + "/data/purplepulse-work/PurplePulse/data/real", + "--skip-download" + ], + "log": "/data/purplepulse-work/PurplePulse/results/real/reproducibility/logs/prepare.log" + }, + { + "name": "benchmark:yahoo-minilm-384", + "command": [ + "/usr/bin/python3", + "/data/purplepulse-work/PurplePulse/scripts/run_real_benchmarks.py", + "--dataset", + "yahoo-minilm-384", + "--project-root", + "/data/purplepulse-work/PurplePulse", + "--data-root", + "/data/purplepulse-work/PurplePulse/data/real", + "--results-root", + "/data/purplepulse-work/PurplePulse/results/real", + "--build-dir", + "/data/purplepulse-work/PurplePulse/build-reproduce", + "--nlist", + "1024", + "--nprobes", + "32,64,128,160,192,224", + "--training-samples", + "100000", + "--iterations", + "15", + "--threads", + "16", + "--batch-size", + "64", + "--warmup", + "1", + "--repeat", + "5", + "--skip-index-build" + ], + "log": "/data/purplepulse-work/PurplePulse/results/real/reproducibility/logs/benchmark-yahoo-minilm-384.log" + }, + { + "name": "plot:yahoo-minilm-384", + "command": [ + "/usr/bin/python3", + "/data/purplepulse-work/PurplePulse/scripts/plot_results.py", + "--results-dir", + "/data/purplepulse-work/PurplePulse/results/real/yahoo-minilm-384", + "--output-dir", + "/data/purplepulse-work/PurplePulse/results/real/yahoo-minilm-384/figures" + ], + "log": "/data/purplepulse-work/PurplePulse/results/real/reproducibility/logs/plot-yahoo-minilm-384.log" + }, + { + "name": "report", + "command": [ + "/usr/bin/python3", + "/data/purplepulse-work/PurplePulse/scripts/build_final_report_pdf.py", + "--source", + "/data/purplepulse-work/PurplePulse/docs/FINAL_REPORT.md", + "--output", + "/data/purplepulse-work/PurplePulse/output/pdf/WarpFusion_Technical_Report.pdf" + ], + "log": "/data/purplepulse-work/PurplePulse/results/real/reproducibility/logs/report.log" + }, + { + "name": "validate", + "command": [ + "/usr/bin/python3", + "/data/purplepulse-work/PurplePulse/scripts/validate_release.py", + "--project-root", + "/data/purplepulse-work/PurplePulse", + "--report", + "/data/purplepulse-work/PurplePulse/output/pdf/WarpFusion_Technical_Report.pdf" + ], + "log": "/data/purplepulse-work/PurplePulse/results/real/reproducibility/logs/validate.log" + } + ] +} diff --git a/09_vector_retrieval/PurplePulse/evidence/2026-09-11/stage9/status b/09_vector_retrieval/PurplePulse/evidence/2026-09-11/stage9/status new file mode 100644 index 00000000..c591978b --- /dev/null +++ b/09_vector_retrieval/PurplePulse/evidence/2026-09-11/stage9/status @@ -0,0 +1 @@ +complete diff --git a/09_vector_retrieval/PurplePulse/evidence/2026-09-11/stage9/summary.txt b/09_vector_retrieval/PurplePulse/evidence/2026-09-11/stage9/summary.txt new file mode 100644 index 00000000..adc72ae7 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/evidence/2026-09-11/stage9/summary.txt @@ -0,0 +1,11 @@ +100% tests passed, 0 tests failed out of 18 +Total Test time (real) = 5.38 sec +exact_rows=130 +ivf_rows=130 +exact_verify=通过：Top-K 候选集合相同，平均绝对分数误差=3.039230769056408e-06，最大分数误差=9.599999998499698e-06，近似并列换位数=0 +ivf_verify=通过：Top-K 候选集合相同，平均绝对分数误差=3.039230769056408e-06，最大分数误差=9.599999998499698e-06，近似并列换位数=0 +a5dcde0e5227b40cc04f0586be1bc80d9465a9f1c91c2e78c228933b1ad98fce /data/purplepulse-work/PurplePulse/results/stage9_clean_20260911_v2/exact_cpu.txt +7c9e01d52c60b3a80063ab51b48153d71632b43703aac1b92f1fe038785dcb1e /data/purplepulse-work/PurplePulse/results/stage9_clean_20260911_v2/exact_gpu.txt +a5dcde0e5227b40cc04f0586be1bc80d9465a9f1c91c2e78c228933b1ad98fce /data/purplepulse-work/PurplePulse/results/stage9_clean_20260911_v2/ivf_cpu.txt +7c9e01d52c60b3a80063ab51b48153d71632b43703aac1b92f1fe038785dcb1e /data/purplepulse-work/PurplePulse/results/stage9_clean_20260911_v2/ivf_gpu.txt +7d5b324cbce9475a4adec9eb54567db5ffc80a6867c939dd4b772004d5f749df /data/purplepulse-work/PurplePulse/results/stage9_clean_20260911_v2/reproduce-plan.json diff --git a/09_vector_retrieval/PurplePulse/include/file_io.h b/09_vector_retrieval/PurplePulse/include/file_io.h new file mode 100644 index 00000000..9512aff9 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/include/file_io.h @@ -0,0 +1,17 @@ +#pragma once + +#include + +#include "vector_types.h" + +VectorDatabase readVectorDatabase(const std::string& path); +QuerySet readQuerySet(const std::string& path); +MemoryMetadata readMemoryMetadata(const std::string& path); +void writeMemoryMetadata(const std::string& path, + const MemoryMetadata& metadata); +void validateMemoryMetadata(const MemoryMetadata& metadata, + std::uint64_t expected_vectors = 0); +SearchParams readSearchParams(const std::string& path); +void writeSearchResults(const std::string& path, const SearchResults& results); +void validateInputs(const VectorDatabase& database, const QuerySet& queries, + const SearchParams& params); diff --git a/09_vector_retrieval/PurplePulse/include/ivf_flat.h b/09_vector_retrieval/PurplePulse/include/ivf_flat.h new file mode 100644 index 00000000..a89c1899 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/include/ivf_flat.h @@ -0,0 +1,58 @@ +#pragma once + +#include +#include +#include +#include +#include + +#include "vector_types.h" + +IvfFlatIndex buildIvfFlatIndex(const VectorDatabase& database, + std::uint32_t nlist, + std::uint32_t iterations = 15, + std::uint64_t max_training_vectors = 100000, + std::uint32_t num_threads = 1); +void writeIvfFlatIndex(const std::string& path, const IvfFlatIndex& index); +IvfFlatIndex readIvfFlatIndex(const std::string& path); +void validateIvfFlatIndex(const IvfFlatIndex& index); + +std::vector> selectIvfCandidatePositions( + const IvfFlatIndex& index, const QuerySet& queries, std::uint32_t nprobe); + +SearchResults cpuIvfFlatSearch(const IvfFlatIndex& index, + const QuerySet& queries, + const SearchParams& params, + const MemoryMetadata* metadata = nullptr); + +// GPU IVF 执行器：中心、桶 offsets、索引向量和 ID 常驻显存；GPU 完成 +// 中心选择、各桶候选扫描、局部 Top-K 和最终归并。 +class GpuIvfFlatSearchEngine { + public: + GpuIvfFlatSearchEngine(const IvfFlatIndex& index, + const QuerySet& initial_queries, + const SearchParams& params, + const MemoryMetadata* metadata = nullptr); + ~GpuIvfFlatSearchEngine(); + + GpuIvfFlatSearchEngine(const GpuIvfFlatSearchEngine&) = delete; + GpuIvfFlatSearchEngine& operator=(const GpuIvfFlatSearchEngine&) = delete; + GpuIvfFlatSearchEngine(GpuIvfFlatSearchEngine&&) noexcept; + GpuIvfFlatSearchEngine& operator=(GpuIvfFlatSearchEngine&&) noexcept; + + SearchResults search(const QuerySet& queries, SearchStats* stats = nullptr); + double indexH2DMilliseconds() const; + std::size_t deviceBytes() const; + + private: + struct Impl; + std::unique_ptr impl_; +}; + +SearchResults gpuIvfFlatSearch(const IvfFlatIndex& index, + const QuerySet& queries, + const SearchParams& params, + SearchStats* stats = nullptr, + const MemoryMetadata* metadata = nullptr); + +double recallAtK(const SearchResults& exact, const SearchResults& approximate); diff --git a/09_vector_retrieval/PurplePulse/include/search.h b/09_vector_retrieval/PurplePulse/include/search.h new file mode 100644 index 00000000..7f20aace --- /dev/null +++ b/09_vector_retrieval/PurplePulse/include/search.h @@ -0,0 +1,46 @@ +#pragma once + +#include +#include + +#include "vector_types.h" + +SearchResults cpuExactSearch(const VectorDatabase& database, + const QuerySet& queries, + const SearchParams& params); + +SearchResults gpuExactSearch(const VectorDatabase& database, + const QuerySet& queries, + const SearchParams& params, + SearchStats* stats = nullptr); + +// 将数据库和查询工作区常驻 GPU，适合预热、重复测量和服务化查询。 +class GpuExactSearchEngine { + public: + GpuExactSearchEngine(const VectorDatabase& database, + const QuerySet& initial_queries, + const SearchParams& params); + ~GpuExactSearchEngine(); + + GpuExactSearchEngine(const GpuExactSearchEngine&) = delete; + GpuExactSearchEngine& operator=(const GpuExactSearchEngine&) = delete; + GpuExactSearchEngine(GpuExactSearchEngine&&) noexcept; + GpuExactSearchEngine& operator=(GpuExactSearchEngine&&) noexcept; + + SearchResults search(const QuerySet& queries, SearchStats* stats = nullptr); + // Diagnostic differential probe for the fused path. It launches the same + // distance traversal without heap maintenance and returns the average GPU + // kernel time for a complete query set. Normal search timing is unaffected. + double profileDistanceOnlyMilliseconds(const QuerySet& queries, + std::uint32_t warmup, + std::uint32_t repeat); + double databaseH2DMilliseconds() const; + std::size_t deviceBytes() const; + + private: + struct Impl; + std::unique_ptr impl_; +}; + +bool resultsMatch(const SearchResults& expected, const SearchResults& actual, + float absolute_tolerance, std::string* error_message); diff --git a/09_vector_retrieval/PurplePulse/include/vector_types.h b/09_vector_retrieval/PurplePulse/include/vector_types.h new file mode 100644 index 00000000..57036f41 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/include/vector_types.h @@ -0,0 +1,124 @@ +#pragma once + +#include +#include +#include + +enum class DataType : std::uint32_t { + kFloat32 = 1, + kFloat16 = 2, +}; + +enum class Metric : std::uint32_t { + kL2 = 1, + kInnerProduct = 2, + kCosine = 3, +}; + +struct VectorDatabase { + std::uint64_t num_vectors = 0; + std::uint32_t dim = 0; + DataType dtype = DataType::kFloat32; + Metric metric = Metric::kL2; + std::vector values; + std::vector half_values; +}; + +struct QuerySet { + std::uint64_t num_queries = 0; + std::uint32_t dim = 0; + DataType dtype = DataType::kFloat32; + std::vector values; + std::vector half_values; +}; + +// Agent memory metadata is kept in original vector-ID order. The IVF index +// may reorder vectors by bucket, so GPU kernels use index.ids[position] to +// address these columns without duplicating or rewriting the metadata file. +struct MemoryMetadata { + std::uint64_t num_vectors = 0; + std::vector timestamps; + std::vector importance; + std::vector session_ids; + std::vector source_types; +}; + +// IVF-Flat 在桶内保留原始向量，只改变排列顺序。聚类中心始终使用 FP32， +// 桶内向量仍保持输入 dtype。 +struct IvfFlatIndex { + std::uint64_t num_vectors = 0; + std::uint32_t dim = 0; + std::uint32_t nlist = 0; + DataType dtype = DataType::kFloat32; + Metric metric = Metric::kL2; + std::vector centers; + std::vector offsets; + std::vector ids; + std::vector values; + std::vector half_values; +}; + +struct SearchParams { + std::uint32_t top_k = 10; + std::string search_mode = "exact"; + std::uint32_t batch_size = 8; + std::string distance_mode = "warp"; + std::string topk_mode = "two_stage"; + // Exact fused experimental query tiling. A block reuses each database value + // across this many queries; 1 preserves the established kernel. + std::uint32_t exact_query_tile = 1; + // warp_query is the established/query-tiled warp layout; block_tile assigns + // one query per warp and reuses a shared database-vector tile. + std::string exact_fused_layout = "warp_query"; + // Zero selects the occupancy heuristic; a positive value pins grid.x for + // controlled Exact fused chunk-count sweeps. + std::uint32_t exact_fused_blocks = 0; + std::uint32_t nlist = 4096; + std::uint32_t nprobe = 16; + std::string nprobe_policy = "fixed"; + std::string adaptive_execution = "masked"; + std::uint32_t adaptive_nprobe_min = 1; + std::uint32_t adaptive_nprobe_step = 1; + float adaptive_target_mass = 0.9F; + float adaptive_temperature = 0.2F; + std::string memory_mode = "disabled"; + float memory_semantic_weight = 1.0F; + float memory_importance_weight = 0.0F; + float memory_recency_weight = 0.0F; + float memory_time_scale = 1.0F; + std::uint64_t memory_now = 0; + std::uint64_t filter_min_timestamp = 0; + std::uint32_t filter_session_id = UINT32_MAX; + std::uint32_t filter_source_type = UINT32_MAX; + std::uint32_t memory_rerank_factor = 4; + std::uint32_t pq_m = 16; +}; + +struct Neighbor { + std::uint64_t id = 0; + float score = 0.0F; +}; + +using SearchResults = std::vector>; + +struct SearchStats { + double host_selection_ms = 0.0; + double center_selection_ms = 0.0; + double adaptive_policy_ms = 0.0; + double database_h2d_ms = 0.0; + double query_h2d_ms = 0.0; + double distance_kernel_ms = 0.0; + double topk_kernel_ms = 0.0; + double memory_rerank_ms = 0.0; + double result_d2h_ms = 0.0; + std::vector batch_latency_ms; + std::uint64_t selected_probe_sum = 0; + std::uint64_t selected_probe_queries = 0; + std::uint32_t selected_probe_min = 0; + std::uint32_t selected_probe_max = 0; + std::vector selected_probe_counts; +}; + +const char* metricName(Metric metric); +const char* dataTypeName(DataType dtype); +Metric parseMetric(const std::string& text); diff --git a/09_vector_retrieval/PurplePulse/output/pdf/WarpFusion_Technical_Report.pdf b/09_vector_retrieval/PurplePulse/output/pdf/WarpFusion_Technical_Report.pdf new file mode 100644 index 0000000000000000000000000000000000000000..8174ced3478690a8963c14b3f8b88ac0c50fa906 GIT binary patch literal 408735 zcmdSAW0WS*vNl+@%`V%vZQC}wY}>YN+qP}n?CSD#-+Ru*+&eSh{5)&r%KbhOJ7Y)0 zlNqu1kC#MVSd@l=mKlm9F)A_#U(v+g&cR9Azz|=|#MZ>Yz{$iI|IZA3V>=^f8xvb6 zd~!Q`6I(|+X9pt_3MdAAdVD)WODJw`C^}IFe8xXe5uZ+!k%}Ilf$cxQAI0B^e>F^h zG>k0&0sd$>{sEZ(XqcG(4M5@lnaNDWfY0<#8_ZOU_{{&b!SZJ&^FM8{;xqq!28x## z%EZ?AuP=X{_@_^T29748c7MF46F0GTF>$gmGQg)3u{E+Ywy-tBr&F`A6|i-*_)meX zfsM(ZwHff~l$;Hn-0lAWEsO@{^peZAE7Cl{Ci>@@&7HMe;xma zM&7~BNXf+Mk1e{tY|ts2xcv!A!p6YNMDQP@&_6~Ae4Rfo;?pVGIsIYp&&qVl4hFXW zX7aE6Z*u=V82LXB|CeC?=JB7w{Ab>jEo@92X%y{j3~c{TGROLVn7P07{d2MZ%-p}z zPxr5uIoAI*n16dO@&Dz5`TNQJPiFq{x&5m#!~VYp^Ovtb#pquxn7@_se=_rr68Wzd z48z}=_&=lhOW404H4J~t^#6?JAHD5gj+_75LjR*WbNprOUy+;twM%pSqp|+War0l> zw5Wx(lZnIMO@XNOpI$F)@@HdX@^@b$ZDMQYWRB0k%uN5c!#g@Tm>AeVxn+5;Zd($w zA%O0^yGOK3LquwQBVhFW0N_1DSlb*$a$}BC&<9kVm?nWnGVLyEKd!i1Vp#$%?3jL@1@Ur~Z`}9iY5eIXnPTXF} zj)wa12KRoaAEqM%`~HXqd$V#1+n3{@4Li@J9)-EF>esFMK$EpXG19R@B_dm)7|B|p z`V|&vID?EjXqsqPWzuMPwTd^)>lOFo{}b8+51pE0XvE@lD|C&pnd6BYQ>u{|DU-eJ3$;Vc*sLnFjG-95Zn}-}^c+zl_s1 zr@k=3Pp0ll)9Pzsd@X@S1F^gN9(?|0=teYbf?|d_<$eN*S~QcbJqE|Zl_^- zH?ii>(0?y>>3<3Op1Lc$`8fZY8u-dxSA+gAIq{L29g74Yofame(DaK>yh(g6EoCn=4mx1 zYpNnQrdxR>*1)$i}#C>ws$dcTzjrr%yrYV&6s?^vl?7)5U9lUQNbDuJalU59VQJapI|<(dX&F{gm(i z`Ua3qRk5hA$ut}9cUz2SyNS>3mR+Im9hZ%l4@AU%?9aZHj0-?d>#FZDrpo?Y9#4nD z-uV@>6U%-duE*ZGP^aZtei@hFQY+4HRHx@vxl=o98hpW*F2Aph>uPgQiCT4@XLYxq zKSmte)s~4z@4jrfcey@%jE{=ww&-5g&RdbWp+B@N6#fg7Gf4>&LKvlQ8`>L+0&p(yR z!8J0?zI(Ny|Bl(+mjAqawLg1JKRws~B;y5-$=}I&4Y|9P0l0;7dwu<)ba3m{ZfQm7 zA-Ep~uEiN$eETH1`0yAhy}Pw5o;;cRbAbIvrT)iJmf^o1lm5SoJu3&}pVQ?(NaEw$DA~T9(a4s;*YMuIni_<0(ffJE?2;NVi%z zWHYj8NP8Un()^e%_8}~Nef((z>6mD(J)W1#R^$HZb~C;`jy@!_ z&U)P*D$f!@-4eVAgw|9wxS`Z1_SaCTK zD)Nlc2@XKGU#zz~ZC?B-jfl2{a_%N=I%90B-&a!9AQfZ~-pVJgwOFlJK2P?!AiPcK zlIW~MVUGF8RR=$f?C~~x-un74ccZ zAjk)86E}B^?P7=NCHoDwCG=jgw!@=_lg1^26Xf#+!Moe(JGleTSLa5vwF~<)D2%XM z!Xgam>@C}d=wO?o_=aYI?;D~`_?=BkTuERo+@r_Y?Um&eZ(k&+&E)C_GQ@IpA|LuB zje+dEhSixL)Ud>_93VU4_YI061KLzcXjwp@SwQ4LZa_?GNQ!1i!fF_*%M~8k79OgM zKzAQ6c|Zn;6yz~TELcD+dO$3YP!v%lo}rwGl|ax;EOjBy^TaELC6R+79?u+ax*(`2 zfo(yCTZHVfE#o0hcTV(a><5!TfDwwwD2Wz7Yz&D>BsI?1kQJ2(*$_}goM;>&ov32W zq9L@B;B3s3PJlfo+7NC{sx_wC5N%DyH74tTIrV1iz_Bj$E5X~CuRQ< zY@ER%D+`}&oWg+wd#Z8_(}ATuB#UG<;d1Qx2+V<~J$y@=m*1A4HDPn?`Y7-S&VlR| zp*_+oZA<2c;3Xk8E;iwPY;e%Vn>GrKQ?Rd-7=jT5!JY#y<<4?3Wbu*+4G8AMHFsOQ@= z&y1PAXnA1Fb9#P58S_002HrX^N>1Dk2B*Wj6Ko;i{A+AhzUr_>lpogZKRK~IGNRfeTd1y#eWN9)z-kfr8 z&ue-t^w@DOeBY9bmE?LCPNC)GgeX+nOXG63_V>M2g>qFa0GJhN4wd9wUvo$o*>=dc zMF!<^d+tSdqwjD(zHr=+y)l!z_zEM<`o#644ZFT^c)F8#IB{X$>4~snLh^GLz2{n5 zOr5+c@I0_aaxT7330CQF9DtqlW<$1qf18iVd}k7{eb2F?i zh=G|M0KJnGS2N^2Qqg0FpROWaR@G`uGRokAH&&Y|56SwdN^%)UJby?u7+d!OG@oWx zUWQ&xxpL3z!L6DFG{;k-nzN8*;?HJK0mRJ-S)BzwPQ@>uXIAFX_GyU{a(-SCUT$Vy zCqUyeXW);71*}la6lO6##o`PqXjH8#nb8&k#P@(avTrZMWwuihJO^cI14h|xJzx?* z*Fa$b4cwnO0Sg@_XcKgaPO)r~jP5s-&usQ}HeFx{B+%74bTgHDROaMa?|L@g&R9uN zfh`q2b|@}zitXQjQkPCnZn5v$IIU!1|5Ob3^4BtKaly+UiQF87sCDPyx- zQEHZ@gVGdh8$h8~P(bjw%Z;T<-j)RY*eq41mZ7tujv76`#rdU=Z_c(ZqEuGGlDwg4 zMNgSzo1FJtWP~2Q>1_0Bg_t|isHjGnZ%DHTKC^-DC3fLNXX<|1!Ge^jLj)RoIu$%Q>L!U1)fS*GsY%zcBAhETo{K0~(1=UO zi6Ih{04Bd8K!L;by(YuOl(YPP_5zJ$c7m?fo5f?pjA(6Zy`x5>VACs*>U@!+S%*V< zgnIL^mIZP(rDp+&(a%g(?ai&nK9lg`bgh8xX^lqmvNx$wk23tDmJ?L0z~wb37l4~5 z1J~_}r_Ax0Uo{m&LsLDxST$wi&ZeRcg|oHjDhF-`yS$uYBEf5P5e0MX=4ue4-3v8( z)L~HhZ1b2<=}of!Sb{Pvrlkt)%*UMO0 zM4er#@#XCt<02%75tM0n$jOEGh0b+RgnLCd%#-7co{&hh?jXtNL7`%DCo}?#mHX)h zjgL^s!Q{oLw3{q(Kj!VGlbV>+^$czf4;l?=!~0jR zft48`+&$G}GHW5Fk>f}k6!{t&6+5sNmNkk(P6aS{WLTx1`x4_dD0E=X4XaiO6?Ug_ z=1cImVb3~aTb+Ld2|yAqK&xMCw#gdD)2*qiInTW=SJjomfKJL6z$_={bBE^9TK@)e zg2KXU$KO62(1x;!0^sJLD$e|g%J0GP?Z;D<=-{oEoPRknSB(>}ybl2ZGY%LpW5}#X z)dW1BGRHN$A`xGUj6WeiQY@+*ipoz!+2@vFWR<5LHxaZyL$WKr#B*T`9oju%;odn# zZjMM94V8KhG%U@oVxXY)0p~4J(O6?8(QGkjs!+^ot_9x`CMdFGTO=CIxtJck{w%9R zUc8`1b(Ma6zA!2D&Y12k#Gj7KkSSpTsNK>hL)gF9Kghxo3eF|vJD1))qAzaY6v7=S zsiH3-Y~+ew+8E=vSu*a@Mnyvz-5bY}J|yU>T>D~TRH}8+BpRq-L`BLQZ`(u(YOaUr zvV&SBQ`)84x0b<3wr(RXxjY%#b-r)%_WvRx=y#UBcI52@MW(7%&> zZrhep@r@I7>jDkH1AeTy4T82pnO2cXbD+O zBAgn)wvg2EDM@y9Kzp7(Qt?gsJ-;U0gN^m1>;&E||Ci8}d09hRo4H zgm?&84m2q44HMFnX$EEhg)~*dZvI1MAa5!#VFDA17ig5yDv=wYmGx_wD>a;q>$&1E z(s(&(=b)7}Hcaz4`WA)L7!RV(q`RG_r#W=!DUD626$dl~m&+7I{Z z3Mn_y!G#qPX4*T4MuYf($-;qXWfEYIx*zeo%+`h92#^`ZW>dMxC9(Vw^@;j8#ho=E zQ>g*88Wc5(7<6@69RwyK@0Am1{Ud?0fZ0<;xF_Q=gfOV}NuWf$`k){>`_81_|$dlCu!%g^3QE?4ut4i|2eMBPk${ z*<_n2sbC1AxFieeUaj~gK#b~Ys|Vre%XPB&d{?GRR(L*tx7 zkqiw1msP4MA^V6W*?CrbK|$?FC4PXdK-K_jeFjVH+8X830DAy^+0djU;LdD~5`ct) z12EWkuP}x}P*5)>B*lgC)F;_-P61FeTT9SbYINpZh60DE&{%fH(g?3x7!vm(CgvGJ zDE86)uXQ|f5So`aau>sYpq95vx!~@ zSw#kBPffLsoAPtr9khtb4YvU$=GG=%#4!j&3JN!j5iO~?_z7Pwc1qG2FZc_h8N+Poqd|*t>ISj`b z7{q-XK=1-BMx2MA6=2P=JyBou$Ka!1X(gZ%E(}*|QvM`6h*!Gr31cL2cOkPcXc5`# z>5srClp^rg)BV$2!0Z$t6URa~kh2MjTymmTA7a#|+@B>2)sp(0>W$yhf z-|b^r;cR){_@+Th#lW%Rph}3kyU_&yp3NHQUspQKroMXeWPT&&QNOMEV*2*?5a?VaUBJ=kyxMzVxT}U@=l3lqhN!+ z?0QL2eXG8^vC&Z?s7DaM-6Io_L1#x?#713JgpYN0Cma{okbo-n%i zD$CkV#sT66F*^%Hi4)AoHe0Yj!nY33;aYg)Lr-Pl*lMZ(@I-ID2@-9vA=tr@@`G4v zvLMGD&7wSO5r{*?yaeL*We{6rC0N0&F<78-a*p>4&P63eHs!K^w(rK)PzSWyUN_C~ zTQ7i*In`J?p(B#9F^CYUgo9%znZ{p{hLWWs)tpsfLNk9dnxv}Xxb_1`Y#NjltMo42 zN%l~}Pv2yY>nSFw(|g2j9M1OUGTl~6lju@3(X7&F&!jvhu+~-vr|j2v#hnz$-QIq@>({Qb$O$58)~>t zaKGm95?EDDhSIR?{MiGXi88o*!V(P@tC{|D52;&$07GTMysnO?RQU;#e6*j>TT;Zz zR?6GN&~7KUllsu{C?kU^D5J&u+Ali-t_@|nC20F-1#IPJOQ{rmyRS@vqh(98@|k_0 zdvMYNHR4Bcn{^Q%>J?;ik5+SekB<^6EDi6Dxr>!op;ZfNzMo*diz|7`vK%Wq6fBUb8&frJR=WB}B&I&FR|!YTPD3JFEtzC}0n) zBn-cdn!VBOr}pHUVn|C${^4;3FPxno_}^<6q8;zlG<0!jG!H&`x4jc)Tg?aiIb}31 z0&iSWQCJ4ehJ^PQ2OALZU=wZ!iGrg=_u4q($R`OdDv3DWcaY0u*N(saE-1w($dNbB zHD#lDYrh=l07-cDaYx+zRp{w^<8OBd+L}}?vtsA>e1{vxma6>#7chXH*w#;gBKW+V z!Bm;8VnZnjU4mcVJG9|cBC1YvL4@7vM=coVa4E7`t5GTy+2ljaq-~rmbO(eW>i`Vd z5>w%s&|Gf?)1z9nU%Iu-7F+Eqk<7GO*7N+4GKZXVO)eICp_fFvz>+sI@HDrTJ-U}SIo!?NeFqQB zNt?Rf>NfIWn^@VqG#wjCHcXo!-IcpKRTEoVSB#_62JX`{>PRl;^y=87vF33b%0-$rzA*)q%UPf@XDQFT z*V{A^k|~3i&GK6+cD7)1#p28MTm zVI_<+DS)o$wu!Uatw!c|{FL3k;`w!$a60&o^CZ|hl5?0cqJ=NaCln}8jwN|47z=6{Qg*)$Z3_zs zuNpS3zOQQPM-F?Gxf#}U?3sUPfE2{fkw(PQ+b*??MUqNYYNEQrhE2c~|Ar{T3vQki zGkk^>HR{8~2`e3?#3AS|W;#5_(TLb5^80>?*hRqh+l`9jLcmRcaTlaF$k3pO7Q{Lh zC1n26-+;AnP1)m1P{l9_smAx@XyY#sUE__^Eh0ZR{T;}m5|`Lb3{GMrO`@Xgpw?Vy zJ!O%v*n%=%AjYVf!dgI?{x(4< zWq)m7EPx`03MI7Mh{x=)P2ll7o$Z(nJ7v&B^*i=FYjv%6r^AP zA+qA>J9l))3DpUotK)FLdxcrrxj~y&OZWL4wrvD8gVSMw(%ef>{NPjp&n%rOn!kzw z8a~ZK*+Xe6fz1JewdLI8m~)^F+T~3!tT0X)XZc;;5)@3tn63Y zL{Oo195tPnCM!6-MyjJK^^(Nk3jId%SBUN!NdR(-iaNiOS>^nlHIVyw>hXOT`9jkr35G1k@#GGZ{wXDw02a3Asb*cQ>j=nz(%IxxPoH#nl80}NaokQDBnem*P{KzXP^RlskK!uWhvKVoMO zKO(RU*m;T*e$xO4V=dJO^*oEg?BTFbC|e-NhNV7}$>}Lf_^pM2NXVKh5G}x#0Z(&p zo5M9qfdmUfzePf8ob6yKfZ3#kaRt?Cz{$9`qrsnZ+jKwuBx)F-GFq-sG{!;`9(xcp zJWKs~i_3m7!^)y%mcC(STNXRiEs61s*2J|>SUhihJktg)=OIobCe9Yt+INRHks!R7 zgI4#~?E|-OfwMZ@YBnRy(9)E4aLn=2JP^68-p ze&xi?7r{N7Q7OjziP)j3sLFKoP{loLs!+y7H%Y3@xqz0D6$J1G5!dPop8%x=*qTQu zq$l}=|AHWR!`JuI`Gu4y)!i>>zRkJ8B&19X?{&gapM4gNZAvCSA^_(Ge&PU}Ir-?v zm9T3_macD+l@Ftc0-rh9X0lTlY>&0ad`h?M)nhyvJxa(C)`%lYGG>*|=8P^*wl4oG zIuEZ3)WLJ2{r60P21W7!bnLNk?g}%3%l> z$}&1QY2q8m$rc@@sktG)X2;C6f-+UQS~ep}vcE|AcOcXf0&%V-HV*}_fS!rEQ_LAA zXdJmeoax{|7cx1^6kQ$KlXIjTN~+9_PW$}@>>;c(vQ~}mF5~*HvURkOP1ve30bB%O z@|FBl>ni6Fj6q`Mz@d9!liZzT&!yNcNQc~1mk|XjyE(zP2e7?Hlu3te$=IzU< zKQ*KaR?e7xiG1CsCvH|+aa?zdA=>oZhR$TyrUP_M6uCN7uWnj`WNerQql;Ii%Qrkq zNe;GJmabrxt7Xk}V4d?Wn^&);v0)A#*kb(6iPM+^2$ahdD1EKlyPr-_>7LjrpR1Mc zZAg?$$3{kDYq4R>32WvaSTbXkqwIE6?3%4?U&FV3I$q=%4=rphs^h$H{+j9YaCS{pB)3!u1Y6q|EkPNd|6!DSP1Q8$R8a1rt0`FSiJpBCPEF;n$R0q6qV ziPqf6=qe&#$Z0h~7wNJtTBbLOcL_)2DRBY;%_bA_@Wat6fk}`%pr3HWm7TzdM{$8Y z&UXwaqqC_KstQdcVm?^mx3EoRbu+9Y%|(Ux zpPE)B3ILbJ!_h6iRo#1-mVMDz@;wq-aLIW z^?7rm5>8iPP-ZOEiB@~`&$n61E4vItm~cujZvh`r8Ab$+lRf~|S!3*!R1I~Qz=vIh zSvKUcp#&1a5-X-8Q_Ow2jJ<)ctlq|=!F>77}*rXx7o$9_2tKbJf33q~!HtHrm*-i?g#5$Jy_ z=2R%l6jL2ITV4ZJYFMaZa;xkc6i3|5gGZ)oTaD%2l`{8)kl1)1Vk%inG?xr0Y2;D` zf%GTt{p9?-u0}K}9u}w$vD7-P_j$9@;znZylS#4f;6Sysqy~v16{w>E6>4gIJ;->f z5ezIGC}Re52=}Wfp)pnHWC}0scJEy_V^#$z_~AjBJV=fcD2i#HdK2E{47Y-njTL1K zy~gE=jsY7pblbxZit@WkF8Uo3O?5yu=!ufQ=^0Al=> z@;b(XO3g{sm2FTAColt&)NufWbO&jR9-VA8PL05VPy^f+Bc`(Ah|`auL!mC_0Uh23 z1Dk3kD4=hOh705X*V1jJ>Hus&yrJRdfRp1yH4eY;=mVTu*Rm{F1IE=4W) zNLy(B-HuS8a3)w5x@0{+h3!`F9S^2MqLh(8J-P)ryrnW?xG* z`}KmK^lDyXi?mfAoyI>yep`Poy^MCK>pC1fTDEhtD6b+X7apR3=U5T+BY&S*oHXaR zex198m`*UZs?7il7W(!aIf5^cDypa3~M%$(Je{>CJbg=NT!ZO zX$|1swx5aH7{ZqD!i@X&GDCK3AOrb!H){&5M&R3+7IwB!7}J#9N_8Zi4FQ5U2q<|_ z%BybeJ;zEdchqg2qWl2OT|p<-rX?WkD6XA~gwLTr9D9fuf@i-ET+E$brvMM@jb z-9TlqJd&V1Jq`9n6q|^x;U1xj`D*JG12u+Lkz_lYY-w+oFI;d>3%4!)x>N(ug3A@> z41FWibAIT-u7I~N_}d)%Ep`Y^V@b6~8O8C~e8m9MoM=^EG1?aK+!JH~{uk5^{Ky6O zkS!00@pAT9e(_A^mU;X*a=6cB+=3*>&%3@JHuTbQs)0vkyCZp*b=q6;9|8 zz%TW zB4lURtC@y3Db=u*7YQ$0N**sYof&)&@h3wQ#oMY{?s+5XjUCj8=|hW+RYw0 zVAM1RBU*(OoQ?G{KJbS1fRcy2o{6jHiV*NjRi{@Xc-v1$8IxS&u!-nzQKCgYKs2)> z0EY_;e~pT>j0V`UPi*&{bc{h*!4+b+>hXVi8+dJy>|0<21_4?gyiaivhS?>=cfhU% z{(1H1Ep+x)Cbb3!n0ky=ys4ElW#5Grjw`Ecs$tBOETZHJr;B?PhnJ%VWt@|fon-=} zqlILiVJPzjOd;h}HCgIStuFg!P(Kxx^qs@!Z8O`^$iIhUarWmec&fE7-Rw8ihcYBb zecn|($PA^Jl)Mo2{pFJVGlU0{H5B2YeAK+RNX6^10(cjfR27gCoONlOOQ|h`S1eF( z3>na_9r2L)C|CoMQp{L)FqgJB*j1$+vGX#|9Tu)82u8;iy3ytB&xmy^ac=R_MD8{J zG&PKj4b%7lqB*R!f!q0nZUim+*r=J%2CTbi=SIbCY(y)=tr44XLv|hi_uN&- zSJJ@+(;Jb&lkLWYJF;LggHywyG4c&@%#1w*!|KWJtEzvwX ze{8B#f8L;eTncG1=?C=?rpS$8AL{ai9FxE4xwqR&2@$ul$dGNOunbR2lAEqX&@Mjq zU3j7zjw4k(XJ)a$W$Q=3p0lOrzc#!y^m+~|DUTOZJ#s8PWh! zhfVQN^^j&55MOeLJ|%XZvu2qM@ZhdCG$qr`<9Z?4FFwRH^cJKziZSo*^K^u1a##&; z*kp=`Xn6n+&YPGEVQ%Qvz)#|x+~OUd3%&vj*4H%ax4~%B54Q@h(F)yFD--k~^vwXhyB)4wfDz+9$K3{Jq!@-oL zOgVZRkZ0*m7*v&=7c8pL)jhqu*76ZpzV)(o>WR%io*xU=$Rc73S5a&Z(m%wtjDPA7 zZ(S79K?_oWC2(j~O4d@qQ0mDWHK`lvl21lO-x&B+(|xX1^+*vt-U1C}bA8#9!c?^js0Pdj7ly{u%*j zRuM4Sz`}mZI#2{)uMCYNjxXrH(*@FQMPYppc$yEV?cqV9@K3 zH0@`ya~Mk>;$z6kLv_tR+)xRiJG|cv1Km!QQkXw^t%#CtVn~3rP@q$ivPEipT5=UE z+sT-1ZSsJ=n{ec{sGx7H=7bDKv8PGnB5m zfHYYT{WFmwqr&Cndigx*Sz9T0a(UCb0?WEY)XGRp{=Gbzizg?WijM@sS+}xm(dtZS zjJA^lY#On;a8G(I(I+^MnG?vRa78n-;Z@lNme9MKeZRIcLptS0wYd#V$3cyuRAnly z0z`H|0W*rYTNHmS50=8^&$&@`TQ%>Is*b$k5RntQIle94X_jFG>XcsB@`zy zkBGs>o;jri){Jfmoy-n%HZ)d4l%*_U-&QY?)!DPS>`&Tek33es8`!UVJMgEjPot_= z++6h+S1TsrF{lg*=j9EUNgkuv14;!AQWOJxP$K8>0UgL`Gl`PNc_`jw5rO0@*5RjH zlJiBZzoo<{{khntx5;>EfI9Ixey^zZo$5NkwriM+3CIzSGp1UA5Yhze+ZvrKn)0Ls zadSorDfjb4-=<2Efaf!tv#W2iziyAT%MpP=yI0YSFE7@4z@2Rds{m9~gu%SCX z;p4E&5K??~?&?X`kboUxZ2|7WC3R-IAv3(ip%5aQe9oc~V(UX^Yrsk!*lgYF;7b>o ze&9Y2G3XO0JE>A5c1ihV-+pSkw}`YOKa+vIu1}z(jWJILi~M$}nvhtf;EM@0z*Sx4 z?j;3>C5G{mMbfxnin1=XY7;MQ#^YtLMWg72ccWl+8W;r}ahu1{65C5R3cF=3@JKx} zI&m%M2r-ridKhEBcVwp4_p&_|Dj#CICs&LMR)B&z86`;D>fx_KUXP!kq6wJVAUm(3 z-@~O*MK_9-gjpnM*#6iov6)z|wZP-(GQLo*BV09`X#2_?;EK&8m`k9v>xBNXIwOwm8bzE*V&I~)lOcW6xju$T2qKk zs!g#q7K#zLdDfJi@~za~4}Ja{fp==2djQ=Dlqj?P#g1=@Jqt176UF(oy(%PWKs!py zLM-R;7nv?v(GlXaXBVuLXBuM7v%2B_+ru|e<)5`nQsd)jjKe#iq`dx#$QWRtro0SP-$K%=J^`YF67IQ8qJzf=R~i*v72`Pto}qE zJ`mQ6>}@Fbq(-lh8o`-SXT+w=Rwze_Ad&nnrPiUcN}1ilTew}8@2mWd_LA}IJ08z{Vs19_aV3z+S zl+cToV@Mo;|FI_*A-Orc9e;k2aUMpWcyVV1YU$&?r=MBVAb%z9Rb*SWaa}1(svUG* zQQuA%p@&w;-#kDJ$9;s4D>6W0gQ)?cL~YXuTOajmLm74! zGGflLrHFZS1V5`{ZrsVC5~O~dh<22SZ935&M$BbejKTxd>DC<@A8-hJ^GljKEC=o( zu70G!NI4>lEWQ+r*Xe!VUM&ovE7$uEA@;HHqEU%9#u=k8r-+9Wi?-4^WUDo?atLF{jZUGw#3+q zQle`u>)MvQwMM`VdAIH>pTIa%Z*dk>*x9(58V+(!Chle4H3j1BL#;)NWq}vM70d05 zre`x3;b5u;j8Lh6&OYY+R#8%FR#GaagMvr8soSFy49wtw*@%dPY%md=j*dZl;V=(^!pnv?eP$PZ(8 zMN^N35uT{c6bA*HIQ+hpEM#IBwlS=5VnNwZ&0-hP$8citA1lHQs$*yn z%d1F{ZO#?DC=JiU`CfvpuBi_XYf%0hpyqirb1I2Qbi9}qkz`prxmF<2o}Htiv;6yls2=wSDivWIy{ENR~>!n30jQ$z(&rxh{q)m5WG_uAS+3K|#? z6*#nEK|8zl4jf|c_J&_N&b4U7sJi`OTWz-%4mY{I^zF{2vKO=OzWrs`Yy!T^3mrQO zjXIR1zn{ql;H9x77uWM?u_blW`{@xMpm?7M-K)7Lna=-xP0{uz_`u>}kksH>51Byq! z9@@Igh+drm;}#>z6=sAB><~BbKBnJoRR527=Rd@xvHzyrj+a+7B&y5Xreu-7FM>4Z ziL$Imdvd{$z%Xv^L;x=eoOj~%J2;F(3m$xam7h$Qwo<*m)@ZSrZYcSkky+wUcnVdl zXIqA6bEc(*quywn1jlSjrAcb~X1TiSCSvxY-{$TnVdUk0ddgHwGovzO^i_clmJ;ov zieAKCFPk5%J$jaxW8~+jj6%K>4)%$Fz(ww8dIOe$4jvvOoeQ3{G_e-CR_-@1-)By) z&x}Hk1%&|%8WjR0Dg;=V{~zLqe<(jt;6+;c5g+<-mAY{6Sdv1D4YOf_rfSS_h8)2~{l{Y6+Ony`-5A?>Pz+LZ=1|0(7a zA?}WT9yDn#D+IMFjO%h=Tq`&p&SL*vxnOrXma1Dg(Wu;v`=%hHS5m4!Rh3uckWUtg zyWsFvKrNKPSVSjO!f1Sq;M3i4rrNnI;feyhnjrG>sB==7e^~>K%i@|I8)Vx?;3Qnt zp5;}68FN{04;5M+#s`6B)RhUVOQOXU zQ$t$v*gqW6)pq*K{>~^>t=did0^+#~rc1r~Xt+E2#H01TCdlZ)+vo{yng>(>KRo80 zjR7-33h8U>K;g2O^)Ey8z6TC`7v3HQ+xIA+1&o7Ko`VOk&mrotd_Z~<1>hJCvh(X2 zQEkO^WehxtynN^HeR!kOk=n?xp+0Z!(ex!FvQ^SVPLOCrB&`ZqN7Jm#>~ua zmvwuk+0okECcxHocXx85JSg6Euv98da=Sp;ofy!`J4a`Mr&>>PIQ2*STXAAMCssNq z3yZg*W|ef3k9o&KW2tOyxds>WCI{%c(gGVye^(|ZZ3YLNBCBOvi+Msw=-FlQVznP% zEFftJ&;W%Ckf7!qJ=QU3mpf3fe5N#=4L_X?V|P>k+?sxx^`GDWFBi4x1bBI{OrqRD zD{t8%+3I;9r?-45{Kb<%_8$!H{FWz+xAb3yBX*#B&G*ps)(J^~32Fh2gGXI?1n~Rk zm$mQnENs{8Uq|)%^Mz8XSDI)$PyW@HH#zyW&p)y`ygio7drD-8-f8hN*Ogaydw<>+ zEWmHBR`psMqRy!BOfB2(vZ)Oh=?tgU4f!0dL9qUkyi7&9PX?d$`f{tNctV>QFE4ir zAdMo^<#t}K2%)K_SEBb+Ql~4l>RV=}zbK+c)<92agFs*tJqwZ-fg#svYSJii11fA> zcn2%uJGD&`saCLUkKsCe zZBwF9PGx@TmE7F<;njJWDx??pltFp{l%q&C?kjwPm0Fe2g+xoB&@jK7YAvW^cl+Yu zx&A1%DIz1De0&)fQ5NO(|z4Zqo(WQe%Y zq8&2lY(e-)iBaC5d?>9WD_qplf@p37z!@OR>}(5gX&CkaB+LmZJzSK(!HAh`{1&h^ zn2xn&``1aN6)X4qa_CAl&TeMAsm!LdEvLyC8+6V=K6=fz{qE8bQD%zvedzxMVL+b0 z_Woi`+eG)*mM-1>>DQmzvGR@wuUS;6-?*ghV1B5#^hKT8cf+P553Kt5b03*$Il6nS zp?%Yt0XEJapw<5f-=GE*bC>0Q>6|PbV|9$nCd29|kgyy*NyirND}8P6QaY|Z{+^h; z=zQ-z;kO?z{X3MMhPpIBn@k`JhRGw+F^qpvj4VF0?X|(wx;3|rPcI&> z_2vdQh`QA)vSS|4OwaT|;rd?!3n=|l>0g80eT`KPewH23-FJbuxR?~8VVoFNj2b7` zf=ZK{3Hyi*4KCr`rp@=Ae({Ct@7~gse{kccez@hiZ#*oVtv>O5=`T+msva2r{v%sI zy>W?T^Jo*EhjG?H%|2v`MA}89=T01f`LAGo!AArHKsu1%8#)B|nrb@vp?hxK-Zc54 zo9`b#b8xCQzwzuyNgORDh{OD*S} zl%M^pWM}_M@*VzXg=>m_wDbgBR(cAqZK7`QCj+Hl4zh89{?ji&-CC0To4SGXXa9$K zqmy)d=_fQ)`pFPhIaosZtOeF#6O6NmQ79ev(nyvT9Pe5hVXxVY|EIq2Gg|*+X+BrNxKFS# z8J8TsNJcz^NG5DKC2Fb%xzCMau04np}jV}Gm{ zWT_-)Mf8ZEI~k*JY0q$AqIu1#L_=3=y#pHeZP?hfs5ughbxsm_?u1r`d&NuZWV!{( z%5+}VgH&bhM^)ButTcy$J51&X4QwbqN)N0rePsk%`y(}$UM?M=W2I+gJb7N0yBB<` zEGbH3<_u2`Q%C6`_~^s^tdB*~CeFZn29oExf`X5?+%H5r%KIfldJe(5u|xlI_NX2@ zDHgZtX^ckn+Xe?pw$j`HES8JU35!9>7cQ0r)x|$c;{xkNxS8SfxHP>u47vrl!St{T z5}m@}QE;_#PwBk`G9me=soJbtxsc10}c1l-$k)>l9qga1FyX4c891j>2^VT=&5B zI9xBn^;Ni#lG_)Ii|K-71XVF>3&BN(6Yu0+c|D(VAZozCX*FOF&}$dc>&aWMT+$s| zdh}#&Xz1wfp{GQhqhYjmWL-t+S>gKkgwfnl_@-tevN5uNRA45O6jZ@AxVSr-R54-< zVmdMdHE4&NOeb@~vpc7Tr*}N`)^{J?_SEJ_*FOK}XP!SXK6wA+jn}O${NTO^ezf_? zZI86Ap{i$rT7NqAbxC$w4P%&;)*qgrZsDuN!aN~--?)i=p__Xmv_w9I!Z2I@QdJW0 zFv`CbCUfQOH4V_psxpI}LbJi`YU9&?`dX_H1kA9c7o^FwF(+G6n zS*9y)K*Oj?mi<*Qdn$SMR4^1Om^~E?lL}@}1+%Au*;B#nsbKb0FncP3!rY5+eHAX8 zy-Hcn6xlxFy82G)VjQ$qWFyJWI08|IeMdB6Z0}l8T+(w-zGBm8>r|?xvTd?h|LL)# zOF!_X*r9>O)Wj9d-Qj@^tu2{EL*vrc)4ST1Tp@jf1!%#&FqUc(l-UB2q!gJ5Ap(gw zYIqrR$tvK-c@gEDEJg@PTOmE~w{3Ggthm7GnL(nf@Cm>} z^HxqVYlZx(A$4kGMOTe=ux8I_+e}vlP_VJyzRq~CsdpKDrZl$uz>?&^%3ZAOFtmLi zw2i|&PbZaG=^%ZXGWuwaM#D>f|L;pmuY;C%@ykN{#SnWx3+wy#GEG}ZXEp3YupUj> z<|IQp4)MC-e-Wu#cI1QKF#Y3)hQDF@&JT>x%)bb3c7=rci{F&SI|8`d#duxve@)l1 zdPX(v^%GA#IsV;WOuY2c#M>|`zowqj!=+nj2AJ7?ss13;UoT?+s^^bMM?0Qs-u^6W z48r@`fXNMxfA)p(UkpIq-=|fj8%mE;&YygMC!b+>vQS&0+yVr+!OSgf^jPWJ)DNsC zD}1H@;*kO22DJ0ceV8_gXevxz>I6c(V5fI0`EMMQ?O8cMHMnkhcxL8dvAO@RzeF2Z zXzqtX+uzu#?68V2qi>FsYRjf7+)_37Ve%$xv-;9g!^H~=P7N2iWT)nQ&&^K_e-qo? z(JOlU@zqxHdtvJ(I3{o$Q{YeKaBGRg*K#d?>i*uhJ7MyFLEkK8S<__h3-sN&e|w+J z4&^R9)Xv<%WLn`_suPFa|7}1$KYm%?ZQ{^fcS*Fb5%7I2=&uH#-s$s<2;r9*E%V`= zM(*;`bAOH-^?mp%?p1K-%NoA0ws-Y!9(ZuynPrM&&m24P=@Y`(#ow*o{phbA*|VYa z?&_0QJ$=;)Xmc5~c?`zg2s|)(X(Sl#Xv4uUS>$j%)_cZTt+Lf8=@eyEe_8M3sr?^+ zaNnsFiepb7Kl1$1HsL3w@9%iz*Pl4Bks8;YKJ(O>V`UkC9KO@Z@J++5ZFa6XNwU6U zUe;$cQ5Y`_9Gf|FdgH;-Le+Ikjz99~)r+sDKaH$C{+TmtV=WyI{^rph9K$@+y^is8 zCn+NxdQh%eb>=vmyBxGT3EyPqUuEK+a|@IDkypy*5qB6>Ry+Eq)q=idJEsIt@h+~$7HJW60(b4nlrabrRG8L8a|b1PBbx(D14}&<{0sZwZZrgO(xrId-nAA_csqd?vC`JvTl-?4*xtBor90Q~ zvCH<{zw?pFLsve0#a(9(s8*`_MV~;;gNZmmU*M+{Su6$ zNrDax6)1)gli;V(Mg7YUw#`)a>mtp|MjCDXVrA*uySsO6ExmJbc}J5!(KodF%TPxR z=pgqo`#STv>zLT`ew&mV2vkit8;1W~{r$qv-4TEyr!M|Zq7P*57V zKzrfRY9LAGVHES3NNqS$6Lt4Tx|04%J*%@e0(E}AyWQMwjMT$e^#E@DJJhQq?f;E_ z^0z!p@%OcqVoph;ac8ur|Gh1x3pBr>^dC4PYrlK3^qY6yk=i2HGCKd?>0&*7eYnb` zITW`7o<&HvT=qQs@h~a+(!|NTho4V`J;tLujLhFiq(zQXA7(W>nP|oA_kiq;Ou>cu zn}-(l4en`OYYA^XyKBJKwrORk*VeIWEYl~9t=ZYz-qEw|?$STg$Z&tQXY00EYFRN{ zpI@{4r!uZVKV78t{{yZ`cqaGo?_*b=HCkAV*d>gu=g76Tr`97OSDF5-`5W75*`J(y z;8zcP{5SX0UmSn@*rBJ6(w>8#03!Ct{;8|3diKOgcz-dh=|k{7@`3khzWP}rOLWdM z*$2H!u6uFcvNL-hynpYp)qV8scgW$~M zt2;BD!};DdRk6C(!f>{2aNkmHgJ_m%Jz)%lTd}!M(t>?AXQc=ODS%sI2x`lGls&R- z3@}FWUl>o%XRMO~Anx3#?SMho<#kJJhd zVGJ;$DB(2|T`owtji_ zU=qgu@AAJG^$tJt%+_Oxs(WFmezOs@rXBMNV3N`HVF((Uwn();nbJ7}la4mz2cNYPmy<{gE1 z8#)w#SW#v#WMRf=lZX3SMKWGhXAlz2Ek1K1rqkD_3w4@!!PnpEq=wpRdRwOZt-in_ zXQp}8rgn#G?LRa-7uAlvFj^}}ZN5HtH8qNh{|8?2`oMs=_^$<~KiolEpnoCOKM(s| zDal4iP0DJTN6U1fi1A1XxC1(?$kCRntckih`+WJhwyuz>Gsf!7zTP&kV5&>dE+IAa z@>ph(qv>mFUG~=LlijgJ>5*@ARZD#<(n%pn-$80iolLo9dl}G2;c5_)`K&v8b9E9{ zS%bUwQ+@P1V}4tw*%4ggw=H9FW36-dh);+p^H!1_!D0cexbu)MI1NvZBF;zoRvlM? z|A?DQC2ilC|8473(9)<)-0n^G~RPz2ZY8N!alXm7K#Fx(I^n`4bb zIbmzbb3>{*LSG6rx$#J5Pm{ki6=_cO2{S64VE35>jX~oySz;O;{^pgaQNJMIkGtTH zzs244*=kE|XslTJR(w?^vnoz)gSRd-JJi7y8+H{i8o3(->|yFXbGM4$6pJwZS)ji= z$>n4}xre+ffXw(R`7Du+E|Y?BSCj2LIJcFh6?`{O7&le0sYt^c(rK}X3G$KOo#k8% ztCbHPTWyhk8fr3&J6Yrsc*GcPpvM#Pck>jF8Iw*j$L$W=`Hw>#3_EN4>uS-SGEA@W+B-!EXsYLnm z`PRZ6k(!XlWO4^;;<4HwKoy}3^%}MQLXFNEO}jNp(c!n7ls73=g0fp7ib_XBuUD2{ zZ^IwQ5VQ_&!q?bn6R66aNh*!rkX3cfHP<}jKR&Gq1?nejsta0c1$>x6qqnr=T&Yy1 z(&1FP};xWRJv?hL#4~)@kc%Gt|qo0 zj)oW48dTJ(QuU-2$}ZgUFjv3XiYl3>dZ?6{uduGcpZaR?^-0Bs;M#S9aK;% zJk@nA$u$q(day=gFq(DiX#5}VEPd@3hfAM1`^XmwpHt z8qO~b{5Op702IvMg4C&o+XQ;qu6@*f&1JiaY?d>Zc_qq7PEy0xu9?#zOyyt0!ng2* z;zGF;o_Q_*?iC)JD;*1lyT7Fp=gAa@VC4A8?QwqK+TYvBDz?%EG{8yOk4~x<{5Y0$ zB*}t=$w{_k%a)C^v5hkrjBU(f3?81#f2`7{Tw5zzi@#0WkFtdiwpIpQBkSE112*woA?hfiDM)-;FQKl?e z2{VS^V|vx23Pp>6jKib|z6~zoN6KBMmGVpe->(+J>O*Bohs8BM9;jn$6Z~Yoc87t9 z;)>EUDn^-m%;m;J|MNfR8ID0ynk!%L3%5_^I3Q!9g-WYG9PVqD)-A72ur$jAX4|9V z63fu+#H-X7`e*#^WPPUAGf_>rI2&ta+{xjlQ%hGPF53Dmr(}AQHkYV)l0#@lzCScu z8VnVZG0#t|syr-f_SgBG)>2E#-+5x$9-kz7P^ z>HJVMJ@@Tzum68zp<;-7#h(|T&`roG>o*4~fokQ;U#32d=G?Zv$kDXitTa;Pu)yY0 zR!T@}hqcMc3C-cMqxb9`nYi@5i8Ga~T4@aCe%)MY%I$0O?#8i}S~)Tl3pX0AXs&s> zZqYMxZ@c^^b1TE3c{yn*qZ#Q8rQw1^n5b?NyESP*wg!GJBQq7uTab;q=A`Ze=yp4Z z6&kgYNf85igL45FFix1#W>ea3O1n*IACYi3#_*MZfn^+pr`&9KZo|X{rz7p?Tyog7 z+Cs2aU=EJ0!NV;C?|Zu`?KY)-rgXxTt|t;ilQwEk20Rb8WfFkvog+QZfJX)ODP<@+ z3f7^T8^OP`IpZw&kG?!c0se!K-TK%qXCB>}6Tp}lyL2CY5w;&BhDK&8q5fy)2a8*J z!*Z=wk=ovF~5}JoYafRDMlLbFViyT|#xI}-u6^W%?LPn}gN$GUll@3|uP{P+U6dBh0+0n}{GY>Hp zk{daRzR`ccrHA1CyFu*odV$$#*uE6$f5$~!zj%s76?Uz45m;xD8NFTqk+u*Cofo^D zVbFes-0L9t&!Dy&aGZt-q6%#}V8NB^GU+W=%SMEQh~JtKSi*=+Q>)N_WI~zV zXFl?6Jt41zy)#&mCJd7*V*;plx#VEdvoWB5qejXI-SQDSCBR?yTZ|^&~Znjn(KnyABea;nm3H*At{#>(0qw>T-Pg+aW67Ooli6<7rT1tG&3gW#?AU|Y^8ouQEkb&t2& zsQaA)mz!BWy+!AVMlaBl<`}gRTBk$hnAhzSB{#zcJ2mRhv#o1$c00#;`N>TzE7z6v zq(7NRD=XJ1=|s{`{Zg&_SO)a61*DgKlnEepEM$4i@&f=?zghq4+h~6cF|M#=!JVLl zzU4geVF0FXiDV@il3+?uYMa-8p}c5NO~8%uQNS%QTJAEX-KKPdDcxa8cbd{gQ@Y=j z9wJh$p23*PM(H(2<^`(D1JLz&WRVJ|5U_0^=rJnzb;2IzO2Qu9R_oFY=qx>e;SCqI zVIHAQNRr)O**0wtgja5i^9t=(`-X-i*|A$k*0XY9{OBU%my&ZkjtxKdm0_FAyKI8g zUOuqusg1aXDyfUDtMbe;#Z#d%We!N{4=( zW;u@Wq&;XsV>zeh_n_qq{;;~8HkNa7j86`>gHBNq-S())r+H93+M}-V!YTliqQ8-R)0#0K0mjA)mgD+QyFet=4z+o74c4v~ERW(|`D7Kl z;-K*QEa9A5urx)wVL`&E-p1KpQyL)> zcH+Q%wAnm@L8?JRGg==pr8`aOY4bf^CdYKlL)GX?fLYOdg#!TfH?!O7f1uVj(}u`& z*)pIBaRrz?m@#_r)hyyI$t0d?Vb_9AvP!+*N^v*6*-P=R$nues-$@lw=MQ?~193_7 zxoOEN#@nR_)F>|`J#J*qddpiU|1XYWK9g$rX{Xa2NVW>pvGEhx%H7*MS#`==lxA+r zsm<{%4QD9dU!Vr$s#fv$REpk?!|U9^xC8!R+~;6CG1=-*qpzSAw4bJ=;(W&yP)9c9 zT=}L-F`V7%QT={*y8p0{6vUXvlanZi;GQ~ghbM0$}H$Q9i$T$2o@9^0XhU| zTOoS@6*I7l;e1_ZhDnnN_6GBA;Dx>jy4nI6D=B>z$H)Rsqg%yri)#;Vz8wg5Z=;Ey_4g%>sMa5s8(54K>s=8BHK zLYQB4$Frh#{E3#-N@#SVMOhUUt8o-_6`Nsq#K0e%4V^%ZZDu#8ey&5@mHH==iHW`v z4=4aiLGkv?9-rHEcr@?r>x@&<`u{@Ptw>AjJAGizYxT~3<6~0AV|6Pvx$-ItJf+QahxcyqQMZ?*bTXs-Q1e8@{%?M~L4F1OS4QL#`+xeP8sS0Y#R zU3wKgyGu`Kr@s5$?^33?HkivnzHS3&l!2_mSG7ORQxg49WL(Qf#zN~q+I9b_eQ0E`l~XVN z?Q6i8x_O(Q<0#6_GwxWs#BwvpTA@yTWZa{9fj=iAD!gd86rFX#l?=cqV9o6Cq3n z2HlBo0|E}r0MrPm$Fnc*-~@*?I&<~_Er&)APFBJ$di~vj2Z!x0E6v#hp^%ejc+PIK z(p3M6Cz_RKPb`Jzb{(u%SGLavgR|RLs?~$L=0fzL^r2zbzzYWl7HXPHEvTGQ-+kYT z)!MxO{#(-wIE5(>&xw&@3@r+*b-ii;giPt6fefo-99b=;%P%tDVhVan%ub|%K4E!> z_PuR+&GI}6?%_PG=h4^lRn(QgsuP)f)zTu$h9Lz_DQQYMBH_%g&KiXNjB~q8X^$zL zM9+Lpm+m#C2h6jNo6?)+eP2WO{o*R#e+BKoQpIoj^;H}|e0Po5Y=G|tT$vxySu~&2 zKiX}U8%eg^tmQ-G;}}wqNJux^-~oeMzXp$4gOB-|eVg1JS)&F7=LWL`O-4TOP*@GawAlbF#DkH@vY@*u zS5g`po?`B*;}pTPCfpz!If1d2n}aN@PKq(;q*C7Q3FYD%g z?kDB4C)oI_@zGG){Yt92Gm$$z;Y44fKs&7V=-5Er6Tfkxx{xjh`V(k;&azDIaNF+n z#e2(n`gE!<6w-R*W0b?6joBPrYBuQ-t(@=Ducp6z>$z=q!5Tky_U=JRb=sO^X_lR^ zj@@zn{?rX6F(hZVHb(ZRg`ng}_w+?87xs8sXZ|P<sDZ$P7DvC@2L&BPnVt!wxd+_3Z{tyIEQmJ`AozbR<2Q>WPl0(vz|5 z?)vnZ3h(6u<(?K`s5CEsmiY=(M3xhvAbybgFQ7uhiAPhGE&BcuG}*xZR)T?eA4w#D z6aXU&Zfb+a5Nc9cy>bPNbekorcY8Pr=rvFW7S;zQOZRHTCa6X{$-%ix|Ej>Fe~rGsQRk1xQEa0y z(}-Zl(77AQ)O;t2ML%rLQ}AWsXUEP_bFRcHP3ax*Lu5~;KHB${cLDkjH?%#hd0$#Fcb-eu7Uqef)oRCGv z;~u#=leu|h)3OSF_o4M)4uU(D*K(~C{muLQEH}lmdyWPmq~LbDLyuD|JH@iEc+*}X zBZz75`a59=_J(7MmA1bgfeQ;> z!)SLMxfZXM7@Qkk$TIHh475^|XuQNzNKcOsIt_blJ6W zUstb4&%nnxgN5m;91d(nP-&@DttyFBxLq%af>Oz@R1(DlHCoOVYg(drB}cuhR{Uk9 zUaNR}T|xJ`SC6;dtkW)~B{dQ7+B=7$#mAPUWT+_y{a$B&$HJ&TG`i#TJkq#((k;$P zGh%MCC1}dn=A6wc z8YODm4qQ`@3F+m1`^%EBCq3kA`Uf)aN|A6_tZRMRUq*VoDXE+QQq%FM7Yt)v=XOOZGwnyXY-wcA>~PX6+e4#!&o4uy*llHH^&jLx zRUVQ5a$MoiI2C_e?Pzv~nv4E}miKUISbH3{Sp08%L5i{mDjC6zY>EpOiMo^Wki2P} zHQn1*aYc)){I<1e{1YXMq+J0Ka zA(V6I;qOBB=!PiC64AWJb<9$XoC^y+fDNS9x|qHxv0Iw*`0 zLZj4A5K@itXCr=1vN8eNodGUxwPK6{EZ8>auDm&<1bk z)pn;b^)TBQ+LsWVlpX9qU%7v#Qkra+ynqlX7KD(>B8&V3EaMBQF2>H;7Z$H8)h`}C za(|a>54dYKRr0y`?9#x*G!Fl9i3A&wmE3!NUOGjs6o9 zH7tR-t4>9}AI->iN{vap7{2tmP+gImq0fCO)RdKa=u7J#4fKTG5t0ebZlk5Js;vL> zp6~5-I$1`sagl6cu-S+?WT&7ZO(EhBtF(JOH8(XJ@P z;{2f=H0Hm~fV!Eq?4?pL2(}TG4hNcNkctI^VVj6S?01rE5e#$>s@_h^ghUwFG|R8i zMk^R79|*pDMb*UNNQ*A0eZcW9w81ON}_&}?>?M~A+4UFMoEZCk(KuzQk&`|sK52&qRN zTj_N z{520AsT|2Sa(nC4ySA`v{MzNI!QQYpA}RI$sY2rR&A?X7|A#U(HX6`8+UAvOnu5SE zZktsNvhz7_8cQ3uyQ8g9nFSLyLE%+p{$ja`0$d4p-|NRv~R=O_=%*u5mk zjN#IAgvJAq*^TtDZq9Bui)g%xgU*ns3T@Aj{avfzY3UH6duRciUvSk6Mpgv4Qtz#T z2^AoA$4w*!sz=`^(eHh)WTY<{4<6d^U`PXb4$0GKUf+ziEwC3cu_3hExrQHo3=FNc zD~Tciy%*X7!S*zHaj+^LHG2mGy@U8hNhlL3NPsJ?>f}bC`nOHZ-^jRNOn$s8v8t&q zonDt$&m_vRcEPThy(2H~*P7cm%@@}{%sW`dDXLQ9%?BlI?!w`5f#dBgl}Ad={{#bJ zVSRYPLDQ~@T?ccygF7c(EOWWNb+j33UOYT~%i$fl!roJNFD>1Da&KYFyYK648|@8+ zdPlc)Mo;ZsN~f3ho*ISOB$SfftlKqq{mSNadh^QlV<#$GvK=*>%>}4DpU~87GOd)R zylTQj%ehY6m9IAy9@zjfHK9esfB5IozP<6JgcIS^oS&)zO!X*hr50$0W2PX_Nf)fP z1^o<=@y#-5;+#Soqb7nrhXbt@UWcQknpa>SV)F9K%%3uSmKs{mw<1mdwB^fG7|7vX z)B7ior#(q{EbT~cHHdjxC^`c&@)VtG1NEoG99S zqKj7J9)Xf+j`vBy*yw{3vY*RM4fUVoS&_B!vgnlLpv3$Ak$fWVcZaEojQ<+*QRL!dCm?kz{vMjryo6qSQdngk3 ziE*#isb$@%Y8oxMk#sGEED2vu@-QyDZ81mi-=nosDG(@?YZ}!`jSLRP&}$`j95`Ba6B>G& z+HpkVSYB`@RSWP=RW5&$d7G&rWq#Rmhvgz#mOp6un&k`75!<*%>El(*`@EA-3oKeD ziGLciemzIrWfr8&e_X;jD~9AZr4+gvXI0kCV{Ibgq~vj(pvO432i+Zm3hvZr+P5tq zLeE{)&%KJC_?#|%o=B+gT4mMjN=7=cX`SK;SkD(mea@tJZeGE-!`QA=Whq(G zW$lnD-ggBod+tgHG>jHfQb1)m+CMZl=wlqtz|h2Gzo?3C8|UP#ZjP1HogC%zWU5h- zLqm+CKCF~wE1IX}6z`&V)@HLY=hWJCGb_f2s-1}_!-7V9yx6d-S=m9kc$t^;>h1kv zespDWe0y&~6chP+A}jSK%aLJa(Nm9=-O)(Md5yNczkg>mx~so`oAyniGBjUssPZ-M zi-`%ZKnqSjGc&)XBFKDU)AYpOORB&Ls5|-s0>gWJ&!gEny3iUv9StNE(dtk5IZ8}v z{=uoaB+E`6oasbuK+VImzUt0tY8Dol365nsL5cNbE`56MYnypDOS`RHD%QTX>F7pe zK^SZvuSc>swY28-m9X6G>km>Khd$^CxR3rVG{19Sk@q@wymJ%p;aInw<9m0^3`bmH zIXS(s6W6evvV^GjQ3{&d3|tBpYBp_;kh6YIlvQP&mGAjbOS%wcxalbpu|Sep1G5(>bXzOf-(lKz6f@a8@_X z^9E1La8-}n5!MMK`I7ibV4fsoyT%-=SCZv$x-kT1J8>Im4kHwE%xZ1$qfR|84a0>&&N^Ci?9AEl*O%U;@C9aTjz)7&ee0GlPZ%cj5?h zk_fl33PCU9FK7u_x3Gsi4Fd~4zJ?2ffqWq@ZmbD&S}qU|f+1}-rK2!W>C$oY*iEMN z0Fkg)<0<&nb=a#a*cs4ph14k9$edTzQPJUJ7!pe){BrQJP|0ir_e*BGjwy|r(vm4{ zHl?Gcblj9~Go=TJRIB@}*fDLmw69PQjn$XBqU7{p9ZQSSJ*$Z-794zfp)RHNA|omv@-%$HPZhm zv3YDK>yhU6)+EUCq^Z|Ge7EKrdFyVL9pjkdc(WjJEKLh;m+wV%h++4p=EBwZaCrU4 zxr43tzG9Vi*fSw#GSqwDg40O@{c@^MXy+4tAvTssXcLp8AyrCB)Tf$@NtbAw`OHbZ znVoS*DoKWuN}0YV9ysZ(j^q@ZmQ?LjuvIDFaqk64MrSSQ%Wp96FfBA^d z?;j%Cz!Le8Rf5K7MRUSYy~nyyi-&YcME819A1RPbkK;n!G%TQ=ZlJp?5oZ8HT<>6c zVfP~XxjLB$E0Ch(2tE$S5`zZMpC{`W@)qbum(w8tIG~Z zV%rB&3NsXL~A`ph%WH1iV)$;Yc>waE5?WMU~-IlEx9 zNs7~zjy=WD1C;)6{kIG=bbNU_n(Ga)9&g4+(GE!~P*1b$ILmEFNFr|(u4G;;lP8u9> zU{&IJVJrhj+>&|CLGv0o2fV4fMC|>zri$-lgq=aYPUny_u03l?51Y~>L_+#mbc2$P zLsC(|lG>1Nm{VlvtEP6XKyHRfL*C^;x|;b&=a##o)VG_8p2e$MHpj#rl$$R^vPbUv z-~m>VPrvcVEgf6*5Kjrd?8J9(;)0%`b7!u}^MQT5AfTSw=%21|5%-Z#4|;~3inQkg zx9m-hJwB0~T3D`$vbFEP)|tfXp}9TBdij8K(elu4N3i4_ zI+%1t1dk|3e36Mfb$4UD5*zMs_?l|`OPq_LgQ3(=vnUD)w~!s$HNuB?bM9DsUsj6q ze7bk4?C~G6BIR6;X;H7FasGXyk#8sk6?L8gmPxmA5htJY%ay*VQha={tutCv5{yv1}ZAF%u@ zB>#*NwP(z7k{Eb~R3H-0E&!rk$RgMW9ua_c0N=%O6|Bc`692^RQ*6?0~^l{P)?lZpQ2)StEc)5xgZ3Dl$Syg$*-4 z$RAkM`{w{fiuF(zgmQAFfjg9L7Phmv6zXd9hO69k?;)jYn&mJIn)Q~KM%vq_2BYcO zZHI?5yIV&$otUji-tzqE9q)pH#?tD}cCHl{q@-*YBC^(w(=12xabDoDrz_%U!!RjAAM}*&XU`2m#ma4J<--l>GbR3qj3_Z zpeQ{q@e$oGk+#fRwgT(#yk+=3Wy;V|Ai}4vLQ%}H#V+9j14F7J35U_=MV#q3L(UPF zLSD~h!O&xqVmG}9WCNxUd=p?6!ZD)|a8fh~?0`p(Cxv}^Q>qdPUk&u9eR_44aTKnf z0nG#KKs=Rou|Np?z(zhpKk-bM6JyI{FT~YUx;CJw^k?_n-)P*whvAl3cIBZ)^WlA* zEfVwltdiZfzArbO=iL1BT0-J|Uf$zi^R<}jamwLzFgYUI9cp^HQ#rCs{f1)MX^x@R zpUdnR8rqSeZeY1-mZd(`JEw81)5#^$mu}WZ%W6QlI~eoHir`iPF+a7BVyxU8r5B@0 z)5&)lyDNq5M|D5b$mJK2J=oG~<#n2(;RQs0bKy(1-HX@ zIt!Blxzj}?Uqtd_`Xqo0Q9ihe&wUc~VXy`vhVXI~UK^*MY$9~*cbdrPk6w`(WhEaA z=!lJqddc##MbTej6FQn$#aY160?b^f2ZQR>O+^1S+QNAQ=%g*;ikKbd*~iEX_o#?{ z!|*qR&LL(2fG0=R@VV_2oO<&Z&M8T>5-m(t zm@(*DbrN-JibTNtkP@YvpG=Osx|+afLX`|40N{U>J%+a0?gcG%L#`BY@age1T_J_E z&iAk&BI}U)@p`Ymo%E$bo{XA`MSY%_V6#)4&FObNRU7m3N~{p+6QlBw(<2Cu^=hy; z!~1-RS~l(#l7fqJ+XeA~!mMAJE|!IWI3y?nwdI!-J$C4ha^?IXy3iM5(9#>MjqaUV zI69v5Kmbi`r9Qo5$EP`N8_WH(E!5mLWcPXdmKp&{8eQq{&kPU$eWEw&6a4w=aLM7K zDZ9f?dAux#G(w7!vsyro230qmpUMl8Bi3pU7kG(fTpYvuP$Qa^vr#G86TI}BIVt0< za@;t_cry}ua=M*(ER~R;rkbMFwE9G{H$yXXj&aA<=tx+d)V7vJ8+o6sx}92H$8`PS z!d9>|C+bOi2N86)l zdl7A)LfhBT_CvG*&NYPnAS!gG3YllamSqW-NaFW{K+7;eZUocw`89&>4XBNf0k&8* zYH#ru>s##tTv^Of2n_IlC?7GWjK#q$B4 zRug_3<%+1kR?9wb$mh)|U*zmoWPY*C+`UVbcQ`%1?G&^UJNo7Rj7#&qfS4&N0gXwCX z?=i@jEAh!pCm?ft;r)*<_|nz!T5YVF_T9qg7sqJ3+u0sZ^0$Aq9c)dlj1TUfY=!*A z+CXOgsE3kVJ!&|gihEs(nw}r6yCrRWl4I1Ns52<{?3ruw5;&#>tIMAV)1PuUCXJkx zOG+f|cT#SjB=I(iS0w65mNU;z>iotxSp_?ni^O`;G12F>`G%)AB|P%)E)?cUR6H~o zmQ+Vz%kDi1m%v%QR>rOtGw2&$y!<2PC1xB=EG^5SD>0)D?$Hd`FpxV28e^gg{{H9{ z)$|^ngp*JhtYTo5%qk{J$PyK`0vX2}`OtBOG3+JsH#sXAUKAtGcVB#K zYV;Fl_T1ZO+`H%QA572x)x!_{&HVKDFYJAw*}Q*WDY{szG&jfTD#Pi4mkEyf1qIFt zcKx4OZU)khW;mAeF=(Y=sY-Ecwpd%r=>1tk{rS%sH0>jm=^D$Ysrvu~pi0yjTmW4_ zqQ8!GGfDW4k?$uacGKY`vCF}<5u~LX3_(hY2s{CxELTZ(UzteQe_tkDy5BqsR{%WA zcB05EtYSAIBx=AhP-qTU@s-yQeGuFbkU|d5I(YhLU_3$Q?BOc-A8;CDkmRf%CpjYv zgvk08%co7a99mD<|Fq>5{Z%taR~?aX>5w6n(N(zK652z#DC8kQg>qB>7!(1Cb8MIl zR>&RkOM*)mvQQwoX%~%PXq7 z<0`1Z%zK#8-($44Hb(IEyppN)pHPi)rKdHL%?>q63Q$NZrN&Tp@kEJbm`ewOqs2fj zGSFLecNQX%t&PNp%0%0HHjQdFubsErg;IbI2$jV^aG~PK<&u2WTMwYQ8O`Ot&a_K8 zMah-gnVwA}y}>Y=ay#v9Or;A8Y|J? zc&lwp1KPGqc}$0)VOedoQdzz*e_=FYk8Ih0q=#ch8J2pEbJFps-7O1Z#LhdAb!%g+ z%qtYDpPhE6(kXY3HWga`zcI4my#QM>2BphSVO&ENYSmZ_aQGB(tdBwX@&Ve&>BY&t z=nxtv4omIw^UT-i{{zLssJ}FjEw|P9G?BddAF$uJQ?C+4v%0el3waEeQU>-dJLHp!H88Q8Poj&6F;f(qlwI zA01u&6uSB;u*ppCM*$?h;!pzQ~LoI+7Y~|2qyJ|6&u7CzKO268jj5UeZ z0UY8rvT8OKonc+9Q>3j-Ak>=JGcvU@QB{yiZxxFZN9QN@Pt*gtU;5I&Q9oU;=kmL1 zpF{ut(buV^Cl5Y;N=F8UcpDaaf*OEb5;TBmt?YmmCtF4!WwrxNvi z((4XKlm7D2aY=Ec+C6z+Vpl&O@(mrzq{c()kS2xvqAl5}7ODTjd9Hy}H0o_mVZXQ% z4FyFd6_4%g?=OvxUiz9%UP{Kbi>8*r_g!4{F)IkR#%klktQvHJw&en=+owoSz;%8ddV8W`feOcPf`Z z#(ab6K|^E~{nm#pFIYZ_dfSavj4^jRNkz=)z8x@X=18rP-IRD!3Y${Nl=@7m-;_p8 zX~L9Fq9S2#s7sf~29CW3mLP9ey>n>uoRQtwskQ^yIjc~yZ3xc%ZQZSE+q2xXeuGl0w*Oq#d`$5 z$7Pj5qMPCwhgYl3*8f~!KczM$%I$Op(!E(a5T4(@JFmrd?b{Kqcp8J_F-;p9Y^hIW z=L75iE;TehC=X5hQYoHxxc#0thPP<$Osy~8yl)S$a68}Es+`^9iC5-^TA_F=RB4=) zl3vcIgxqpUrQQAzwT!$aZ{o+7~u0#YO$JMs-!|vERj(Xn@W*g*av>i<=2@9kWG6H?0r`hQ3Yf%E_fs^DPe+m=C~qtLG!gf6{?^&c<`3aGnd z#iJ>9!R`i;Jc{H|A}^Br8FhkCf^Y)n;}Eef6!Q8U(tkn{u0C2IhO|PA?Sgs4qAwRP z0*u!Chy+^gm8Ax{(RxL#fnU0+^ZzsZwd8*}P?numWErcLd4nhq(Z*xX|tTnYs$T*cs{!aw*E}o`DE9X}~ujii5UVfAL7&D<`&}Gs5bkXuK z)erDWxP~T8mc9ja3M-`FWCwu<+$ReX#sE4_o$he{n7-~&SMFL!c)?oux zQKOr!Zk9?H$Y(>DHkurDt2SpCz8wHZp!9jum|-cyr*5D&8u@-SH^!izCq+`VOt_Ru z`Dg|$ZIkc3^A{@8lyjlVQrsodYAWKS9g?5rY}_x0c4<)P2stpczdYS!93EF~B;}#4 zR+^$OeNU^Xj_~PlOmK^$PmEU5QIFhP)_w3?a(gpfJ(RvUp|!kJR87hZ+zBa_PC@je zj;Yoo?Z3P8&f+7FP)qgQJ-mla9owQ!WSwrFl~|j~yHuXxe7&K*2j2hKNMj_)({9D* z$Gr+^+IwV1ngxH$|v4aJHO-k4v-uCI+L8V5w|oQ8zsyU5;f zG#??R5VY6x{NW&!l);p?PsimPxgytpGZ{Jw!trWFz49Vi*|CJv_#j?=qzcXz$T&G- zxz_A;Mz2)ND>LZccbRTHY}G-0@sMS+e!gYidGm&kxOu}zbc@}{ILJ4wVn{%cu>;Yl z6)4xFLtLpFlBAxz z{exx3?JjQJy0b?8s}Cs-fYyJLZ01=e|Csu~MWU z#jC#GO>-PLoW1G#bh9;EOG%KYIysY%49I(@uy|lA;|6!b%3^8y#=@R@fX}4mho_m&^WwoD^>kMz~;8o z;?Nz}g|v|0A*3~XB2=5x)|;C%JM!1L%}^Lev~l@^%vTvJiCK9pE7VTtW=T?7gabP@!r6vN z>oI;POnnfifscx2%J7!2o7$8 zNVr@R49PH2LZH1-L1kof&zRROn9{N-?IRL!XtheTzbU2sgLEs!5Da*nPt+}xGAifQ zZh;Ik(vTu-{e4Ud6z%!NFGhcb{*C_P7pJQykL-@d7LMM%{ic^vkW`#`^RByJ&oGQN z`TXgnyN)i!qRWSlS7R$DuC3&RhrZfom?f5NeCZ*(te*c4_5BkkDE^k4*L#r~F#%41 z0~{+CJ-_f2WtQVuF%$SjPYhWw2f#Tnff{9Jhk9J$%+Lomum6W!QPi?bS>~UbkX4~{ zzrjKBGV?w9pDf!gHyiZS7F^c^ZseBkI<}Dc4cD?c(d-j5wJ{G-A~zx;r(yB0LR}kf z309184J_ZdhD2u(Atl=;+&*>G9czZhPjt4a>%U>>N59z2j?3Pl65d>l zHS!6M92NN4!C1R*yc{n~^fyD1fr+Im#4vDYI^}3tvZ& zL-k^1!tuZvI-u^_^c%>GifZ5x=qwJVb;97s5ol8Xdw?9t3QHnt-*5B@tc~;c5462P z%n7H|?}LnEGTLom8Euy(`?i zYue^dM{~8XixH9%ua>6!YJE{V)F6A~a0O$9Dmcc$`NJMZeP&nHDY9IcrTP5kNoTk> zqD6a~jYv{XiI-k!Z%_R;RhGd-0w6Aztb}~)U)JIxD@A3OFS-1{-o3?r`%cA2GUWN z#)yxyLVzY^qJ17DwIFtMeoQjz^9?txY>q@=NG6Fq*<9O5_Su-l17KFED=ZPiHtu4K zGA70-PKX$!{`GFF$X}BxxkBk=W-=yMwd8N~c&pVsuibj+;Eizr zMjl`P2WN7$J+-~!vpPM3$0^3zdp9i~sk>>0SL4Ol|5&(RjyVq``T{9$euTQ7qlS@3uL#Rz>i+?M$^DvD=ljMvW&kP~r&; z2A4mC*1dm6-;-jw9e&%b1gbi#mq+5FM8JVap}&i!7rLfXOW-5mC_X^WLF?;aT7wyN zO~*cjIt!@S2A4V}Lb#pq?OaD(?G9x8YWj_2B4H3laG2+GBxU3Llqt=c(iT(NV_vPR zVZc3&(@Hsg?!ap&b=6NvU=;m|+C@s%(_2k9u%E>D0XW-C^GcZd`?JBGN;RC_c9;rr z4ySkbsgp@w68C?R`p9E{a&bYaym;=$7fT%5*cuf34?lh^np)l`dlZ-3D)YgR6w#9E z;+wlAg`c~5|6h=l2Zn35ijE zdTeYe|Gi9A^H}Aymt}pKD0LL9T8O34N5T0!rqf2DKC*~x&X*W8#4S;@j{FJ~P#hqj zNph8X6`z>FCkTqGDjE?H_<2>5a0jXqcupWH2r-h*Dx_4^iE;!-XI3G2ZX8h!y&Ck{ zv<`EC^Hn9Hm5t~%V1Fq136a9IWyrjFnLK)C4No;QV%MR!i?lR3#h5?9DQ3jvH|CZq z9Sr%QE5rcBt{+6Qk6HQVH@`WK!4-Sv^ZB_F`n|ZZZ?3;Z%RLKyf)*OsQd6izBNz8b zv1~1clwqV03LKA?j{Hzh)+^EO^i)4%<(-WIw6+NJ%B9~u70kq;TK z)0xrIR@#2sw_du~_PM#-wNHKL9zc|u0!^$uQi{5GT5wrCiM}34j9S-*=JKM|C5obp zvWENW)Q7;<%3uBp^KBjPv4v*RVatc9o59>Jk$Ha$@wsBEnu`SHK8DU?#6>_=%&g%( zwz4n*4R|BKmt{aBO`$7rrPn->aM%d^rHfT88^tAXJ=Eh25X}wtxWpR!Y$0g}_P3K~ zU<@1xG#ht@v(J!L1{OE8d zAWnU1lLrl~;&fO1k)Wr}>vgklS?(Yo=ytNWK`!7K()!`mE|wyrr|#)~vmOpJ*I_e2 zBKCsOH8a?7^P5_mo~1{^)h!YY`@Mx|q`(BTsF0d)lE2sbS!6WZ$d0DMLM7wc^vvIX zTy#(me*3}YmgY+L6l)niB=k+l(Sqi8$So~h@FIiku{Yirc=gr(d&iuTQXFg7Bjtq( zN7G!er!i98`9L+$JF?gSkAOcZ%A7~s_5N!0{kvLoy;&dh<3OoCR!O&e`}4JEtx}CL z>^#fTJ&gnTYuuv8%7Db&jIk3U}C z^Ju{(xPv{tqeb+@$Ur9Dj(`(yH_Ji_G!;yS6gT(>9Ud`I*E5=SLp58Zoz4#E0qjsN zSB^Eco&mG-v;R z;;oQf@wi1N`b~prpF{}`8}(RuX0Q=(gm{-#w10&RMD&d-hA1SF#%X#`a`sc{N^jQq)GOUn%0?>1TG|301AlR9JSrs8Q8bib|_L8-(L~8pO zq<&*sL@SZrvUo)tyt0rRZWb|=<+#~)H@e%9=`)2if|CBwuC=!jRkgLj)}p$}k7h{y zYGbY?rQWkJw*Hq?@VdjH`j%KAA|VrrPxyaG(=4TC0}|_UI&XbE7<~NJWAEG9kUYiF zy(9K;qm(I!TwJK^3gpLIV+%R{O(r>W%MM#4wDY!^I9=X0(~-*P!0)ehP?u}GHRGNq9K7R1DLaB zi7S8GTCJYB0tCSaz&2(I1edt7h#jNg;1D0(ManhmUg%_rqF<=pv$Fr$TI1>cEBDl1 zyY#?tVs!boo%_!(4}>qRQG9O8blZc>4jaR%y?bX^UubSukC$Ps=!;wbb!V}YlWcZ0 z%WI0j3N|bAALXH(YIg>51LgmH3Oj$%KkB~cURX8qb$;?c;H&RFv1d++=L2pgko8+B zPgD(0PL3$BWb8V>-1adHGSyQBsM#9^O9tnG{sh@F|B1fSCK_UQQ3oJMaf+m}9M$({ z$q@1puQ9J*@hApJ zPk;*`{&0(4hmK#Ymn8VHNZG=)OMxlK2y#Oqh8(8{&fiP=Wel z&h!)eJ7(I2((bf2E+SG#>gp|&nPvxmX6oqYK1Xft>@I`PxVp3bEcJ=Rg_E64jktg* ztzxx>f%>L)*0X+&a<2c!uVbZ{Y`4p3_ZUrCi>Gfm7@|EMq2T8jCF*rDHmh}OVOr0h z3vBJFYz;t3l(#=gzwKhISdahjOFv&H458U3zt8} ze3ThR9rR_(7pbp<|AZqQ0;`_8(7spyL3TkByB<8_2$*O9Nfhn|)oz#|+&!vm;~3Ux zo+Qy2(wHf2Go}5ev|>s(n$lTQy4{qXFr}AF>9eNv$3()a?U&aGBFJWNw^pIy>tqT& zyCF*mC;b{x>y7b`Fk1gUPuxdX!smO@`Cif%lIh_3F|s$b;2P3#tEgj6xM$>9GomJDv2RN^sTRC^% zH|Cap^3+p5T3Yz_!=YNgnqFJ~vtT9kjYt|iRM|3&GIuoeNRPmv)j{C`<>|(Y4Ht{X zyWP#G?E`Z#7M(2X=H!qbW6VVUyP`=pH6h#4%esE14o5sf*c}oIm9jS~rMv-_w+Ug{ z>DRI$D(ezmPHRa>$x1aKiFOWYk5-r8fMc)$e3AnV@w`izn7$^o`5`kQ~$i4=%_}< z$u#*uaL@BP@F0F7(rgkiTK>S|vN75Ym2Is=E0Hq{95gzVwo%k;R;XX4row3L&84mm zcmQw|ouxh`<@*VY` zuh0}Tv^C_;R_Z~TW7S%uH7Yg;kz**=5%5^o&49hgB%5#T7cdVHFzwB4oCp)#?qjT7iqH`Oe=^%m7V~-B3fy z)F9-EDtVvM+i3^vK3-|ot8!`6^i-Ocx!B^tv-7Zk4We7uZX8WbX>KQ_ZQi+@vfDYD z=WSV6D5%$xEnI$ud5I|_?dlood2oW{N%#v}HHYtCOd_OR5v4kCY7}pY3 zOXK{!dA>`tjx%C_M+ZcYD8i{gXr@6$=2GdAp8qsS%BR8}VV^s_*yd ziYDgoG1A2abnZI+73$_!f`MC&l?pU?oFR?X$ir(mp44TceM z3|fN24*RXd4BMMw=+d_uM!aoYWvs1#OY$NK>lOHV##sMJilIQ4dEE*#xn_xNmb z(dVR{oWt!E?&;lDDR1w6?uHN3w3PLUoG66*dMi@m)ZpBGIbUmZI?ZxhI5xL5J>d5b zPA}#rmzyCcWw!~Tc#;0?+y}N+uANBq7Bq*qBsc@vNU++|(6XZg{qaj*m^xYy*P5Lq zO?gtOgd2_H4?y6BVX53i&Vl|GINFI`nB~SAGkZ@C;`U4|NkMMcTOcv5~JA`+&& zFzTB#S3Pym7PE6fxIeSMsVxj;{s@RbDV63YaCR6D3A30PAJ6wfMaoV z#@Cqj$t^A1?`M(Cmnt_x`TF4mm0*1Ni8>#N)MqLIrqao1UaywvtDv7kPRdAy;cV>i z&Y_}Lri9E`lXiJZ?I=$PyX;~054a9HHIAA?bEfV&?92aD%;vLVExRM$v-6R2X9hi9 zwzzWh`}Y7uf?naa<6wYeJa&4=PB^r_E1V4ptQwPTQZhm(R6 zK=F(HKy#KIrW+adbsd}uhaoC!Fr%(mD*8|6FeQaZc)rW(MNKp0+*b3PPwzW#TdpNX zNOgQb&%r-W42tn9yh+E&OXV!H`b&7p%5M>|89XcEaNkyR9X3GV2qb3YNJm`}yQvgV zIfbF&0a)w24#o=E23W!~)YU$wjk(2FLK;ypZ(Z$hG@)g326z&8l^`IqH-f zdaPN-_Zhb=NKrM{vC9$3$?!JW@eIqQ(h}>kaXU^$g167NwhR@;=*~$sCk0RwPfYTJ z9(RQO{nK(HHkm19r?d3IWXW%LFxlIu>~@CsSy@dTor^k=((B;lz+RRe zW~|TXbx$0-ep&T81I4)LvleSXt4)f?RABJZuL|Db>hONj7Lasy8Gmj zd-lzhqP?x5#iQfXXRkd}t8CwWbeLi4DG$wisb5oix)&6#9EK>`Bw#h+f>ZDv(GN9S zGE&}MHa?5<1*4b$jQJ5$L_NOGGKB26rvcyMNm6!b(ThaEAnem$8-T&it#zvaAn6_; z(7<`Jq?aFF(4o)pz<>baHnHn!d6H=FPm0|-t^sm2Ebvc@I2{kWV!-M)?zT-=l408J zBS1Q50FzNacTU9QUH6+u;Qo)3`#(YM_Z0Yx0Zr%r>iJ9l+ecXiG= zPxnkucTdha?(EL&&S-N^+SLZ7m9(oI5kd%*5JDLt8G`|V0zrb4urX(QFir<^&R~Nr z8=L=~{rBDW-uKq`RdtUf^iyrUQ@N|Y@W$uCH0okW{Tg7kK16yq*7-j{B3fOfBBxjJ zGc^k#02mxHBSIU^ux$(tNOx-U29&RbKJe4;*}(J0XbB^#&5&2I1+BM~@MFScb+Bb> z+l`(mE#XQXEHz=ruOa8$NUn2X9bXqi>y_L257$bngkRVyXdO!&QD+qD!{|+kws;?5 z+e~XIcF|_+j8)VlsTBf%oXke6jH|YNvOg3Vn%mviw{Od6I5;r5+&OqA8oOqY_RHFkk#?-Ae^e_=&KldbAclguI0F1jtz#}WeD?%yxTaz zvArx$*QZ{eo_@AacJQ1@Z**7-v|*TI8$Y3Wc1TGv>RItP0&?x_*zlcgvRkmHi=83G z#(~0qI<1r}XWQL8Yt-xQVtaXF0(`B7D-W^XWV1-eokD8=JAm7I0ewg|uOK{}pni!K zg79zx4G<@&ELytJGL4osv>ZnZoN+%|om%^^D zQ&5YbYnE^6HevmMU}!GSk^}q=-KBpM>)=Z8E#FrJghs5k0=BH|S4>{PxB_rFLb-!oDohUwzql%wcdQ zl372a=UoX|;sxG_zT)`JUB-^FT1OOVDE-U*C6BG^p*5Yu8Y;vBg3BW!wVOA4ve9@f zRPrZ%cD>E(wxr8dPju2(yMC6pJGb25A-e95#*L!GBiZ8)i__ttS6Cq!u=H=;UpCu# zUNkV);_5u@jD!F|g*Ui4+ThHL9+=>)MypQ>@+N~Au>&t__Y-!X%Wca`UUc)uV+PsS z&kOqKM1s@n4cs6DFcX7!4(fC?ZM5<}IXhQ0yM$i7$zaf>#=>UNV%BvV+)`zG+?`IQ zJubUZC+HmUP{dm=`dmUso86}C>~)*$p82+Tr%w*ptzD7a0j2)8^UBxQ*A%X~0JRs5 ze@CeYs7I);Q|~e_y+ysOl$!%F?_DH0n;|t7n8Kn;<&<@w!b0~3^|DA1Q^aQFJ;-V? zEbLd4fj^vfhU+UJ;t&^`P@7P0{v{-Tsf0-na0M9*-d~~yl>ss*s$iV>2n=O~F)qN> z2@d*^b%=KyB|$zoYJMH+i|{odO{(>3yVvph06F{cI_RnDvFA1EHcfgFy&1i7{)_0& zk8h}8oKUi_UK#HCeegvUOSlq$zJ!5KUoL@iiZSNCOZa}&%<3GOla3M%l_5aTzLr;P zZF$8YtFPAA@@kKX&0r4vG@M6<hvnvhP_aG)-lWT;SCIO zaFl4fuvQU9;pG7;E6iVqlovQm!1v%;z|nA?JeTUM1NoK>Y|{g@@xAM~7W-4$Q#_+R z#Y<@aSCAFva8{~XEF@1W+4wj}(o1(Z0JoaVac)h&3bp_<#j9~=U4vSL;gL{bRO1r5 zas}c!T5St5(@0_*0Trz(HPRa9y2?vLaBr$WqEgeQ*a5~to{%WzFaEp>2qnaPLevX|tB<`X)#@6P0v^?)`{}-{+;q~+k^!wdyDKDo-LzO+^P|B?TgZBKx zKm5bP@qwt>#OeetR$`edsI8;fPMf`)n$&f&ti_%3ggklgGuEEifB7P^g?Gkeo-sIV zbhvTTbk83JyJR$xqq8aiBr*#B5FFAM*mqfg=om-S>LKcnFzDw5F~d|(fl5^Y*5&RE z)l>r&af#ZmG&MVsuqzg-TS0*X87IXxE5&w_>p+8tZAuL*SOPN&$kRa*s+uIFIK4#G zo}ug#P6h5INxqPwi7~i;6Q2fV#v^P#E&f==wj(~=HcXP$q6q>GB(IU(_0S4TKt(HO zrGJ%J+<)V|Np_|8?Aac5Mcks!5i{D+t&HaA{%W$PdoU;24e`m5UbnxHNjW;lD#bvc zP#Nv)7%JzH)ILZX8q@Uajp-4V)rUL#m-?pm4x}ZX;f5R2%-g;DcP!T zov1Bh-XUPEQ2F0hl(q-cD&7kY81=Y%%PSde`SGJIKfd1b{_hD`!cpoQLbE!N`sf zjS({{&B?&g0+sq9BIQkp1+4(r^xm(3z3Pj(ZF;@U9rFd!(TIzq`Rv(q2YrlS6EZT# zIU|x1S{Iy>-JF?H!t3tsg~AF@oT7ou-WR9;&#!;|>min9SWhlr3Pnqqlm}_M-0-h& zkJsHl_ZGZXr`)}0vIjg?R^lc3rvB}r;zFV0?5yDAwqBhpUsz@&$I^vzpdL*JYs|ZE zybe)}ZnS*gU^W_bqE9e}q9LO@jOLp^jtKS9?;dZ)`_B%y1v;S7>G>JzfeZ9M#;@vb8;jbky81)YMkCLNdd8jP^?UCq z#fq{eDkXX&xxRqxG&^c3Z%-(sIN#}|5fjlv#4isq0rHK*KfLO6101AJVC zYEj!B+Wr^r-jvxRAAKli^H^d*gWVH#Pw?!mX2F$-Y)zX3Vs558rIS2#X6{g~bZ)IU z)?M;A^Buj(^2|Uj5b7P@(cQB)-5(0}j4oDczTU}g6}`)~a_8as=ug|nWm{!%DzSKT zs4$%wy|LO>zllEMt49Qi_nnzc&Mo<@=xMgqcy0o0ZjK*6m`%+E zFNj?EE%x`A|3E{yOr4~YI57y6+N;H;3uSf#KIaxv{SOi3v5lrG4C65%HEt1cwHv70 zkgJ44u@HrxCopu~M8f-3`Zc)T^m=O+Dl~%DBTvDY3@O-AplH*EchYA6N8W0-A z6#C~6;Qe4=a#n@LS4we1BT$XuoVl^bk!)669L#-oj$BdAdf ztQx$Iv8%WBIrDQ<6N&W9!gAT|M9Q(kP?nG31sB62=|y^;Sjf&FoenCb~)39*PYi?QmMr4=H?#>UWfh-?5Lh zig^<93ZS`Iy&bd)s8`OSwNu2N#6Im-t)_3eJ-KPnxDk${Umh5A8YDo~{&%vaad)h@Kz}Blm7yp{ z?(j8ki;qTk(tqAqgF4vmE1zS(g!Fc(W1mr2zJ7->0=?|#lvi~p^{}$uMH0XJsS#3y zz$&j0bTZ(&?6>$q4IiaMW1^7AFo^PeYFn?6m-kh8!g{5+wu zg)8>1Lt=<})D(J_U8GCmZAwJ{PRfOr6k2L%nLx{Sv>ZXp^=P>VEsvq)^Jw`7TE34K z64Aet`WYZg?*={fVS)vHpLVy8Y0?+TUTlJYRkb!gQv!$06UuAhNyGLL;GZl}fR{DS zQJwPIAkL{$Ncl895yq|05bFf0vw7ua0Qb_j0iA^+v){OO)D+p)sszJpQTHF#q;r~d zAGtF8H1!@YYaa&7z((>#l2@s(RO2kMo&gjEN5}Ln1Dn>qTe*Ri=1;gs^qZJR)Swt9 z9*MOvI3$*8jfkn412yaws6plc(h#K zeEJ$>U0;0WP+bmWVhNASTdqoikcgyW3F)WZ6?3ST=gp2_+GC<=t3T~=7VH=LAYaq! z3Blu)?k!_Ufw90Z5DGf8nWkOJfn}- zBK}A%{6PJd_CzTpFmh7hSX;oA=5$Qcqe?BRrxs zytpRiHK|9FCNycQCha2%7J4%mGc_Cy%q)j@X8l3)Mmux|NQS}QFk!aq7 zUZcO#SZrPP7Dwxyp-`n~x>IlB2M;b!CF9LVSdU|$iyG6E5{Q2m~5qcLi$sW)f9BEmRQ3sLQBx2Xihu!f{YKyL#l05qYI z4IHaoK=;|LH7xLXK&Ix-4d^rY3BbvXkAZ1G-F8uXzDGq|iTFqf%b!_?Ney#-y#oCS zIy1P}0ez&V)Btr6?HZR{K1*Iofl$Cq6D7e<_`YN+Or2A11&A(*8}eXg+i4U zw4*_(Qw#0kM?rXCw_*(R5S@IVcGfZNEXW2`Q&OKr@TDm#N0X_C(gE(+{T8gS}{S;?k2-^99B(6z1X#l_=PRklkuFi0+7|zL;ec_}V~icuTdm zWvC8CiD*-xm@E2OUw0yv=nK*>#Yf%Izsl z!ePbo_F?~H?p}BK?$y=e_U-hTQd`q~aIS(TSi#8I48NoeV|u3XP1$El2qvdHV3#sB zoz2L&Dzgbe((^W!F*s9U^gC?nLfKVX_Ni{k;MO)*shGCGoeZ&xE9a?2Fm6uUm3P^< zSfF_MU2rJ))d;Yk`XyRO1lW)IvL7jCe$>_dsHysqy5y&hqvZlx?nld$Xn6%KZ=&S~ zXaVMV3at}b6IS(EkRee4@=j3!!N zzTwohlQzrIh=0#*#WvO0*P2(gMdQ!oOauL@sPyJ@x zUg(RPNuobsN3*km#ob3cJCE*O3f%4L$HuE@YGj%kuXQ&S9Aa#nmj`}>HF@_YI^eyU>#Jx+SzduPn zoUGK+LB*C#s_GyKMc9BLPd%zvlLpWgurx;`TxNYyNhz1f7RbuEf1UW6A$z7v@z`EZ zMpW!WdxUz5w4yHJEv7Zeph;#Ty+J*NF6vpw`3bfS7>U(m{OC%LkOx(_%|R`oa-Iq* zrjEeilLNN(jJlWe?%Z-jw{)gdIcVc(X*%dkcrt#qOQij9e_YyMEp_KRz6l z0;cedD>UKLqC0LL9wqV2e7WXMc#?M9Tp zPk^`ZyY6ctbsxYw7zHCMcjbBZJ4y{>yHZoKmpVbGA!!qraySXPT_@?vu;z%3AZ=iG zN{HA7;K2dSCQJVAAjj9p-(6%sINw%*Spbiq2E~(uQ8ugu`FmJN?SUkS8p$5R>`4`| z8JP!JMceS8sQ~93w-E`O3TkBKg!s|bBOqfAMCcjnH5N5#t0t{#(qSS&KTxe^mL#&L zC5@%Ui5cz{m7HXPRu!nydP@p3`%z&JmO!n&_wKvBu9Rd=cg=O|J?Iv__CU?g&`hxI z13h@2gm;EG5NAGUrVl8@GG(!@E+61EXfQJbNPLN(7w+ zo#YAIUCDr~en9DjDg+hMRw;O zu~jqz?MD;vi}c5VviKDB(EtM<&TavSPFVZNTFqR6bJD~@1g-im5!<1+Z-5``1a-Ue zsrI5fgvdubrSN{?rf_y0z~E0&cPU%o+*{TGw*#Mtw|pFJc}zP7&U$JcO7ZX(h#+DK zwcu+@z$Xi#QI@zo4-wrMyFAO}hZr_66eO?$!8R2)#!otr++rfAK>gJc#${B^1&|@Z z^N`rYTf9N}@nBTnKE$Po64PfuC)uSOc*-9?EiO(qR#GFK!&Pc65RNxyGClT`!!%nbtetB-l<*JUY z_Dr5nrLLXe7TngxwaC~V3wDPt zG(9~LXuE5xCEywS*j#1)m8&X^zqZ6O*~V;f+3Q^{(t6-Ia&s23u;B456v`P^l%oA1 zYht*Qw@TSc&=l;A>hp!|S-rE<=PmiKf`H;6>NeK^=N6yqkvQ66;ACr==73t|v`S*g z>_z>i4|rFS1KsstB5!f{B%?1Jmy%Nv$!#-L>Z$!lXLdN-a;Ye0YM~eRDpW_J$?Cfm zZ+beHm%M_n-P3qIn06WEv>OuW95m}LdXjdz7DgjuHyXpg%=`@Xu~F*ZVDRiFb}L3n z>n3A2WPVvTuqzy#kudlHR|LQdc8g8ioZ*(H1?P;c6Qm08xtocNm@7f)ph2v*DrG%(0u zeP+L}e@Ay%zxcr5(8rcIx)%Z|zhqw34Wl`jWrjJX@jszaj4G+(|CLxR7FQFre&NF5 z)uQCmC+2ouGsSYF9Cz-aqW8oOc zE&r5=`O2R}PkTuuydeO_2W$>dqY3HpccJAI%I&eY{SfgxT(6`@fbWW9AwyTCGnO> zbsnkkCP?bomGNh2Seng^yW8gTQy$M${x11aC?^^uo70^i^>{~f9bu#0VD>uWJ^7G9 z%2)-)YEN}}6F%8&wtEu6|IgDAFU7lj^s0HV=!#^99FosqV;Z~Z-o|^EIL@DxJd(kj z>?)?69A}CSRPF9UEof@IH^H-Dxs}#>nB%=Q9mkCr?o*;!f4x2K-Xg~&yVoid6Ynb9 z?$vh)9+N2QVq4c%Ge(hRMZIAw(uD@lH45d)csxB_+(S?HxwyoxqTt{Sa^tQuXWGu5 zrRQS(ZjKQRG4W48Yl0>$pCCgl8zBoYek)Eim4mB#Fb7`Aqjd)PJK$lELO@*Tew3N z#fFKYR`avQ-=OCAFaJu@|3>GTziM2+^WIw5UAt&z*WLB{-Mdy!JrZMCUHpjvWk8z0 zb9%ert%X;f-z`~~)XJ7IXJs+!A6q>$WeuPIoxVU@KIaLSuc}$>`qNLuIgV%J&zwhB zej;4+oI2&MMNU2G=?oq}9_;iSe}Lmhpl*AF=XTOOD<(aE(H7?Tff={SZVtHQsLWYK z)cr!%U@q!27yV4GV)gjLj(n=-k^1`3IdabS7YgjOKjVS8`;jZJv3IaFr0CkHW$JBk zzJNDdiI$3#3oR)nRZm3Pp-62<%Mr9(kCuDT@)%k^kCtzs<@;zM(NdB68Csa#IEzm$ zBpw%=A#;!+XLANBBW6mN6+%ST7d9Fz1Ts0PNi`y&OOeaKTBR(3+l;R(XhJQ%13FA= z)Y}9@2y-_8@s09l&8~R>(1&i8QLxM_$@zFPzMN^cuZ86BF3rjgedeoqT>()b5Ms8Cv0o*+zjtQGaiX~(G@9AIAYS9pqs%!29x=NSz5H_pGV)4CoSS18Rl{WFO9bz ze6TN3FTjjKb8KjMVNY-G-g!*@rodh9>Y6(~RkYjN_dR{-=qC;gxa7*@?g54}I%Abg zcYWLh!v5{?mpAM*k9gIOehb zBy3{~@vNw(Kqetm zvNAwBRIEMaoGP*0Tauol%?v0f^LLF~coR=Q(fF6zL}$+H%~!{(sOvdWlEheb`nJJR z)*GwaMUPcv9lV&D?4Q1_qwkTuiLKRJ!UJ7xXo%KLH;zrROnOHIVB z)~kKNs{Pv*zu*j*4NlP@niz(06#RPK7>lko3z6kzDMeH0aswPo@7Z(U01;^+Ry>M)wUhtcdkj0TNiGLQ6P`hf!5Ux+$V8LDa21Y2pI4o?NKn3 zDjgqSH_--YDd|Ybtk8;1!3u1gU=%u{IYy_~q81RXP|qXL+$bBo{2z?5@u*Tq!7xHw zrDAtBv(O%V%Hqb;y`_k)h=dUuV2KTK>`{x}xK0 zo^RZfse~-DS7vD`9;Q2KddtNVd&+`9kAL@p)$d(kF#jUs>Gq{&m7E}j7vov>i)aNtO1i`e- zD?eUV;%Yb~sM4>tVx07fo2mKmgTh0pXF}pL-uN9cA7~sbjwuawC3SHIj!bmDw+f*{Lo%+SZ@6 z&5gvX@@r_UG={D!7uEtoG#a)9+7t5S#0c2&XoM_vv`My7M_GiT3x*!}`NPq)MfBUv zw3#YGrc`o1=DcX>Vi+M^$onU*Op4fEcYb41f{c} zXAs--7tFC#O4!o&=9~1{zO}r?#xXXYF$QXFkXSoxfeS$73>h*-O>z%7DuQ+)Mjv7s=@yuvYy(& zRZJl71P?3d;(*z@Lt&Y~D!+;3xVz8@iN`j9+{#2QX(1!xkciy`7NXq2en`fL_)4R8|UI4C{_ zoA#SaGFw^h;22!Cfn`WZgKMmkwMsiWBsR-9L5A$EJ>q8c9<`RaiR6Qi0ob(TM7|nY zFs{=1Fw<9^1fw)opVkz8l|D#`)4H_qmKH5=cEE5bZHBI-!^xRUx4py1^9wxh8!ejs z_L05O%9=m8qk@JOS6EI2^`=~x&)1csKjoh7$a}JhxMSzwzU1`HL#GnCTV|4T6Mb^> z()i6&Q#WtBsa&~l&z?_omTq05f0^CUnM-X=G+s^ZE0_1B==qu3h9y}){gt!GnjPl2 z%0xM8({nWA*U5%Yqc#nQVH1g&Sk4iT#T|_sSawMFw=CPwLFF*|_iuGWy2d{<+#uQj z|A5ZfdF6TbHRd1D{FXv3{|xnOXjUvqg~bi_xSnz}6qGiZO~xgmyQ zIdi65^Ezba%RE1di@!#Bj=p@GFX6TE?xah{7+m(J@=N))_-dN|vB4wu?Y)kj?Tk z+mD9JdFp?{XPY9rpp}HU$CNeDX^$!|9l#Ns>zE}CZ=sZt5u?!IEl||D2(6>11Z5`U z#9Jt6WhPK!rs`0hjpn2pZ5ijHViPXgNya&WjB6?nMk{K0v}K$Fe7VUDydNu!Si?$d z2)Jq#hM31{J%dwVZL}J=VM)_b+D4o}o-=MKKi9)rr1C(_diQ(t^Y7jL>`&Xaw0VlX z3tjd3-l9kEbUO8q-}$}Sx$oXNXG?kN@uFk#^kTPf=<<<1U(eDx-WRp=j6dN-`jXH2 z9;chqaT7PvTp!l{yJ)@1%%9IF+v)#qPe_$ij}KIxuio_W_n^rDzX0m@UuQGajkNU* z>Ou*3EQk%l*HFEK3I^=PX7WIoQed4WYt^p1AgW2ZsH<(@`3xuQLYG^mWbIp@Stm>q zaLHALL=VU6VkGX6Qw$5N^7z)9P_&@3w=3m8t4dN3Xx*x`q)At4(jHAZL?nC-7=&^n z#&(6+#@TfYu72Y>VX4OSryt2jvU&Yspkp0G7veg-Q^oX2rCZ!8tU3p&^9%hys)u*Z z2Lkgu4_85zbj2K2#%w6>u{@esN(=GaaNO$TSS!OAy`^+vz!ghow(YrQG@rS= zXb%Zv2Dd#TIU=D~<5OwLVewbH1`;qUp-XVw0LLly#c9;W-eG=%dTtbHFU!ak;A>_-2dc(t3Z=+OY_3%WbIx_q*%mxmX(XKaqf zMrL|+D3S>bPfo?@L$soxk_1#mXq*n_{DK?{7#Jb!V~#>b$O!xc<1BLz{b%fRtcYw1 z2Q>@hDbo^V#6EtQn?bENZIGxTjG&NnY%Ag7Z8eDsObzhE+m)*RYG9iHhH-zA{sn!2jVsSHifLXxn5(7Rz~+UxLGnVZ%5&(n zToAGKIb@waU8}~c=-)vNTu8Kq@_ohkSI*7ktVjoh#O%wzp!YQX?C$u%%Hx$euVSeRg+(6+IL!t_8QPyx z98P}~hMmy%#DmUCrA-{exB$`Cm^&oX# zITSo2U`F+l+uTsX95T5Q&S}FQ6!7ffGr&uuN_#YEza|~lq+^@z0sfVn$Tcr1T%_u8 zpVXwswKJX~PlRhFft_3P1Zq5?I=zY-z_K44l$z}Gz}>B$f>WLGb()w3Dt@gx{SZX` z51TmZywcW8opv%mj*iW4ub5n}UB9ztj-}J0J74nUa>Zb} zyL+-)ndt6GXL`qXO`GgS3vbie4T8%PbpF20ZMW&AfYm|^I=|HQ#T%^-`|QVhpL0ee z#%Q)>+sdabPG8hjj90^riCCx0)fr0;2G9=?oi+}*vf@1+A>i!Z9V$=x{ZnO{=NuWy z###*e>%-NKvQKtd0w-5Ysd7(9kPN()^}0iy?!~VTI@(W*ZRnMxM7a zv}Ce}!fC%jAzKpc9B1X2fM?;Eafibc+cRWgc)i;vaIvoWt8G4Sz@80OqDPg?xgGJb zXv*sL1ni9;WDd5sAI#9*yKasNHh$#ot6612Ts{!wIkRAjxs`mZ16ST;?_qvLWzg_4 zL>-|{Q@2s?!x)0co;WFQm>_7nKD0fq1)d-wqL*xeGze8%(4-Yjx}Zsyv}-`7-5oF< z?J8m082C@1*n~$G94>{nXp&EpdNpZ)NEqsE`$p3X8Q z^PY^$${3u?-8^6QNDR_rIBsm-UMLkEu|O%8b2pxyxH@PSg|-_8IX(RZYN5X{h6g$s zi;!#g>i)9v)j+weJrL}lKQzAO*i<#x7z}SMCko3s`jk=7y99Zrk`DA#Qbvo*hXz!R z_H8fzLvq#w1E|1TV~q<|x-B*A)3J=zpmSOpujRLRyI-|uQZY+GjzwaQz1eBEh%Pi) zmK^j2)J`up&N_T{i_Q_1SlZ%q(s!Og7we}LFKFX4k)cpNI21d;+Ze`V7CWO^A)QGI zf?ct*f>&N*f1gcI0ctlefe#WDc~Ar=3WWo@Dx7Rnj0dZiS-ZW^jIKPHu_-;!nwCHfzX3nfmMosC(amFH62ICfR zTl#F?#f(l0s9;k9!QEJCiuFs*o(8G6s>?mbT*3|A|aklSZ42@D+w$dQ28;_e9a zoGMye?!FUOUq9JMhcj-YD`PVGMmzS1CY{ivYc=VjCf%c56VSIG z+#uCfkhhUgs{e7^>?lDtq|lshR!(F!NvBC!P3q9386x3mnF{&>7=u8kd!hu7r5<}x zlkOqXM>2IQfeBhi5)LsBX;o?+N^nU8+ZOd#PAI6;;~BTt)MHfB%tDkl$h$gk<; z#${W&khk0Og|zLp)U3>NG$U+bKLEqBy3DfM?>V_+b|U4p>k7waVtwwm@?hzjk@BuW zKo^@{J%MwFue|z7u0T<)>TUxS zUCmYmSuIU~W1w6Cf)Nn8fSL=Ho+T$X9m@bkR$0fXUxTQB;2LSzs<36Rm9USdPnm%x zNJRuoDg-fCKvZ$Y;}jDhK9v>UK8p5=Ud~$5;gcE(z#_P1u_;<1_a)vRU zXBns0*1eKH&N?X{p=kgudi2(hV$^L+Wr`Z#7Eq(dvWYKG+<%nC)7+n(cS=|t$ zIFH!7o(Z!DldZ)c2`Y)hm!N$>Hxr#4|>(~ z`TV>uc-IYkA4*0UscmJFF}p_>(iYa+_!$!%xq6E!=w3N9?3=ODR$nYIGCmm-LY~@z zZ9}pgX9US>?%F;x60c|G<`(G#9IK>bHa_J}cm-AtyU_cN_-KCSrsF&FlG~7;TRlzx z9V7DQzK+9tx5s2BpE-2y)-h3Re9Rni=2nm2x#B!@|1Jf0%|PX@Jj#BX?W0D~3=5I< zZPe}5{nQVc{|3;rKU7}Zx2SiN^(!RC9wp)DBZSt4N5^3_41*@QiG=-fkQW>!uhmX$ zSy+R+^Sfd*jPs6&<4^C9t^oVgu+F!{CZOQAT84jcqHRTb>}m8=QAtgU$Jih+4X96a zHRQ205}VmA7BjxRPOscePmqS1R!6^fS5;-GGTRKIDHuw}^+L#IrVVwJ$}29Hyb!lDjJpLXf)*+U5>?;M>&3m=SC05EG&zxYxHw+ zUW<-r&|Erq7d~PN`dw08=^Y_i~AEFdnrUThB-(ZVqP`jJ{(Hqn~ zB^-2t3{W*-4L(d91!_{Y1&|Vhc^gBT6w{;vk??t{?lCH=Fbb`ERlrop7lM#9tPhg4 z8d`?q)MzEhfHMT^3kuGVde_^DMB$>1Vz zbCZe0(8OvV7?ESZSa4+PWGcO|<6!UN!QPOGb=U&&67yY-1*mZP>A(H0I|pqtA06L* z`qS@RWVvK2>g#Cy7wQ&+92Xq%d3Z)Ja3-5uRN|}g#ah>pz_S)3=XCY3TtCav*McW@ z2wb(u3l~RZ$j$gwG40IV_w#p7*(9?omXCGzoz)Tt*8l(QCp~6s5_|_zyppa zz$%qi)`y9{2GMtwLk1=dmDsSES^??$XQ%_p=N!?bWlh>kBzz7w|M0a`$0=;T8)jvQ z>g**mETfch(6ZbG!v^y9mpsExgQ%f;`HMu*22%cqQPXQdlcg2pVZpx>%jK98OQ4^>Z+Z4lpVh0(C7lM_i`?WI}B>q>u^ zd{vvM(jNsyuBC+ubW*3~f7e}PKcE#T$kVDPyv32=V*c}~;8-NIJU+S79=#D@D0gkflpn%pw}~;&B%IvShq}#{d{^}bg<+af_)V4*3CXnRP8<%o&>!V+ zr5`bvadI=~YoF=lk z150O?>I>c3XxI^I8`!ty@`GlW&8BDtL`F`CR+eRT-NnJ1Pj2_u4y-OFje^Cb3rsAo z6fy(-BgOH3LkY7@cpChjGc3o_dfq?X-FUKiBnegwI|r1l?_&(OS*Wu!>gC3JwoINl zPXAeToGL!h;+1FEZ?awJJ@=v!;t=&i>c{joZ=o^sUS$%y7roFY6n4vdsV9_I43M0+ zDj(Q+g`Mf~hKxE0i6V9TrP`0THc^><(nh4Sk45^hGnjV^LEX|#J48e*vVf1A{= zvDCWqZJnBw)1+~9)V+!{rEQxh5~c-Kak)TNJpcgAuIAK&4A2UP$T8};c{Nek6lHB4 zDjC$P!cTZ~9s5+?7MoxTZ)-nZeUj%j=_{J_4NZDelino~M8`MNH7G=7EK%WF=&x1& z_~fQoI9qomKL7^De*{giYANU|n<7qX!J0-{L55^)uvdBob|k2Xi!T-~Y&W>%JvS92 zqoJawFJ_PUW!_}7SPTN^V0gjnH2Z{LSMN~L>M&$SyE|SJqp6}lUfPmt{E?Xx_2Gb= zkPR-$n@xlTFFoOI?;g#f1KZtA>9V;{lij@)N#x{+!^(=R-XfaQJ^6@W3m1pVYv-Va zOG&$tF_?=LhrOc|ZI64649}bFky?@Crg*m4mGIgavq6d%`?A@tl-I;#MtHj zbRs?Cn<;P2I^5>iYk%>1cCNvNt{6e2)-(%}YE1jaYQfH~xp_6SRs!5s55HYPvRt40~qkRbpgMV!! z;C8TI^#;VmW_3_PF;+l#!beh=ny5zDwGGJbRWGjsKte*ln)?GXP=6c9B|#5vB68aZ zE|`{D+{`8nHM#oObX-!veia3yhNGx1OgZb)rBlG~=hC;K0bf9EyNRjvp->z1+n%9F zDmvy3uUsGWonJL1g8kjR9CQe6{WDoxz$I}%`IO#jzOCf51?)bkjNbLo*|oI6V=*vx zi$7gUGjtt&vDAs-U)zgbv)`E;iHmN@!pjl6jS~%`?6ytvy18K8S#Z}q^pBO?jMdvK zC+J<0g&vt?$7TWpvsZOBKJ%c%V?CQM$g&|az?g!CL|Eil!6nKOi@^#_Nym3wx8&{} z8V;d$^ZrO=IwiS`nS)c2&QhDtf^-$sXyj-)b~%HT2OO8JT=^pVb+)9`Zrnya#^_s85gp=s004{gl`YL>S3djMR&zuy!OEu0tDdP`fsurZq|8$kcE^KG zo-bh*qWcIizpf?6PgP_f&rbC+JuEh>G=>yHmsx5^xilOFNxraFYbdeT33937W#mal zW=xaXG^t&aI*EkO2EP>iw51JvZxz3HNK=XcwHM9M%DD%%tDMuM8#U=BO}ePv13c5W zuj4h4&VHPDt8s}ru_s-X&5AkkH^qMwm5@sGg8oW=WJR?$o6s%5WKG~4)f`ZhSXz_6 zkw#sd@N}M#Jo(V>m?7Xe{OMZ9p<4Bj zBW6!5w`Xhi%U^RRLVlagA4<3Zkt{BHSea+TS+_eIb{0I0(cr5Etg+sr6@1Qy-VU8; z%TGq=*DW#sfG=t*Pj_W(eT|u($R&C^>gNNxwml`8V=ZP=+9!ntju~Lt#S0}@XGlcT zSAU>5>~+U2*0zMV=$zi4pNtAAclOlO2G|DUpm;4DaUn&Ewn>~XdWdHEaWn~h&+HZ? zmuT?|jlT(|JSN(al?|N9EB_)IbcgLWpFmIfYLQTCFbm$bFJE`_*I8ggh+TOCjq71F zuE!L&)7{il^fZ{47zyDz!jSv`^+}RnHABo!Y>p06qgqak$`0a0=XOh^{?teP5-p_u z)JJ*Il0{25TBgymhL+=Kxqz1Y(efl(UO~&7X!!wJfc+I59V&$*$SQt!4M^-I6*U3J z$pFz6-yPBhv4o%QY3Rj|mvD?aO=_BJTAhP~dR#dvNFb0v!s%z<1BUNtlHEs>Q0_9JXw znAA3HB)S>si&@!Tzwom16}oY*^W zc3Ec6J0W<^3^ab$2e2AU_?Y0lya8!}*`SClTwZSvx@;lY8R&w;udD&b$RxT~gO{Ze^K>yfmZt2!`o;Cmf95=}FiWb9h z!Qm9oN`8MxemgM~qVH&27a2(Yt|#VqTlI8gEb7mMeKwQT$V*b=;pyuRY|opmTxNF9 z%{%BJ-Yqt^753*-dn@#T$`+-|2&3r8%Wn;@BoZsb4DC&a{Wcv3@sZkMx3Wsl{RM~ zo*h3SR|mKcHMb%`+Mb`>WS*!@^I{W91mXfXz5pH#c@h-|9%ov7@*5FD%K) zKHlKTbw!2oL(`p3XKwDcQ<1%W&fHjNi>v7CjdVD7-n5XnSo2dS=Nd;@+F)~s<#3xT z5p)=lB~o9zjq#v%)u)2Q^s_>!oMiP}3dX!Q9cLQ<97@W1+7%e~xJ-NM3n!oa!w1myp}xNT%G2!YY$w6df!TkE zzKoSU;k`Osjj&qb<*p-}}5eqC)=3jS50A&Lj#D z&$U2!2o4gE#5Dq_p(i>M{5*ibfC@3d0ffQ@$Oiy0Ss*o`h;}cDO$^FbcM!<9A`M>g zjbvnA69JF_a@ubs<8dE)F8mrs$ctNRc}eHA<8IZYJG3+48C8rJcwvAIb5MCLsMq13 zc@ziD+27m;$!HzdD%rF^xU+>&N43H?C7eTLBz6>O8Rx8)ao$RL;G}=?>Z|m9FTFJX zTe;q1!Xc-s(_QtsT3R-`vqKewWDx6UB;JKcAVi}9pYG)vm^(0 zjo8AmjMpEjxf|EyX5H@DTx!%;6|!#VZ-7Wo11Vm>?ddFMt=)4z-+ULOgW3u>pPa5< zdu{PF`mcEHwe;<%$MveVYCB^Ncen8t-pK&xV`sK;*pFrlIp)%HdWU%b$g}iHa)_STda){6Snu4fqZL^> ztT^78v2i>}9OtQ_E5FBnf%Rzp{cie)P&so0N!H~^CaHB(gH#(LM;ana8Y28HlD~6g z4dvUATdJ~e_lnJA6PEO2STBp=B6r4qFtNV?H>1fm#LRlCV~R4@I$A@o`a^kD!!nJBd9H z5NR{O$D-IwWP(0+U;`^=5KmZH$Jx>#tM1v4N&8p_JK}6 zxB*e|vjj&?b-sc&4@wZMA&CNxJhXxTh6wp5L|n4;n22G%;O{%g->+@}w;oTus$3EF zya4`Vh?|0i58*iQB>|`0i`18uAA2Wpa~Vl&{RFjF`SD*C!Ki40hZ;%WX|>^l)FYHd z`C$=2?^mRfwymm3U7FOZNh9d~Pb-gkr>JI&KvgZ?kMFPY?7S#8+2l`1>LfJ`!*Y{wmX%@L4RLSS?o^tt15@Ss5*|{4CGvczeMr@)m=x z*pmqeR!^$01Nhm|49Rl~yWL)?FCTfq6G?t@;~OM2rn zy2?ILWuC*##*q60+sEGn9!~@T!zY=PnX&nYyi0lj@q(i>`TwB;Kgojt~hwxTxl3y+z%u?Y~2l?$++| zAlZg1Ff^oo)Xj9w%qk$E&^j3bylClUNi~gAaW;eQr)ZN9LW#%Pn5v{9sVGXHp2XJw zUu$0;9!GWMdu!`Oy;kq)R(DtLn|fEbmX=y_Yqcd?vLx@eY}uA~Ted74Y_Ma@-~>WQ zh7e3laEu`gn-hm2iAf+3CLwIYa|TadLL7$SBRs;)7xE=c=7kr_@1Ck#t+vU`_x^aT z@2Y-Pb?er>Rk!Xv=iIaKvgtbs`%?3LSudn3Zg(ZER@xhE8J{uQZgnkY18sK3ppOj%Xj@?{pmb=oN{hD z?Rr*Mr~FHMpIuWQ%=k{Rtm=e6uC2qJ-^8%iphd6HX;rj9G8cVcNmGYwp0f*~#~t+MRt)s|44TsLf;1GS0X~&8W>f`cUhbRWu~vuPnax?O|VX=f1r;Y9Af8$0s+e z^7|G3$^Az=bsCE1F?-u}^&w0T!vHmhBwsaHedmh)t$; zdx=)+z(r6Tm+uZ|fPq^>2usKT%A9-1wII2?1$|!33;gX(qG_>QF4W;9(A8f`OQOUw z@q$4U-(CZRj%I|qPf1Ba!txL+x^=uLY3!^O2_cY3Y7xy#`snh#4^;2cZIn3Mo5=$PTH!cR zyp=uNLHt?bS}`2(&r`y=qPfb`Mr_3b@2inU^27kg4+{a;PV|38eN0v?{BL4bNuLq@ z;mMO7;5Ib5W=)FbSjOI)O!nSKGups{f+E`2J2$QOB)vMnFVy5WdRxkU$z)$g%iN!a zQ>kH=b@+`MR-;x7rL3Ldn!R zMWwD67N`B5Of0}t_M_jlw7cD1R)(>5x!vs+8UjAsxAk3lM~K*zNuSPOr=OgAd(VA) zMlfYTdj_}Oxq;d$8TruvqH0&!^E{YQ^C~<8SXAvSyN+er!Fs5jW~sTCt^RtuLp^Hs z+08D!wcSINXjaJ%0lm-n`hOkh5mL6G&tn9>wsiF@bB;+OFX}@5nBThv{HLi%AjqSl z_bBL6`-tkGU5phEl)Q|DDoSCbA_JU!tc;oy9^C2p)oNR+zVp!fC>?)zysYZ8{Ytyg}K11 zg6;=21av*%x*IOfe|;q^Xb%NKc6%@oa)6nFVARqY^&csYws_4d2G4nO|NKY##~j>do=CZ<#VfzsVRCTZ z@y+`>fGwjcEZfwdoA>nhmv0BULUdzWhn?j();rMK7St54NaJq z{CiA0rtw_+pD+vwLL>B=bF!*P)ks}})kWsmc7FE%36|Jt8 z0qnl0>1awzeqA7T-!Q(-8Zp`q6mBAM=iOBltfEvE6{@IAkR-EPa1p8?pPQ(n9R%UN zFz>!gexJn->c8Mhcu|`ye4Q!zyj>~Lrm3n9$ezAL6Dx*xgtQipS1CDevHEa0YUUgo z2gH&|>5X`xQJ$oB#P zOD1zFt~6spAamm)zqxxieXlu_j*WGbFuhskNw2=VS94zxJ zL(?V&XV%Vbj6arNk@h%3%UX)ZMy$8)s5~)iD;6_Gy^f3T+_yWa)?xx&#ps3BG|OS; zy>E7MG|)2Yuq0xjHJiTr3iFI;TLJ@78q?-W@V9yty$NfM675Y9KL-l3Hsv5uZorHY z!q>@4Q4Zs^(Q6tmgMcZDlH|ojiL7lKLDCh+pznY^63oZoR-7hwRI7;jtC1MBFl0^! znX-ax>&X@zLBsh*vY!-X#%?rGU76ViQd>UH=L38`bgfh5Yt_WVl`65lZT%@e*DV6Fid^Rz`Bl(bptK+G#&3xINLX!JTQB5Q-?F!6K=oJ zsB)IJjMrK1q4D90g|}-kiT0eQy?Z1d$S>*X3|)EKU5du-evile4L<01tKD9nGAVgi z%-goKBjXITFFUen{mm=7{QguZYtg`n2N6z>j!_Dh=0~e}*Y`>zk z(ZNo$O{2@Sm0i3=p*DQmnM$QySx+X>=%k#{Hebl>_6W8nb2i^(Z%{Bc6{B&bBOsUR zVx$5kfICC<>UWvv83FUA1#}NOP1)gNmS(k9vNY#|@F3#L0N=Qg%hFbkVYC3Bt!i%c zp$~B)nj3wXUF^e5QlD7TLY!8KtKNO+5Kgz_^i`Z5!|55EUV-Jab&{#SY)QD3lns-Q zjHat-rixC$JtR30D^|%WR{Rk0rjaHeg9#s(Vvzf)C`ix+6eWETw6%^IrC1Or@Reyv zIwSj?g;9ov$;a+bv60lg6o#)QudE@D?D7seSus3su^@#sn#b39y}d^5iF9IIlAXAc zdeL48zjgfvclOU07wFk1?4SK5j@&UZ9* zG;N4<7a4;i(-}5miidHO`-egro5R0y<*-v>HoBWzyCQ1dA`J9*Idoo2U@)$+S(cp0 z4<1kC&#YNaoYKMd4qkWqE>UfhO(wKJ8W4WbKzdaOAxUc-H z%HvY=Dpt>joD8SZ>eVi%I+e7#RN0J*>ZIuo-2XUdAzBJ73AZh63F-I-r#{q?vNYN2 zi-o*RZ`Nzl?G=wDYIA3k$$Fznql;w=^?HxN+k(5jCcLIXF|2H-<4oJ*-64M}AXxPE zoR*{Nyd`_9p_&qlSv2zBD<-8RcgsAuB37e-g7xj z*-1;fKsmvBf$RicXPl3zD6hPCiykzkE07ee3)Z7{iM|rLiIJb=g`UG4^>(rU>8r?G zMd2#SS5c85iPx5$(m+MQGG|JxJj~ltu1q#0s{P3GCh12O$qj?lrYs91)ZLdj@BrY(UHD3i!N-?svC07?xE4>ieDY!oVL%rXVZq9 znoz1xFElK_*{}EM^vt6N?+66$IC%6seZl2BPK>XwCX6Bgfn3f{*YaZDwUr_(p4 z7%u<&uRY9=KuC_dC(U;4sQHTdAIZe4sbf%X?&FM0F5T;QC@W1H)e>#^U;Q z29(ErRRf^mcPW~KKdZ=Xv0EAz?6R*{lSOv--NpHD#)y%es)yXe4Uo1)xx*bR#DXOTUE~W$LphsN4Im4<7tHG$3)FX zdx#ZFlG6!C*MN>GbW6#rxS_Le>S~A8vrPHIj&XRm;*@NzD;o?*EVn`;BM?qoOd-PALWB?5NAPt$$5+;Z%ZdEl*&tCsqB=X(f7`1)pbGZNe9Sj zT4M=`)^&1{`WnaBz*sU*oB-t)~kd|2+8IWj&4MFpu4_!rdOuH2yIR z#VqZh`Nq&^fM(}9cY=L~%igX#0WMCKt@4ecZlO*loZ8$$o#9sCyHmEp97soxn3$b! zbkvtAdgRj3!(TUj2)mpP6eEv=k=ryP?d%CvevdnM&Oo!i2y`Gs!$u=2z1R~biyWVi z*4rYK74`|%MPzgO{pVr091;F;V?p9dq(CSw*{PmOK#j`p>IIf=AsocjrUny<2o~HH z2Q)Vh`bbIQJ$q#fg3!hTdydLj;hJH(^#B-p^J;PV3aS{e>U+((1l-KEUvcIR-aGpN zrV77vPzkq|XZfxtaK#uHI{bVBe$xLrk2HtwiMXyjejdR`mOvb&LJUeEth8(vvq(L+ zfGd~SCRQhls+^0{PyeQ>HIAxVt3WRaIk4_U8h83srctuatBd~(Tgo@bi31i%q2c6x zdNljzdN+O)WOJj9X!G}Rh}}&-IxGZmX1+M&m8pp{wtKMb;5{L9XzX2TD!AYY#t$8n zHV_^JT{}cfWtVQ}^+ZH{jFP^>&eFa@a|Vuqdx)Vqy*UPtWSu_BR!^W0eoCkc0lgg; z(Atcw*D~);H0E@J;_C-ZU2G`2HT$)vlj(CTL|5zZz-fidX&Mw{xijz1_4h!M{c38A z7cRoId?p;`>19?8@cG$dsohBuke-$GtRgeRb=BNm^O9VBDLmAe)4mbi!arVu8P>+v z`We$)=g|-$Hn{h0DXUuQnX@&FgITn&_`EAKLz^*T)&9<@V_nF}870jYQ`Rb)EO7IZ z^TLhgCY77k@HwDkg;CF+DtODZO2pgm{Hu8O7JAh>MWQw@WaBAVWnZ*Ww4HXb`Q7i) zZoPSye>2jqz$ycGe!YdTj}uEMC22mIm7)y^4N7D}sVEbO>xB7s3XPJR$7D&IAu-n7^2XA%tym881N{5FbL}N*l z^*qE0J{?ldYxuWp+lCCs;40|HL3B1+8S^cRwQig6WwZU=;S@5l8>%kJVpD8Kdvi{7 zj3}{G^&|W>;uhbnMPDO?xyyW{k~|7QQ!aW0+FPzBloX7ixFTJcDU0oLBazf-6eUe^ z1r01|W2p&4Z**3qpj?@1^tLmcCk_4fyQ=@g@vejX8lEP^V zq(@*nZP)XkgOV6c2#(CI$0_i^uH@hS+6$P{=chK>Z8s^4*5r(HJjP?u1R8B8%#|9_ z@UhuBp7E<6gQZyu_2569GosTDR2*zG-YTI8R1eBY)Cxe7xtvOYOFR; zqCFY@mL1rrL^e#d#~!y8Jx+AGRxOlC+t1tLf;?W7Q+)N$M*U0d&aWB!;;P97bhLzK zn#{duuS)P55lg?sJ-*@VR)IM`GT}H|DCBKZK}4D7uIjOzjs>>EMrFX#*&Rw_r?w@{ zcv=C_N@cpw3Q*RDn6?H37t{4FXHJ%u)oa9YI<9IV?~aFG82+qnICA3?VNP>`H3l%X zS1%;i2m9N}0n(40Oet<8qXUY$rZ!HbS8yYyjJz8gJ1fm*m_Ik;?Bx+(xvtyh+$fU^XY$6SuKLnJU!l-d8pB7?wcNhsmIZiU-=L(3PALAIrY z>d_vhocSIna{X0%rvpwp(+e$v$0SQF2a@KvEWk>M&e#LM{2w$lR!UFS9&moHQ#H4?U^8J|?h#Cr<>NVN$tm%AeC(2meOBQ>aV_ zGaA9A#JrYU*9sB*-MZTPqYctj_AvWAl7jm*;2ob!NvT6kC0#?qrA<<*8I9ItD_gR>=b|)jRp2s&A=1?~nG&89W4uE6~gE-pd9%y)swk<7G|+ zJUs$pb;We4PzHOmYO&(@93A_m!vb!g^N^L&gR?6YMm>4;?2n2as_}?!^^bY{x(XV2 zRq$$OP9G4bpKh&;`Ju?t)8#Z47R{`9vaN18ns&CaEeO-&EF0V4YWzP1lae8jGdzi8 zLJ6~Q(nW?w8{9gg&s0>Xq@$5~fn4@6Bt;=%R#A53{>eu`)8Cqh?DKj+#?z zK1b$sEolEX?RPioKUx3#!2e#CPPT{be*a*7a#x$%;5zypW(cRnA{ips;9^-SmX&p@ zyFo|uk{t`1q;v`Uv@B~1ck(qb_v6-8&YXRUPG3ouNXb%QC8)&wv)!2DiZs&|$@-SrrJygdMNm>F>60%#RJW z1&Zlc?}A6sEF8+9hkZX2K2I_yq@+<~3O~V#o{?sCghc4(MGuJG?lJ`iFs;OUQP)dB1rw2w4FX|B*0W8_kYT@uTSLa(!0sSveGH=sdL%%`MC#(~!N8`N6; zVpe8G@P9zSI`6HY#5^qAv!YlY;R5q3tyFNR7_U$X5m30_MM4IZJ+NG`hOSa@>5-Me z^ooCDQnaCS^NK8_>fmg4cKD@p*@&%dB50_0FX_EoK^uyROIs0-%N`>Z{hU)D)lqE8 zFwgfp;nsa#CfUngpU9+f5C%30zrjy)xVDFeaa2TzGaTN@G|tsD!gF(bbFGK(u9tS3t8)s-A~)gr+ziq0d3y_f z1wLQK8w*Y>(R+zzcm0nXrKAz9c^F$ZXFdqaa-P40g!?|NQ~l1{@+#Y_$;$N11ZMu1F}G&4J(lEmjGv z6gex46x?3pmzO|S*iZ1RRr|4>JqlxRJIP%quu=pEo)9Hi33(kP?|@1iLF%%v4fB|@ z7ONpNQ79`%pS^{{Qe}y&p)tD)vb-TEcfe(fRb|rfL?E`y@>_x+Trpr-7yYAwsAtQ& zXanWAr65^v9L9rhPa0Zc1Y`cR_s9;+nReZeae;J&Do<T za(+D$3hJET?-eZVdejU!>&7PBVJjE#4O#O?T|R?Z6cCC_9Zws0A)p^|+ik4M2l*hS z_8|Mt(ie)|;Y__!DEGDY^TdQ~b9BoQ=>EhiC8r8^Sl9wku!fwrf!uGiaWh)?|9Nd_ zd5)QN8Qnt6Rozq8kC4&t7BHDzMI8a5B$gt#mZzY7IBhQ-dZA%Fdr z)jskn^QS30X;xjyBx8?y_YOvuE~}pderJz-zppvd{)FPuuN#f{ol5u$=~#38X!#e9 zLKCE;XrO`B4yBJI`d;5>=GDkS-)*U%2ZMETpOPL-Y!s4zE5T*y^#=yefbn(OatwC( zm#LmFgpo5843Eu_x<8TI6cF%xzr`q_(hWwMmyu4Ti~A`5QghisW)2&_6e80SN^A{D z?Q<>LP7-;QC*(ua;oeye+L9&lCVU4D7qU``C81WppKwk(Wb62*=1;rj_$K(y-gU2- zcN;x--8a0SzldiYYGbbR<|s_R_T&19>fwFj!>waJkv>_NvD``gz$Jj*LiM38h=z(U z84)ldb|7|Ob6`9PzX;2T=_c^zXT#-2^(1=m$%=m6)xPf7C8Wp-ZwzP0Q828qci+X2 zu4V7eTp*_&4d3$~)h&{@5I?axdA}3z5PzAy8^=x>HCi%3FqJjUnd(e_Cas)=Sy^21 zY@jzebO1W4yeRJE%N?zq+{*T4`AjWwm=~E(&Cs!a5hnWaTh*=U$^jW2H&QCXGa?ql z_s5%3WA)&cOVzpJadAxe%|^CqJDrMHyWD-&<@w20bDia^W+Wf>i(9yh+wRD4A+`8p z6!b14y{|c0M}?)c(`4=vTUCdpfy*B`+$kHp6(@t+blS6}UErAORx7acws&=x8^5dc z%w2zt^roL!@A`a%)al80*quh~a#p%1Q4)uU$>}P-;P5_O^S$jrjX}Niy$IxPS?9b9 zZl||uJKd!R|9m)f<#(Q*t<`FO?4X$TxsO-sd0oXvwBmc8!ge~iQGZ`4o&MO*uL2|YIpjWEmid8f5~pSRHX~bb$@#;&Fy*&O~+K`_jnSW zjy=xJaz9?o^?8bTSTwBNczoIYbbIjabhICsR{m^ynM6&E`MlMh`yox_fqFq?WP~Al z?pC(+hKn7Ed)-PAJ3U_MPC5IT7Gyp_FN*}HfXRkDVGwcN2jzaadA}q9q3C0)3 ztWp|~1NE@B`x|3 zrxlZ#%7AtV#-uqc8tXJ2p77cYyZ-4#t!&Q+-+9i0DH}jC1m6MrlVoKU3XO)+;(Xof2YyUI< zK;Hg&{iZy9$g}2M|@s1$B z-S@?`lrDd7vYRAbDb*+n>^G=7{%XYY6*iBKT>iP++2zmz`|sU}?XGGw+>Aj(FEn3E zhSa^QM_6BpfQ)Ntl5Ogi@EXRS1p`({0E3GEjPiR)>-}DzUJG2Y;fDc4>$X{I3f52SNhKxs^4`=V)gEAb+ zI#UO$zwfkI4Vl6GtQ1*IA9$7Mq>ZSa{QS8ewX?LC6IxpTT+0_MQIGz0iaqe!B+4~% z*d>BCYpC(&GxzWt|Kw>c?vNT@@8Qd@UE!w@)tiq}xzG74th=ZQTV_O4os`Eo!1;0$ z{WIw+(0gV<>4p3I@o6lvKVGP5ql6dRZb75+vCdOhJs;aJ#%~x+1IU%7=MUExQ@ftb z7<_MYH|e48e!Pmy5PF4Ud6DI?Jk&%N5z{teJ2GIT&g_yaOG4Kj2kd-0;2Jo+ovijD>0D8fH ze$-0JKxR(nO2V|#e|gd}0_Y{2^sUSd1#L{NjDh*+Wy~EN&23E8%#EDP0E`R_z)MO_ z#@4C;b|9^?yB)BszhM62bTGGbvUPyrsQ=<}HgNg}kg|ib@juZ1%B*awVq(|5?)iPWS(wiHVf~zyz#sW(H0GI}-zdm6-+D zPFMl#tn2_rPL98o%*@95PheqU1+X!&0hrj?0nE%y{~TGE*#WGqEdK-`>Ayw)cgMe3 z{{|rG-;(|+GBfl33CGBGd%vj8;# zjB_&oqpZJ4e;NLh8(0D}10#T$g9E5AP9P_ae>B9w%=m8r{z~{)uYUos{MG9}fY|=$ zrT~Qd-z)t;8~ZPui5+NS{v$E}3ybt0iSy63|LQVn|9kt)@$c+2u(NP7b^uz!zrBo; zv54_MHWS#BNE_Rj0&PDFJICLw|E(Ky%koatjx1zA3%PtpZ(BFUk}ePnCN5*N((n0k zh{%!Q@>MB$LFaDEcGECDI9R*y25T#er-Z`Zb76SJn&#s;wf-kAcbogoL;L!h*-9mE z2PaB`jLkZM+#R=02y4W^-iOb|**-^Z%8{Mkb?wzn`!n~e@~%bBODg@6b&YLqPy6lK z0}Y?}H6p+I<-x75@VmA4*7@KR?yiDL{b){qz*flpt6e?K$MOV!Tk5MtdXZ^;W$d_7 z?J0u>|AZ+nK1c0u!4s;6+q>ls<#Qz+x!tvGLIr{vsye7Dr&T}W?Sffs4(P1g60lN# zW~GB|xMz(ez2MGKCA%1}<(YNjI>rEtlo4u@KGqY`EGBr4}{D`VP2_~=Q(S>V8~)WRf-Jg_VS-|bStE7fUxd&f1+Twz0S9ze3XCtmUzax zL$jkBhVV+l{1R(1M!9;6;YqxsdlkiWHNA#+I+lP(x<9&g_8c-4{WY;idPH1FMq81D zmkWY6nE7V5es}V9X2L11jDI%WriIYdT3dCZ{~gzYAgI(@v_~P#L>x<(na=90cm#^K^S)7fRnIb)j zXlS%GZ<%0d{0WO6RWCp~44N64eC$`=23VqzqIAY6gF5-}Uiw!hBT4D>a)Gs{I-Bnu z*Q(U1_3Tkl@Af)|X*UzBLp(iC=0clYyEy9Zz%3AXZa7=Hf7;sa;|PykKm7{7ULNt` z&>xRekpA*#?0R&$bz<*ea3SVt@oDjRKy5eidKVI*XUH@O4JJlwLm{$fh=wM|uSXLD zwHE#o>iB9T2ES_tA@=-Al3UGH0QToBhK-VjAB2q%Sw0V{Z}j*Sy^V&~`Iy^L^UgJX zz6dej^=j%Nrta&F8)dyA#$}7CPH=Nu3pGHUr)z||ppqpSVBgW)4@zl4Hv&qWDLD=OYR3Pu`j0lfxsiqhh z6nUl4XXX2!Z)4S#x^3|7wkkSp@2+=agP#-Bu5YC}ZFg(BZE)TxSIOL;zFzw^thIPy zIfgADi?$#chk8!x=|j%n0Ub`-*cNFG40o+4j`%35qW}gJ#%3Y(8f9BYvON(@yc*@k z`sgN&Gc?W>k4x&ujbU}4_el7&Y<7M}a0@oI&FOcdW$0)RJ(?Kijx?!>*EB>%{&oc6 zh5{B2Fk!TLQ4>jtN9IyM*N}3NrSuvTUbE!{(l%eOck}6lSV{B>J-25KV`N+htb?oI zg?)x+HfLq3A#AFxmT8V3x3;RogYZ-&I~Eq$%#k_=U9UYDU9KO*)dA4~l%uLSc|~DT z%&OBuX<;;%M`%n=UfHHtg;outST^5ei;@F4;)6ouZ=SPd~wt9>y zuYPGd>vhm>F3VNQf)T%I;@WG0NsenMZ^6u}?H?#2ykYFOi)CjP1-5Y5dg3FX_AqRbXxxsyPaUb9! zKt3f>ug9!d)E{P9hn-3T)hSuyWY&`tRv~HoTIA-aKT>{3cN*DvQ#N;x@KOD+?)gvw zT(sb7$0I=|pPeaLXTI^U>^ub%sY$vf!RCK-WwNCh{bb~2>+0%gXm$5qu+R&4yM>){ z%EzHnWlO=z``jD5bkq&4k25Auubl8qKTOZn3Jn)ig(`tYg)MHOIzN*e z)ngAcy05+$GU7crn=AvDrVXDReHUagNc=5GaoXK#E0`%j`Tc zl=qji4VbbIn3{eW_f!9_Jw8$x7^R-ik>g-k)rH7u22zo_g9SEb$-pa_i1*$2gmz(+ zbPA3M{QYKviOG`{lEbh+aipAwUdXSlI0IVFA`TY-v$l&P6CI2VkDT1uK-Le^OcZ!q zUW7~{VB&FdU(atgII{JzyQH*>%{_eEZ$;hWKR|>?zmk5Oafuluz@@iq{lHJShI#7w zH2ulTgLj@XA0BtS7NC-3t=*S&%&JDVm~xWTvx99}d2vkY`;hEAc!_f^dpY&^3swgB z{jZ%KjESfgyR?+R&K7>zJ^*$&#XYWC54%qPx*ppc2PY|4Dce2(}T)th6~4B>0WuAJ!Rl-bxs_EMi(L zCy=jc6e$jEOX4GtS%Xu@4-dx|2})n)rq}C@clY6jn{6mMGRd%<{rOSiOH3s?zlat7sTDf(gA6}0P&-&v(ZqQ3Ai_=i`p5dQBw7fr zXet&KA$se2-stXB^gXL^!`2CD!S>a!&a=V6LHa+vy@I=RTpu$EPPYNQQrMq_s;L;5 zwjbr}*wK`7IsCHOm=AiX%`)ZsnU(~lSV$KK9hHdx*rg6@B7`Y66 zg{qYHsgPZa58b@dtRW63aU zrsE4ZLJIJ4_j#pveR4%HB&*;X^Hmmcr)5jN&>f^D@If}x#qeT}1a$Z}at(~Yl-P@Qs;_r28etRXo`%|X)Ad$78Uv(bD z<*6_v+6{Y*AlVX`WE3mpfxEyB{bVG=+XM9kR+O5u=(#R9KOUzO!gcxe4QbMIFJ$$= zbLWi{uP=^IN$EQDi!am_1&A-))i9+G`c)-svu!O%H~idb6TNO{zRs)49+-I<3qmL~ zf~Cy)wlP-qV*p+-y({jm{un-2(jC@!wKM`nofqc=R+HMB8%{1-=b7cR+MV@=RCgJ7 zb8vi>{4Q%{MM6_)7!C3Gmob+=N1;yaF+lI@!tI!I1C-K&9(Z%Rtmr-P1p90pJ(yQ0 zMN)B}Ev&Wo+Hu_w$4XcExwDs#MEG45Krh_#r}K9er+2cskjG=-Sq3U&q3*Krk$%Fx zzlRiJgnF0`oc&?i;I3c%m`PQ5_2QtCy7q7;QBKWSlydl5(gPZmWsOD4_}%4I8Y$J)#3f@1N|A;QJDpIsB2qT4rBLgS=wL@KfAULu=NVd77B_$whY4WJtIR<8wDt;!%_x#@ox~<)et_Y-@1uKXIw$mb*Yqp~4^4YS6G6uj08D z*-E^sy$q+2<-1p=AMeU&Blk3Y{wliNy);-(8wljisja?i*RiGZezL=+gWM?(R z!1&nLz1W&dMF&L%`T(xccahkJ?b;6lqk^Sg6@3Qjzb)k9PuG<$*#(VXGQEA}>Ttmv z{o&)J$FT2`x%8RZ*jw4U)U;({UzhSE5V~dzF08t#z=Y)g!%=*Nd1Q(?^IQ675{}u|R%%+=enu0!uS(a@z<#&V|8nq`H@89)^jP zH1DCt_p57+Rghv6vVFHBZ8)7`G&L?P5$2k|A5V2|Jc7?F+x<($j$UtikB>0=apjm( zn8x>na_S=0&@7v*4-1Rq?FNrQFd24YL_a6QqSzS_J2&_p{x45Dq#jrXrg@8=$*PQx z)^V?FW%YXV&sf#y{GaWqjRXVR|4xq4FUJxd`gn#O zFgXPC6zLmd!b4;#3yl4NIgnP6R-u;Be!PCUNy?mgF#FJMAlR)sDhciB$U_Br(D*XP zW+BAQ!JMrQuQLE|U5wBc)AvUS;CjJ{@E#*Zq}*g?P|}FZTQS`XZGlafl&Q z_*Pj8+0$)F_5Q+=i-#m4i>W;F$mrw-MZ)-TTyvSLUk?2US%O$Y0%FdpLDMxL1^jIK z^HTl6S+*mlRbMtXXF~?3o^Jqlzu_<{&8F^E5(}^@qnmBpqEVT!8}(Rm9!%EXqZjPL z_TLNDWz$evW8S|Rur0>2IcNvh1vje~>oN5EoYoI(+KSEzN>!%jCw9@84tIqnjkHtg=- zIsd1DBcNfMVS0VztdoOS%=#kD$Zo|a7d0v%t7)|$TT_$&b7iq#E4UL4-(mNACTG$f zc;-NoOYR;xA>5hLKK%a5*mT%)DE}J~D2lPqKIes09wet9Vy;OJ@zCi+;RLUM4$0SB zwJ#q82rkp#p+tf_)4t$=b9ij0g&0EOQEa!u#<=!hv)~bi?mF6e!6LlooKBz=#Zv9r zBYJ&6#w{rjZs{mFpAWDWNlD~ehOA71m8%N~tKzpgDbFZ`#$<5e%*{gJTQAMZ;?o`< zA>i{0D0I;pURmI9g~F&jV3;>Z5ozaAd49Eij!GhJtAU$fa_issi7bY$NMv0Zvnn(( z1U(57yFh_r_t2}@F*5cvG1$F`ZHFbkbWvVg+lUFHK@HuWxJ%(fn$W;!EX604^gYZu z=0}Bk6~C4xq|W$dSKl+nMRRPyj0U&nwNkch1Y?pZ;W?yomLENU$qxXs|;Kl&Oo^s5XD1&{oj?S6LhIGMJjTYupX*!K zyQnSa12^FtsOU9%zSi!`vz_2Ne+rQ%yO@a|r-T)DqAJfYHK^CsjNVYNV(^owe8aYks{jo&3p?(aN{R!!4{@V#)CuQ3Fw7C zEF~&d;x>lJ?T`hE9NwO;-Rd4Wo|mj?pZnxjw9MB_j*1X#JE!osaTe%!MypKOm@-FE zsLWU`ATW}gP+4J1(ZOX^u4HClK|JfLG?1)VL}z6}KEe6c(v_ zs#;Wnnjp;G)mfPVHeYNO7CfEiJ>dno1e$+CzpyRMzRJ>9T2IkbKIRlZZU!_Wy8vA_ z(G!Fe7X6rk854FMXf)&!l_C8Nl^=b5r?N|(o1i=|!S4M-nuKMMpw3ffxVp(vb@kgM zOz62)-uYA?jgVg`Y*MQ0J+)!@xs}rgM$F~xAvo}Tqc)nWYPfvf4-(;KC$_1^>qwaJ z1s5shtUfvYpxDBX>i~7Yp8zxp z>s#sIS1-@1;^PN$b0k-$7OBueYjQ;@*L8=qgoVt7R}1V_u%SG`gQg=J$7e*~`{Db6 z`gM&{p%;G<2N4Hh|3V1bOow<*pkx!x=}cL@tlWVq7&CG#_BE+p3ar@nX+oQd8KT8p zAh7HNf}u=W!+4S^SGLS4Pvb)+^h?s=@4h8OiJE&QwZ6l##n5q0gkSD`fT!2F9w^Lt zifKHAJobh?ic`@ys28pUco$aX;_!)qny4j`&bLyPol&g&t}Ah zt>C5(z9Dkx$Ag2IAhBi~Rk3^sy3ur0g|1<%6BGYE4(%^#bi7~VB|c~ALecqa`CEV_MCtN?x1jD))l5bTR0@TR?*Q8y(DkJ-PxxUn6KkLX*|!2l=b8J z@)OApg9FIVB2~Z{uGP6F>=5|biJ6hfw23m}*LU)3cdi$#kUX%t8PFwf1t4bQVj_*r zyS?GuC8VPaj+fTlpHHZVpwGRj+CJ1cUR4B_hk@w=gt$px34T^v-Fkq=EcLj}jYfs; zzNi}sU6p=1usw*h;>I!BRDoWeQd%=jRAI$-F_;*dx7Hz9oBq)uY`slZN-+R@0{Ug87&hBQcb zdxE_|_c+W$L7lAs_@tKcHo>G*B(r9K2$gq^ z@rCl79232w%@bB*vSbYo%n4<6A+hKQl2CCIs)piwKCj2hCM#kjR&_&~#gbYlhBaeo zc{%i=qO@)3oy&-)Jv^#-)r#?yn}YP>7l~JMV>>OcZ5m zxhdgnlirtleY5E=RX*0^Nh{QU8l@p=n@8+=7VV1KrhcPH1xt@vyRJ&_brznmaYS!ga9XpvpYx?7ZPg&)#XodT zPNd5~u(2&D)r*>hXyQ^gf;@`U98q#;+xzA}A4TS_ShK-M(Qh`BaNp}~;JI?Y)7diV zQPA~R?aOqUJuIL$rlLGpJPM4cQ5RdcIF%n5s>eK$@KKv4+&0=gTID!rdiBN8o?CmY zq{}=m3oklwZr9xsPe;n?vS~s+xYq9n3~QWT-I(aw(r_o#kBLa4r2s5 zhEyrCB{rycVaSO;gJ`*^xJn|_}yZ&t~bcy;na^f|KMJza> zBcJM~7z?SSsj4JHy3;56Ez-?iU1pX&kQ|OhXH!+Fo=yHj^$8pgZR=+hmu1}sWQu|? z-(cOCfO|$;bej|*Wp0qiW#C=q_=&Ubu=|BM+QiOLdH$7(8rK`zEfV};n!;SI;s6&S zF>VPaW$sSeYZ9bzWmO4aqAES?TF~IBcQx?qJT+}$76N{XYL|f7)d~1I>~BGE5oJIO zP31jH?1Q(Z&PXsT{a$|trbBK~`Sr4Y_#=ZHHT@q2;AcEJ!RN?vix;LM`TB&WvJGU=K*#12jqcL^wqd&Yc8>G}l407MhHTL?=^n-xtMs0Y zgx7B0Fc~J99!|OPbK!I&5_F%W{-Lw(1XRZS9|Fq#RB9E5xgQmN2*4 zDhw}+OR7yzV{~2fT#4%{q6O~mw%Ol2o49|VGB?*(JYpBAH3Bcj@>F6XRF|mSCUXzn zeOIjyt;F>2wv85%-KY0AY}c5#Z(|93-j4D^ALWXbP`NKRCAzsCvBcK_F|8o@_s5i6 zD3xc^NHzDT6CaXZG$ojKr)A>eGTR^@RXvL0L>LUCF3)p<0x}M@SSjZXoDFSY9$dD@ z;cFRmV`5la7>P&|fLko1XN9K6n1&Ih?P*GnOmY8~TY#-sxTF8goSu#82ZHuqg$1V| ze$M*@7XBf4Nq(efI<93hxXKysR~|4oxT`BBs!FquZw$iiu^6K`mO1^5ZMYTfb^^l9 z=L|vDJE*c}Be ze)Y1zce#FZsq`5y6_5kwY?}Ql!ANuvhY5MtvWva%8y0**!D2*4W?w0E5qDs?BNtB$ zjD@ORrPN@YVq40e5jg)uY+axHUUuMbd&7SEVEpHH1CM>c z{xds`*S-huMtt6;5l3Rd_2y_N1Q4Oxp?OvlDmEPQW|eQKR*|G_)uh*sn4WE-lcXfr z4^3Wah5%0696@1ZQJFacH{Lif13`E;)S5t$(xlpes{WK47oX_Kwr_-+iJjeJUE}*| zH#iFI5_=TEB{~a0@<#>B>5OT~_D{AQ@R~tHV;+4&i~}!c3uMZ#`26ZvygHFrG}v2i zp!gcCVjX1qk|w(!D9HtLTqc051-iVLqEa$BgBat%V@W(%&1?`mx)F<~ef7s|Tjq~$pXa`1-8iQ>jZwGPCuYh#-p5Je z(A3-8&aP$oklHmtY?}~J_c!r`95*@8u43JvFLxoTr_`i|krakYET>jmyr*%mhZZ($ zb1zueuID(7WZ8|LxDGBxdVu;5>a)OOMqird`u=M}o=5Q!3C+v9Vqw`$?Q0En%@G*~ z)JGcpiV-3u{l{x5!!3Pi@jR-|cx`HW``8A4rMs8O3178EQ#^32{fCZ)<5!#61$1gX zv1i#6zinS(&Cl2-r&frovOk^y%8H<7bF7`<(+f%Ttkv9a>~lOS1V~F7XLo6G!K>O4 zVYQ~*VKyt+#Lx-?1FwK#a1HRAEsbF3vbc^la>VLP9tKr=l4`jYnAeu<=#JN)TW#j^ z&y(5&eZMp~HQ#3t{@B_4)@i6$JXNpcSI6CA9)ZD7U$@t%l7i#+D&@bVP^F;Us$zo7 z_?n|i9rtK!#2-3O3e!e&dPKjTX!FZ>DbqeJEbGd*#B%&cV>Lm`sLSlxX%=4;U+Sw` z_e8edC(foJ#P2E6+ws$@Qc2q33U!%C7S&xJ75ZZG-}+gXSlf)jrO9cG`Y}0w`ew@=ytNFz!h$JVCwm-@`-Zzl8=g9 z%fgX*@#f2r?MuF|&yMKVH#j^{)(d3<=Z@ zf@H?_)zVxQa)0lRL5htBgj8|CC>CX0%;qpu9BzweX$>7QRzHB?XD`ljcJV=U>iLbM zueQY=JM$G6%AFoLM{HA{*kdXrT;nhZSw=^l7vQlK4!WA(G!3~Qca}))UVT=$!c!N9 z%9Kw$N3CQjU{3H>d@s5-j+=3wg~B}+Fl4|^WNz+`THMXzz-J$^8X1Z((OZg-lSqL* z^=jZx(%O<{J-B6$4ev8!OS>sblvo}o31yQpaUgM|U|*T{C(vDGb#hvhZm;v%^6ogN zYaEdbka7XzdE_j_5T85KGoC`82-%Y--SM2qNr=o5Aw4@n+BkIObp!4nkj?L9j7vXp z)Bm;s$!^@}2zj9smV_dRYueJTxS#V!P00;W#coNX*u6vJ7(i^fR zrJie=Fm3~a>1p|yd`lY6w2uM;xR661GK}b+T!B1qfKYSko&=<4!%N#rKQ3ts2Tx6z z?v+ZH3n2xEdL%!ly_k?f>_)tuIOw3)HzXOd3D=6$zBe7P>@Q7$1)Edusz`8IXQ}WG z4W%Z=JHpRZUR86UyMCr(s(ig&6UJ#c^MiQCHF1xHjcn4R%aW{jhEB>81-6{Ou=1dq z-)U-=0}N4OtnIc}OMp+^aKo?LPn39{;4!w_C-uC*??;$lfPJVBa1dD^ut*wbg#s~6 zbHhmXKr?zr4R)TO3P0q21X+j}d0`CaK6kQ{yaq7EoNPmY@z=l=rfceC_VBja5!{R6 z7~UxSWtoSlS)x-f*e>!*3O+KOHCrTVs)Og=-Mru=Zz`zXa5Xut!yl#@%b-16Cs@BE z^V?>HlUCPPe9k~=c5g7+OEl5@mZ%n*0@U3U`AF487+FoR{E=k99Q6RACDn;ITWNJ^ z7?l*=0+zHqd2-}$mkLf;OIFLp_kyZ*fM)T8UUG7EBd<2^b;T2adV=zLsMl6Wxctp+ z0S7Mh%({zT?%~yQ!`mK(^6q)e(ZHr5w3%zwv+!ViP(jy@)E65np}CE;bkU|M1HPKs zVbeN#WZtuXsz_=NFJJ>ZZU}5Z4kcQv_lV}ZSN^6;QTI-KcV=x2p%v^RuC_#qYmp)} zHv|rP3yqcZ-n6bY|7_wBr;-Oqm^OgclVQI(PGHEG4U1Ry<|iH%13s>Y1KZs)Qm9~D zaSt4#s7R{*{HZCeY7BaG6`G}yO-r(@E7iwNOtNX)@`|rMkMvj4*KdGlcN5s~JRNRH zmC5_Drf?7QmmwoE zW!?Rkwi>E74RIA&H#-%TY^^HF2TWnj>V98K$d58+7rIp5FEIL@X5YX}%GTsRjIwUz zXp#7n5QwJ7S4zpNx6l;HxwP@ZZHwSkaXfIA)p(vo8k<08Cz8qjf!a;DjD#XVVF_@p~LT z^vLUhbF&(nRk+zC3|iBo#uR&Ll2}&4R#&w?dAI~mW-pQ&s)!a* z&An|MlJ-NvWnuSMG&ioNv^y$Z9~dL78$I{2FA!Vp$z1f-G^WdI>fYTsY1ZP=I=zVtK?>gF(oGr}sRC-Lg z;k6hPM+wx;0;tdyjaA@P&+&fmK@3S1TCy~$h)1I3=juGutWu>8X+VH{#V;*yj&OoJ z6W8uq%pdlt-1MAWmqSRz)OaO6#~luxU#UN)-EgU$jCYdE3S?f@`m=YgDdy3E8og1v z zi^-b4Sh#kUhI{-gV7dI)z3H}hwn0t8PeZ6V9~|_#JXirNuCx|vdi=I8ZTTUO<|SYg zs34JOg~_-|%MBJI{p}Uxp-RXDzZT0OX$@u+F(ezBHLFc5@k=eMO)`%cm(%BGrtg)t zl;b6>V~N2h`-~4QzH^@71kXEhiV?z3+|7q_^_|Jj!BJ8@*&VPV8tdH3f3PC%rm8N^ zB{Z$R&t+UZRv-IpWn9R8wdE2oR}?n=);^S#!Z4$G5B6GN#iSyl8OelC?5pq%2@T~U z#esF|F$G2^_YZ5ytvR=%nUJd5f!Qee$jaEO^$Ti z&mU*sih1(XS_lA-h!Cf|O4jE}-qwP7+I$O3+#8!MH5~*2=C>7qG?YT#U*u}s0U?=k zw%g!k{jVKAbvE1XbFu@gnN$3lHjRE}1dran9?q?tQ(SCuI4{qhuNMjDE4!hH&ufpn@G(QcZ$s4DpCca1D zAstThvT=Ld&$%e9`y8Jfof#e;X!8B!|GmVSgo@f-V?CgvFy&cI^DIzctH-j&cQ^H| z<_A>eCc9eqrqv-$3TZmOR;IUgTyR8 zg`lgKKdxTJ#FvTu%rvqrScWTC#CtW%aT|sQH~wrw4BM)jePcc83q_qEU2*V;1(hUE z+p-;`u-Vx_DL32s)0~CV0>|SjlQo{=IzQf@P?vO1d?yK#QKS9M;AS+E-=x?{aTGdL z#KrGcv7lYnm^LLR{q0LL8JjN~OjNOg;)5ayRm7QQtsGZ5=8}5pRA6STa!&Z&?*&hrvPYuvpW-Ja3d;7;>7 z-9JG<$<*egwDtB5pfW?N%t3xm z%boiTz{o7!2JjB|+-elnnF; z(~HqFIg(g@Asr<)O-dXw9--ZNHDg}(4AFQ;cF5^R9H%_E^M>v){-##TL#E}|++Ss( zvt@G8d$lqqdQ>i1=11aCgw)ncTAU4ZN!aeF9^-{@0MRDh7Qj$Ek5 zvWG05B!;)^9GvE|RjK6)Ki%K^32RIOM)>|N>*yfVs=zO6CXSICiJ$k8l`BHhbt?vN zCNVL9kl|LmmJ{eRdo})?gyToAEg9Nid@XUuQit*4Mkawl&j71iTS?key)H%rH)s6p z->@52_`fpzFiPp`;t_T}(D|H8{QMZ(fIZBsX)fN_XAUot1BunR_u4<6tjRVn&Sb}l z7lu8Wd6LFLKI60m|F$%_pNHp&%pEkZ6Iw9x9_-uI7A#<;i~WpGJw}xDu^;f0dfwtQ zMCVbu*yMYAH}HF;IgC`ZF7ibp3V3cZ+^wRd^3x(psd`(h|&alloLh z#(W&csWNF*d;A>VOC!V{qp@!o$wXV5o&rOzDq6oCxJVoQQaJT|h zNEc3no)mMql{*~4O)r_Rc3(C>g^|ic&o=5Zud~flNU@jnJ^a=HO8g+jf!eEz$ zJ@`7hISnBp>PwIxR)~jo*pGvWaVdqFW+WJ>=t+{4Y2cg}loo~`6$q))>*|w**{tv! zscY+d-ijP0!>^0nFVf1{A-Btoxzd+B3s0b&wHTx?mTIhCna-YHdfq2J{t&bGqA?8Zx1vB0&jU$Q+60TmWzMBX~^fy_0UhI4ZsZV!G@ zXLgMEJ|!@Z1mxmv>hq&CCgO+GrSx2#W8z8!72nK9g9_F{hOG*UI`4$w-R5o@twFZk z=GW0n1?3irR+mWTOn>jSbCPJzo}-FZNFHBA z5IHi@1&pQao41aBcu3a#gK+4hhLE)9Rktuv(pa6O&f&h3Lq}bj#=iReh0ZgL4*p&> zVSyR3W_EG~TQXo!wM3u*BN!Mb?$K{&Spz*#KCtE}jQN>p%bgM#o{L<^bRdJ>O=#oT zj3j@&E_L=Qyr(5q-Bcw+F){q+Y00tb_8767w85$P1eKdAlNsw2)L3u3hfD%HP`zW1 zA{a=I;}{Qir!BR4c&EJy*YU^!=(oEQ=ZDsafdc#}-zuPpVp(|@gwE$ntrABfMR^`* zP~R!b~M>cnICq*gMF;~0}^QHkD^@~cz%yr6p zMTRD+uz^QcQ-5CY5+M6@H_WGIe|Ia?9?H6hf}sjS+~eg!Y#M2t;{NMneRh$p%J1uB zU7;go`jNaSHlgH$GV?Hc_U3|w^<9PHvBi(=hSU1Tex)VKsJGj6R6;%a_JW+_Th21T z#!3wP(9?Cg(t;$enV33V7bQMueV@sF1_jCXM7rZEAr@e{6p2_`Yusp1gK?e2oni7_M1~V*qJzM{a*cNZJ-a}_OfX(qH(t- zHnRmNqLZ;T-_f7hUIqfY4$!|z{UMVQ&xSSo7wb&D2S=3yW!5}6F4f`Scqu17_lUxl zPs+3(G$!&fZAK=&JY(OA0a&mR3Bt@Bw> z{oQyt)lzpDF@H03ESt9Vbzjos#Ar!Nr}B+) zGq3iMd-mo%tP=d86s3Wq<5VdZRT7Xn{$81UlVysOrsj8>_{kbCv*#=+$qUGfVhi}q zPViv=8!W3N&dR zWbCBb-Xr=iC%zW3p% z%)19i#r78C&&(RJ!jl6ajE5cIX4jRSzu~Lzvb+B0?$-|vZswe+qhV<8;)4O&@^w1+=)7SA2Iu%}F)PN%4)&33J0;;Q(yow9b!KX%Y2A3GDzR}e--16h_3nOy>Ae(Y0N*K}5xz0y8>JQo~8 z9nNhlP9(NjonS@TKlXq}E9E*UKXG+C(B@LnwrT4V%f#Tf6@I5uz-KMIj|oHImbSLD z;zx2)Khl{#DYY)O_g8NHMxITW8AxZ7TK;xXXu_?Gwi5`ySeb6`M<;~)=mrDXksv*I z>T?&`RSq}vj?tLxk@1byi!Pl{paC>3{?SSlR3C_wgwbQ(G-vX63%q>FDm3$Mu7C^g zNmo-eHe%@*;o0qAlCk#(de<&8e}09z337*8jW=>HCGTiqxJ@B2a_TZ?foP}QSf5$t z8HetxjQ24~kkQa%DLWU0^!&Kx%d@so@vc6Sr|xkv)ysWkdwsS4YMh7CJO&+zin=F? zMz!xGf@`)GClI0%CgE488P4K2EGcAg>)Q31%_{$xK z;1>6`3W44$?lW914y<6A?Y0NgJxXOH7~BJp zqf+KH3?DI~$t|^`z{i%6nF>n&%xi?U=$5<7#xt<9a2@P%EH%&HPl##jOG6)trT3daOn$Nfq4V zvNIUNPGoE=wGA4g(1OY?dEf%huyihaIwo(TxqSq?RHd73>5~8JwQu) zX@)ZM$bfs-6@NvDd0z8*ksR2t){)Y8JQ0J4R9YmkXJx50IhH8N+yJm1@ga!J^z+!p zlZWC%PxKv3rvePVYH7fWL%THEI%BSP78gB~gH!~stJK`hKn;)3r%YbXKxY&7NrJF% zg^zByB4T{xz#LWPyVDqD_)wxqX;M092^;2JPVQo2ljbhl<%A2H;oK=^TKQo9t5IrL zF)JfA6DAdUk^|thTmLs!i0*r`qpmX$M*G9DosD=+GVT4A64Xm>ryIHW^YmO!gO=<@ z(-~q-BI8kiIcba@u$<1l(^}W6Ml(y&B3f0 zfQ^xeMe2Z46)%I?;qs5I;%SLj4T5$2a1$IXFd+RKgmF0o<_#qVo?)51>#G^p@!v_6 z!7c^ngs+w4C@J&!orB$+Zh^U#BZq53>8|d$27zKX#kQq)P^@L3-P8pE9$#aj5a!vM zt9{>9Q^)>X20}>vL}AFk?AWf&z1z+uk{^mloFtClM1KN?lVc>zm+>b{7YxD|cIDDK zor5hqYb-<@@gjH@X}9E3(|FTM#cS~sDYt2z1_`ennrp5Zx~dHfzwA8ton~1Y-=%~V zvIYwSF04v8ET7*X8re>?9H9k@WL-6qDV5eX==tO`*934#T*4rYJtX49|Jrk;m3QBf zKSR}u&4Ia0+M}xqpd&sCe=t(WFT9=2#SpthHAdls2Og2u_=-yLIYcU``>qkS=oYKM zR6vt2L|lAXA^jrG)7UDnt}Iq2l0UJMkWy%u-1oM|J5G6b|0dC^rv1#@l9P zXj;*UPj0Fi8S%`R8`L^Zo7-d1XYL7<}n7y_;U{r=F$UA3Z{ygrhLP-p;rO#h_PV zrq}XK(T0Xo6DLIt%-4g!$01P3+Hm%zXyx!7f7jz>sYyAT-cQA};1=sOiV+FZ>RH%w z4?5TT+kazM)*4 zX@-e;n*;@=m}^nGKP^WmSB$iRr-bTFp}H5Tl0zXybaiwR1=a>i{qsvhG{}e9?%!n$ zL*&`Vbjt3tm7lW?o?Ozpypj)faC}*UY494IHIfGoJwWm+AWRy^{>iCqRcqmh>p_ov zpR1jGpIT5L3TjznLd!bTGTF_OFbf$j|Nd>78s5Z}P+D$f0-#|8gC*NU7dOb`-H8$I zbB0Z9k~{)ZNbF9X;v!B3=AD~OD!tBiHXNA^OI!aERU^n{v4WpNjwT**4?2#&Mx!nc z+qGHq-;9`M@kIPi7r33zpVmD6OXfZ{6`rpZ5B^D~>Wr-C(({SYy(tc`rRxHoSrMsE6dZ=1>@1AGX%S; z83p{2S8Vit|BkM`E(LboyB;oLQuQqP`L?oZR;U$_RmF+&(uwdr(^crsoX^cA#7TZv zabsC(j6jpLae1aYt4IsRmR>rnvF|Q_g?CKn-Hb!{^i*W1>ayqoXxDrNnzvlJOjH1@ zpF!UwdOIi*#((#`#REr zsqdc@i%u0xdOLVhH>s~D9llT2o}ZHxQDryrLIklg<5ZK~m$EY4fk6yu={nZi7_prp zUE*qphuI^tje3pj+#f-jLeo!8FMSSOyQ-u~8Px8ofC{=eU0y-VTztzqVJVHDiBJ%! z|8iPMoHl!GRoDem(APPVm;*xbg>HUdj_0}VP;WR{rhU2!+p0^VCoRzgbh(R6)3`Db zOKO`$(I?FHH7A#~yByL`x=jqqay*`OeRnESJ!3nW)X@-&ZP-@#-b#0r`%4UV(gF&` zAJ(}nKB90-9VYG+J5Zpj$u=G&(Wu}pD`q5m6lGB7_%o|yr>VvM`cF~|Ghq-%d-h(~ zOp0jffqSaKvx}HvsB$uyIeYdJks?`{!*P9(k22~uM;g{;z^kM%vO_L(8h`UFpa7qu ze`{B9;NulCzGdKXFsR2H$zp zo?>18>+GC#l!PEC@U?1-`seh;dRIoX4EESXLkR2Lp9fn#$JKdV_a8@g$jP}Bp$Fn* z$yt+St(=pBmM$zr1Tzxj$xZ+bUX0oUmFb6%KfAu`OFs9p*So#TGNdH%OFm`m1p7oo z1$DTX!R{IfK_)X|1U2qW^u}XlvTRNs%}!EGn_BGBX^xZ)wMhnA8M0B0zg;g*reS$^ z%f9*hK1**+WYrAO!-U5%#z<)Qo!G=*P+?1LROsGC4SRM#4p+@t_r`LJQ3L+iB=l%3 z<4wI1A}23NuWueo^K4#4KfhcuMmI5>$#t;iH4|P$^+(v;5ss`p8zSROD=09=#u`(a zw-purx$G(wjpnEqB$4y(iX{#gYy38uV^n{$K-{~U*%C=zHQGpx))STL`iJx_cUfjf z%Y<5K^050b^G#C@J5O4qJW|U&lgPbG1aUmX63eT(;ZE?%trqsdFVQQ*3uSDNlC30W zx{%|qA>Li5xt**T10jixQH*Pw=t0cC5MDQ!S7Y9+&Xl9hS@=<&Z-=D##IpsnHMr!k%f6)eS8^p5>|bnph@qJ@)bYFjK4f z)aXL(P}^#^Cue(Xp@&9Ig=`#*3|gzdzskipoje&F9&XJP>L7oYCt$iXz=;Oj<6kG@ z)Djcm;BdPq9YYJcFCVzO+2>}CJG|YjzMMakCBB;26IixveV&cpz#LmQoiAe%k;xuk zyqLv$%89f8oQqTWp^V5Uel{Y{QXZ5=XBDpY{8&7-N=j^a*Jv1@^yL zeIfLm+(4l3&8@rPqIDhOlsc>v_Euhy%k^Zaf`@QQ($Lk98I+q%0fk=G#)>!-mp$bZ z+L2_X4lP8jAm`eynnd5=WG$0BP1Xi07Gn4IoNH%E3KmV?B?48yew6C&biVONa41)Z zbP&G1JFSYygftF`2YpF|kY0dpO^rTz^lC>nuW)m$$O@wQ-p{x-%&+p#WePekTJy`!{(vc|Ipi=B(e{`SAAm9lT!Nj=c!M_~-`-m$9m%i6l&tQVWR+7sXMjJ~>En4IPiVo|8m` z7rM7_b#QUV7fm3uEX3^d(x!qh<+B^;J`()a<*M{as9BQ~b+5Bml|1qPz{CGDDi9Qc zO)GU58S;##OS_e}{+4G#xzEy6&3A@d$nFlwpd z!?z)-ADi?rB~Ty?>mg|u5+2SqCW7O&aNP#$_?J1GaCzC5uq_Kxe-OL! zxSN=@^W_O-Rqfzd+32&YCqQs=p8X_+R(D3{Wr*%Lq5@l6mD#g zuJYL6{nb}Q>l~y>TBf9VX;IU>_|u=CxqkW=GrCFIWbWKl6yFogxc$sAXO4qwch55+ zO$_{|qU5u7bx;c9vl+aY6JNv#BV>Nnt`@HDEAunDrqPES(YGzB$N(FijTA-nvNG(r zN`DP*6#Ba5SdJ8wpsEK)w4V^cvXpjSsFeU6q>oc@Jy6)WPFbqrk|aQ57rX}<8ljt@ z1ErQ)GP&q?m`L3->zO2(-5W0@*AfE|P-9arVl z(g;Ia*@ia2K(C&1TW70hJ9~))`8Hg4u#EIOVPp$ml$4IGs*x_9F8y-cAavbTrb<*f zY`k47+-KchKGr?DRz`F1hht_At?yct$ucjZ%(0h;OruN9+KI5o(q4r-invo^G>l&& zr_a}^)`ctJD->D8Na*E!E2r)r)X-p(AD%jeUA{p>g@5(Cc8VY4TU?N!5=B-|cz+7Zr_qasBZc@-iaRk$kMmxhyV9fuZMtJZKx(cN z0>AOl@;CgSWsVdXT0B>w#JaJPUdpIt-}a!IMi@)=OsKTR8#&oJ!kcgpJFV({a@TGV zXX0GtuTo}#R``fLENZ)|w7tPYItvK+TJwojsA>`9&&4dx%M)|a=hPHVgUZC=|7_~Q zPuf}UV4lC-Bg;5ky*WiQG<4{K#pGIiW4YZ&i+6BXY#rI(IE%H5T%P6=)Ul9X4o!bb z1~>=)pwFV%@HtGmFe0MXEULQ8bDSro5~oM=(tN){+F^yuCyBdp=H8Fu?7q+(jM?e3 zxBqv6g}eQQ+U9Q2srvY8|HMHtPtgU3yd*ltN&ux?B$OFC>$|1|o_31s}7oU&(TY}r$)|+d$ ztw+yYob(Q*!^A-i)jf8H;Vjz2ZzvA}soHg*tJYID*h%ya+F=s7O<5KgKS6O8l>P-i zTI>RlDkcFg%st_dFEyFH@8mXrMKw%QMR{2r6GQPeIBXVU zn&w)5Kf)7wE(+3LRlMp9+8X!UQL5MMbD-lc*nED6kE4D3au}pOBytui#oZ=*uEQMQ zJ8-l^ZqTkL+GcKbJ*st~i6iPoogtwc+&F*qh61^!?wgLFR$1> zPy!D}3-`;@6&Q)S=mlHV7)C6a2y2J;DQFajQ?aK?q3VPgVhPc^7 zzL%7>Igdjdot2LRc2eu_yY82Esos{QJx(^cPwo|S{v~$C(Mi3`8p@5Qzdrt)4pimx zI$lYC^mjmp$LIX}_;i0;TKpB6XFAf-XeTwT&+T;bXY8eaDEOEur`_?kKeBXzR}g(= zf+l@Bmlx%e+i{Pa|9Ex&x2kOU{@d&Qj?dA%++p5e&!Z)<&X_vRl9>)UAjLg|2ZoIZ zBs2GG7y6%xu?@vTQ*gjN>EuL>Vp;?Ci!a9TqfS&A$1#~(&xufn2dRY#Qpbc*bUSE% z8NCTgr$1teyMSVL+==R$l5nJevUankvY8dZgBOvzR@^8jfu#ssM(Zaot!APKeWyZSnz51zUi~YH-ReiD@dhC&F+hNU{cUy29fXUnL-w&+K(P49%Gpe#1j7(A0{dG3chp0SH>wqgJ@T-6B-kM%sR5@4@lq z66|ZcmsiSqd%*)*rh@Xmh_|n==%4*e?L`LU2xqJeQD}P^!M-%EEH)<{UOi$PQefYz zwTCsx4TH|Yg}bss;&DP%EF)jhQ#Qv4^|C{&K0GBE^-Uw*=sOGi(+f_%AmPujmj4IE z>K~Z+{~5aZA7Ix16x|g4e;M6mW%*AB`F}$<|EK%@e?m9^&olO)(M|UMY;OJ&K>tf~ z^FN5We;fF3Qt1CL&CUNM2l(H<_%C6=|19sZvi_$Z>A#ZqSlHRw{(E|FWyhM33lsXr zM`*~PHDz}W02gCil4!*^pT^6ydl9z~>fKYlg5&i*{p?2_DrzdtO)Q(0&u$sV-gdcv zck9}HW&hRQeL=E(es$`}tGB$8-ZtUC<)__CtjINdg*@{4_{jz81^DytY})6N0DBBp zj^oSA)3dv}wl4EGNDS-J9lG)@Kp%HEuPcw90)ovb`O5>hm#@LEuSZYM6UIG+iiUvA zv%O1SkdL2%;>wn*3!s;;T5*FxNFWd3<7?p2_OlJwZe4hC=^YCDoQzqucltR;?ygJN%@QH- z*97Bl{ZI#cJh#154#NiJ1BV^ybbB7b2$OCO_5_RNAWM_OP=nimgLENNIY_`bG%+L~>y#zal+zRO zgGF+3Vsz(MMb5rBGYQMC&Tgx{?Lt96|RkWu1WYRy~R!e zH)&iH3s634uR)vT$R_%4kf%nBrCpUthMj86>$(KX@`C$NMjcTU3lmLm@XpxvuE!nU zOYR4<|NmCW)EmAtalQ9^=kwC@LG1rultTCasTo8H2oM@2*+sd7c?tdi{~t=Bo%H{8 zNAU+5h!zkcG(Pi&aTtFi^-A%f41xuu2=$Zv7o{bU^c~Gh@&_D<5)dRbmcOg`f0aTu z3Q0nJ`3H(OG_M#RKSAFB|C3T%2@`k^bvYpKA^)aMZ8d3^jMGWb6>4D3zy=QBmB`l!yw^0$G% zE9X397%}8FY`MPM^;#<29Ps9R@w1e-*ZT^6T{t1xEY4ma#Pew-!R`J2m8r>ar;X!@ ze^28$W0yG-aU7+w>hafaXNBZ^Awy`r3I86z`f&2WyxL}S6?5t_*<(Jhv-j5jsOZax za8Ki6sQUtfK3Wp}qGkS}2N)4>E$sa*Fyo*a4j9X#cpsl>niB0j^s)^E@co?=1LWVz z;T$RMc{!iv$PTSDirIu7Z=7CN$L9KT8J+M0B!RtKx4_LEzL|WtKMQ+`VQ#?tPaQW)O*a%YlLwvC(axvFiO?etf+`|5cwkQ|=P(y3zZv-xKve;eWwN$@*z zG44L%gb#x!WDJAzG*3V)4*`v3qUu)LTy%y`nfj^!B~05=)f zcklA=wfT{v*%XO>$H)^ez$AYQh`QbT?WQK+I{$iy{Jm|${9SHO3SEf*X3UAdW~aA! zn#uB^zZU?;A95=HKi+w;{crr=bMkO;bNx3zlZAzyi~YX`GzWWNTy;lMxdiXw;pfi9 z$b>N=NJWCcU?GgaeQ>CrDz~M7wpK1N$foI;=w3NFI7RyC9VBU$<+>$pxiw~U8+&V8 zaIMM5@5?uS&mM&M@;E{g9(Q`W@Br=biXq4*8x5d0Vek z)w^~*2%7sYx7+L{T5o&xLT_yJZ}{zJ(oEl9$+pl!46lRVfIYfTlc02`d?^2Gap?xX zFJN$ZngSVpzk^?--VR{x0(6{ARJHrub>oBd_l0DCBLUW~XDl}~Df@sBK(bUMREz~s zpup55Yp%TU`MRAzCMOXEf%W>IqTbz|&&dO^U-2#XfO2_~0Hcs!wG!~P;*0@bJh<}U zc~!zPC8E}9LF`Im;SZw$F{2@qqYyol zgt}M2bddI0BEn{w@kNTQ5Mzts*dxnk*|S7!=6IWl?Su*DsGITZM0Dpkn~A21c$y(E z#k*#yPE?u+E=7Fi{yjev38xnZpP=%_#1#~u;PS>XO3}<>olx?I)|Lc&LFx z%t_6a}{RKF`Q_)5_A&zVQvLpiF*`w%?X?sxRP{|2m}y|qZGy$hR;!) zC_mu7aeEUA1m=k=7M9PkooGM6JrH^m_XHS-AJ593m_NXKWBH-`!S{qs7q86mdqeg_ z3dVwvdqM?+1)~MSLByYsZ&DAOZ~Si%Z!`}xC($ReCqxhC4_M6sz$joCFb)_9j0A=P zV}ZfIXka)9$`kZ{VmQ}ZNK!yj1d#d)^bhfm@elHk@(=Tm^AGfo+?MDSy)OI`@0IKo z>6Q2j^$+)t2L*s4Km?y~?Was9ey-q$XpX&Dpy01;(^-yX$3>l3y?8+47esFA%KN_FwPu zzh3=+z4j+f(THBBUjt;%)ws^&)}9F`5d_Dw0V+Y$Kc~c?ht%)YkP>qsTLv`)yD=lv zf*=req7LvY`ZcMNuzSbyLqmd37b&NYI|RG=05tZzZc$8LUnn;1rZ@&X@=K)BD}-@Q zb`RRKV9LI}SfgAN(wR^Rx(U$;5w0Qyio3rw(SOc2!lB1qh6ugrrt5irp(`C%_?;?) zeqPa3UqjC(`kiFo7jR~uUdkxtQ!*J(4gCri+5Y2#%C8te$TZp;d(nn>$9=GrpgD3Z zlFM0ao!kK)f{|iTpa$b5P}O3MHrPbrPwYj;F*ufCc3Qky5PR5w{);#enD{S~6ZHVf z#~k5+F3kq#z?taaFhQ3rUU<7HK{`z3w1A4Jd9UDmsQh=;@iZAJ2zc}LXw{a?>Y6Z= zvxdgj(228#DYHjs73Rw}C|$>OHJmulS2cuT6hSz^O&$gfqwl5GSF&ba-ZjD7I+2I3 zd4;dp!nS*9Ly76JXL&ddGmRq~4v>=!J_z;Xf=Gcjb!Ri_>v6m1MbBSLA)5;zf+>X+ zRDgTvMGUhJc|a}IMe`N#n_S0OuH~C1%uU0%Bp;?rpF+PSOd9cC{GETnW&lmv5xC<+ z6WRP6Ka_CDfdcPo{puUY}MWW1nx8BZBP znPc#q(G*|5m`Oc6c4d^`YznJ-KS@ZS*u|X@&cmD~xKijljkV{HWE%*Ml`&7|q55qf zD`EuNg=>s4_pEefD*Hq-`mqVgRK<_csAsH47`V?5sk+cD*wu4X1j?)X(0mZmIbY@b zu=C`9tndj5osro8!h1sCKYIP6Bs-xLu;1}v!;4XV|LV8STK?U{r^Ca>a*@>88eo&f zwmM;|uoRlkdE*#QdIIsAxcj|DFm7MCoyvOI>suZUKS7ms*^R^aNToz7dV7DhU7S zpg@m?pEqEakra0;UI)P?Cg<}*%OntJyM-==#+aLkTpEoW83CCS_4LfL-g$BmF+K?S zq1xd1*f}X=3#n7HaO72S)k1_J2sc8%r=_f!!?C=Z*Y3>9KB2sr2_ z=ylHwWD@};Hdj=iy*DsDl7LPW$R4;8`Iv3^2Mx&`peOMZ<8<(M(l&4^C2yj|c1Fep z%Rs>i>zzj0^es4ZkH$urK>vg#KU?LiLsc`5WcVf5TJuak#8#v+5@>?Z1KmDGvJT=O z#eBb>C!*AlHup{Hnj_t8G1z@kT;0HAt_oMEaovb7gKS)86Fa!QQbp-$wCL|17Z9BN zISw?%c3K;8wgwH74!FepU5u?Fa#rs9gppQa&f)pXZj7pOqrjPt;Ql2FGn-4rE!AfP z(I1Fdmm>esAUA;ZkPKiFDWf%)v9KDRDxkm}8bsw!u5au_j;@8Kju8-h%h&z9P zkCohaPRmT~?(t33q!l;7BH(Ags3lx)t5l(iAQhL>{yB!kD38AN2LT#TM6bP(`*0dy zDMF&>8Mu$pV{tDWrFwIv<(68!v4QU#!-?M?mlcuQ>c>s50-2 z&(Iu0uY)*zBwjsTipr3BIHNm2jclp~OSiyh(Xwu!Os#!;{jHb@3wTgyYD1x()aCHm zcXhXv+k&4uYAg@Z{?Gs}5HrSe*Dq-QIur(p;P9BPrhhdwvFh3P?e9f0W51xqjT{Ax z5yFOD$YT+iFKDi03Q+BV<`||jjwOZV!>Gx%3e>J6q<$-XPeIcqC*Og0u5~^o16_L( z3{aQ%RSL`?O$C4V<`UD~u(CHlpMk>q3B(>(PYGRy&~=P%yxaTE3L*W+2tqqnx!nS6 z9W`Ad<6HBfl{WHp@G0P5BnlO!>hT;;yT8!p`v>`4-*fTt=_By)xc~8cJAjUH;XKfQ zB>i@Vy4?ZBbW1;^NBh-WxQa|@^vyluP>Sr|+KwxjH*eG$E>WiGx#O)q+72N_u^zhw zA(&T}3!);2v?gJ6{^CM+ggY<)$Kco|4^pM(UP`FuY zN@!Bh%19`@qtMCS0ij6t#;B0=&@52+#yA9n5KI4(J8_koP^`m9NwgEL2`#;hack(q z=Eg%Pyv{6^qluW=S8z6fXxkKOgjswA(?m|0K!TI5K>si~GHHa1wpR(jx&t_ShPm~q zqGRl}K^EI!$5gt~9RVBv27)<%FD8^C7 z6Fc#1E3gH$(uhSc9)b&$1e3ndYm*m_1Ozf41$W-$v~w{^pe(CoTEw}Pt+LK0JM~ZX z9MnG+o6U~@7H?@+0Xt3g6{xhEOs7)UIb2egE9DG>_7@-pk?&PDF)rOYe2U<0S|p|* z<<YJ!)o(}ztxG}7MX`gniJz_9$G&=hBl@cy?lL|TQKGAdgf?W_4v zi~eY2*txr15_6bk3CB227clEI%MfZiX~49b3nu7oaLFy$;RJWG`&o5Oag^HVup z6qnL*<)qE1)|Z=g863hUW%VJL{NaAM}m)2kf#0*6tMM zDzVh;;|tS$1-86$8jXT*(o7+{`6o}Y?JKEK(QhoHwAsOnS(Z*P+hdL0s*b-&-0{|d zPYk1|^hmA&%>2a@fsB@J{lekeJb_vzKcKxhLUoRnu@2mR)L9-1{293P>FW6h5|(I46V;c&;C)1rlCU9#TM-+rk|DW+ zjnzq{FZg1#oD0L&;j$CV7D}w4R%<0B3b7D0T^x?eu83~S7}zWOHxyzqHUAnyu+zB} z6{9z>is+doJ$T19%W(U4XA{c`2medu@yi;`R#SRyuXqz`ySH!ISk|j>wFK#x!z#DUXM0_?PU(2ht12_4&7s& za-FIl{92@M!XqFzwgZJ1b6SV(Zts;#q}aCW!8YVxmyJq{u4PHO<*QELCA(`x7kIfv zp^w9fR49D@g2p|V5ifj*yw*H5x-97f8;nSTaborzqbQ zEA$mB44JzKtq4t~Ekv{v_%Xu}Wyxugty!>Q zD*E*s6&jf@gPJCBTOnx8@e!kMJYE1*UhNoh!oS6itE&<}?86n#9bt%fQ>gGINDPaX zB7)(&S~Y3stg&8wa)H=AGcgk}=8W0%Iu46Q>X9Xm>N+(X4Kk)T6fh|HNE!PrKhK{! z#d5>yJN!LHuar_!yk$A?TQu;#PTZ5_Ih-sF9AbTPh;>W&t>7)Gr2KGK!`qg};c4bi z9N8tsoe*eqc2N2GsqYFX+kcWe`R8}Flx|zsb9ywd_N)oO<*KDm@mh9bn3t|KBFwee z(DYkp{{f#y&W3(k>37~+j=ZrX>d1ayeS7z5;gpy<1zI@>FIvfX8ZbmeA+7jaIO+b~ z-f6y`R)gBC5!1~UnUO;kSkP0o4_b=i5TDru|I7gsSZJxX^lWxNRRCE4vD--r{VFaNa_JJj`r^E+7L zOgga>D7}TG231OIR0H-cxRr$b{I8Cu5=DuH=O}|C#;HjfnK-fkj;KR3v&^uQlUCk- zbmKa?ihxXsw_e&wE`oEe`o%ifL^v5q7_V``NB|J(E;ia3n12X^GlsAk#5qrsm_%QA zlYK;}Srl3{N8NlfAMHLJ~lcgX0D%H)1(MlH! zddphu4B;|wNY(w3qraiH;@RxBmAnwfs>;||*{Eu+#y}@akOZGRO^~Q&r+V39uTV;N zt;ex6t2%c&k;WPFmSSD)Oi3qi&@+7sCnGu1RYaSS6NX;SbjEY%LE3iPOsfzPL}N;s zmM3HGnrbQ%fX**34nlB|EUg$S>gYy{Vh|~wt5ivKTs^(z==heZay|d{S(C^cTT^n7 ztnypC$}xm0PV;opBajumTHE3;M{Ol(LD*?2%&V?f7UQo zKbXG?&T)@^h44$&vRo`+q}SEZ0Fhn<2K*g#g;}}KaiS>{suM3$vS~8?#N6E@2Wiqq zNF8&)$6>ywcO^n1DbcBwkjEr}YsakfIQw-#K}?(#G?#1f!j*n1t&`APk$12sBC@>N z7hcIyG2J88D=nj^9J25695G-7x27N^HJ;S}QK6oC0IN$8D46skI*6nFktD`f)X1Z^ zZmie^RW=(|v-GJATuiiPSTUX8k~=h-)O1_3<@q^-#}2LvTB%DCI`^_()nzlNqBx#) z){%&mG*l773*!{Fu=jkF_+86s0k30YZG!FFHM*f%+r6%T*aml{ksZY5dGW$GrjCUs zKR};eCBw{Vr6v%$=c1nv5QTpQm+YaUVwjzcTY|?L=8d)bAuH)Qm(_gyZVL|R@CzHA zzL;nDkoBb+ot8H;0H}m{Y$dc3n70WzDY`>$0i{2-w<>bS6f;dE$!|oM0{thK8UIK* z?`ZU646U2Uua1o?G<3y-Y^d={8saj;iEN;Z*JSb09|ALs~YVbv9 zU_n5KID!#Zz@Scwu*867XJN2|B3PiN-?AXo<7CB?oP;Z$1)os3wPP9zVaFx(xSKTmguV6utlb4r=oL*I2K0*PX{cSJ0h^ehrp5(#3 z+U725dmCh*-sH1HcRh>K4vF>8Q!vs!;HJLklUy{e5Hv7WwO%40)}eBMFQH%M_>+2# zUN^(8^pbZ*bovl?=5w|jhZsT`zr3{CP?oirO+8)cvEzpWN89iK2FQC3gsv9;tZ&>& z(sU22u{HUPIBZT6M8z2VsLovM&djDU*Eb~`=^?UBP3{VNf?7x?~S(DNFfrspmq=o zS5~hT5;o7UWABeJ-hdUFTzPwaJDzYcFOPexu~(9ThiaN& zdwWZO0C~pU!vTe4R$`^u$^H8r;jA66{<-Z5Zqx{j&V22tm7hVc0p`-Wf~X47^I8VwR#&De3Y<^c#LM#UeDc9Q^5WWC@4r_4YdlEk*=S z{V(!G7~GP}<$=haGy_89A8F~xYoYVAubc%6yo-6{sdOp>64vNCCua4!jF6t-b~%(T zJ@`kjV5wKitth3XuaZ{6{ukiQS=&V%jC^21zFbl5ScmH+vQXoOB`UM6p{+}UDQIg* z6s`!f48kY`={;Hl5w^K#TNaBbw?@@8|KKbKb*&Ffz(>BzLl z7z`mPNXZ&#@KjF3`YW>k+E=@p>ew{PEC3@=B(myt4COdKzhCN_=6MgAtET$u{B9*< zz%}b6uMnjnS=vcIl9Tju5VyE?$(wI8Qq$6^WuCycDhHj~B5FC2y6>I0A$?)+n?B)i6rFgrkY$<#(x>cxq#^SK;k%vcpz{x|GV%O zAX`Y)c3-UQpl6mYOf({Mb)c3MFT(8aV1}Iul7*6DqHe^o8bP|TtjLX<-k_JW-CcdZ zAJK*Jf&X8;-?0ls@x6WYi^2zZG!Z1s=W0k_T}n@DB3^5#KRA)?|gKL zNHctaDzT#TzT;}=1RiGkFKL004Xc6RBAYbuHg$5PwR%;k0+}k&Rx01+)*BmDS~x5k zNBY}!ciUjW9`y7NUnTsQ2{}6&p+DS*)YuI;@VfDt_uUO9g)sB1M&%@K-0ui*&w!my z$+XT|y5wmZN$uf8vd=f=UQD}9#EY`s-xw=%B9V5amid*TB+qK4cmp zc@mrT$|d&vtbSOrh&I`j#caywcJ?}JF^-s++2lxzF5$bP5R5+<8RiaEVW5?77z6LK zsI-sljIk#B3jRL;jX-k0#%r5-Q2^so z<%eHv%ew=$ShqXaHUE^fEK~doIP!kR8I*4x84LQSEqCt zJ4_z+4y8e3iCF_d!Cq^yayo@oa{zN#?Q%O+z)31aC>yJkN^27dw>KGug&2w^y^&m9 z+~LV~jAXN8o%Np5OzY~}KqgjUwMAl?0QW1=tGjSja3)$c6XXW0{<1O)tRifhZ$82G(GdD~q+Z-Yn43+>NNmuLagsrahIwAvT+tBIkfRw#73dpR)b))L-6&jAEn z2a=Zi6`{8B6!;7LR^sjW6-a8HxJMFFnEE%UghX24U7G(Us^N?j?&&U_m>j zG`|%+N%>G%!G~sJVZWS6U=_NRiM%Hn)x*jF`$5eis99>1H;PJ`!+?}eED&FZfV@nR zmLfB~=T@Wa=6w?KXCQxu#5NG;J=vq0rCz@l=7jMuTX2YZ0Uu@CjE~a2>GSAOrUdlG z65<(jE|AVjXqG-W1FgvJ8nV157MBgu38TEr)fbmrx^Ge5>^m4I-7VaIx=204s6~gd4tpA~1h3)ebplyZVmUw*DC-5bnbQSNZt#}K+l-uUhmm1Uxjq310(HadogAP4! zj!Ig4s5#Zz8&s*{HF`m>R^^A@dW-YLY=Ty4bL!20uT$-b7)`uY?XVb~c1izTgHP8S z%(?n_jq0nJS+Bm#RH85_%Ch4*g)Lg{^jo!}u1eBL!TK6#Sgnl5pkdQ^y4R02rPWpf z2dvel`6uiiONmeuRhLNJyPR34y-csta9l-vR;^Rm5M9T zD0SLZLqqk0gA2S$ZHrmdMw2y}N?MHu03kYsBw0N!w^48PllC-U}01l~Ix)ZQ+0I5+BuprpDmjH|Ggmj5< zIg+GgkvJjrsDv`tPO<`^q#8;_*v6`!G2YG8=T79b7;b%VrT}FojYq+OsQSfT)h8 zc2CNlVd&(8oR&86cAYO6GB)}8ycgbFFDk87ZdJe#?R40DN=Mve#0GO1x2gk~)hT_A zBfh2~qHdTC`{z1DL8bEp76JWh;5mgjF}Ob-o$cWDrB2CX(J0JmtI-w8C2Q;5O0He+ z_j$GT_6GL_ZR}`0o+$6Fv_v(D{SC=|F^y3WHCp}bV_P4(YEQ~orV33Syk{1R!e&v# z{WqnEaKzZ4P?ZPQwPA$;4y~YY*TwB_T;EovQrIfX@iTH9XL9M|{HORDkQQ_3 zDX>wAtqN&}ad~&qK}8Kh4mt&u%>J#Us&F2r4Ka-!V-%ShBr{8l`ZI{Vx`pjzIFX)K z=8Ke~NO_A?d68-;Qd5jdBrW6V{W~~VObY(%YHfdE_`4Y3ou~Aguth>>ts+Nq^wOKbWYCPUjwH^e3)?JrMxvh1Q zMkku>c4f<+dWF3zRFU$TRa&dvVKSA*M|!)~giG+J%RAiWklt8R*WhYD8Fgp`lZMm! zn-UjBebbdUSt}(|l}5CBLUw$(yvGW&$mvyLX-WP`cVpLJ!c?wRXgI7h+tmY?CBJ8p zw7kO8m@-%m8bw2EnL=fYc*%T7ZmWMS=d0aN; zTG&;2P{}|Ru5Mc~So>BC*1i>kwQa><6)-6o@GDqe*teLX14#z0(E3dT6|iSNLWUNe z{V4Q)ica|BtR5M<0`)q)?Hr}vWMy)kGd20_QdZt5q;_8stn#LdVw$t|RIMDW#{X56 z>ibmCHy~(OD z4yv?@)w^n34IM+t=drPOyDm0a9gJWCpdF3g{FN1PmqxIKy#m&`C4AQ)@u#gO z+v6k9Jxi<0myw8b)n4y)w^#s$LovS^ni?Dg%Xk!s{!1ic4H~p6v-wIa@T0tdU(iTe z$!FF$^!XpTeHNY4QzZ#lZPb1ha8ha1u5;?i-g^V!j2E~($fzc;k9*MoI*HEUw+O;M zN|(w2>SF3bc8u8gtk*gl!}zANF|C?4xS)y zo^-IX^Qeo-EOLf{gmaO0FIkXL5P2T4GfCjufKlE|?`Mdf)VrEU1~st=<0LkWtoAc3 z7|_Hb+B)g#Kf@Z|vM38!v?#}$ktmaWK=&__gAC+VO)QF433zOjQr2Rb#3(tuF1tWp zXk`i6pdx{WUF>ywSa*Bb?>=^&?0}wOuf4^%9JjJ>#jEdIG1OaD3<&H%Xp`p*IN<|vz(pd_-m z90x{@rxfD7$Tb?13VAKd$$1&0QQnLo*9$IYLN_fCFfZ?*5IcAb7QlMwnP8IRVr1&# z_^updv1X>eNKX%vRaC`xNRp7SeIGvl%+av06f*K0+W(tJHT9)#$LkHYaE@|qe{ zMIT8@JdEqf>yZ%if`_NJSc{aBjofrxu9;yq3p+1uG6ttz5UEnKg(4#<&q+LDl!B(=BN>2ju4YkcKS4&b5w~C67 zaMoydDp*z5RcYlbQw~R_5*R2HwK$J#*?ib#sHhw~15B3x=B`~Y8OjVQv%YoERGpAY z*QC>{OEpqyxkKf)X4*q01yv$zc8!89aR=H|9b?_UJ zzWzG{{navGuL8b)2y74yDnnUx1$NHUW6oq^$+2gdzmGG=buEg~iEsdwq*QmYY)g^a z3-=MKT}0o5TYmwG+tBU?QFwR&p>(|(mz4cZ2&`t z$b7<`&2h3s2y{Neq7LJX6YE(An^8IIpiIe2%eO$d=m-PC^-$8iAbTp;8<&q;5L%8x zAum&CX)mJ+)-6w?N#wYXVmt|RnG`}rXi$k|+0-RFz8H8aQ%7NuwScsErNX|_%ne7v znJgc4HMe*A5`n&+Hg`Uy!)C!^_HOAdDc8rk>Jtt_s6{lY*DK7L?zz?V($s|qr<}R* z%SJRcg9nCGE_=_;{+PDBx4G76NO$y?3%r!G=rszhDm&Z|)`{e#0k#C3KATQqsCDUG zRlSWZ)tFPM@Tk3}BB*UNmX`;OJCte>>kQ70h(1_i3x%uPde;-BImztp^pP;ES7$H# zEH836-`F}g(&_-jq$_Wn$Z>)oa^pmTMq0vzq4jDp*#I{v1$$lBOry)7a>XMKFzD(^ zRo+0&jjM^>lv%o!e~B*xo2dd-qgnJ(JVne<*=u+R_)q3W8JE~;n8xy{Lma=@lV~@} zrw`3>nRDC=#Nk0U?z$*f%3&D<{1XR8nB^p@mXt?nK^nkdkAwq6j9z?A7Q)%$5=a*|O>p%js9w%g*}kc#XJdP{nd1~??M=L= zyxO74&*A%5q(Q+NDUZ}RwW7U3GP)|e8r%AUDgevaQD@e18kM4=IZE8DMb#|MmNH7^>yE~`xpM{S~@cT3OMDuPZ?6fjn+M2oUTV>TJp zB@Ty0v-chR(f~lwMm3EN(8k5OXVy$y>qDre8;P}-%4=|y7=@$Ou z+#l)L77~A+f;pQ5G@d2_)Fu|5u!iOy$~fmy&?ui|A-qaXvUpZ73PA$}Pa&bn!B=Zs zt|M=>5Lcg`o=+jTehuANnMaqfGPx#2(^eMi-5n=*SB5n~wjen)h~#6z8g}FjF+<_# zhi)$w-1;@3EUYH^kc>>pAs)nuKN6R9dLbx>&>Ml?$ihI#cGEUS6Wg1#d+CC_wulL^ zmz?>5qgjds?%$yRveWcQHDpxGv<#4BFzt(e() z#i~_T?3}5HOl&*&=Wcv|x`{l}*AXorbP*S~bErJpaXx+FtrCskv^i^>9;atjM(@7G z==Zuxnst)o&;@NI2v71$g4$v_!Sm6bQBIBl=WMm9I59f0&*(6!tsW!bi2edprUbuPhatOcy zdG#XmR}*eTOoIB@4vm5NtSgY8xz|WIi7esRR^N(4e0Fk8%+8p3979$cSH zZlCHj@FFj6$dBL`FHXbZo){O_`TS*Cjx1|Pwl3d=#aK^xb+{;q$!kL-N~z^?vB zJ_5_u4J##yRs)s_6798NMIhNcV4w|5EED}f4ABE%8E8o$2wfTmcAdoh7HYOHkf1c# z*juwe9J(9OAzEiGqvY0h6VGp!uCxe!*cPZj@^0u2S(z)%jEHeLF1N-g#}iaBnt1t> zY{QrM!AM@{1Ubr92*D(DVK-~Fu@KcLhtzEVdIRh9o&~w=zygWMk?)hOT0m9B&~+Js z7uF<}3c)M*!vm@^h1dm#=YxN>x3Kc;=Le(Nvl?tvIqXuE!x-~6tkO#lq?>dPG6)_?_nJ~Y?cI~R`3!r!5nRV0ws>S&3#%|?=^CnitFq6gRHb$ZF+GU(h^ zTU9h{hFlTy#f^aK?VM5Ie6jeZjLs-_-#wzU{W09ul=Z}N zd)uz8ZV8k}Is*810@>G*zb7+3}4S;Ks7xL?h0q8vHS$dd%jCasY*qhK>^JphJOqt_q$al(-DufsI z#L0n)Bh0J12Sq8P8H<#oNR<{TPmu}~scezzWE6mSV<8lG9vHV!dc-InhTIV+A?t;^ zt}as76{)j~0$DFd-W33Nfo0@iIwHiEcP;~AIo8Q;WqcN58xnFfH(U&{BNcF~vf%pk zVj7!Q48kokDBh*#?P67~%cCoImG?GA73M1ZdtGf)qpdk}c`~m!dA&((bq%bY%goOA zXzVJjs26mex`~rh8jId$R|oBu-Xyk`MoKLzDWtV>4o>UK#;je#PgGU8<2wy21v*qH#@Z+fn^&q(O_dn#}P#2}&=oYb~|7HJ19OjM?V< zytO`CqiIU6_i;xxdcM}r32KGbYj6ci^hy9nqDrUMXhfx2!8uI&sLg6EbJ{d!w^C!k z3Ui=T8;M4ZCMjZy+iH_>OKZ@p%P+Zuwi3?QU{xy>7V|fFxLdSk%)C^Q^_T^nN~$sI z20$mImhRwR1sP}nOt1$%g}!?pJsp=Lw>==~o?an3GK&N_bf65AV;$@x0z_=o6&V7_ z2!QNW2AHk47H|g1_o7>w_o0bK`x9<0P{tw^DN@aFe}JKnLfH(XWQnvHMC$=)eXU78 zG*qZ{Wsy3|DBwSZT`Dq1F|?Ni|9{xlW#O=S`a}2JEwZ1UNhn#;iOaDlqLQev<>L!0 z*q-ga;oB9M>)pA!&Mat4?2Tjl@jn=Srk3y;Z=cer>gBago7QB|sZ8aP-cc6yC$eru zpJ1)%OqJEu_eG7qaEYi>=}S%Ch*Kw&m21tMRo&6JS8XrZP-5xbQxU&IXf|*rL2YtZwN`&tR5h>bh-wABM(2&x`hv-FSBaoi>Vn~jsXVGT@g`p5txxVX zhD=(k(V%eDlww|O!dZLN>Z{sNk<>Z$V*X16Xe({uGK*bNk#wt77N3MC6dSMIz9tfp zq}J(U!@PnwR+%`9HIOKG>-3PnQQUGXtIKpZ zLgQ>b$=5FajthSgZZ8FZP>gM4&g|l+L)d3Sc)GsqnO4jfDo_#5cA%LzkL)+ogWGa= zeTIs6Vh8Y{8*UhOcV*IDZekDeTxmxp(^*RJ@P*HpxFU^JCXZwFxLykReU{Qe;+A%# zB(q+jQPyt@+Bv;1U0G6E7Z-Hi$}Q`9bzQGtfd#I#+H4T@qT;H*>(+O#os0Y1R?Rd? z7GceJ1LKRKaRq%>UluwdhreYKq%$@UIQ{dB8Wqz zu)4n7sw%CsITFo-nMF&rwW{LV@ilE(XJuLN65#vVrCa$=@iCa`Ea1lLv67s6wlZhM z0p{1XQfDg>wFSz=C>iw=f3@syk$)3}U(G-#IS5g)NI3a24s0gBi&xbSb)$b*w1zsd}O!mUEXj#lXx<>jR}qA()h;WRM)g*has?@ne-$*J+QMCV0hD)~uD+mFSIH zlh0`pN}LWWU}T+WR%+aCm-aWU8$D&5RI{p1;j|7;bI$zdEg`!(?)0~Nj3&{Ol{CCY z6>|J4(CYI9>jFL9hhy%9sq*eqfH(;s_U#UUF~mC+TJ#_N@)N~_vxFY8~`Q_8E9BIW>vlo?(1^=UhPN8m3!6;7Ai6`r)D4<_&* zLaP!QtD@}is>=Ry&?1~#qiRo(n3-X;8GV~y5As=y425r>0dH_%OIU)Y6klt%DgxoU z%%>+NqMk@awxR-OxbTG=BlQ&(^${9(W8`+@5N$&VeDb$GpaxP&8H@xW;gGgXvqRIa z;n)8f{?InY&|dB&zX|RyL))0gga~;B3Db(t)9;Y3R2BsnCCN$5H_tpHZpu3@HPt2ht>@IY>t!eFW0oke-C}DM)WaV&A2TqaQ=!Hps1} z7U=FlJ^BSCwtG;IoRBK%hga)SKcpE*yC5BhbOWUOAUy-=bx8jNiR>Pb1EtMGdf2EJ zf%vmZ=}SrUWr&zV$YzqyamW<$BBW~o|EGwiNJnaLx(-%lI%V_1VhcZyuFcei0}88M zEUXwyp@e(!!n>pRdjp5G((GMxySm-m`w#8QPeq%W#+sV&OZ_``+`eN+ANkqX$Y1jQ z*SK$*BGG|eb9c?!%Jcu)INsPa+H~uVk3zMb`F9)Pp3x@CZ^LLk_e(wjYa)bV=u`4m zqn9}y3g68LE>b=U0v`q8?SjBZLExhx@KF%>CX;G5veB=c_@&X@ufseevM_%9~FXbaI z9Z*mRK~M!jGG1B27nxcztV$^>DU^1cNs|!-ey~J{->^2r{qpGAnFHP+bsuM+j{-EHKW<`k&ahZw}gU?ct>AC zD)YIFb@``1**esdtE5Y_p4*LXq4G;plwKhHKvLbJX`@0){vh&+r8jUJe;q18GejVf z&->7^wR&p>(|(mz4cZD1!iNnb2>lTjJWnCiI5ZF8e* zCmPcHb-&F2EdTyLei;kbV$~k#hadkM-vDE0LcQ!Q^l|z7l5#YaP^6Sas-#HiijdXmh^G`6`_}Nc z;D7w;H@-3Q^{>M-GJuj6F;95~8jll6kw}uxF79fc`u;zk&!GWBQ_m5d($t^oH;(aZsWt#eDw4A zo8SI>eByH#h91V{4?cKj{;Tl3Gbo4G@D7+2X$2}E5>cQ7OBSvvm`&j|6G!Xr&2iU{ z%l*`YEY-oYt;jM@KNwEG(Q0MyAMyjXBTHHtY{7J;8?c|#e@11S6PuUMWdlG-gGndH$#S{oWS&+Q*xFUJ{c;2rJI2Q_gtFLN5lIE`n*aup)j3VPq_P}w&8v>4sBXTPJSY z+=IKgw_nlR@Qr)+ZqBvsSlu;UiI0$R$kFG;V9q6i9kuj-flW zS27;Y%<+$W?sT-``+axh$T*DS{mLspbJevQQaf__PuAcq6GzCXk+!Jp;2x6Kxn&u4 zEQ^saND#LyO}K|ZoB(LguVP|_lVusbUklQiD0i5*VlyDpSPPp$vK4P{BO(r?YV;+p zmirZ~KO2*SL^hDKb3A>+7*2<5I?UBx_%o--@5i@&>D|6xO}+6tjnl_{AgA|Jd%^Ny z-&F=FX@bPX@&bOJB(7{E|D6%^SI-sWGGCA{yNdG2ifMSXNS=#t%kSqD`24$*ufH+% zD|oIQ_2BRDepnG6G$11-Vp$U;QaB$#%mvrt^1d(mT{z5)5%Ia#yuq;J+|!lUNUjIXEt?zh*i<0=P-n!WqhM>{u1o083a zsSU|>y)WlIo1u>>w4V3!JLm>iFCuV9I7RDPH9}O zn_8r&_5wAyg{$>qfY^ZuVLpPuQ?*hy#RI&OML;kB7)y2(g;)UruYYd8>)UT`#M@u} z(vzE4U%5|Jo$J}$ygfZyUDDaenbHSdx_Z-t<71C*S$$bk{yPmFbrlW6Ib1R|weO`b zzP4-n?kBGsj8*IyZaJ3CW%IAAmDX!FAAV@&ffw)H*x1x^$v{oxrW2j;-a3$}-{RB2 z3nehBvdknCBE*S5f@2yd;#UT7$`Vm>-&%Jd|F!*_@LJc_efx92|2_Ao{MaA=D4#(d zg|-YJH*~-wKJ@YP$WI1CMy_VSN9F=PGQ+T%0Uw!R_{@Nh%z%%~fRD_8kIaCN%z%%~ zfRD_8kIaCN%oMp6c*W1;y^;;Jc2RZ=+8O_lgYW^uFA~#IYUThkfQ2E;gR+7=pxUYm zn77`>#LAHU<;+_C;Rx43C~ zv#q{)+l4QATiTLg3t201kcCTB7o$*N&TH+GQl3~}mRmaV9D|Bs!36V+5#8OBPlz4R`(1<8S`p2QO`YVyL6}mb>p9fBnWC zcbk5MAA9-jpW@7a^!B$lwGLlDbmFxGTgPO*J_O^xj*iWMe7K#$zeSnDNYn}ugTkWB zjo2uEMifzXj|v`?<<+Dvt|s!uZc_^WPI2od^N&tV;Z2h{ye|I)x0wGrj?riLfR=v{ zp3M_Hw*m*s9-=JV|KQ|g4u)CAANZqC*C2b6baGuiv1*9ip$8{wNjO0Yf3$37$5c~I zW3C~go5IzzTbq2Tu4rkrvlsuIK8Enz0Oh%CVY+2xrC4CkCpw3`xQc!y9Z~r&|1$w1 z!%djJ9>8#J;IjNzrl+y>NdDm*?#jOjFZezd@~@%7v+x@QtSc;gz?eRxxRs0(ej|U^ z6t=?KUN|bRuN=P-Qmh7 zrd{~!6FJUu;aBkBpKvasuYuR=fY-IK^koa#EOR+iC^Z-a3pFpOV8Jh3`s$9km%sYz z+}umMPCozQiBr$Kz(;3a{_4wfvoC%1mAMP!&tmk#i+_2cAY00qtd5X{B!g8O^^hUL z?nJ)Ga#K$uq>6F|q>Bm0&Q4|J@ZG0}JHw&w?N?S$PF^czm$)@tqs%zW>PM|2F&buE&quyfBr2;oQ>GM~`<*4;{>P1^Q=OTgbE2u-+bpaZ#eOf;?vHO?i=pL~L>L6>+eB9(`o$ zQG74=4;PM6OB16p&}}(rLqb`3WNmCMIGFq+2UxU=MbsuQ7!r4HD(DU?5z=HoImI@a ziDcq(v4d-xx3-r~s=~do%#bvJH}5+-9O<1pfY0aqdRv=PZ-wL14S7^gKt?oM2@H# zww2@cRYgiuY#;MPg7EMN_@95^YtDslIKDh{^A{X%-$tJlyzqT^8es4Xf8vx>Q|Guf zRF_0PAhXm=J}B6;P|^M*B8onN=hn==_S)<({$=**XSe@L4u|vK!^iQq{C{Fo{vlbG znxXv`XkUe@3Uf=Qm8Tn?be83(igdQQ;#xIaSh z1upvG3D58{#<&<;h_Ie)L@+kI{PO0_yZCcs@BdD2%ewSEt`F9V?29SrLJ=mK>sy-- zuM1N@49xA)J?K01nN=5Witvjo{#6;CJy*$|E3to7<~#7zmv@cu+A(r#0=>r_DDv8} zTwj*yg_%lm2Zq1CFu2t59{xhUo{P$TMoVwu-z_bDz#m0J_kNex;@`Em$@N0q%h%cO84>_{nEalJ-|Y`)6Q`N&7|H zG((Nk2$_|l$^5{?dmn%DmlGo+pPx8-=!L^aINio)-g|a#9Jqe?)HA1!ymAGS?Zzm# z7W!+W{9|GKqoN9rraC`re4X3WyblKo1T05Tn z^(T5-D>ofIcWN^oiyCNqmi1Lyo@re0IFi0E4~1#PP|S|Z{QPlwD2A>$_~KzQ6wki* z%ogmsu;$1okDq+z6ltIIeIxYU2-2VQUSJ>RHx#K&^3Nr=2os28U`cN}` zfhhiBkdkh=Avx}W$PNYjmukote$B=6N@SZ$LAN8IpYYNiOcoUp>M5#5BwUlo829ZP z>+a4rfl2ZOYp^oamyh7@_O-0)Hpt_k&;PA9Z;fgI3BvVWP* zDqsXqTPboVgnk(gh9?`Q+{iQQwl>4fg$8!DezSMF@w%3+6_LG7M;^xb>7P!H4Idnv zK6=^92j1MhvoG}O&cyJTSY7kj-h&_CIGq3Q$^FM4*+=}}-5~FRFm85&$)pw7Tw1XQ zqow~Q0#VyKcIK0eq@$qOwaja7gjZwdje{iQHlXJ`C(xFh- z*ccxLBAg0?#_-cIM@zS$=V%*NWF~Fsi+(M#YqQ)8S&(TMN6?*nT%mAnB-~Zr6Z5%B zNSk$GX!HG_cej~V>s)azBg>cw@Y-*n?Gn`dKVth2!>h85R*-igA_`2c7~|g@T|#4H zzdVuuM(!s+fx(!42j}01XOVwT@@6Wl|Car7^XO=fdk@KT$?pL^3!=_>`p;Stb}nII z`Gr+2|8oo^Si;7Xu?K3D$l49X3lY|A4vFbU=5U!x*{GBu-WR)#-!pD%*f8oFF*QyP zg-1twb~S9U2Id+D_vUbAZ$~V*WA@uv8ttpe&g}RZ=U=m~xxOK{_uc~EL0|2t`Tqys zeUQ^^1x_0^`BVKDaoXDUm|f;H|0rcNSuSmZJ{0yDKCtbH`;I*E@n1jw#L==utOsJM#wTZwi&mcqxjvP}xTGvY8sqtc||{-W+rpZ%H>L%cAjP?OS@cb<}0* z!{+don?IMjMswy<|zr9%^D+D3}ky$aX8t zk2$hn99oR4I25~gl*JdGO^7v+_qcmNF}uR@>jsKj^XR`NAV5x-{Z~UAHwrwtX_Cl!N&M- ztR7JktWNy8_=mGM-?Z)1&u%|8?|cWBVQzjtza;a78rFp!+E!jHIISIlmFx^W27fbO3*w z90>F--GO`f0a#BqHkL;Ebw-$T7BLd25hCf@76I5SNF8VjC0PCB%v(>Ow?o33SS(Sa z;3}JP0ejG|4re`8HdUZuPh#}d0l-VC{x9~Ht+iHW?pp8Fr-ojQ5wwIcU~+spVDb;w z2L$uu%g4t7?UOxM=wmzWBk>d{Y5yeV-xI5Fn)o;~*%}h30>40V3Mt%PU8BK{j-Ei$ zujk`6l@Xo4+8*d`b90KC1a9LhSATLezEx!d?UWWXAV0J_TZ*6 zOJzkfa7c9NLH_UfYFK+QSchZupL4hoJ%=Uo9Ygs$a+~O99mtn+_Ao=Jlzjt<97gF{ zl)nZmfAg7qz@?OZD`_j-w~11ZLD^wSJ;BQ4@7%nw$f63$=$V8WX7A{WakXe+o<;}@ z?9{^y+E1y9Vp*zK*2K!>Q*F!LDb(Crqz)IU+l$m4Me5Na^;nU5f>G@I(pI}v_?JlJ zI9Oi)|1tI+;Bj2%+33`5vpYMpJG-;J_YN#}QHc%$AV?GvAi)L}ks>Kl6h%^^D2kFS zOQs~tk}cUPP9(V|mTg&9u_a5k+>$uOv0*1pe19X=DK~Nc|4DJ2|G$Zr_k45C%r0c- zxmSWr-#X1# zmTY-F)7~JMNVgI3%pfm0PFRKg_ry}~y)b)Xw&u><oXxMbx*u2`9m{>hq4RR5sQcUa=7l5 z9KGEfx0{Emo$M;lwjQ8ufof*|BO~sR7&z43SZktvrc(Cwi51x!6MWlN50wRPiM6=G z>R$z#fk02-Zwgf_B#5>Cy3mLSyr+=*n?lV3m`n#gOcX6jDkzKewBYx;_yr{`Ua?1` z0n4PXq5i_*cG^vS1Xuz5-pn&S{c$-Wl#(;0aXXMGR<1wKzR7|un52$SpQ3;J zO*AMzwh5b_BCNGhvJgh4%@}nZ_#N#CyJ5~+oCgi*Ij~Pp-vWY?qZ@d8c^$)Luadu^ z_}*o-eg8WC8z&j#V;)EHrzEJc0`(<6M!iStuFnz)-{Uu6l)(w-s_@`Aze6S71)VMT zQ|Kf-xUsc1uZ_mdh<3H(#o#3d);F^+17GgQI<8s-V-Prr~N@FO+uc`C0JS*w-UEZYbLr*$SaPGSFgdsgiJ*v$QzY6=_E;kfm z*k7g@HDR(P+@R-i>jX_&O`%SXwl*!K&XA$!su)xmdL0;&Wyfa!^FLSO$L8jaCLs@$ zW0gYJP&zZzRdBPAV)klow7(IG_HUUk_wE|+^{bV}NUkv(i*4(Pbr&*jU%3}bdi7L& zYO&gg4J^dt3j?iCa#Xd5c2}{Oladp%CG8Yh+1ZFCdljWOS@u-i!(v8exdR**o{V{7 z{$^L+UYiaCr)$ioKf8_PIo4U~?)Ijqf?BF78vLQDw6~{TYHbDfrDc}IIjYMrExZ?s zjIs1lhGv}P2;9wZu@+{iF6P)?K(6;4Fji)Wx7Aw3P6WvH)GH+GIiE?bR2epW zb0ql|F2t^5?J*~9PEOGu3}pOJ00S1O8LgM@L|02{(lK=Xii{J-v4nF-F;fw&lK319 z_=nnBw}bwx^*JV?uSa#cvlN!Ok@U!zjufpLs=g?(9m$}owpNjRDKBJ0X$X9^O}fwC z;H!xC%;d@WsUzcAyIAqlj8GdI4F*StY682|YPFs+(cJjanI46)3f`r)BVP}q$8new z%cW37jD>kIG17e4;&wTCPgRs%tXT?o&D6X0meHL5wmsnYnpm4PyWFs&&c9P52x94F zhGFhX*Cf4kR%H6aU-&}nd#}Gf^T|&>FXvsNPgNa(LY#FfDm}@Xcu!e$KX*k^IE%oD zT{r9&q+Bj(cX)ZT04w0k%{w@o*_CmNG|QVyIKmsDNg_JE##zx-1&L zBy53DI9Inv%lo~9M1`@#sY2$$JG42kU4w%6v~v$2(Xr91*O}n7%$pASHdUvBim+G8xbVI!NTR%+KsWWx)(PYzuWFDrOd2Vb-lQJ@!+ny z=;b5BbBFKy%q^U$dL+wn3}X{5OXHAce-J4#%gam2{(EK|ZnMQkJ7l+I_~!cbw7Jpn zC)n{BuO}oVaz%G#wd_w@lKcBIqsy1pkgi&~{v!Jp8$sX5sR4_RQBP6NQQxKi<4x){ zjq4Z^5%IdbLfA`yiTLa2(|o;($M-IcSN`kNE*bM4>r%~-h74%}T`Q|e^G4fAL%NLa z_?jj?Zb%;?d*7g5L-%@X9dF-8+qbLuS-!Q7$%(%Q^9+zkeir?Ev5G6&+(Yt#7f6Mi z*QmF&?bp#Q;AIw!&iJOLuTPMdxSeFvWQ?|&Aq^PP5W33OWu2HsUxz_nfrrv^h4m|) zG^ERhbQh73(jm301cUs(b$E4r^*5=fjpOb?`<_L|y@j^Vuj6W{Ab(;VS4D*%=aWeO zB$0oE{9v!6?d#;&H<0`WdbV#F@A3OYYL7Er4NiAXXc5{ClxwJo&>`ZL8jTy1p+)1o zPE6C5HA*#9G`==`hB5tZ%1PS9VBU>dwZcHXNtR^FH#!QC8d*yuC0@TGy`hu?^-#gx zdOvM4FPNp%ues<@=6BLdW}A zZV$`364_czNQQ<^(U;Yln)T(~KT^BHwm@B-~gX9D#Mnr|7)JO4WV@|Di zMHb|MTku%bWZ3UeDo!Sv7G!g*A1IQbG=Mv=S}o{qHnZDh9zJM8?^HrY71BwjS#B-V z9bgy-#|h$xX^xp@IJ&Yeo0=@98s)qyheh`;r1SS7JNPB7{_z%SjCzVm0tJSdsKi&b z?Nfxt`C(Ffd{}BTBS9)B#Owvlt7@iVMuI1#eL~zGGP5C1FG-HhuLE0!ek2^x(2BY1 zEueT~{y9t_jSG42MrV(1V6q`x#|M~gA0{QQpcPlD!%qkN?V}Q=e%URx%XdB{wQD1T zfBh8UbH};iSEV-N5?oQQb`G*$035FHL888YXdUl+Mz3HBwKb|$;E9C7?YIuieZ*@q zP8i^6>Vp(!vpH-8#ULgGEO;Q}pbeI>0R@#{4pS$zIUorsRGQR{u8+%5HI24DbdPaO znkH>+dt)E5J)maP2PMd(g;&?lylP0-4Cw>LHBOM1g{o9ftz$eYPn3C`muV9fYqQtX zW@6P65Vz2pkD%ELrNy$?{J?Gj%zB4i0c^XpX;Kk6!^o)6n`)$jHRI{VrpS-!y{+GD z{e6BwRR{7k!*ju`rxq8?{;Vu#{iOnJGO1mDb9ku1$m)1wvWu};W`~*6KmF-ywl^CT zIELkxf4a=0CED&!RN}o0@x*-ZE1Vz)(|%{pA1+2@Q2bfe5pV7s9$FpF+kq4>usZbE zeD>y9X5i$@9KXV|L$j5z1e`BHi_`II==c?uqaSvwYC$qfw${H#Gd|fGtVRS${fl@; z?F+g+AAjO|kPa;8f)56(fqYM(7JS@S)CgtI&>X+SarBw(myhpWTq@forns_pdNgtH z%*8%ZB!TgdCzH)2&!4d_@jOJ;uYZF5DqE)fXr*``^#b)G^)mHm%r#g7G1J~P>Je@G z3P6;Ln1~cx^>0wGXrCJFiipM-lOk0PJm_5PUeav}upi><7)1&)@ag2LSRK4?16ZRg z)P33+zymq94xJyj09A~Bwfj28EF8CE9dGqQjc_$c1&8gybs$WrP(#`^K1ilPuQTpN_|P&zDRvR+dfWxUfTj5^ttJRr46I6>CAqd~qK{XFNiQLjJC_d6y$;7-vi$39Lb zxepS#{iEtJPH z5A2|~&Hyz)%D?d($M(!hsYFyj<6p@p7)elk4({yPk9q5bLdC0=3#qCyUW)b)jHHB& zIP-yir72rkvnx8eeOJ{LGw$?WlZEWEk*R9EEyGJGtavXgc=o?;zV zDKDEmUHf;Ssck#U(z)3HXEt#b6JKd?>~@y>_ty0nbMp~8S zD=V|KJ)rjf!SeFp?%gzcVUF8@-kR;-k)IkFia8>D<+-yKCr#Tqdu_5Bwu)V@=m?JC z3|#+h_C?l5m8f3gUk7UYm==eU$&?`)lFyKmMA9{DRyDXBwC~?`cd_(WW3%b0o^6e+ zD>AhGhN);m9^SWgAnc0RLh-b_=hA?Z3uVR!djtMTCdll`k1CUe?F)PR58QoVJQgTM zi;Ksv9U1H^>|UCSEZtvJBZ68Vn8;@i9NMqNe{0u2#JY( zpMu$OpJth1o{)pcih}L1R*U^jEi*-ju!_WE1PB=2V+0SVmm%on4)GG@5gyblVV4!` z*K4-d^$<-Fj-WFrFOZ;UZM~#5P9c1D3%_QX?9S8as|N65YwwJwz)l6y^6YL zvn|t6hrM&ssc*KAv{Q@V{d8yA{H+x>8Q1IS@o0`>=4p-%=R+bVusq8w+>lHiUfO$q zPvibomOIL^XFfW#<(V^UcO-bb*{=BFfy{I?d9Y-2@k&Fs&=yV-zF9h&%^ofNU7+GW zDJX||Nti!9TT{H%!P#nS+#W2)Jr%z%8}SG<=dA9$LN~fAR+h1v8H+pWW4UcCdl6X_ zyO3Hjwh)W%906uu?jmjF)vAjXLOmrQh)0uJO}4su2A%4aIbe)0O(*I1*E5pZ41pt5 z^NfDrhrjvoR(H%EpIJTo=&R?!!wV76%Jr|YZ?Z|v&mKm<_MP;y&m0CkJ@S)p}!16vz4DB_^C zYP9GZVCLiFAS8BxAVKbeehh;{D;HIT`N2V9zMEyDByAyaIvhb5(c-$8Z`PLEUM0)v# zi(9wecEheTmM-3~oNkTNCN@wB#8km{*Ow;xA3qYGTsk@B^jWr?KDZ+)G{^s!XP>8;t8;Fs={xzYTgb@FWdsep_6nq=qy}*cLS(K1}L) z@(M#5MIXMr4i*R5wE?ey_d;I4ZsMa48Eth#8ZoYd9~^piUMs8%kBRryx_Cm4+K_dZ z{id3uItwb@z{*%hDKfOD8w!TR$>h4(fHdksbyNeQE$)~lkX60jSimEwrC6lf&6@%h zm6sjQIP=k+YS~BceD>L^g;^iRI?_{pv`eluf|k}B&an8N+HwGxl9ry`$+HfhL(VAK ziDq6@hrEKrBsje;uRAA}lj=l_o`Ej52aXPmW?Mf9R71XuO#f4Edm*2l&$hmO(@o|EO5vk{O9yUqW5rWEJN>dCuN75&mKB)s@tye ztd&O-bs9ph9M#lnRo5vMeZz6;*Qk$B*f9ZlwG)IncY)MI1en_gWXw~iO9ew3G^DLW z!Zi<1Y`_8!@CSYWSz|wV0)S<6Cs9WrvI(_kU?K-F1on<%{zSrlsDG2J*OVkpgdA6E z*~GKx?qs@auc6w+#e$H0H1k%Pz9VsiT0j}Gv$eUeGyk;+!64wgihGqbWZ?YF0Ia#v#`-N!Q{7TMMkRr?Qa zzLrnke!$-t-O(*5vatGO>3{TJTN0x3!o7w0bA{@Aj=J1@u)m(OayF|Y6i`f_N-oSt zs&)^p3Tdwz4fv!_S|o;dsIJUVCFxLNiYYU)+9xZECP|PzR$o^($R;DUn|3>V3l_WT zclo_O+3gi=>2lpChG?&p@nw?>$sxgQ;{tg%$Ir7o9V*2GHX8(!5*L={&UTAYIljM( z=Gl3ki)KYMcXG6nQW%TP#z}=zUUaDpZ@2MC>G8Ql3*?C=uD`_o4r@VuD~o#47~!-@ zYF;hqO;r-B^BGdikaC7JPo#I(uZFKfzu0H7O`Jcn z)OvaQNN)Ve$pKF-V3uWxSL-UTcyE4tM)LE#!@`C<;h7zC7#lo%{W10h=BH?m8>L>x zG`4PnHU(vOOu}*3Jn>mmCS%Da^N{ zV_O=E(ij?#^88tz|2~%=+tV!#1{w#Z3jF-T`}Q@RLZ;dX?E4;l z8DixJk=e)mWBOpZd@%j9;mc?C$2?h!ckAAp25A#_5H0m8&1syr49BiPc)j(HwA%Uy zq2hgkC4xZu-DBN&6^{}jyz}~nHr5(O(mG3VCljqV-Qk?*O=DtDwvCL!fm0I9>4=$XmrRTAW|#xlEz_-Y>Ng3; z95!+2J&uTbi<)%OKX8@08o@^Ik#g&83oWopcBCo<-QH>;ZY$Dj!*|Uez9(0>dh{c= zw!Xu=>_>&*cDw2lId6~4%2*kb;BuIguEIk*xy9cYGwwZV6O=-CSUesTKBfUoG<`q~;Y%Jw1MGCx#J$r?~hbeJsGCuqw)prIQk z%*Q4wd;K@rZ?N5Hh4fQtt;p4vsh25SY3YF~#3r%T@GA9LZTo^2(|nWKL97ajdVrLu zdI9x<=sI8mF{xN{9sKy8rC!x~%{JpWupKa`J#cb9K#E?iq3i9?q;s3DsMl+_VjTG( zdZLUbJw@8Id>QR2560 zS9WfLhs125ARkPR<gUyAWEcxb zVd%MMuaYoVw-1(EpZCR_tjT0zOk#R^gcrq9GwR?_3s3iKYowhT$1ANAy&`M1m}t+z zT-S6nCnr~O*~P@SylK(uwyQ;zW=)ne{;e}(9)~h9JLL(7whZ;Dc6DHQ$bVOUY4?F1 zLFPCIANI^$>E`%V)O)#A;J|18VCrNQ-GX0YS)Sji>u7n;F3pW9-TevK>M!^?G3;~q4)%M2)YI8JJ~P%3c{5F0&Acppm!1Vv z>L57RK&PBQbLywr6q-{3XLOwUH>eAN3x{Ng^%K*oOSXCSlT`rZ7RizyLi>|imn#s@ z5_DjQ3jj;!2@D z)}v&=iI16Jp;a$V3zr9O`8Jn4RAI(PfQs_(EL4Aw<&YRSisIO=r=X2C66s{-It$n1= zNf>jMu3}k_5rJF-uI_alv(ZZdd zA~rLY0BQ--s{YsB4SY91m0m`Bo+0O=`=i#6qrIO(dp=dge}5XupC-=-(V;IO`4uF; zgyfgtDI`)M6kY=A7(7`8e(wnq7=Jy=t;sOB@@T+bU`=Sf0bdl+5&UQN2mXggy_7mX`j(_R@~S8st;08l8feHHEbnz09h*WXyL zb@eupCIC~t9{bR!IkI{^s?IP?z0spPM;XPVS3)6in*j{v_SB(vj~9DT&0we#=v|ZE zTu}k9CYl}Jj>6C#dT3jc+tDF)n7&MNR=atI6Kodt%32t-o}XEozcGpIbjIxQDkjwh#r$q=ow6ZgjwxkL=pd&z z=vrK%)j+qc6n0COK4R-&-r+vCAUzTdxc=tU1tCU>sr~CiA-)W}0PY z81z4OdItc?3kMq}%a^@VTepZ(V03&~rMH21V9vy{b}5xfIu$XM$vAgKnj$#1BvVv* z{9ImTSgXLgDvM))7!$iZt4h*Lqmif}jk~|lk~1exu7xGu!ZcSejygSR$z}5TLk`c7 zYNaP=j$42NEeky7uZ1|?)pY((C8$VNZ`ngLW{2$`KN#p$13iHcemKzQ_4Efm+kE$8zL}t3WUKaleF1>ot*l z?*<<6_e$7Z5670t-%t#*X>@mRS~N+=o<~y3a5F%R0k-GK7OHaIP44@ib%G+RRl)If zH}QJxBgl&L#9>z;In8%d7qlC=k-acMB0gBcm(G#(p`wLCY6gN#Kc>w}jva6N*YOr= zeIBmjMej!Pn0mzZg47P!fDAP2=h5~)z)!SeN`$IEOX5yOO_I9CxCq^z{Z*CDHT0Hl zw*9vPB;hoYbig7y6^9tMtF?fd_gE!KvWlrb&v<%=5Q!&5`T>(i+-(UsTi<&H{kNM% zdas=JI705stY3+k)qYu~W!CESN`Er6*b}zef|ad};!xgivHEjE#o|y7Nxs}*X>g$) zvD(7*#i4gzd4S4L{|La;aN4ciO14A##S1hke3B5Mzpl7NBT1z^}qhIHPL?lGjDvdlF%U>gnn*@WGh0ZpH9mR(K9HB2GDlTj9pV>Gj&z z5-1W{SMeHB-jKQtX%yc% z`uoBx!|rNLt*~6M85RQa?Z367Fn(s=zL910-^jjwjIX@i7n{q}lKnn0WZv>?i>SlV zGH(f~ET5m)G7t}jcv1Cw9b#`>ZTi5;8=3rWu1hVA1^dEupG7qL3$mvgj!saTE%*%k zO^xGTpn}vv>IUj@`Wa9UZY7Cp{bXXY5W0eXEl(O8>lWho*JA@vDhIM?yUAzmLEHY0 z85U&hz`@J-Sv2Eb+W^qWS?U_mi}b567;Oj9li*S}N5~x^J?vIeDCew%Yet+vEe{(l zx1p9hjh2VCDXg>T-y77;63)kgXV~?+G$SZ<|F|xsAM}-geB8P7nzv-GOf!UI@~P9APTRx?6<>Y{eZ*dm78iC;AL}jcLw)Y-*+nvr=L#1XlsI<7X8LdaQKNf$@`fzbl3WxJ_SKhLK7Rkc6O z@M~zk?~XKmF>h0S?qPbs76^tM8s?2rdgznY#mS*iIB5%1RL;VSiQp5_-pu4ex{-J; z8Q^WaltA`&B%_YaCx`r2iI44BTgwVgj*rnaLkt)p9Np^6?H(Rl#r33W*Pmx!XRE+ZgXWsk5W)y+9R+F#y4SIX6%s`J zxUMkFNUK_pife3?xK=ild|Q*+^?#v~NIwiEBI`qFjjZbXJNgB#Ua1;3pA+EW|JAzXXYQB#lVv8|=9ESYpfn?BYl{4;i?^8A{WrfU<2V-QMJtxHp^ z&2$6DT>V)2@LqQ&S<@jC|Z{!Z0o@mYNbLulrVpgC|_{44#0m-!k~L-18m@DN!yF-c>6`P z{bCiL`;~S4)Zd1F1`h}pAnb%b1l7y{0SB*pm1Hs8OVEK=h+f)7JweimarR7rSaQe) z)O1el>wkhw_T5CrJr-X74w6hfhOU2AJMW1OH3DS4!f+Q+PP#f2mP1k-VsWVB$A?O2 zY3*wFO6{x$upaMU$I3oj`N0i*H68v8Kge_I__*iER{v4qI(k;az2pb}Cffcc+P+DC zT*&xGZOr8Mh zazXoEO`FosxoVtqZ^ys`6x0Lj7@7v*A#ez3mAPP^fI2YX=o%tj=jM+7J%Yg z&B5@@ro zsz)h`*~QgkgS2ItM^A_Tk6zt}G(BJ4`>)~tpJK=bDw;xc#$I8y^a?URwDCS$x2{1vnd=bAhbacdrA}z$|rx z)>UqjaLzxR{SoN*KuQj{^!HV<|Nay;t=;}^vZAh#nvIvxs)l1~GlUSpVn_ly=M?b* zoue+vIB!tjn@3l=o8XyFP%9*|t4kxsaa+l`IGIs@<&}Tf z*OJ{q)QKaRTAU8==S>{zO4KIn)zM1A#R8`Vv$yrB{=Koe56vFSC3p6}cP?{cjAw13 zpof!Wur;wevmAL(cv#WFeeRL)NoTfLR;H&u{&D&-r10*w zqJOAw-^-W}pv~Ym1;&iS!po=7-UG-2(J-kWWWU1vr{O0$Mh`-lDHAWzW|*gitSgR= ziPTDFbtt&iX}JTu0z;A^fxbWu$~tly02#1XuTu(Fn_34)zkU?7>?ha1fgF9~2Fbz( zjPs!?6SV;SBzxNu@};^g9> z3U2uPGsiyiW-b5REs2?>{dLvH&)zuK8*sDyVV+}NI{oRBCr^InbnDvcwVrU?=}gsz zvVcxmME#1{&2nlv_&1@d0;p4DqOzW1^lxKj8OkAZiyU9*O}f}XJ>;zp?dZ8&IMUm{ zR=iAjP9I{w$<~ks@NdwcP8j_MXnhjsZwv7Y>e7-S9Uu~(FK+=q_xUOyg3&5mh52Wc z%y2kGbBT0TE5Y9D>N>{Pzq?K+KvdOr9B>880O%sNBDt~wwn~{=(jH_Fx))x# z;lWQ3HRdGQ>MJ?iNB4MB+5@A0pl1OBIM1ttxo9&X5it=|ADh%Yf+DY1L567zcbTN= zSM(h%->{uw?4kcp_e>-_+dVyVDp5ob zjtx!Sn#tTc#dB&c=wJntjro>_g=gujlb5S0x7EQgvJe_g(mb~l%|uLpjpKGe*z^R? zIfId;nr^05NiZpm0LO-g8a%Hg-Mr{a1r9HUf);gnX0bqP>>)@)zN(?>V_RgFvs!q! zmEx$O>tAF)hwSc2v<7XbzQG)Mlllg!7yGgX=Y50vhJ^DpAgl->Zuona{0+$vP#PUf z#FvS>wzGSN{NPGvxld5eV|@qz(82-Xd+^f|(9Z`h*r=wGl`CEWymI5(mRof{9CNKLEVw@eRHB z1o#wf)LyOcegb|P2sdxw$0~2ML(Y%@s9W41>pmio^#tT`bf*WkD?LblOz;}%90G7( zATNTP+Cx=LOawpD(T-68$DAYQ-AcBYy#&ppg$q) zq8m>uuwF2UBy9>0Zd<$WbGM2P<`WxF9xvucg|Ig@s5+wouW8&C^GgaJ%!GY*UW!SJ z<4Q~p4#y&cK_MJZ$niqRWw9mczj)}OhjN3}oa)I}hjPWO1Kn!r*4==7VCdOnncNLC z$$ByER?>xfGF8t2!Zy{D&J1>!RJGJKlFd9;?28G1Y}cg2yy z5vgviUy77MqTL=X1x)5}BR@Up_6gyMoTpmOim}*8Jipe@usqKtLrU_C9J>w82Jvlk zvoSQ$aO|8GZq1{M~60<*+4Xgs?$2^sB~Kp=xwr)i$u4O-ANOgk$O3LAq^*aB{2 zm??&xK(7-ki!38pxkO0A1o`E8(-dm%Q!@$;IXWyAx&A!+`&w;_W$IS?2q@Nji4lpD z*L|c=%e)pr!x;~-w9HGo6ABdVz0{nxgv1RgZ%8E~VTLXka_>&x05;^2qoXpV(NlDUF9 zYG|S-y&NK-rOT7(RZWc_Jq`c-A4Zws21geedM;N7>(I zi|C8rOuLb-Sj9PuU^D9QXq{F&8~Q>3+P=OA&H@7uvT1Gr zJq%c^Ae$yepmB>No8|zui}b!BErSr)Q?Z0WZaRw`$T|@*=w5oX3)(SG8|TpAGpD3> zdi*Y_UDzF*LpXB`CkBB@4ae$r(Luf$v@l(P((|G9>d+=*de#UUa0rU;$jjbB8`E-c zvIB_eF*#`uQQ~N3X3kk~=T>(w#O5zF2JSy-b2$Yx-9>^(BeMCsc3(ZZvrs*B%e7U1 ze~e*Q(aiLR=IH1Et2lC1pQ(FsvOf^$pIq#|{cT&OkP`xzR#z_t(w>Vu&vyrU1}F3M z-tB$vxWzlZe58TqvE3X?dy3hZ)g;)=3}fYmonUtEhWRkJlx>}e^~KQtgjCTl*}P33 z=ZuB#03NL+m=yQygf!zFXg=j;tj^T|n&p;I&tp(Oy8ZgI?5(U9SwuUj-^9y0d?0S4 zUCRoLDV>cMJ_VRVQ(EDSDatJCECImJt7ub+-U^>0w1M|7Z&3Jan+a-V3e7H9W5+y% zDpjLo?KCK(rDGcFMl0|FVva#QlODbt(_>`OdwB?}qlM8>tM%I7piXO+7dmc4`?$S8 z6-={-EhdIPgGL^23ipOxrq=76iRa|*nWLlP@27_YwAJo3*@Dg0Y2NEx+N+rCf|>s9 z@Q8}0nb>1@ny1d}9P+ZBsa^YaPB8z$Co}K7TG`zn>MEC1#^1X=VlpwbV2x~Fn2)1h z`vI1BZC_mp7MgL13k<8GGj1&mOrb8I=o&Jxa9SI-)X!;g_BW`DQk!;8*}z3&aNyX6u5eEKIoG7NR~TfS zH~)&%hU~$0^lI~0q&EKrn8Z5jsncu*9HFjI*2wAx*VdKE?dUmc+TFKHSy)47p6T`=LES!u%2qSgn#vATRBU3+iX7}Pv6Dg}DKHf*ma=Pp@ z|LHcQB+;HOpGaGH%StRINGUZMHrZW4W#HV%&UY3UJwvN^tZrEeg}NtR#aN=MS*WVUvkWR7OO!Tm$B>wl z+EDN`I(rwv6`iL}YsbR=zKymcY#iNbzqaQn>Fm2C?BW7CW$>AvmLPJyxHL@zcipwOg|@m%LSLwRlQzJb&xAwXho&^dwQJk5c@Aj4aR?xCNw1m$Hp zYHc0r*)dR*6Z^B-eaTo+VgOlcl2Ti`7*!eE#+ya+zB5sl2gJV8t@>S}^Ef@%Js*wE zcMtAOmCi3MoiF8%PxY?E<14*$w-l;ZckjMh&E7End!Cesx7aP7TEJwb|2#bAPx`k+ zgS58RaXi~P#cYZ* zC;{fTlYG!|shzDnF6oYO3ZvJ;1i!zZXy@*ZVk&wmQFq6b4YvUA<9|=QcX~U?? zT9=z(Ce4KoSTU@OFe7N1G3p3_!DwUlX`M-0*KnE;WL7aySqmC#td*?hjB(Cv-pO*dZ{amnn%-+kh6IFyXU{c_Tu8c^w%O`+;kHQL)X zQwws*?p(|%#j-s~n)6pABxVLCTiS79&YSI%^-tuq}j-`DY)eZ~)g zj{&ymiRxx>s9nM?JuqiNb8DCnN4n;q6=*24cw^M-2mc%}7aj-Thb+~GuGtuTDZ=ZiY+{RPH6 zZ=IlNT281xZp4Vq%1G7NZ`1KXvJ>VF**F3>dVh%1l*rC+LIf*n|}VGHI4gfn$M zpi7d@2ZZHL^HdLog1mbj^EJY_1GYuYJ*(y3PEuB_19mEp@Ko=I(FwN)Xe(S2E_JJB zOvMO>YJh5IK?89ec1Z1`j%%;Xb-Xgr%`@u|6SR# z#@Q+3OtnM91^E*EOj?E|J_71-Hj;dYbA~MFw;t-gQr>}3G%3D^rST_t*r>_ zzMZUxP_QHG&hu_mVn?i3BhmQI$3IU0S8kvX6o95s2;~QI7eCW?ChSP`%r%;G^_U$J zQS6c0w!Xe?wTNB4b?JfceXsRqZL6P0mSA|U+1wW5Xqr>1v1mT%wpiT3d^9r|GO^6? zrIWjYjMXNky(}*T0v?l{x4Gn$i~gU_e-ylu$-RZb&OB(VpX0f`paO%s8$MW~|G2a- zrvJUlaUcDUS6}@C$3&LO#lXX_FO%u0lwnLNB3#fK#RoM=$%BH;}Zqo<;Ub0$23_&+s zLGE40S(ww39tVckUZkEOudPdKhIH7F&KS}~L%Iz;XO_ISESPZ-kEEQ6LGqYQcERKahXA?pppV?t@SgZ_X=_`utJ15pS&Qjj7ZLVNd z1YehEVVrhv{osK^Jq!yzF~;Xgmjl+;Yf?4vH$HVN(C-=NZRYE~oSPOM&Ojg_Z%r%| zDun~(cz-}j+ibMMIUE`4Nyp~0=|laF*x?o4X*=NXTJHGL)clV=^r1hQpZzw|&5EVv z866@aMgupgVGDS@EDp=$_FKE{CYCn09&i1?Uk-e`l7x{Zr#vPbL)$H;utrRp4*iK5 zaEUf=zsfV5VEr9+Bq_zck?>5Ugx*DEtY&v5U=r12kRA~fYnKp`-C0!*K+>`)DEa~e zN>-Bd{;1dYC*GXo4>W<;VFfIb6~OzkROR|B?3>7*El}^Lo}#;86~l(+!em^_dEp4B=D1jY|;z(mhd2el5ALXusRy3pAdHL1st1`TPHoPh&j;GVzBXaTR! zdp0mI?csGOm4mNIk?hPW^md>2XoJKPyh2?d3$c$nYGJQ4`JwTcn zUT0K-Zq-9}dhi}sQmxe4>1nfHB zDZrM}v(dl>1EB)|UxLrMThd*45TAR5cn%@746KjgDs)r570wwZ*BB*R-Tbh2XnIzn z>Fwyy@<7((al6s8dW~)hWAG@t#|5JW;30rsy=4RcjdQSZn_kKWgkJKasYLC4s6!ofZs5XxkSzx;UK{xV z;TYhSgdCC@y52!DSxk^u9FeeDx!jPAJo`yG1Q$vUvs>C@9Xm$o8M7je zn1a%gM<0FkFP=gF^{(B$(&y{Bw0!usWU&7|oQ3P2uEzwEJw8xo+{$?E8IE0Mxks0f zuE^SB-E;j)m#;gMY*lEMhx{QYFPgbzm#TIp(MZ_~K^gDv;_?{k^T+csLZB zj9$3q<#XrGjom{37YwOov@CEdd}^5o1`C!+?QCrq{fd+IRTPIBpYGn7jPI)dPS;wU z?oW=o0Tbh^xy!LEQy8hHWvC7$I!t!!(;9C9*IHE4#YQ^SET)xT^qtPQV^YsYFVGyj zi-XF8cT-jR5ju$!LBXgcj=l<7&U?M|BSR-n*N>kXxOt&(1{qSb*B@t}V^w5loTeR+ zq`{I%_n?tXL}@W0T!1bh>xgmaO5pcUHvmel7j1$)sdb2V>&L<|T^jd2Zh4oHo52uBq(zduf2-AW=xI`I88VUh=2mYX~bcqxLY{j`h`-UX5hSTdw; zLuwe(ERnD$7DIoG6{g-v1E8-G;1y6HaILxNGepVK(=&`fq85-MG#Z&U)PwB(m-<3D zyihms!Gfn67fgj|pKrR590@Z_Y$RrhkM?t^Qg^^q*y^`>b1Ow=VgJS1qRXA@8Z9iJ zj6vUH`L)G!yGEk1(OtJJj2@h-`+Pl92Syne$L;30?FYS>m|b$*i8>C%5?^x!#XwFG zypCUFt0AYkr<-FeF6WQrlsr%od?Hx9(qIMKm@oO8{2nI!82QL6W3wU@Tas|jRJ!xo^bP1Ra z!Oe85gq`-FF9F}y-9(1;O31we1UEeGBj`yUGoBVw1wINr8}QtKXL`Jfy=7xnP|I*? z$EkNQtu7KPAGpZ%Xu96p-BrEE18*Ai`Z4M;((8{CT1}wt9F;M;Ti@G6_rQ4YInst- z5t!70Ee*WPrz9MoI8I(2?hl~2n>KK04(5K?Vh;cgy4+2U({(lsW=Eq)PXzsMM$x8v z3C1wOvp8g^^4i2syapZQm+?l&;-b0kGHAtsRYOl^>VQ&(N$n*9tollRTvf;OXyD*1=poe$6vSpt++#g^pw#Y=1x$WAu)(cBu_HeALJ~Erx_Hcm# z(uvj!4?q0yA6~l_>)9C{KC$(&t+}y4a4ZYu-70P?YcX?=p|OW#fkraYsCXb5rgvWS z>O~Wtf{B~u=}%c)HZ|$yokHtp!KfnI!o?^@o1Eg0#jrEf?KexpCmy{oTFYnDpJ&Gu zVDR8Mc{uTlVntKq=P)5IE-__x^nwx>5m($#!bD){SyR!W!H72G*=b zNS6Xv(h;3M22`wrT10Gupm?!EPa;rrfRyXrEWV@9gZicitwF|oGVl-_a8_UiPTA)WJi0!s!TO3r7MLfw9`HiIcb8-X$k3qnA$Y`D$zL zAs(&u_Ws2i8eK=1hXjV<_(SwH-FjWoT&+m>yUM|RyOL(NOENi!w(rW;>Jn$OnrOz( zaOU2%>9R@gRYMaC``>xt`RAwhCHoRz3uLq+Y7Fg2hQF2=jP23*jOy1PW52|9BSjW; z@*w&K7XbZV)WQN-k;G*huo0)To zd75yv>SfjaM6>15--@AkdN(lFY7`<58#psQxUspsCdfLCWRN^6Y<2EOI0y2evqT1u zkItuFBwf0jbX5#(!N)-?Lg`o-;F?S8fOm?Lg$AAs$Nk&coJP@Bb9{t}n~5jWOmtMT z&9-a9^~SaEou0}GjBs;D?G!s z_S;5#9y)mV!%b6T$miSAcs-WV+)&AG4n`w_jdr*)^k+-kLZNM?lh#l;V67_M>Pxmj&~L3MUEaGs``O87 zpZzhhYLPJvA=E-z^w0ns)c~kFsAzwql0bM?FdXCpga5NKjrR4!w|SOX*ee|8;%_TVx)=lO%^i~_ctU_mrEa**`{^a;4I zu#t)D{$&@%FWE%~6?FRlLy7&>%P-bq|IZ;t5N{LW9?>hw9sre9$~M-*@GKMCq9|Kp zf9(%=0xrARl?~ec*?~$Pt)MVjMazoSWHa0HlL2R3v?``#*Kp#Uf3I#02Detxs$0UV zZdFmLVPD0Av{r~9@=lER85>#?|m)<1>fXaSB@d^~McmG4MNrLQEYB92k@ zBc1mp<{zjK>al}pW?R87vLT{H>Pr>$KA5}&wB53SHP$hh6Tz~9viB8@Oc3jQ!$e)~ zqAHX_CiNKfs7cSJGEh1IAdM49^lUnJT#mD`z7L#gtV|DwOeTAH%k)a4XV2_#h~u~| zjrXrkrWfXCVl4+JsM(PNqjEKv-??qXY zZ|n5Ldxr;tL!)ESFViM>a&UEv92IgUM&RltDi9DUhwUM1GBRjlTI{T zm2i|9wtAc^CE;YV1LP`TQ|i)DA_4v%eB-?C?T)uu4ApkZJ;)6_L6a!1nbf+d<6&ji z6MaOQ7T-ma!Wy_!=JmohW!#pl5A^rA`&~VeTz{bTH8~-w-IW}gzUxiZY~$VKQrhk) z)T(Za*&0p8Yg-fts5Bh4*ova>VZ(ebU=lHgGf1YX-D2D zwTnybCF=wv7~TS78TkH-1d+H(ih~!41_R3xAR!?4<)#gIs8#AF9B3nY2CiI>#=49H znnqP9)J>YbPyoK8IFz=hSKb^I(k6XYBPi6?iuRuw@)xa_q3>mUf6ubvqTel;O$>9G zV_HAhcW$Pv0wUetrOGyw`Dd41m2%Mu46LyshsfAilZpK{OzIOk5q)5biTx4B4`v)3 zZ!x25AbW_97vs2k37~I@>La-uPAaJV*r31(S{+y?N%D)LV=>wtQ0~PJeRfIvAQaXb z*(%^-@B}tWNUYotsL^Jo1tUE3%V;j(dt!;I5;dd>I@hh;wx4u7dxxI6&|$CiN!aNO zTb!Dvp#$-B(vaJAy1>1(xq^h-$P_erx-!r49HcS-qZOUk`d#qB9%XQ46y_-EZs?aSptbmI zHioQ-4CKQDz#Qfl3TLXofR0z&6+i^Et;v{rY>?EanId)qY;`z@4r9hlG=rq3%_5OB zBzn72wr)1kP-vEXg$UioZK7YAHB#V5G$x0RbLnp@9=m&Z+pe5g@XlYz1d^gX6!u%} z4E?`yiEHzTL1kI#i8hYE^Dn-1Bq;e#EmUvZHzOxlSAMd=d)%U)BE#rfb|XTm!@_1e=4Y5xp~MyF z^E+V%4p;GL<+PGu3)-V3$Y>oSo-~(H>=Z_xsuf4VEpSU++DatAg?Zwp?Hq_zC-Z$A@gM1Mcl6R^k@U(KoX z&K~FoC(yqCm$CPNljJJzMJsmh>ZIj5c3o!K}$o0GJwWp|Zx4oG5! zA_@p3KnM&b8H|taDVJwWf@aAV<@bmKn9v5HyY-~TK_noiKshXYz-}hFksjp9k zuI@hJ3;*x`F<4V$u6-#}4vOYxCr6uQsk$dMd}4Vb2GsQCSc%My#rM`0EKnuL0zl$KZzK_n>mx`mL%GLE<5($WA5+)CNNK>A_Q zHm-5GKo>}=&L+qM!9JimLGe>$E#*{w7|*HN8@JD|A*3mm@mQ=k5?yM7rj}FR&dr-b zS}~u{Y^g0Z8aw4<^b>Erb+mUep%$BGLVE{$7E-{H8(*T2Z2mdZxB2G>(f0?Ww56SQ zt(|>*UqygogsYo>PFue`MYEhq2?VVVB|1E4xnq@j{6T-0$aB|ke*2Ns{u2Gy?qWV} zf!_zztk8~pt5n>X>!A$P)RjlsSI|7=r2Od1uT$@*exLd&xPTu}KQ_2vfdLlSWicfT z)c<^dxNe^$68b)$tzZ<^lLRAk4RNoRNi{kXbu;y>dXb1)$^;~!wRMp;Ty7L1_cIkN zgQe}4+Rei{$8L7!}0kysmJv5->*y0koUuM($8-(!KU?$J-&QZX>U>^ zht`Vb0It0=y^UBC+$XxNtCrY74egb5+yr~3l7^slvke-*9sF_?9~NCRIb1%cJ?3$T z{3%Dc=wcmABD$FFiS5d4o(t6@&2UG8nKd(lttgsPUSBLT=N40~u4qOy8I{43oUl7- z*%Gulg2J6j*6vITg2V?BX)!vI|B6*M54f9wDc)+>ed}M3#`c`Lz8|83$DyO#_|4&w zZ{C=nP*|%~^d}ny*61*wW(ezkKGA69Oo6~uweFg z$}YdbU}r>cRE+mV?Z~=Ir}yMyR>j62=lH#_?7ADxy35tkY`STV+ly9=HP&K~Qxa|R zcNFQcl5>f^nh)4`_G`wYD!`giyz)Bx$7}}8ws+AX@EXg+NwPwqUB}gae;jp?u?;}9 z$SOnBNC1tMEK=RFXnoHjWj>2$=`6JeZKu$76WSg^+jD69INJUYZQnqfeQkuNzK?`| zL>se)BRxZEFk2=ZRQq)4ERiUPXk)D}rjjng4VByu;clj$HX8)A=4V6Q4 z@DMddou)nuY0nrzISM~j0a#;ktP(i5A+zzQu9gD@O`JL@Y1ts->yYn;TL36{lpNEd zh;XG^IkZ>x0d0YrM@d;ds6y4NUhLPUG&+Xso6qZQhlqsz78v}5dI6_npPF5gmLLM_ z9j~(v5_Wsga5Q97rc7VOOp|D!`L*6rKKtlrF9}xW+7I8de`+8a$Y(NMSJt<$)0^v&aFVf$4ku?7!qp%{^Ja%ESUG1tA%?^SH(f#o;!z`Q3eSaql^z@D2~_)U z9HgI9I_qUA=B(ALu05X|892FbA!!x(`ttFIP7fS&VGDs}rJ=<8LY<+y8uvJVC#?ZI-pC(bm^4-zB$#AK;g$zV;mJ+qs)cYbc_?iYO34#dNB!y3~@ zLM@lW>~G#Z79G83sE|KUid1(OKDqfivw!n-wAwJ7tFia~W5*ua*D0!n;F;CU*Xe=4 z;OI!msUuDSQY!E zd}vXQ33Y2wqW^gF+Kc%eC8iM`4QXZO0~4XmcRlG$&X=X1b|!*ugP4@~Y}8%yIWumX z)096qZM885huIzPD(77OpfOvJ8fU%TUes;Xx&jQ9yK*1z`QZUOB&Ow%SZ_ArxNfA3wChR^}`S_82$+;s_Lxo63rE_fmKyzhsFd7?}Txq@(nF#Zu)4`gJ z4EhwwusaoYbM9`ZcmJSJs5Sk`=q&w79hsBu6IGMYXf-6PDKPH=3a|`blO1q;5oX6l zkP`E$Z!iv*V(}EEaM>@XWK%G5A<-0~Y0;SJoh!n{i)s;Dl~!d3%&lq+q~pp9>>r|j zJ%HAx2dK;RtKf;lc{vY4l5GL|-2mAGw3}9RY|9%5-djkM1!Rv_`U~1q>tMKOXM&|) zUk4;JZUJ*eJNBT6>nn|*cN^DRCdhp?zX8a&^f1h)+L^#`0SKOJEBGovZC(~JwDGcP z(oi@;3=v@n8Cd8DuDYySsvObgtXbV$YCOpk4cbE&L^Ue$JkR{(+o>VH#p4S&QU?n@zahJn_YYN#9w{;B7sGy!F~K;^dw8~Is^GQq zs7EU%(f&9xQueSM{j$|7C}qj&wzU2=mh(yGU@<@=B~$*67?zcOuf=T@yRXaVPmHuG zv7Tl#!Z?Fn0_YizQney^Vm_ByDJZPTVHaxBCdW>lTIdZVD@M7f$c<{MCt~wio$^3L z>YiH4p>?_M$_LorXF-Z0CGp1qVZmOsDk&7;Bi=OdpltW1{ns8C9RCJ|(MH@|;K=ME zdgruCQEMVaFW|c4pi6)rks~pzXmw??U#r+c7Hcek&yt>ZoEcm$A-#y>*^`s&#&iV;R-_J*v=tlhni;bo9Db~${^;Sm-k4tc-gD1=Zz;3)}#U3}^_A3^5^Ue9u_cYHPrGxG(_Vev~r5(Dohe#N3e-yf>N_J&d%hdzDu6E`{UAjq^9w!noZ%Xy8D(;Q8j>WoV zNXme8!A_M!Lp6UqO3JgXDsOuQQOrJzFTf`ERk2%Ee)rvdo=l-5p6;$>-3E^5l|r>U zk?6>!)cicQHE!f)FYWGkIUJs`wc8nsJ&`NOQ#*IJ-ik&{-rP{hUJL|#>v_9WY<)@a zxFoZjQB1Ve80a}NSDS9;oIFqOxiFEm2^KM$?MTe*l6afRRW6rZ{(>M1ydyr;WIojT z{0b>7&-PqDKK{N2i!%&a&%pR}Y~r4;N=3=+c6%)Tt{B5wWkGYPqS~Z8Dv*YQ1?-fQom1qi{1qPja8k!K!F6}O$V95w51Z- zgQN{|5VL4ris||+E=Xl1Rk2KJzl0=M+yd?xP1;SaL;bU}f@5c2P%-1X)xP-|>Z_DS z6$v3^coYqu;SjgN6Ed=_zo5R^SBYBUXq#3+)<|&F2VTdx1CWzMs|9Q<;CVixrcd5T zeMHj0kzXL|{BBauU;sVcwd7-HZIik*qhGOCZ#zJw_DTjAVXHYIt|RY=TeZsMaQ40A z`G8vyKHFm(0A2bjlJ*CR=N;!On_JM^GN;- zlE1W$_iv*8n{9PO2=qw#uM`ffBWzPRG+U6qN`FwZ5ZeMUD#0qIJg<`!VK!mFHS)R= z+bXbLUFtw{5MR~Kh-SBA&J-3}tCdwlH(ID3XrciGO zV;ACS7e83xMQeO2$tc}tvQj{ZpB!wy|FA`I{@(G_Le?(X?dCu>5s_TUY-dukG3BV8 z5jeB(NT#cplBG~A?F;5tcPNw=KuT^Sl$?#sE|$$ULg zu-un*N~!u3{XL|6WaSR0M zrw?1*f}C_&{f^ep<5|US^c4aOZB^uV5yHG-v)X%c>ZLGE&J- zQ1{Y%!G4m|N$U=x0iGZx2anpfY$y8$R4G4P*#fADG>Dp#A+D z1l0gs98kmMR0ZDRB!x+EG!z;J=3GzwM{WYZERkeA9<9=KNrPQ#N(KOO+QB*4kp?yp zJ{torussX#hqpEe#Lutr0FjJ7kg;ud;)*Oj3{`ll1zwkG)lcuv*g? z1Gb|k=&Ml^e8N~~w^!-Zt0}0{1fI(@Rzcui`u~82|EEo|TX-SvFk20DvFdi$B9VrR z=L2Q8!DoF#%tsfMN{IfO4=ca-_Q2tE{r-b%4|de=Uvt)`IG_+YqtUBG@HB*fWEW_~!oOYM!KqiR8B02+ZdXiyQA*7^C zCR+bJ&>itrtn_y?Q|WSYG=1R?I!p818gJkY=+6llkl&_xb`6Ys)Xwqr?Kp-*JDraB z=6w%6G&z^-OFR>ZOAPSrFhV?hdtxBofyQkH>7rj{6G-FhL3%A#BxCA$icwAVx0iJ( zNhIRe(Q1bgdr|$JbnQ&9h(mrL`WMUh5-4{+Dh>bnorxc{}^?Tjr;7YI-_}TJrKB4%HNWEc3n#BVL4wPml7>73yb|+4b z700^@lDF=(=IXt{`G5I5BUqmMzrMEf{$_CW)(MCYRIj{%^qZv06|_RFQXi&&@Fw*V zWcH}JO&A<>o~Z6d5;suDWb%ZFbGNOieTy*>Tm(j!T^Dh?wS|QH^yz+J0J`_MaKX+LSh*FZKM>@|+-C&=so6gV1*ImkfGe38g7WWdLGhnk(L z-3^xeS7GiLtY8Kq4zNuH@cn7!B=?bAR9@ZR2ubCt9dnSxlyUL}F!`8o>+N4br`Ym3Qc0>3JPd>>kymLBzpc2?Q zlJCohY#ht+d*0c@^Bm)FSi{c1z(9}Ji!_0d!wxqm~@$un+FJ`XPe7(0s_of?jJ86yy?<*GPLxA@RE#%Al!f5Wf z8OkQ18R-jVr^!_iP5wyOrqq4twiEF-(| zHO!L$egBfU72DDHE+K*=ZxBj8)sKjY9tm5iT-n0!?;urIu2nw0_?%~>Sq;Ft2*?XzS z9u73l9NHOvu)LT_-!e6ETRO6Pa?gXc?hjpa&CA`*#}1C3OqXuix$~A%`oswHf0#@x zdnP*MmEv}#FA|${OTpiKEI(KLmA~#Oh8^i_PRdRN1jS^uv$WBZOFSE$iH`a5$(U4M zN+@Bc)H}1B2b?~~@%1yCU#D4q6=d}18%RC?@&O*81?n9AUNlD=RK~q7WL2G_{vMoD zHZnb7NEiduXX?+v6N#kFZ(va8Gzq$53A?laxzrBJ1bHubhXLw8r`nnO2s(6%#8}Ma z&F0iPQt)PTq@on4s+zs&(vK^;WzQCFWX@;{ts> zCm5utAeBTVVG4GM7J749>WnUX>S6k}wFm3<2iG|6D9bJ0TCU!*WA0L+!f0d?lYe5dQ$pafblG_zV(oKjAnWrwCnnANy^zek*F_-y3NoB)-;( z0}BUBgM?QXO0`0mbA+dUgbw`-ZOqz80R5yQ8tS52hPVf9Gick7wrkLKE85}Ms#VXF74N)^F-Q)U}XuBmZBF}Qd4d<##vsUZgjJoHA-p$S6y2pS!=T9_-C1J z-omg-VXSJcg@+c47K7Or@R_aje>d=uY;X!q32Io{PEKgOC{`&?S zqQII*}i55xSCK=BFL4NL{gnn!j;+;U5StO3d)7Xh42u!r-iu7*ZDub+Pn(eU89yV3q`Rnyb%d%rF{rAI5C*V|qq60$?E4afej zsKfvhQ>%=5REU(iO2c#sRPoT?J)pCx^kfplloWJV7vOFmO)z=70314o
f;0rx z1K3ngk_?m-#LuCR8MD@M=i__vb3QmYbw1-z*Y`FO6RTfXS*IohF4zdYf%A}SNxNTp z{HlywLR^?ItZRmpTGq^S8ZK(FE)JxRbabp}Zy%*N&xJkw{nubBv&CQus+ZKa<1`&X zKYHEe2n5XxiwfE&XUORI2AOSV8O!a&Ew-rhG$LJEwd^k-@5GrjKg)s z)(ElkTB9MJbeji#(%soKJqbT4-JN)e!?Z=tpeGSa^~t$#sGL9Dt|m~)pvrlh%KGYe zRTy4M_q=|)62nr zak0^8a1C`g=L^AbEka=Y0Z%{Al0^TjW!(zL^dk5b32P=4XGt~uMN)-wYuTaWBz3>C zklSfV1YvnCv=X!Clg8vbBKs(m+uZWGp4qa-O1sqZ4y1f%SLcOhUbC%DKaugDNXJv`%xEq;dSxa5CmgKJjjf^f4QyVy! z-noA(v1YB&l%wm+0&b5R?DiTW4uho}YXJDl;1Ev50TbhFZWXJe)UmVb2x7?!dyv-E z1fQ*MaN5-|@jeW#d?^f$%49o+#C>WT&pnt@jA&?>E9(S#;3Z?Vp83=83s_BKB~ z48I6t(w&fXbGFTH@BaTLlWg+jy0Mx)O4L;7|IoHmcGmJHV-{|fP-Wvb{$RGL)UscS ztY;W&u%?b+F8|ACu})mN2XLcsrQ5x-D~0(V@$Xq>_3Baj)*j2E`_QWlPMX5f1{DGZ z#4i9Yy*8=c30F2Hem1!X)?2lkI*R|k;Q;;eyV8?hIj}H-uqb~*N*Uzysk|(=@p{-4 zWh|_6d&6*V4yCb`O+$7D*Elcz`jz5!MS6wcqVNJ|vO-NudwzD=We;9)MH#1Lr!RsZ zfWge>U0%W|D-;{0r#Qc(IHy%L$HhYa`KwYc>dK8k<+EPajiNFNJR+&F9%nN6Z!{@Ax1y?YD1sPjnZ=Gmv;`|Ke5J( zjIk8&`zWVLFP${HCA2v8Hms~Q3p&2X2w@KlWe5(*>`tc%66F^=tu=y^*XS%-rmkh8 zSl9I7i!XxL+6%6Lf4FDrpU$Sb8h>AJS7>2KAe8~!<-a%`bz&sJ+@zy;1BdhY6$+e z_HGTKq4f=LJx-zU;AVP%<5+og_=nmJ=wFXC5u8D*IlA86MX_()?H;69@lS-)yEa5h zNzzm^aZgOsYSITiu8E9Y71YPU;3hhPt(6YyvWX~0V)VK~J4vst$_RyC0aDY$&^@fhy`r1l;L6TE;{$jYy#>G9GmmQc8)94W_yJ&LPKJt#$0KO3KE zre@{XTG4zeDLkWiwN#u9@v|g^@@<%dUG%eByU7{7Zw~l%dRaRvJR?k##pkUs%*!)r zzbpw_g+EnV+da8DgNrp7-v_Z4Z3a@&Fto}w%%!qk#xwPWu9c-#Fm7~vLKeXY6Q)=l zb8uS=$=M9M&3VFZ>G;t#*e#Fe0Ned6Fk3hj*F3&UXT+3u+nUL6-$lXT_z{x7y42lR z64-qpAj_wL`r0(ky<%upK}a%&4#k*mJQ(WQ-9_JgNL=uM$|A5`yI~A9B<)E%bq+(< zbXfGdOo0>JT(+O#tu~_TDyPGx(P;wCghylY8g)TW)JJdnB#JRo=8}1fWa`|zYgL0M zF1&2W$>}ZBnUCIJN?Y@|z_hI&i=X=zP08|UKal>>*h(_*SS%i;(@v>bhTn$r?)r9z zdgxA_abeD2;#FH4<=%*WdKvsGOemsdB^Pi~vuN(3{3#0z@F`@4nbBZsR)_P3VJ*$QUfB`a$Ybd*Bd{m9Fv>@KHei$*B)tkDSAyj5I4iL=LCWkFW zDLo!q%UZyPwP}PT)!z*C0F9QeZL9-wOJ+h(HWdfDsvbQ2uE)uv+|`(tuU~gHy>?3--e7WjB4HR_ z#@GRh!dGOA81rlW_O;?d@!K1MG-~|oc3Jc=a%>x@RY{n-9pIKmlGMO(iru{;k|5X< zzh1w)AxP_6So_Lte88yJV0vgY-(oURtko>uoTxP$aG3N@MiNFFO1Ou6Y%ZX$=R77x z?X`kYOfUi(_`mnPBYkw+*?+!_4xvpquaKS=ui3U#+th;-Z+@KnEWvB2EWe5hU~9B` z>rnmPn%a~%fKs^fh}9k!9E^TVvW}pZbN~J6r>~lEidaqdocHOc&jdnI8>dXQEV3k1 zc567{nEII|5%=<%h79a&aoatGMrbsl@i^VZ1RUJn$QdJJVfnsWCH($FNK{x9UPY6v zmxjpAP`LC?Kq@`%tF)e~+ra9q9;yPj`4rS0z?N8h>0`La06Hlg``JrTHUe{M1d(53wRW`iKqbha9kO!*2SbPnhx_*TXOMm?$`s}-F z@3){h(il_*M+w5*@U+*so+KS6FBq@sWYl6pV+OQqfM6}E&P>O%P&vh*-D(JS2}OlvUmUeqx_1`AF-iu+SGWBltE$U%5xshA_UB=-`m#BFvaFT z4@yF)^OV{(JSX{T zHj5OoW-w2QX@^R$IoUB=`K{^GhPqPgpjr?6<=;QA4OCqV25v4W3k-LITm4NGt&tEs z`md!mX*W-tDUG4VCzjvWCX@OFBGZMMBYB!WY%xJ|ESUuHVe+g2!LJ#9rC? z1utLEvO8irh2q+onf^>^1iq2)@W%mUFz?z@+SY|ucJ^ZIOFgT*U%1(4Mj+NatyuS% z_!<0{yB7gg{L#R~g3m^Ij)t-1k+bQ;N5MfFsn?^(I1BTfzdmM;$92z0!FqTh&TJj< zc>znB%v`a!+o6r_>Ds=!uKjlwlqRvz(oMY{rT3Dn4FvK2gvO#L8jHv4lU9T*{p|AY z?rSBF*j~kY#l8~D#CXVyDxZiZAMTttN;;E7r#bFp0GM8QC|j^JLiy3Vz?k~B{gxR2?O4knQs z&mztev^YhER4QbUp#(=Tg>#Sh3n~;Kjo)|>Iex4*ykVVFZ{^tFZ~{C%HirKO&@qBj z#@o{>nPeg!UOl1Wb%u={&EUpDmsI04w^K|akznpq2J`13>qJ)Zib3O zeuEIUh%c}<(?~P9ApYm8>>3k*2EfU?T5=RQ$Z=*^9?C?~3f#k+N+iLdUsOgD4N4ryc#?0jNSBkfmI3{3!eIu<7Y+UGSv2dpLZF`Wt zP<>%g<4R_NfYugd(d;gUE~MK?!IUIN`tpF+t5c4SNvqqs+3vEAUbj}2x@*vdGN7>y z@Tk_pcJ(TC4!^fKBmQlhlns4(T7q6Raol9dXxlO_rAo#}?DTT@C8?fIC{~sF-IaFJ zTrJ%H)ZPOXx=HD?YOd-+b*Vru2<9xFWoA%UQ_1KWa7Sz|gsufpqkzF}Te)4W{)Ky5!~>D@XE z4Z42OzxG2#c^~6YNe3@mI=k&No{-hZKJ!FopUhm#UtL*^c}Ax>tuAe2;xn8x@9B4* zWVwba^9@#`GnSCw3uT2H7j*3ZlDT1Y(W6x=YoDQQiu*J4b{NXC4iY)e& zxS2uO9}bogn_JPZ7(K;FtZc_zm9Z{4eCX8kndwzIp#(44#V>>Mb*I5D)c16EkVX{V(xaC54MQX%OWxyz_ax2gih`~b&6K?zAku_KED z7>%=+_75@a#mj2@;!fqF`fI$pXF+JW=M{Z-DQDAw^uO$*-T~+@Ds`dF~Fx z49xeI1}NE?H34Yu_@tynD*=x3@MKnlrgFjM0Sr7(X1szx=6jn$Xxam)l@Yzm8`+I6 zXIXO~TubtYFGMdhH1H6$5^BquG6tI|?t#DrV;~S{qbE<0E?(!;8H_45 zA6VDGZe}zIUTIfrY&-T5&xGA6Dx6f_|6IwE;C&h$7M#>CLftsyqMgAuHGq;m$ntiZ z9sP5kcoT>~c?qdrs}Er$!7r1WqhlU*YG*teV1j0jRBG3n0w|VEuEg*F0yhr2gBU4j z3IY|);1sG!PWDFf3y|SuE^TiAc(#dJku_Rf=Agc&Eg~#lVlvT$Iwf8&2iK$xqcr+> z10ldvB{E!U#l^h)?q=nE;g)hh<(_}2TXQ+{D_f~;Mptl*IZ~U)*egNrxenffo9)fO(AZ20 z;<^Jzsl^)*FkUn*6i@j55i;b~^BgAtzRpWvO-o;to?&5STM%Oi<>hjW{CBXRiFRaM z-|FgPn7@AeItuAGFE0{ltV zepie#^j+cq7C?TEcEZ5??&qU9m0)VrU=*4^UsEitvC!h=%|e@yF9aO z$D~!0{q|4j2wM169AgRLXko!Bl>7tX4Q8UgMWLxhVZouYg=v5}qg=0(#(TRFkdtn6 zDnn6(Mlr)L+a={#AZ3KriYrg25Li(IoLe8BM8~WA@aX@S#BrV#DHa z5@W@COUe2`cj?n|crnlE8{=7mJ2no`v#)N%Va@YDJ7-Z{>!9CdbafWIDA*SUdj+* z>@L}<(w7b6G|8^X7YRl@`h1vG2|-@z86IAK7QOMAd%_5dvRgwr_iviBcKI8}(@kW# z{&G!Ez(0WTlK$(6bW(T!hnbvPSzWP{6?RLfxvH>RGMb-1vYVYeIV;IYUCq&_G|7$^ zd{V(xKRD=VrAQqF=Y=*yq|2t}VYQy_+!pUJ)^c|}kKT74nQBl10v-lI@kgevE~j9? z*@F0Kq2CjQiQMvo#9cf_QZcJGEcem8jh<$TF+<4N=Ok$c>D*cq#b}J(@3htmZowx0 zmx>Nt;Oz7=n6`;CMnUN1$2`jHy$`Vllb)$-UDcY4efHCzg7-Zwetjf5(qSvMgxxC} zrbd_14~ICmUKLD(I2(miDU!m45)VYi(RMd_9ZGam9k}~mr_<5ndu?Gm3p}H=lrSF zZ?`J}YzK#}9F&(%zCQ}!d^5)VQschR)j^U*2(DB8nh)_9Ou*u$z~F3@a=3Fyl@xy) zvocyCrc&(aO^-1 zV=T$$nEEpC!l;r_!kDG&>@%qY=ee5ukD9ESwl0{}Icw}ns%RDe)!!%GgpNLf9qFve zVhk?)&S|YtN|o|O;^it{#c3DxJyopl_vdL2E->xxcHYDL;~uB@QM#Vm|3g1TqiD#V zBU!x8Z%MgWr90s2s=>nvj6}$l4qL6t!TGEzrAC>t2XA-!xkZsu_IgFgIzLx2K);28%4O}lD<^2zTz__gVz~S z(u~+|X?}8&BO5v=;;HBP8qHtapDm#)C7twr-`(J8Bdtb$MN9EaB@UxT6Yv^alCeeS zA`NO3M!@}l&&j%C5BsfN@0&uG9`-jv9T~1Pg^7aWnHGW<_fkgH_TBX5*{MZa%z8!x z3yEgv{+bq&!px@I^PA5e=G_jGAv;1l>wY`+`pSif)}i0rUIedft!rx=ASpVAiJyy$ zGr>sGu2kU^2Of}uS#vLiAI3P`_U2ml{@-cGE*=T$l#92llyinp!nqH@R7pu`x%3d_ zOh3!xFPUzf#MnM%9zLtUnVi?np1+KT%Zy*>0r5oA0;8*s(L0a8jm2T{p`gJlm8F_H zf^t{!31lTj84JlA%H9^Zb*Po$teOQ&e%#c7apLj*0YJLj<-34nbtQK1tLievx24vo zw#FtNXelb2R=5Jz>@Xd{u9trDRXt8v!cMF5?F4!KUFW7r8X$%Zi@O(KR8M0ph0`ea zGrJ|hJGp=Fm%kcF%EKP!z0XF|UYsUWd+Hh)>iU*mQtrTl5pyyTV!6TiG4X|U`&_La zy)&d7X>~I^WJrSkL?Ca8x<~xK<(H%rfHA9Ttlj~C(uUGlkl<0G=KLP=!ZRrvp>BLa zDcDw?I8f%SSf>|C&zh9tc)1(it2j$>bcWSfzF?<`uGlQ8v8*_Zp1kWZ=T>{?jZSCx zx95@^XTGJfX(@Z3gC&e3dg~~8Yk4DQI^wV^q~Xu`>u_A_;NvFnejv!1QQGY2q+8i>DNCX- z+&zpVtIM2<`%_IA_bf7DD^j#U*~G!9HwXG6^dw3hY_S$rUKd@tr+C- z&eOM2Y3X;A>Y$_3OW{kQ&*e=v z2_Y)fbA89E%P6OjJU@wptJD0ftJfAl$dDB}fm2%TLKO%%gQ21z{xxLuc%#5znmVC| zi)s`>CmHDTaDUk7wdPlwCJ>5HO&yMeGQ8JE^O#_Q4oiha`b|z;xojb#cI!8HG=kj7 za2zEiJR6=~1^)p0Kw(zc6>cdGwn|m|-qJ3}DGR2vgzPGDv8vcq$vD-enbOWob#zcI zJ1|^Z(HWm{1cK*M{LqrT0d&&NN$CuLUjV4WRN{+7KnKI%sQ4mi5MLA@YibZI z1`VuE{%M;PWz;!cAOqLA*n7qIJKFUoCu0m-pgWhe)#3Iq6<=xyH#qg23f9xO-Vi&Q z*Pvr8r4|417!>w)xlw`{p| z`*zT2^w?s4fnJo@C|)ZVceJG8R9f>-eQP5@Hu~cJQ z6q%B@a<5`d$)#eu;wstmOkG-9^35(mq^gIB2ia5KPRup$&Gc3qiZU(jlyBEzwgfri5zT_SB zm0iV%Ph+)tfx}&-SeqyM$8={`80rl9v$Q(l60I8tQG> zDT}R!cd|y6t)(z&u05}%WodJ>FXSE^Rs}VQrr3r46iUO&m_=<@S~#(XYiJ~_@s5=H%nEM_*$tJrMrGzz zrDHJmTC;Yr!9&C;@I#65yO%gU!nT z!gCy4k#E5{Q7ElF*lOGhSWBYkuDVu?oqAiU9elC=o|x6l&^DD52$3d|)IzW_!Hd7A z5?Y9W!AkM>;9E=2pn&Sj6hULuSuF;X!f5dTO+d20*s6uF)^2>J7P^-W%X>r zh>+=Q;)i?B@n96wy0?7@Bk2(OhjbWs&R-rmH;I`wg7^*u`s&+J<~O?sSTz@} zkHd9p1U`=(3Ed7B+|6B*CqOYvkBZM{x^z*UKrl+6vYCuRjf zBfJ+2vXj3{zN*FBvr%WhlVg)St z?d3mtIH<~f>*1qEvqJXJo@0xd%m){c@m zbug%+Xj`GFKHFFq;nagmd6Qvpr^d~OyX#6hJ}t%Jm!N-?>V_4k%aqdda7g7-(gsWU z%~5_(rF?mcA#aHv168Vla!oJ&dl{R82)HEFh2stg&MU=bDG)y%6%&fhirtFCC9mLW6{7t=Hbg4+VBWH36O^5yzMLxN z2*6w>3$84OeR%$wvV6DTiZI99D<_pkS~>2OP6vkMH!>qpf2l*g1sUTQ?S@pI6y6eb zt0D|+wS)$B9kh+DA3P_fYoyGk_AT$ii?6kjusWOt0*_L3M5Tq-dcgU0MT{*n=EMZs z6cV2gCyaU=;w_iowZuqZcF+P6HT@>4hVyK8ow;lhR@=23I@&LOb{BE^x9BMP^3J3e zB_*u=%R(E)3R*MXMCM-@sUaywziS^l)||j;I(X)_Cbc%gkvV2Wyj+8mUS7)Ofo-{r zb*igv5T~IxNSexQCAaM|g`53BL0BD|dbZW0qT7n{dt#I@_8Sro%1UT;rG$YF%1#yP zc>{o%$F7_8A(^>bV2*J@iOuVslVAU{)4Pm%W>j*ZdHw;(Sp|?2-5EZ;lpj?p)eg@M z$vnzj3t>&NpvJ=(x?t4KbLN~m`5g5ey{<$TJOlv1Kue=c_8XU#>(+i|?aY9zk(7QBrQuBG{N8#g8H-Fw17|#(a|Iz-5FqUQMtq^TO6G*4BdztFD79~C zu@9qL?7ptLNY66MPj<7n16e;ayQc2K?olc8hS9GFYfuVRt#w-CYCCVGm7GeQvIX`8 z(&FcDydhq9)6uY0^K-$}nH@u=-vD6mSWV2|T=Yx#J}|uoeioXNVxT-*kZuH2LiID# zRoE{#QEHojZ^jQsqn}wgsY)_Ir_#Tw;SXBJjqcMiY6r#B=J=j7hSKz|m@2GoyqG06 z(1ZcTUh|ET^i+CrA9us*F~+721S2FNz+8z8^)NYKROE$%g zk-j|-?NSr0kJ4TLLKvY?WyhKccV`n$LB4bQ=r=%Hj$ldSqIL~}ODO36Gsbw%Xgsq& z0gch|hBQ}98Or%IIK2YC8=96PUb|E+Ei8GhpKO&==HXOF$@8S^WHGUmoyALEK&!WX z*3-IL8Kx@V1rBdd6$#-347sQ_L-HTn#a|+OzNWq=X!7}@599C!ev4EC-vZ;y1V#@* zYOT7SM~JQ>GLknVlbrYzYieD~FSCn_F12w6m}-Oe32?mt!{ao#rm35twnyumY35I> zx%}6+N@u2aGZjYjCk_O07H4x=daB@!B$TcQC$O9i8rzbKVN8PH@%G*H^(*a-J=&;q zqJ04)e!n{jD61B2>ja@-(t|Mav(rv5hkpRm3X@dBj41ZVJ-6zZqrw;o&q@NH`lafE z@7y)`3JR$_C~vui4$UvK=NFU}j3WI}zZ+gpU(h+i+Iau%wFQj1zs>H2U^B3GFQ)f5 zEMV-E|IvIK$-m=v+cZj)w23vvGs*kcv@ww+WuVJA8Vcw;1YD@i!u>_H_*sdF37qgU zmgid=x}^I}>~ZU89>5W3?98R?!!Okk464gD_F-hyt@7FGYM?W^m}{O%VKmC;+IuLa z;PW~iS+DqX-KcQi!&yq88%P3WaQ3y85}tM#pof|s5`Ws@4|}v~PE8705T_#8hDIMo z5%|iUL4r4rr_2F0zoNCoD@jZ*hM$083IFX@?3Xf`0=Q>(n5yCcvy=x+7T-Q;Zso-A zxu#chZ8xN^oW^9sQ`YG5WXruXxFk7%%JVTRptOHSws-rgA_xTvq&-MVwH*@Zxu=7}1u)f%eTV_dYE94MG+XyaSg zaKQN}rMlr09*|pXm;9Pu|2p4U0|97QeEm)b{@NctdyNL03`RAE1$`c$&JpAHJ?F() z)|qKxXyB@oiZ>ils=aDAZ`Y}X#PDcKJsh%rFV}jXiRI%F4 z9(KuQKfCi`i|Mi+?%;m=fe`OF2)c?+u=#b3cDNTjzN<*w@xl3Cl;$@!w)PR!rB5&3kfE6fjdgXo>d$m? z+t1$*JX*~&U%T*^0CDDvzF2qRa|uEAEWoKt!byZe_$zKrd0k4IQBdMS>f#H&|I#LA zA;8{@Rd%lXqJo8xUp67wXzEkG(&E}=dvR!J26QeT0Xe40%!SGnNw=PlY+uY z(<7yGmftRe*RuMrzX0jWhO|i=IHei_*JV;d(15YwkD(jl?OZzT6?{|6d=_4xH6R)Q zegU8+7P=Kc4H|q!Knm8;T@rvL)wd!`2~6ErCr?=Z1~&VP1gS5H1D{7nl?d|22&)nx zKDq6V&G?NEHr?Sqb6C%lC_&De*zo2i%@-$7E zrbc?kaEgv%(51FYk4jsB%K=@=+f@Dw=xk20BLgJ8jqrbP6q@dg}JWl=@N54r6B7BB%a0XH|N; z@|CZ2ubDehtBRtjO5QcMoUO$vD(8Y%Fr@Kpcr{I0jvx~ZDZoghaO50#sSN+*J8`do zuvDXtfmg_;?l0s+5_pckl9P&;>uj}?wq}B`KyXgEvLNLGf4$JMX4}dlxZOhFKAq+2 zK*o-zPOc^AYyi*oR7{rb7?Y=WV|05nr#ERl$t!DyilVO++mOF>_N#23 z(~QeY)AH|G5A6b0s{o_));?OJ*|!f{zCsx`*`3`u+-|bDmO{`{lGI~6YqK=}9*W(U z%+tcu8VFq|A#toem>`79NYD{Z6YSlZy_b>um+@P}Hk&7W5e7Ggtu}Y$U`xMJL#n8l z39J_H%(4JQIzUgb9mkCTY2?K_!P3eLyf`$j=hQ~DV^i<=Mvp48ybI7Fq_%|#A(AV>; z8quxOSd>(#>-NUlfNW!lX^t0cY0fNsud^jn2h(NglzJ|#!YRgH&F z)zPh%?o}Dxve!FY=>g0((c}uMLMD{{G>8SHJC74u7|RLb-^idI(D17^=9ZAj<%!+r zucI_HZn4E`%qAMu+UxJ4F3l&_krUt-_PakK&$)7;6Z8lMgdsXOTxpmL>ZBS8<1EEZ zO^s^ku|x`IQT8vvV3*R#fhWZQK~Nd!kySry$%$8NJwip$Sc+sAgY~k-&FeBz-ngQ8 z)p6#bhX>9l2flaCJHt%t!P`!}7a$luaCOLmQ3&~`fJ!&G6!dnP2~|q)1#St;`}B$F z{qPe~9+3SRyo#VAs>mru70XH*5vS`}R}UxZ*2>%uEt)L(VQJ^gV$sti-Z|{@84uSv z6AhpMKlOLwxE6x>m`d~&lyb+KfB9vgeoOqP^y$5~hWgu$HUU)@ff3fJnZ|7|_uA=2 z6wX-N9!=5BeS;>eTT_Tny$-cT1f8KSNJF>hWEP+D3#YA?;{?X5H=1s*cLFNq?{*l1Pqj468Jn)d#f! zZK|~(pS#fX0Q?M$NzbfrZjJe$*^%g146>E80ugc@3F9*!BIPqEokv;UdS>4h>}W|h zVC*-~Oy{LSxKpt2hH4$UZQV`ayNd><&Picfnr}g)dzU&07ST5(o2;2OdRMb4#OF#e zHXzkF--rG!p^ac^A9iOc#yaOdu}YQWo4?7A%B%h-OHIv4SyjiLxORrK;5&KY%y|+g z$6Lvs<*{k@B$DgQat~w%hI}U@eECtx=JCd3^|b}oiouMAJGnt`m2YUM&uTABqPs(2mszC^ACe{GHJTlobDUG?FLC-@lz~#@MMJB4Z4x z;esjxycITi{juL&5H@>!@taSCB~%rIgw9?@)|MPS=)q~qyUguHNQ`c5>7uE@LGO`J z4MAh9S7%39J#P?rLBQNbQ!0JI0Ng9yQD2LwRkRK`^20{?x4CldIMJSNA@KG>7eW5C z#_vQ(lolV>^Ga>DL4s-h(yLv1DNrez zLw+d@wD};?(C1QYX_lTmRMuV;s&HWbvIHz- z;YzEeoSo+6G9Xob3zjjh84cPD-i)`9teJPk3>J%0udFgbHO|`rzh8U6b%x%M#{j>&oLm@r?M3 z&1<7LuijW}m<;)Y7cP8kk~(%``8J`mTpJc8EPIc1mbB8((kG$)t%`kf>+89TS>Cya zpsEWhpMp~;nP4_zWZ`+XCDpRsKSRPy&8Rb!e+i8oy^EUM8|~)xOR(6dsU{qLa3>y7?2+;lWkAlRv&hYfNG9!W>k zeY{E=`gu*tVZteu?%bM`M@!>^8hY^4F=p{oXFG5nC*O`sAlZPQ*b&J|XTYNSPG(nk zOH8mD2_AH;Ui`1A@2?+5bT-|Fj#luK;RWJdq^y&Yp8CAGi;{kYUUJ>~06{U{#V7Ks zTOz76#xjojGYU!VS-mx7F8ONsCFnD${tim`b&cZe8OC!7QKHGyA)?bweIOO8t&&Fm zqwg~Zfw^ukb0>`;d)-jdtD*_iYXrMRe;p$MB(#wgKcT}ED=h+O+&$qs%3?@D`vGH(W)A0-NVk$=3`@Kk68bEd ztgBJg4OxrvBu!)d--3}&m5l?>%O)>s6EU*-7j4;tA(T@2AHVmq%>gdD;X%MG_3;*K zjg=A7;Wt;0!)lvmZA<4jTN}7VS8k!hSSm&Gh-pmxINa;ylzx(R1)oS4?9zKizT;ME zJI2f0D0ccN{515ggq5mJmdes_44$FrS9a^A)DD%mPE^vFUHyq$3cXe~nW<3<0;Q2o z2^k*UnTvD(ODLoJ&Z7>TrG}uJ6FHQ@)l(mZ?F`NY7WTJHq)7y0{hYS38>$t5^VXi_ z4a&xAz!^WymTJY-@T<^26k$cCEn!W)-o~3V4(G zKtF!<769ITzKzzHJNC5!drAFft!7*Upv)lDA8io7T^8|Q1vc`6PP=75y*)bhanG2; zOk)O*9oo;SquuQ^kE(I;md+7^6#rfuO)I^A-pJ~KQnm^+5-bFpF6hE8YFn6rP|uIQHe+FwOKtj;s%=&6?S{~W!1kW{+WviP9{wO+YaZpElC zMHVuY<`Q2AMW_&@MVNY$iK|cvcK+9O*S&S=r61lRPF#20dzW4cw%h`_npd|-Cy5fS z>e`x%pfn|XJslz_LJ&E>E5mxY;SA4ddCf=<_S_Y?U;d>L59-&S7bJskV=neat0 zqR^H2!wP^a=gycb6NGuUEnSV2ZodmZSzD~ch0s^zfKKJ9!H1NpW|D%rL>B67Rw5JJ z)Ypgap}Q}%o)~lk#dT z$#(bj;x^8pZuAX7CSH8FJp*WqHeqfDb^f#)rW7;SP^#lfULje<8H#O+OBCNc$*-^i zA;9e1P*?U(pQAdjF3ac6Hb5%sM_<`N(fVg-#j;ROb*bZI){pc^t3raat2DBhV3m3T zJ#6zZB=+ahaN&MWYTP=)T6KvnnIg&zs5`oM0+Wq`ab}e(cfk(w-TtsgEqG$gCvY1_ z@JNh%Q3;c1fJfRWg71QRBtY>q+7`@jp+-Bb#atb`cPJyB*r3_YU;-k=LgZiL`V>QR z8`)zjoWiXtY7NlT7szB70T&e$xRH_GlyKqwU-~hM={r#D$BBO(+ZW>iXC$|^^)2@D z%dgoCxagCMcKnFj9!NXn^_&U24<^kVr^8U2$u;b?nGgzN5C7QQWRbu8=YNwNQPbYf zPM%RO4z_2gtKSTf1WNRKS%gMe@xKNK5sl5Tydigs&qopjbxj++-^p?rPOZLba&1q7 zN*!1ct(P#4ZF(d8L+E2gN@ik$(h=9nd>@Pll|9;0$G!Rn&lS#*+vhTG(WwKjhu_;$ zfDNIV4?jb|x55LMO^pRQ6Wcxk?kEcr)K9vu7r%r*K6kzO5E#3X+SNdVoo7VEk37&} zO-*4|k-gG!UVp5`3JF&d(P(j;&WcNqE{qq*#01$qB{>0QP2OfG+ zKC8pitKc6)UrG@wuV^YEkIh=x|PX{US=|*?isfA zX>?m<`Ddjp&HA@p0G||0I6&7~TM5#pi{&J2^4I$Jb2|%)<%&w2+Hs~?}V{v+8`aRN_&%PW&v_N)R75 zE2GSYo7UX;EuWCN<02}A{;{RLTl}w!F9x;~>!0q{TRe+f!A))1npgy*FoU?l;;5DB z9Ls+vGM|*WU!Br3IwHloQuIdWH50Rbz4Q#6We=m7^}w>;{hT-d}HrutYf*$u6PFY~MK@cwi zyIKM!la3u}Kd(2UZXFHsBRB1eJ8K)1KHk@9*|&~a0g!WQaT+tHY8o*f(zzZvFO!qb zz2;nBph0b6n*sFu-$his3S2P!^3XFTq!5a8*|-3EO_fMybPk7BU~s902a3ml2mJPhT<|D|KH-PMAB+Iw-_1fxwV@qYm-y>9TZ zT9nGu-Xq(4;{*evu@g0_h81*nTdg_Lyhn;KBg@(Y7>5)0J2vWHg?0OGNwvNHP2(vUlqv;-@GzJ z!zv_8d7|k%xffe;i2zT(>685#Akf@{vEqXm3lfdp(y#U;-@42bRN)wRVzowXe*6!_ zkFM2Qy6lNtUj1OPa^u@4q%TjFy>wTX z`or~79kX1X5qhQAsS2UIPZ>IW{Z~V;lcF&^%Ypdh}T}4443tcerKcI_; zG48BImXPJDORzKdX%Ck;W7jF6juKQ}4B*v*i3IR9p%&?h@!2MvfAQ;tQhX9X1)Kpm zGD%Uo|JU4i$H!6K`_7r(XJ>ma+M?~Nt?Io?R9plFr{X z_fWhO5I61z2BmYgzsl@(aox9qKMW2hrwzyoWDVTcXQVxIqJNcB#W(~rO>mJ-g$79;%1^5j=c|h51O@fWYA{_H%>lZ) zl@OJ^ZHMUN3kNy7QT~e#O<1r2`3VWD)M3~F3+`#GcN%yDs9Fv(z80ovye=s(G6_9k zurD7x_|i3KeMdjV?b{%H9Pk;M5pC{Te)8-SR#O@J#=3?u!`-e^?r+Nh&SAn^+rdaU z8zpIXu(l#;H!i4sry@(yl(V!H-t4aaulDzhgP8190D;pXq zfwC!#&y|vW;!)J6+BwIjjzXU09|(i^w76o!Mf=DLp!R3|T=OL@t!1tW)}>qj0x0iH%ID$d+A7WYBXGr;Uhw3_6LkrX1)8dvbv7WHu zsn&jm<`U$QcohTpvpc<>tidXj`y^2|rk(9ihx5(BS zF%GdHgUANBmLHzx>!_m6dXBJyoNq`tbEYihpG%! ze~Y#$V*cIUKg5Op=(ZG2O&_n+}~Nw-rVLrc(9yqA># zwu;`fYBO{B(>KRSi8Bo9kyFvU~;c2>X)kJmP{!WzBV5DQh6Ih6k% zYeZ#}vDUv7T)`_DPkG4T_UL$8GJ(}2LkUv2f{`iGefj1te*L3+jgbQjGfBJk%put3 zj;X7#*U>*An~<%@4rI65=9OkkpwfSVHDKjj*S?R80ac*h)D(8g7T&1u*g)UD1U)I2R4dU`Q z&p1!FeCnl$6$RprZO=cIlU==ThR4e<4>sqevQVPS)x>g;Z5|!HYu&mtYbVbg^vE`h z%7zlgoxd{bobjx;H(}_j5%#~}G1-kdck>l+{;H8tD6v1}N|^1EwK^AX;6y?)96nLT z2~kP!aIgd(@3wU&1-hk`2U#{wW|OqY$Y9CW)X!hDO|EpPudV3c;G7!3UW0s9chMi^ zCge172XYqj)6*_pm2>r!7spDS3Bp)(9%`J8!Z;lT^&%W0|FolU$rBTk=YiB}x_iwB zrcg-~ajXrH9ixh=GXFrgB@jx~Fc#ea%jx3lQh7>vZP7}68N(REkrX8!IE~$oS5pG) zm#sF2C*9Al5lEhmSackC?FDx$RRl2*zLzzkI0nq$YiqTtzLzJ`c~aQokJK~5wRpXO zM3w)D=!2=U8^5J~&@Qw8N0htzLl}y$;%0 z@d=-a6srUG2{apG>>l7rrAP@A)|g}$Youdl*=SAp#tNY~jvP^NuoISC-AD_{V^~8S zB>=v+8`lSX!^19Ih@lubFgAAOiWU3!8P==)?;7-k@z=O|KEg-qQ z%#L+8l=)c{6Au^m{+fE_K&?KN=MOh7;;m-Nu>Ka#>Zu)m2j`T_gU( zi^K*yIk2qWvwOg!srbkM2bwDXurl0DNB?|)}FkJ z6mhK3zrfF*xw<;{)(D4Hc})G(F6!N6~o#jz|KudtblYvp9T@ij`c$Ec;tAgCb;KG)|j*`W5biLE=%(CU*xfy5zy%07Vc~Znmk$#IY5{hF-51e)B93`F| z%NSjTtP`!Yl33x0bbU=Hr0>fUD~WYHZwC`w-{Q7h*Sd}hTUC$oKzR$}7u^Ht!hO}i zo)Bo=L;h%w?jIJ^QVG|&9qv@YUD;+-zV}6g$GV}d`7ftFMt}cja9Avx$R|ji6eq7c zv{9_zwe%j)2Ix4EZDe`IddrUoYA=%{tM|-Z-5=~y&d>OAJ&*niw!rq1TrSf%%gk2J ztdAiYtvGNx!Q*<76rpmY*gT^M^l}U zT*oQin-(#qE1#8V{BA5~#ExdR{G?zp5{#b9->tkq{FUBz;qj_PI*Xfv6dK2hc#R4t zx>T<#H*$x|^D&AO^qZ2u3TKlP&zN6M)>Wxd`mVb=9CYaudYN%lO$`@e& z1=~*L9Zy0kU+w$ZsiQ3)+UhY+b9xERQH8t*bH$Vwgq@HpFEak3$cwDMX5p66XZ-sP zqMjf_P|~4#R*)Wgm_K=w)CjrDRFaLAyWu!%qSi@Pxa_ZXH`9z&Uud9&udO3kC6H_o zyHbPjCa3PMiYh`l)0b-^q^Ae)UV&#+sK1+gU4%eam?eCOXABG1#8fD!eSg2>R5^$1 zE$qbEkKO)_g@&P*;Zj8`I|13xE4D^pOPVW|Cek=ZnZFGgPr>!}t{p&HzL3A+?GP>y zWPhNlh2);8Z-?yt2l^YsnKDwKsa**;8ch0@g409NFeg72&INIv5?(g|Kiqx$#e11=Pt_%A))Ow=XD+eZnn z=D>HOC!G)O##$@{#UC0pdZ=%Fk9zSQazof22*nQb=`iG2Vja|&Xko$OSSv+)y;Ti_ z@ZIAWt6bI)a+Z}x7vif0)>fa&68xD5h`0J&s$}clzwds~EscS{ zG2-&8;J=A&h1%l{Ewdpb7hTy5wy>`l35jR&H9Ox2Mr1jj?_x`bgl+v9*~TR=(5 z1m0*SP~|H{_Ybh-QSfnPbt!?2g%{1Hw7cztibNt?pmJevW#i6lr?a#AvSV zBJ4+l>@SxqX9O*|a%x5p`9H-1ER{G{HmL^T)#RP^0GlhjpAEe{YkF7oKPRR^MpVNN zO70MCQF(!{mS)UtCJ?B?qB-ksFtChmuK;FqFp$8WTITQ-jBd>MwRO}MXN3Xai_lQU z)g+QE;W_4tL*^l#A(l$KQ4Ckm;u`QiNVm{F58YGn=UGT>8adVjxkK$m#m36 z7?}`cP1sR3Xg=IykCl%PtFcU9bcO`X=v8iZ? zhBj82R~jiS`RNwbA3L%HUnU8*@?;M3M3E!)tvG*PEc8h{X*Cjh)=+lSO5I)e3cKBL zo}uz#qU*s2a}+~+J6o}D+`(eiix*yNbF%sx@TW%$(n951>2yLL2=X$^$v|IY#cI31 z&P0es3l^m)j^ZCwS-O3l3n_O0$5(rHv>o&rc*CI|Kdd~kS3cmWb1=1*Ch*(-p{#$P zlBO);iEX2MNZ&jB4;1XHB!>c)y`Uml~7EVWS?c@ zG6t~rnya4$r&29^YnNv+>*5`*|Eg)HglpNQ>+i%o0UjgR)3KI)hsewK=9@&;*3`^^ zzi

B2=Zieik9G<4ID8h1|X|BLbtF-c!@to0S{Z_U!>q%a3 zIC2*G#C%!*M}zU@W{`mh{~VjY3$-F@_(E0fAGFNav|u@6esrc;+Px?%!c~9Bd67kf z(_tOfT7%(<$TEhaEwG3Ir?Qp7dmMJjcj)`Iqnc?#m7rW!`-5)t-TN zYxz=Ug|%B}GsuYo`2LrTA>O?TeGo4H^{p(|(h|?o45j;Q%pNz`1h7zfSrmdkg7~U6Yn;5(W3G%H^3bcf#7dx zBf4+qj)!b4m2mNeaz`*2Ux1IZWZA7f{cJpNv-04*5fzi?ai?m!j$*aqG5ra1HO)$b z)!Q$lXmtiG>ad%bTLglkG8PVFQM3u{%=9j%m_IKRf!%%0t$UTvvPLp2BZ{yP^L#SN z3mAylMG-(|LtcyH>;nXiu)ik+zDC{T; z#y4QSlE|-&=SblTP2~wnyb-i*zonr{Xj#32*Oxt`+T+@I0@^j$zTC zsTNd|3p?`3h{r5(k{#VQxJcQ_u*%)uV1^IbAxr22%b&YzRf_3;f+`?q;uWa7ItELg30zo0>bfYr{WP2?+>`EjptcjTH1Fayx;M~D9r|Pu zRN@=hV#r>S>tsZzZvpTs2aMueKht1S0hF0nR@Xz zj0*S)GRyEUZ?DDx2_kh>$zS|t^10ER!EMU8esRQ3+vnldYJ3i~9L;`b1YH1Fmd`*=N zvPfIa)KIuGt`AiZG%k(!262N?is(f_65`HWN0V%H%Y);U+ffSZ%GCBMpZ?URz&D+~ zLMY#8-J!f+-5+cb6SZP%&TY`)f(e8pu0SF#T5Dslk1M8@V$Wcz^t_<0sGm^d#a-wv zJ)f|cLjaVbZERZeRN}YigZ`Ef7T#Lw=%8r} zidD+*F%bh`P?uNOTGcDKEyg?DO{~u2{Qcqn`$EAFMxwQp@cAZt4NdE*l)kHBnbE>h z#!jqKM__%2nHw`LqEnBe_@ag_VP_ziJc181w6SiwKbIlV3Zw3^Vx6+d3Qiu3sm7`lMar4QW@bB~X_= z09WB99YKsG4T^HpO*gIc(wu0tU%7krutkjQxbn6N<^4d5#~e0Amw2Y}y2c=1BO@*( z4EM~1$Rfzy9M^CP^wY%jMKZ*aTBC%1G^-H7=6BS^#N;&g=Ui2|sv*BHnDktOm5--} zxxW2|QNd-&grW>B<26PC2L~&w{CXQo`M_yXnEXT7uHpoqt2%m~3#t9n@piD8qD^qK zGn_wv{#rcYmJA7*(1{p$D^tgXdwJ(q+)X+R3-_R{ZN^jOMRefqOg*HxKH1sNYyy@8 zDN+#9n_4BDrUd2owL_T6V_984fIcdQtr?>(=R8^T$8A+!G8>J?q$3#!l!sKH5mNL^ zmWa+Rw5<6+CzREj?Dd(tlrd4o=$h%4Yd@cgdPSQ}uOsyKPq0z4p`(aljw9Bo<=A&H z9#Zs&kegIrp5Vo@#~IMT2Jr7Ckr&FA{%?`Gr#-+`b8CDWLtFKz+(nEEoDs#V<-zBv z6ZV44(P@Vm51R-9PA&mkDzN6>HAT4PeYh%KUa4X(zX7s&iP%z#lbfz;qQw8S22Qz~ z7zQ%4_0s!>h(w?ae(SK_OaP+r#9?}5SCd}XzGI2iETrqg|Lhqa^5HBk4F`R-Mw=<7 zduO?{#gH{IH9D*NHHNZMOhhmnZx3Y4)Yvayp+FC$E60|`RXxV$mIVy0d>Odfo8gw% z+(8RHb(u5?8QH&iU1@_`1gDg%b+Rm)hI+o#QgxM&Oj^o!TZ=?qk zCbzJ>&xaJT(an(N_$vA@ghjHjzm~!s@CckYr!~B!(%b;^LcGjEAyIR&kkAM7v4ado@>V0oxtIwEt++uMy0votGT+jI%%7lP&6c3nLlgo;W9Jq#-lpm--KFRM_ zK74=F?vF&l;VL*`{-|rzVR&Q#)>jZ~cuU_{Khu23`S&I3jC#GfFRVQ9y*4K=kbW?M z{>$MuRf_s*k8-b3hp`E!@rAx5&EsVA*2Uz~+9bu|bb9P2i{2bzxGK!F3`7#OJWE7; z)Iy`NJj!{(19r^vW0qwTWc^>bI9}HQ37&6zd|}z`G|9XupJO?Eq`#7@zo*8S81D5; z`Q6b4OI#+3@92fJ;>oGi*lp-!5t*Vzv!R-qWxly|_za-R1(aQ|q(_*+sZv8`{u|pu zH54EuNq&T_U>G9+;Vzo983(MS6w6>uJ=V#`N*3W==*PG+cjW=Gf6e5XO>0bEM}PjW zY`J=>?Tena;`UV~z4Q`z?w#^n86nW}pYl~COY4>4L^u_<8buwc(}O3JvCa%8SdAH1 z6ZlvvoBTB+kf!YA+>*yVvEj<{=C$$3d&*^Me`TvEMg7LlZ4Iydd-cgRu9xsSkPqsrGRPEHcUm#%>sDY-ly}L^k0a! zv4XFsHqDkfDu#z$04?vv2MebAKL6YfY%i!jtK>tjU^ILMJ^+U2ct-bf=#e9CQZm`x z6_pfE8q4Cmth^-kgdpxq8&#OQPEQJV9ZZT@-9T(EyZ)2#q7OMt6P%dqSHb= z)c~8C0p=>fKdJr{nD5`n$!Gl1!Ro8=>u^47qONY?aP#|z1xT7_JQg|?j@bj{$ptZ~ zmE(-@)9D=Cp{DwG_ONys4F{vXl z+NoU5R;XCp1K_Z^pQFquQLj@j2RBbAa6ph;Z91K74MkmgHg6f^2|Oywyy{ur-X0|h z3}@Oa>uLUAO~g#Fj4oKHV|n|A{SmHuZ2ZcBazk??$Kse7&9+vz(#-A=Gq2}tE3cUR zx)1V%`kcGzWO?4_kB~em-5z$v^fuw!<5#VR{ga=Xz@98(mOGIVWVAR>XK_O309t4U zbwpvtONX>*z2nA;F4C%Q^g=U=Vc$`x+roh@Hp5qK%uC)#}Yn`DP9{&CAa#uiHzK&bmj*o!zkv2H% z2q`L0{+l%yVPy|uoHH7I|_nka$59*?Ss^k2QXAG_tHEpbLbU|0aMpIPbs`8d%sG`M*0?)b81Cu?yoOHwJ z7$ya6Tf6G^a53UTJ7#cJ=W2KXFumP3};2DGRV^{4I)UZ zxOWAfC5^B>OjFAsZOR^nqD-;rffLH48Jw znim#6-y5N{f1qohOVrbG+wyOoBo040_?j;kcR6WKMOIih(8cpExAJsFGR;WGuQJ}?6=bNpuN1CpURrG4eT(|>*LEf;j^iSYVAIpFW;S5krhFB9`rT#UH3Zk_y&P*7 z>sGN3uBjJI61qKA$8fx7@0ET->gL@EAyV!aWnaD~Anbdsnqe?H^-8XW;Z|m{Vw~5- zTa^!%IO3ecSnXf@&pWPSzWgSn&i}*6=o^~bIR+J#9qBrn2jPcru*lI*-e&Uq6MRev z=XHg&aQL%+IDq-0?^UK4bNeSnjByn;PqoNIz;sC45`kcvqv4zO`wVIe+&syJ7S`}~b& zHevFUrf|#^weZg?uXl9$LX?yTj{|+Mho&8n($_0*N?}@$Qz+P`yosJV@v#MQvGUlF zY^NukrwE+Xf6=OQixCk{Mwjh_HbZ4~*vQr-tt*sc)f;k~@j`i0ZtW9(xzWEwdE>qJO8A4Wsg2mT(SIQ3qQA2$XF{e? zUrLm;*+LcFfkM;%BKQT=cF_uE*2gTU4e+7?_B^Uu!-bfZ4oyqm8$m$=8VuaFb%?N1oWa*JrCY?nuzI!MH%Zs{Zt?sZOv& z7>uEJUiH)vcj&bH;*9O)mBjMlN?Jl)=G|vR zTe%Zws)_pR8oDVJ6Y~X_nio*4_0V!lU89)ZQWvau$1Eb#0Yvx8rX{N#Znk|f9K(sJ zQS3|TACL&7|9X+dkW$`Mv{RaO$(h%}YTHdWF?uN*);ZYRkk7Qt`UqNP7Mkf+29{0T zEQdOb_SSZ(U2>RfCq9=BvGdhKbZ`@zU{|MFS|I(fq%-@4qyqaJVyJY@0XWG8OhdLou%S;jqbtvJw^XBo^P zR^8qllxxaxrjGG!0r_Ns<8kWGZcl~5Bj)X*WHJj4t6y5?Q#OM(<=Icuw9av6xzO^) zAo|c+d&XqE{_MSXR8!lxHmqVtu%MuV(mNr&5C|PYuOXC#8bT5X1d`CChy_tlus0Mz z6h)d;MT#OKpn{501VIo56c7X{7U10pc#e2I=ic9V-+#VuT*q)_hrP4cvz|5QeCA$z zW+!pDLRz4?nfN-PQm1$+aBuTvpojW{wUzg7iRvn(pWW1bfS_X;Zd=}RB^#zdfC&lg zwju6Wb;@C<Ja$g!lEYy5JAL?(`hHN$QJT*vqwiqwJ@H3_@(Opw z->3-g^vLnMVnl64-$+=} zq*WQ|%G)f3H&<@#+ZBCbG8us|U~@Qq)D z+54TI7JjPB$|SEFL%q$y3Ui_(4K9N{gN+TtTa6K0obOZ}|49G1XXa$GOJdZ{iz?rO z)SBMDZ`6C0j;M@m4^{G8DOPu*$Vc!Lsjo(y2$i9o-x}$e)F7dIs9^nb>K22&&{ro# z1{EY99N$^rS1)jGd9PRAI$c-fotsk1;2S>LL6GG)Gxj~oF{)UH*>dB`gNG-U)W+Dn zS*QF^nbqD`(rj$qB6LtocV>l;0qL6dz$4I2n3W+cEj_IVY*_4Zsh_7P4e=UJF zFe^TIzT}?3?yjJkB{%mcZ+$5LHdk1{YC5Z~>+qU@)H(UKq$pKpBOG7mhHoks=fBsbGrmXXG-Td3rHd2J#@iOkNV^G}X3(rYT zL|7|md;OqN#nZY&$K*ys)jQi{0t9+g$FC&bOS(8Fc%`8#;LaB1_T)*5PE7g!HJ@dI zRhF%Wj=%NM7rcQH@wcM(3v$Zr9Mmi#;vCOc=}6kS`F#2Y!OCsV^!G5*yU+Ms+L1U( z#E8hf8*0tTv3=NEqN}t*Tk>PKzIkq!cfp6pdCmr(^d;?6w6nxl#C{jq94GQ<$o_{NhVm;zpY#XgeE-_V*oatDoIJwzaA$V-Z=ZQq#=+bTO;Df~X8paUGw}o3GL~?|a8;uK&nE5;8xE4_? z)SuP~ik|WjxzxJ2SyIpLyFxUg~M~N|g^=E}TA`DTEk=K=F3tQBQM{LDc;Q<3`xc}u zux!!R?!3yyv7~TS43%_XkC6CqnA=*>O~DOK*S4PA3+kGVitoQ{*cqyP?9lOvM~_4= z(v`%AKKW%^5#9x@Jd^m{bMrWPkwq{cgrLB>> zy4Cj&SK4eYcShBHJ^c1je$OTKrI}I*c1@DsD=(Mhmwk6NsLu2ML=4C=p5oYQ*{*u? zNR*&3HNo_hy4#%|;QmGbjOUbnNA#7*z)hAZ}lZm53gH54~6r~k!7wP5+=2RGn8 zBk^6MOw$l>U!#fHr(u3@7x~nsA7zlS7lcW*QSkOF7Myjh39Tf={j`Jfn9kvG5YkM0 z<^aR`rO-N63S-BT>tkJ_`@C!Gm$>;|JzL^1R5?aBEC##iX7_|&Q(YP1FF9S)$OY9` z4l+d_pa1Gs)6 z7?>UXrr*n)d%2-$YQ_6-eC`vE{8*6EortevM=ymWrk?20FM{0-e-atp7$mhpOPQFscO6PePx3}l57j^ZDGmq8tm9} z66$_%7rc0TuFe6OfQIefaJCTp1-Hm{%~ARqG5Yqk-IIvdo ziH97$i8uAmZ(Wur{AHB%eHTVd5}SOT7NwBcOZxr{7YdGysM~Ov9>g4Qne4ONaadFL z%_cvO3s}bErzuOVqw@xzR*MR*at!2}qH>Mr!py@fufcrJ7~Ww9jyX1H7_1Xk)xX`? z|71Ja>w1k(;q^Ox3JHn^yWfXGFKjuocA{5u-*+_i)x|eQo@Yz-pE#0zc)LZ16)E`D zU{Z;TzW##~T$MNE=ZeF-)%8cYE0X5{`=;KHu@}0XHlNQl&BEm*Tb4 zRT$=I%%mPAbag#5i+^+bVSBRwvdal+-T`y#vO{Ax@ z%G1%J9{)JaoX|Dh*!RHC(6qjpX*lqa(CK-gn%kI?xVEQ_l0#gko%S&WJ}6aR7(Fx# zss1cld)fcg{qEsfoo)UmSE{M1wedf^R%b4$eNNiA>NU-e;C4h!I+EyOcGUPQS>Sxb zZCm||{YRgTpE-75oy6XgHxin*UtJnQdBn<3Q`hmPJU-pLQrteoF<13t);iM3@mnT7 z8>{O)lx(LotR5;~J5*>0+COm0Li#9bAOb(UlcUv>I}~pep^D3}krzw`esgN=J^6y= zHO}%bY}5T3KHTr|BTT)$-3DCQO)&(gi&$0jWjT@Rhfenwshv`e3u1ICx?rnS?;dK}Sjsm?A(`+j=B>vC%_@&WvbsF-Z#ZqODH=y7#5UV?UfL+E z8h_gDdpY&Qx0z4H9hw=!GlRF<-1G6>*7;#kYtF2z5_m+pd3o*v7BR7Tvh^JN#U)1% zQ^))7_SLUBA)X-$3Tn?!4bfVELQr9I2E;;nxP0{qu?#f}2+ch4tQZGMr9_NBfnn#z0ek@DhWY6`rHkcM-t zs7-CIy;@;!p()w?=@4W1Wp+hEs%sfOqW@+~i=gMnm)~@*ohn|M0rsN~6w|Dd@^60;j%lB?QCRU|43jSi=h74PB_khPCTD|lb>*a|r&OYGr zr4c9L?DgZ+ALI8;gZ8~U9i7{uEcWiwz!`D&fsVu2=_z^pm)xY%N0BGKww_LJ*@9}C zWtZ9qhU1oD-p$dFk1YSQ9$pPCKU=Hx<2wa3nYT=%{I%>72Tzx~~JJ zZJA;AUPf>Cz1#cl>&wSm3>aE7-=BOKs=9!ivY2)1@#vX46_FJ}oVl{gVPeF0XQohd zrQ?+M%;}X;)d@#HmR%Vy8XEgTNmj*OFNpMr%BWC$dEv9Ky8UnNbPYro(HKv^S6mZw z5+lDszhn*R;B26yWmUDJ>RZnMMQ)Q9T}PFs!X0# zoMP$3?8Jn0|EQ{2wL7}zgF&o8TpoGX`zc`^?v>s;0eqvFV}au{p^rCQT|TIPlY9Q| z$>&e!{Fc1#Ei2YqTUsp*N~&`8J9oOa=MI zNGleR)?ZXE>^2tx0#^^t5uaoc-LKgiJr=;US~J2o=Tg`EhOhfj+sFu)Xzc$mxW+Ip zqFC$D{?WA{Yw4+;gyM4H=e_;K<%bXM-(Dm9tme)JW6=6R{qo%Ro~{Qa>EY8Y53Gx{ zvIRd^o{kHaPIuYa68j{wJvr0FQLg*$;akBk+9r>zLR$`f7G`y6L)jj+w?n-nJ2Qqw zNQ$VPqa{{yy?t!oUaW}$%%dX(7;ch;g=*lnl z3r!s9)GCP>e=zB~L$kJJ3*0}s7G=eh>}c1q-Sup`ZhE$}t7ivg5XE}ahR%=j5_@vL zthqX&gyn|LkAB@z9`*gfCp(Rt>`0wdWZPtC^21R7F67AQbYp|aVE;~MtxC>R zUoeGKQ616psl6h6YUjj&Mf7V;P9?>wzr?lf+D7p4Juh9iXY4=y_#P#vPQiQKt3;-u z@cj$hDfNOMW*bK9mc#Z+=!Qu3jvoAYsz^A+F)1LHA*+?%rMXA$80JalxAF3+J=4MI=ENu;12d58 z4AS-6MD>fSQ&Z|6+q|-LRgZqXI=8#cxOy~TqO`0h_#>RugT)E&9@!I`R5hrsJyOvf z?-FKcBy4%VJv**=8@`~1;yVl*wYW@QpAv&ton3X6+Wm%U$_Q(yD=*trQ?^O0;(gr; zYF19^?C~57k+}j~9-#OnFYc><4du)h5`EdL(YbP+S!@3vb8oIsOEzBE<0%(Z^W4Fq z)_wcdl-a9YS+kX4eb?Vto#Xg1`p9>$aP{?BVNxV15EKrR0--TzDJTSrl+u8sq+I6@FlI2RFe$hO z(373iTB-RzT6y|YrNDZ;e+H`d1bbEhjmrSp8-VPnv|y$u8-$Vqn{qvw42oU=jY$PQ z3$|o%IE({(|5b=<^w&PJTGCASysoynZ2tSO}eS3=EFi~gAn>Vs9wVCDc4Jv zx$mz#NO#?YUE#P+vCADZ zHzKfV9RW_0$pMIDi@pZ6fWGux_nC=o3vgJUXQ#G$+<4gu;0L;{eq=P;6%+j(BARU{ zjG8ZHRc3!PZ+3KWcbqByn0!n9)Lp`?-BHqLWhjnmV;a={&_6S$Q;c>{jMC|ax7jLT`y_)PC{H8Uu9uxaU zvh>E~Q~xz}f&V84kAXvfT0ayJEDac9kpo;5ioxF4|BJ_-~;IfBnbNA+NI=$tH ze91wXwtTTWFIR|NSrZ?#*@(GOP-d-dx*o8nE#3OaaK!^ripAwQG5aIuFE9Nxsso== zS36XHW+Wn;GcxoV-hbiiMHuZ;fF7_b+k_1>Is6_34#zZP_f+r?zeIU?+zZ(@yCqFnd$fXM;pJ;Q zVm7j8G-^}ioRYM%%oDi>)rE;Wha$Y3jA8AE1iHVkAMLE(dU|Vc)11CKS1+aE{<|wD zUp|~wTALPdbx31V4XsT)^QcKkLN0{uQh5TZblf7g3{-GHRjV!Dq!+_|t9a#y&^EOC zY9ZsD6Na1%i3TPdiM#!K0-MEdZQLYuV|(L8%Ll8+*MyiH@ec!w&E)OgiV-wfUeRo_ zFIyKc?k%`!=$HW8(8?s?!ZAVLSn@G(Ha_RM>UAOCg!Z(_>jEEF?+U+cWfF%?SFaNc zPtP52cqtT&$JtoF6r3>D9zecJ_)@oin~mhVq;Hp_2ZCQp3h4?wm-@0Yd)Mmi@dwg+ z1}qWFR-_)IZ(eJdZo6TvG3A*PLih+?AmCeeL=9jIO zI7}aS$cI8-)TXfUQ{vJY8+LhlnbqZ#pz)o)QGRX=El*x1{wol0WuCs?2>l9Rh`B~?9r+|iK$I+pXi!L|SW z)xj$v{_p!feR_C0yCryROh)p&MbpJ)al3P}fbYk$yPEAY{2q^iGZDtp2A4{hrh0#KVw5!jgpt@0w0g5{ zOyRV}#Lj7B(#=AhwvC>;E_=2wkJuNGDq+8}=xYZpw=KrM&@sEt2NX8c=0U%owR`J5 z`@tP`>rcfB3aWmUZ`q1QXhWgV^b7f(D#I0SXuPa+W9KV+zWbn{(&B%5VE`s=_s))6|_^yj~ z4n*mCpo@o3`)VwEsOnwQ_nxA}3)@OH*v~V9_DpXyHQ%a}jkz(G?K}gN_+FQ**#GIB z#FY7`Q}4*(8i&1a(xw9`r4WPRdlj8eU%BTgJSaX@2%lbB71x^m=!lC}VE5bWuO0mA zYVUS`9G3s05mg?&W9C?;(;WfU3eh7+!gh=ftk+t0w0L^m@ht*t_7rVJ*pO9Hmil$* z56KQ)<}AGwQrjjT`nlF=U%fEtL{82c$BFd)cP6W!W(Mp0z-7!_7pOmz2o!)lI}!K| zSs{|xslE1E=s;`F?YIjJuV-4_70WQP+06``P>M@M%ZpPI7U`X%If%%Fa?^-z@yuz_ z;nQoexoMI|lbzNd+tz+NFIxVEc+F9C&DD2Gw&{?>8pTvDUgE*c(V=(E_lx8TX*Sl= zt$y66Dd`c3pK3;i9&t-Mos>%)vbI&K>}csV)p)0GwI6%hbJVIg3ThH8{Dr&R8t)o; zvDnJ6^$`8yg$kkW((-Hl>2g`SY7>3wXX^TkhN?~|u1~dmXb_|`z3!^whZWNo%H2&- zcblcU<=a+YlIJdU6}q%S?ctQJTe#p*w&(RQ-{1|RF>7=Uymow$i!5Jl@MzP{>lei%`@D=H%rUYbsxj4`2(frb)~p{7Z6g z+)TY;!Yu42c)i?HWc#@5aXRNh!L_4l`9_}wIIS$w4WH?3zep*oCEGl(%%dFPn=c2> zKVXoPIY&b6-I{vL#YI~Ped_H@_qWr%rbqLYs5LDsZkd|}c>9g*usZ3l&5aa%b%Izd zMQL~xJ2FRWOx*utW`_jnBIqe*r)0Pk<@vWqmz!?s&5Anp?z$KIYKztiwRexojYe*~ zSQ~W(qjt(p$hQB-)W&F8;&m3?Z{O65L(X5;y{O3eoQB!d7qYCQ4t__=*wT+4%@fqXh5T!9)AIz-aW8a4BMI6P@PLi1*6-@W>+>B*YCt;@{1 z!eqaHir*3nJ2z9Z)T&pPxT!66J==MHY z*DrxvUZQX-uezoua&DwwzR_Rj=%o2l=dJfA>pBtfdr4&ICrg#>m79I9hB-uZtSAk> z1ebjCY&v#pP1NfhlRoU!~5{&{5PVB`J!7qdYJb&ff3OViV9yNcemkB+v#`-0n< zm`4WJV>hlptKXooa_>lSoXY*CYNYUG3_1G*YJ-g1XL8(*0Y5w5Gjkb!gjqMhdL* z?$+tC-VPREBGb%YP7<+#1)?-7f|nG-5)H(n8ZPe+GID7`YFIG@B7RNQ*X^$6g;EAo6Sn?4g4 zss_KBQ}op}v&nF-^NbHO(JYd6boY}hRnK3Ty?J)z{6|T5o0D$4%S%J5mC}P!ot8z9 zZGTX@s%iU$qg`T*;+&V*QQ>_=(F|%@-o(BiISMx6nDL6vEki}|sAnVjr)T%>fW6&4 zE;;mKqPZ)=cw%m5=E9?+k#ffz|U0b0XSM@_ABI{K{`V?++{$Scixn zSUxndPSE^AYv^ROvE<1+eiYZ&7=;(7Ue|E251A3qJ%>Jhu!ddF-~2}P!IW9%#Ct=6hisAL}q9bTeYaj>}0dvHytqP4Z1 zeq2v)|Hyh+;ySRnf&7P_gdJtKax;={_J3DoNYA)9aJ*`#?=yraOE{?gm!_SXEsw7i z_1V&#W!?=JZf`b{?E>&#A3 z2l95BXUyCFKOf!2;+o#`N6DxYozEtSt5SyTQ_VF6gzRRa> z*>>(MU;W(^S7Da-tVTX*chrw*M+YHS>&G9kcii5)O71Q5kem*Kx~92pjEWIU?N|}A z!uskVZ6R8$UK72*&aAPK*z9e7mu7nF-jYB!5&Tlzg}Yk8KJ4-m!>vjyj;UoHJ*HiE z-0!6RJ!zr9QIGFC1>^26TV^5kLl@m6SvZ37lwKlwZJVg#Es5%4&B6hEsn~J%19j2r zewqVYPURR)>yK4m z9Ef7acGX^NdQi<^Or&p}6FIVeKP#rI^u6}=Q)i4GKGDI5iDk9O_2;GL)R#ZSKFLl@ zPIfG{yKa^>_w9$kwxrtBUSzjtZSc!N`08=GkQG=tU)FZS?9z!>->--z>sg}5E}m>1 z-KG&uXmPk8JWeP$^2H@+SM_r*pVN1&!-wvVt9|ONGkTnavY8qvvRP8qZ}f?%RV`|r z)mWU?-!z~hzDaFIOd;v|&8q>TYZUZ^YWHs}h;F_|Ba0k3d?;mn;AGk1lO6q{=A2;% znn$=Q6`_D5IirQMYSt%Lj=bbgb8}aRAbX;eOc1=Lj$oE{g z&Q(V&uH71l)>;|*eNAX>)t4m|%Eb>`tfP1B&H2E&{k&gm)$64dZ!rP6Pu`p|%oUI= zqvPHWRP4*!;+}#YVMAw9uPhIZ{{AfOmIJb0a?2JstJ9S>6)NX^RaJbQwIN8Wjc4{Y zE=ygZd35c~O|!33C_aEL~Jq&v|TIesLf2NiTHpp-PC6zgXXOwtmx+ zXM4iBTS7w#)6eB&r(23Iwf1%AitBv^B@^>9kKZvZb3o38mJF4gAt-cYEgMaWE9MFi zW(^mpS!>*qAp^cAJczhqb)Wnti|duSjpID^X#Z)-|^DmLT{#`>WI?N#=TPSrE9z#;uL$6i>7?a4P(uTwyYnQ z%+9A@-u7-~sk*~4NM_uZ?dL!aoSjwoC?xw_!db|E7st1oG|6^rGG&~fncLTE3OVkH zR(=-u`13Wi(h5ZhgVSsMSF-6I1e z%$_@$%^QOYWu`TrMrsS{S~c|BV?@sPuS#7i+D|;ZqkbyT?Cn{~o{`(jOlMiB0M~sa z+MFZOY1bKvW5&lflo*RtDE6U*-_!hv!|z&jM0N~R&>!9UAb6>#=XP#ASOii|uf2&F;|-yE0e!eLMX7FGSMvT*o<94{9#9-S(aOp~N)C=awOPo7vmZWSur9|k89SE<=DR0`f|@K@S7BZ zTHvZMougO#G(jd~U&h(bHE&Az(1C|BAG;lCcXOBKp;s=a`Y#lqKfk#4Fv0wE6vmPL z$}xZpj%0^~4#ypzd-rLbLLoteKX6re?-^R;aQpwA;t2)EE8f?pzwDa=hi_1hTQK%&znlyKiKS z-I!E%P^T;UQ1ZLCi7S(f6Y!RAoF-fLZ_CJvZB58M>vMq6?C$m@S}A)Fo1M_pzV>Zy zz2v?fS)&yeyM~*kXvr91*d*p-V=UfP2_5q&J>KaT5gq8$Cu4Gu$n-$`i~u% zWR37}@sk4EHET$98~QA?3w**_j&^0OeDe6Yy+z~})5~eIgvcJLQ|(GODykI(@x6f& zg<4;=R&G?vxZm!7R6sBa*Fp~)emAnd^wWmi4y%cs*T%*jpH^K8P{`}ug-z{+ZcpF! z3{q@x5MethSQ*mo*-18M;A)@^35V@99yD*^w6D-j77iIvyL-!SL&?B^>qLC!S4#`` z4HAfuhkLubKNpH#(Y&p&Vr8xGj@P1A7Up6Ep^104zAixrUM5GLz}}T}{RrAlOibCd zNdi;c&U8E~-_blVGOV(_akf6&BQ9zK$>708vto~)cy{H6^KZ|GgX_s5NR6)_VP9P; zLs=S#{;#)){?5UEm28onZtZ1(Bl6Ln8i#T>?$g7koVa-oR9M`&?m&e3Hce2ZR&D82 zV?A4HNL1h?838x^X5`?y5r#aXsq@q#QnGDj!PcEi>5&OPsshb6)-JWmp=J%Vbv)>6 z`($`l&j7mi}(Ib6c-?8ETGH=2XTFV0&H<3asutp!e4*5< zBtKqzLp%6YiF)YJ;W|@Uh2jq{sim3OC0_LpoP+EI4BEqlPRs;ZQ!WluUDw4G3x|`jXjZmuq1(d6v*gJTEtq-mC27c z6+BDq0@c3ae)f4?BRtshT)+J0Cdo8e(^YS+6uWUvlnCc3&q| z%CSrj?M?EQ^OK3K4>7eB8flk@?ppIP(LL=`*r^KeX@f_P%_)JDWSpRSh3f9YYYuLS zj8}fj6{CZf!xO%*b5A~&9};${W?b~#-iIX}1>xhTLn!+!HqNfdbYG!3xFbPgVtj9< z8E7?sINTys&`^i62N;3^(-n})c> zdD*OU6BriZidXEL)O~+b*k2Dx7jKV6rKn1((S_i zogxEJGQ8W+ciDE43n?nyMYa#D1Z>W&O22x0N#_BV){W-^L4kJ@)t8KzKg)`l?H|}Y z(*Y`{xJu)DBAR+dxPte^kJEqMMLe2FX4w6}J z*DtXayI=Z5M@wt=NIRx&`jz)9W`PE6{C>%ioFxAu|NP;N9knIFA#Nc(Ay-4*hZKjj z4yCl+%AYDmx6?YrIwT8p{C79Zrp$yE4BN=<+PgA!jmgBSvV8aJ8^<1oro|bx6&^d5 zRG@v}MW*csk?U*Lo~(>{7!&)h@*p-X!puGP!Ul(LzAu+*Dn$wT#zrVbsU}nw8@+jW znU!#-r^@V2<7HOfow(kUmHTR@Rj=>!+5bAOCb-#9C2HM_-Bs(Or}|Qx#ExFMT{0thl@+U%ZFrS+PhP=)kCGgX)vYxd^3K3Y+*v#8-Lg(lcZR{O%BrtQm$C8O2-wwmeJZ%|d(oY1Gg z|LXm7sjBfY*kT$t8jVasQ0?pI4SiLF$SB~`ktQ{>)$E5d8X8@Qul*^#Kb+vyuSH`Wun_CQD30^;LBcqkPyeMBn%sI;^ z%6K&8aHNXO(%nfRpexFocMg}jdaJJqOp3W?oERi`_zFAVT;Rp^Q-K3Brz-BajIrMO z?)O`M=H#+BGNteD$M~s9j>q_MMRtpPxxYlrr&r0z{ba#1=T+1#x|bf{WY^eu{^%d0 z9cIbY4b%k3mps!r{?zCCm6Gm}s(yH`PnU9J(K*K?oraV*hO&cWo3!)<#HTInuPE+) z@MJgUg#i8dw9~RnltXS3fgUDk+1BhRcwUN!^ntu%vODTqRdpyK{tsQfV+?9*U^dF@O?3tJw>lJiGH;Bss<3ZZz@0$cNFF4@!S*ar23;543!%z4>9( z1Fe91hm1qZ{bf&PQy)r=gNfgsw#e7&EpJ5}?l72qrj#Nm64r7G?}RA)>iH<1^B7o%~QOR(4y*JI{jFD=6yARMFC#;Yj7(d#IaV#@9cZJ$ada{PpH7 znU|T2-YVE)@>`trN zz~$R7>GvX68-6iFtT28O@nZSF(-lLTZd2V$l8Uda?o&Pde)coW_i?|-*Be{EsYx1E zo}5iW7|b1L`N5P+I)c#sL?tv>I~Ui_ZU25>?aMXJif_Bk4|N~=ueZVwzuXF2sN&H; zq14eDD2-oBdC(AqIufIS`K6u*28F0=z|la7&wNo2Z{wGu9vDPJT>}NtkOCoq1EEM1 zObUeoPJ)KOrJztKS{;hPEUxoG@NUcfvCcX*oB9n57i0;oU*=t2pjjVFz2$*XAuJ8-C0tN>tx_a}}NhF5m^R}3`p1PVG?v|o#s z(7&8kFPKYbv89yAa5M_|k0!$)Fft4U4JJe2Xfm(flh=@yc&!bu0d_zjY+e({YnZ%- z%4`1K%l{kauN`D%Wk3&s0$47C6<`QdMoB3dVu5Y|g@dA?Xh3ctst}AK1fux!I6fey z-~dh_mBR30P`&x5FsFuxve@1nDJ9@IVC4VZschIR?_dg0k)>ou4P>#o7M@;GHipJh z77Q=8Cp%mUBt=$Y1yTbztY9{U%Ig9!5}v~NQYqYD7y-s+z#EJMuQB8`dc1~D44${e zA31-o6oCwZLCG-SKOBmtli?6Zl#Ca^DTD?FUJyuv8GxO@-~z?9e^U;e!jX3#PhRsk zwZ^=+e7gBuGI(!Uye5D@Q4lyVQ82U@ze{IDG7JiI3PJcv{|Q3~fC0h#EFbkBhvNnu z1BF-s4V(-EjDi1(fLf6uP?QI7dcX>Rl*Jmb3y3{1N)&G(Y|p?zsyC3mcrrQEUrmf> zqdzO04OqcI*)JSWa=!LUA?Ayc`Bl797+x*(U&i{=$bUT*@93YW0?LfJK!M$Swga4w z%S(f-s9{_wpdyxk9ve?Kubld)ll(@tw-cWx7zC>POBejl{tvoX@T=dtuwhd}{u!YFN~d+Z2^2OjD4O_ zHjEED5}0#%G#L&@(}1Xe0vZT{k9@(l z{q=SM_5ojn{@0qwKZ2e|wlIkc$dEsS)_?U4J`(l6siX!{0su%51g}B!(NK84Wlx49 z0MiA`7mh?)nJ&WgYuVbrU|PsOf5S9S)z8fG-vG{&#)E3aYk)~#@M;Qgi_2@I7MKHo zA_1UicOIpSu>N{o@h?~xrumOp7r6bWY2}j%=cC~@ro4v8Yxp4t^w$&i$lKyuw)rAw`-8eHHj;TuSgbBT zd7VeP!1=!-oi_-;Vxdra2w=TvxE(+mKqH@JxEt^W^(!vE9sVXs3yl2cNsCBZAoO3M z%ulN`FpIEgKoc(6ra{;M9r|I(wysODV?Sg`oTr54f zYeA;}cBy|h*m>9_G7Ju+CcM-KVD7h6WfA5-sL*1l^LQ4k&`*#51A+Yhe{q;aNZdGp z_@E#>*Fo{*2{1$;n}#f&(m!a>pAaq9pr45NruLK9e}y;i6eM60p+UfWE@X31G&+C} zZ~iX`C=~#Vq5=j0Ws>1AaQ`eie{(nQ2Y!^W01rhzoWH}}&sn9; zCq%aYg}-0LWj?S2Ddj?RS@6L{SrKZ%nD~Jmy!r93RZ`SW7T^X9lf^G|*kyyc&O{9=y3P`V4w!w2(FTm*o3 z=KleJ1yH{Mn76)vc1}Jmd^Gdc$7AWAGaw-20j@LWMFzs#_+uV~;z9Wh@j`rCnC5xP z7K;e#XMo{l<){TZ|CPVLwVE$;I39NXc;;n<=E+MoP&{LW^Yw2bhy%EJX8niI{Rc^b zF6hUvz=5UJUsLnNz!#?BUxCj@L>}C~f%7jm_`v@G{eoE9{3haj3tpUu^6~w~{~yP( zI7?bQj(--UzfES|r)-&k0s%`2zeGXqyITe|2h*WK0ZJ*x!kGMM`}b7OyFZXl+1M@0rqF-Tvp?cm zENF{y{j;F`%^PnPAyR-TLSWRudGCfm!+9Az@CHcWAyB6NBEtS4TfYJOZQABJ`e&*7 z8_RseU*xEN8xr5HXf}ZFzWJ3heUfZ$Yck;Ujso0wrKf{-`($4zZljZk^EU(_%j7e3(vCQK!*@CNBvh4_^abT zdz+6JfU1Gb8NkBw&y~gb05NYJ^Yb!qMe{u_h~R>fEX*x*9x$*RIe+VszZCz=1w4iq zFn@OY_cw+0>u@C9R-R5iG)I!I7{1r9N=o+ zM640jP>u|S`8skv+!d(qsz$m2WEllprlO^nkB$e{4@1^8HN@L!Q?<>E5HO}2$`ayk zhxMVm+u7LJgE-Qr?#uvx1$9$-FSHpQZKZ(164VvkEr=vVR4bXG6)WmDc2HT+BW!14PFM^(p zG&aDKqJX1Y8!&8%WNoG_&M8pV4o5b%_hOn`Q*bJJLGF(Fatu8!DhDs)MnyU)InYsR z(z*~YB`qySUlm(|yT7%Uk-L{NT!X1+fRa(fYS`-sDYdT6a6FS3(~3r*A6 z5A6ZN%UT8L8(L{IX@=ZjTW=E|W2%-vmlcR{GgdWIHD(!e=qgk%hBizE_=FeQ&|U`Y zrDdrL@>V4hfa)kOXD<&OoF^DBtp_tOwX!6+;k@+~?2Sw?>MCR%mb4v~Du-a9$S&&I zR!%NKbYFzBiI$t1buhxpR0g5Wku}gV33y%2tObhNcHNfT#h=wc z8ssXi!64wMFbI)lPLRWreNA)%Jn(9^b`%EO!`a-|2j+qbHr7=MlGgP>xKLFC9gx!2 zx(?<@I|tyo-*A$#34w%=!!taCO=w1NB8=_fVHybX((yG7Lcj42>FWrD8#alRar{R~dbzh6@y>AkXG04T0m4<0H;}Q{^5htTpw?>c2D0j&rVi55ftDIF++c5< zy}YTKot2^)5=BDb@eWELeIya*tc`QAc5y>7xpMM01`rnwJVAzrVaan{;YtPsbFjUY zjJF{QW27zzL3ki>#ZDEQd-^k;a)o15IFbb8Utk6+#E0eXXDxoM3q~T?-={ zu%f1>9#IZq%~T{Psd^d6QPCijj}_5U-oVH?h!UX70R=HtNU}~etctsfiL#E4t(#(y zp|qNvv;&ju$#%ryNm|+jCLV0=DM$20I;ydV_R2K4jva=rMWYz`xy#599b8DN2s=|* z8j8-9CF1046ll(ha1BEnISn5jcWEc@Ai9|qm`K%t%c?8+s+m~3+T*RwkQI!Qj%+wsTZH=@XY%C#U zM-WR>&4sRP%2GE*G2KY2E{5bl3ptJ-#@`$u-cHL7=C9@85k#|gr`oE>DM|YUa@;j> z4t8Wmk~15l5MX3Y@uu2KL$EuONCCJChK9CIOy8BDp>jhYQ$l#P1(lk}1wuc-Z zVh*!r1v~g6)h$r67zmXLc43;k+XcW(6!pDvrf6ktjxyHQUXx&AWvw3Qs^DkjYp56C z>E)^!H$@-3ufGWyVJOY@$J)Ds=%iprkhh*9k>~}t0vW0} za>beBVN?c}vBX`0?@3?-5oT;In=-rkTNj5dUVRaCWE`ZijINOg{mF4)SJ zX=!KY;2>>jr)xrC<26XG?ifF!9n!#7&&8erG9t@z(p5D$bTDoqQUV(}Xq!v|8 z4y$hg)^Jr;(9;M&(>O?^8`%x4u1VInQ8T4`5#@CZ{FSv7?e#bW2U!e(#3bm*B2C${ zPHJelF+)>RMG*-#N2p?9RxA_?gfa?r0jAOvXNt2_G`1x>$YW6iPhBslv7wKQ4=q?z z8E&e{anUi6XEA*|a0spe-i+qrNrf4?;Egm*ycr}{FxK6|4yCJTL*xL`V2=lGY1x^Y zg5+3=e#+7YFmFq^nyq?(zlW8ckp~{Gh;bn@^a*mo{s?8PCeF-+=ttAmQU#nz8_Lmh z;@D~^E9$uEXwtQP+t{+9;oIqz$JZV zAi*?k4YHD+4jOC*rh8hzzyuqPJK#OKaCJPH8>koHkM#2-DUoay167T@QL;8(L7MU; zCP`Y^AFO9>Z;pYeQc(^RtY@%`GsY`G)e1#)kTauLBQ<2$L7u9fzQA{ahz59VG6So{ zu(tJ=)+8!POZ&N7dzia=c!IrsIM&j>svZtZ8iQc(KsAREbf_v2X=5ft-$%~T(No!K3JFHuWyA>R$+PJ zjZ`(5G6b}irnG~zG{cYV;YNX|Sefa;^eMJ5Zw!8La8_y zSk6ZSEoWql_5wl3EVc?%JCLpo!ja)Bu6EMCn({=Dk(`dSv$YkSj3sCTu?eIEH8jyc z&~XklW`MUIOu-xEA+2O2O`>S9?7#tdmL*1(YwqD_8f+eH!bHex5EU(KkQ&x(GkY^x z7Zqg_8<3eA!N*Mlre_Eyfi&#Nc%}x8U<$?3AzEAqvYd*oF5p{KxPmjz#s$K0f)UVk zEo(@qAwXQrmTVvt*H+&To&!8eC`S#F(|cWSi$F{Ch7t;BeV}TOnvbBKx4m3b>t} zt?nB#m+>gVTb7r4qW8J9v*1EGUoJZcVDGw$!dHM4?;*m*(DcIw4+&)N^1z_ouX+65BK(m(N1hKbR85h0t9+R~ z_(9{P-N|sx(!al78ywe1xl>14<78FgwEZ>^%>dp1y_c2D-=DF(Y(t>%I2y;F3*dQD z9zY9lKJK+L?7r$Lv={Wc(=4U%HzYO5*7mie58Y0`et*L8L!Nw*P2ZEmFvk*dx}V`w zs@m&La!hEz*kB`N0n09@t^09t8${N#DU2(pOol(Kl0eKwk;dZ&Y3XgQdH>D24>x8O zJ=$_$tCvvo-FP~AF1;@Jvc_(^mYOS)rgp${c3gw)IWA6oW_@+dJ{`FG+d){h^J=y( zZ#d{+kX;DhCA)$9U(^4E8;}20h5HX1$bM(=uakGm2GT400XQ4fLZ{!|B>nM}xT1g8 zj1kgX^)_XKsDC@CPQUMQ70r+iM*7zXImz=s1Nig#-v7FDnnP`P(v05W`hlZIuy`Ar zBm(t-ct#;!F8_8<*^)8-i+JJmV=vx9}0_r_h(TJM^B7yNYt zitHZMs=fqwE3mi>#9-;haNKIBj?3H{B_;Q|^lPiv`naO*fb=MCQ}<+b#%Xs1KJW`( zeK=s4c73tlM-~Wx@G-JiYQvayno;x9`J-;`iE8irMR1w^J;-tO>YViI=aB%9Lj-;P zpp2Ncf(n>*-@t;TIhI>Q5(c|5CHMO>06BHI-H1rr}kdQMWgnh35{j z`mC=}9cLQHUP><8bs`XPt#(Kb5@ASJXx%Zx8 z+X_8r=#KsK;v|!9yGmIzcjz#n$99(l_$Zl%FxFx(C#!bpFaRtz(S*DZXz&deJp=8I z2FbIOS*^q0tn+BhZmm@|SnM+6#ka7w*y9$Q%6awVyj9AEaZ}~~EY{tuQ3#QL!u8Ts zdX+`La!B>g+>T-uZ9i`hIsCwuRvjz&DXEL?du31oA6QS*mRlL>)c(+C%U|k)C1Va3 ziAGHx^YL>NzoqRgpA5+Qy*9OOaW_`rYTH5yVZzO0={>O61KS`dk<%)d~!S9oDhqkB$9Z zhAM0NZpQvK(CVswp6ROpGRm=@p9T{#(;$$|^sqGVY~HY5@|1VE^PR;Oe7&((u7Au~ z>Tuvq`arM5Xwb^B11L5C^Fz4f$5k~Jg7^Hl`k4IMybRFtw5 zSJ5aSzkOPFr`vY7vRQ6J5r<_WXQ-<*Nsc9PHlMhD&%-+aN0e$3OblRYcuu?-i;uOYj$HY92#$0P{qeR z+kQkcsQ&b@U<&iXt+dc|bk6#XJbZXQm4d{M4ztR@=4VF;3Nqn5)aoPyU4Bo}yT+}9 za6c*e`tIGI^ei%aN4>lMUe6J1h_TIp+Ck@wt}t&;XOtOQJq_fGb>ROFq}6PFb>(++ z_ga@(e9J2{$hw#MpW6SGmw=u<{U*g#GGRYz4feY%5#@}tL@haH2b;W4^@%Z@&xe>k znUlBOc~;(-0)3k1H-n;U*V?P{le-Wn9(H8URzGvJ{>CT9-!dbVJ-@mR-(t`N_p{qU zZcmPE8kx7Hq+Xd5ygbt~xy)M0<{JmS79`L$B{#C@+ z+E3cGaWGpC5X^^$FkC!#6jkwhThOJ3HN5-R`HZ^yZBog;587;9>&5ck!c^!x(!<38 z8r8_ka<)Kvf31@>X@GhYol4*LqB1U=yTI1U?(L)GP_`m#-$pT89tw!(&O6%Q-&X?i zyEG4*2eX2SbR(Ib`1=hlf?#eWbD4KeKdLpvd22nhobsRtxJzNzCqOSSIK!d zzmWn*HX3zduD$T;lzL9LUzi3D{foXCpRZxPOQQWqD5kYJo6v?W9%0OZS|u00jB2*Jt_9; z%!PMvsVQ|4dC^|y!C!C>^?henfU^e%8wY12u3(|151k(RK3>M^M*}7`&L|p7YC((| zb6`-p^xwbNLsQp(t#o?*Z2ESN(vr{Bw5=uZpIpajYh?X3!@*taH*V zk8HukGhMcX_ujr);n)M;f~)>Y9ZJ=^%G?v^o*aQ}yScS0;lu9wkJ-JM|G^h%ZRh>8 z*ZDXqP)Y`@qF%xf4b!>xdc0VIPM9#)5VW&N3QR7a{P{`X(kYrM_VP1DfVYv4M(`fm zTFJTHd!ru~PJFhq?v=*E<(M6r&*`heV0#x^=A(r-kmD8eX!~_)*H-oW8qz*)1<26P zpEA_h!Rm=kf~JQkz{adAf0DV1rRUsg3(^YX1I#gxA&w-Kjyi&*uZk?sdY<4*S?7;yw zM@!d1EyG1&*WVl&TwJS8#UeOvHG0d@BDL+%T%`F^HB?)IMHtQ8`Z0= zy@#CCs)TW^12eAg(=LWRC+QBe?-()zMqll$J(~5XYo2mWH&Cx7^(Pd5Sxyuh$o79~ z1x84D>CjJM+?FFKGDHn!zA8xHTo6C~-CbbXO@;#-HQSH&ef8r-9KpXa@oWt3Ps%lh zD$+i*I)pKi2}P^{eDJES+WkM7ZxYo~x=6LU$KUt8xEMlbdA(4uUN?)K&*X(+W znOvYT-dO!fs1V+m%PGDjCp|gSi%UNr%#+`8OX*W+G_S>Ti%kvcAvWCX_CDC`vC_i{ zoQDpGdj-T=@te&Xw%c!jmu18FRUrh~nYZI*r{wGq5no3(@c^w&dsW}o zCvx@u?-YZ7C1d_m#h^$056d~I=$|EyY)-XsH++|k{%FC&haVz@laGMd*-+R)|?s; zp&-0%TP!uJJw|j!^pJ-;Cs=q^%Bx+4q)%`-PAX9{vLEDcEzl&fcF716vnMXwcCB$p z)R)%L@8jqE%7?nF9tPK!<$}wvv21SGIi^&TWLMM%5jjNRbj^#*K77Duvorn#w7SNiIaUecXLYEHyR1axhsgDKb&&mn!OioBsGQ& zvEKbdP6tkPo^wTR+Qv;kmr7H(t8{1S`u4Z`R>b@3;7IW68c?UqPgLXOR~jET&vQwR zi5U)pog6Y;wi!}ji1my;UtJ`dA-4FOUldYpvNec3SF0|N(iijIZf?S&I%o`nXdpLw zcLO|4=XaGJ*2wm!T;1ql&<{J+w&Tu@vj*@6@}$=4#HVpS>P|55D)~)(Fbjm=5Gd~o zHQb+-GeV;9n(w31j9WI6hJK6m^t6_*$QwU-__4I#Qq3>6jX}gkOb`bn8<|rWv+duT z4HA*jX#IM4o8Y?Cz_-#pX06PbO|TQ$L@_N~Jw?rWKfgpv3+IH!#{$97j zzIzS;>mC)mA#&Yn&e>&L_*04yV?k-ulQBBD-aRy%gw-7T8-Aj;ukXS^?{wPt@t}4V z=oso8t3%If-Z%QbT`~G!_5?s47?a@ zAftC$vF%edouKqQN**INt@P}*MjYGxcX@IeDs|3dwMXaXt`j!ZWKYfmyal;=fUaR; z@g`oiHs4NCvHQ0@pL;ryg{i_`vT$<#{8n42e;tK4JKc4dlNgMR9zoXTP{iIlMznq# z%QPYg1Kh?*p!8uM0hgOi7woM^>1jLq3nA^DTu7YY;?Ifw5jAo%7-V-eeuDjOJWevu z5l(X{8LU0vy}9`3+On4hMOiAipTl2S#9R<1m`5K-P$FUIiq*rS@_jRA^C{G46-a^F z%UbDqx6kALsxsS3Z^P|IYPSs>kXt2Jqu`Vn7Y=KF8qKp5x)_s}^C7xOUx;U$4fp-K zUN1YDwpW+wwgxNJ9*dNE?pHl7TVeMntI~M(({I)S$%qa%d>xvy+Ecd?+|vR zm%vWeW{uJO-xu?z>^-abwldeMr2|RSN{yY#(YA{9L`7%C2Vyh8K`FDdRU7y9FhaSnaVSDQ?!#BpVYCx-Y+1`80M-o^XDeN&esxO zjER6F`OeqdORl|IO;H#op26ScV3_Ob#!Hb>UD1=8k(94?Q=!3q@GA6Vh^i}!*)Ms- zcuzt;r<_fj$G@Zg3Ywiv@PC$Fk+$8>L@sQd}+(fqZd?G#jhue;mW?Kw79!K}NYVyFNIXrB{ z+RObT+=*mfMg#Y6&^`neWD1|Rrt$g&M4$iXC+qNfG@S&`TU+pbGDxl4_@>+Jc78OT zBv3lvPzyW$ZDSq`uo>*s0cJhyur)N! zu><7@ZM&-=3R92yJvN5fDDGg7bg89hTua$Rblc%jq&kQBaXdHVX`Qgm&GF4HQJ(YY zZ$1Nu*Zs|$<$zZ<_T5Rn`k+^5Y|H3CvrSAAD?PrSR(%VOp!#j(Pj`1@xwBfu{RqwV z&tl=;=haIqXP?i+E1ofPKTqaSzgn77i6dU=QsQCsGldN4$jAVv;NT2YMxADr0CC}$CV~>^ZFfsX&0=n<8a z*xY&n^7pz~E+O4UQv2V~ON>P%^j-n-ysyFMMswdl?s7Ck@Ym>JAq{X6aTF3G_e zkt2WOw$<4*_@fZC;VfN1tBS7`8&A*sdjtACmkNHxlpUisZF&-w_>#A}UJbb_6J$e? zHoJXezXAWYPqw*!USQ)`ZG9G&t#UC*fRom5RX2rY1+0oRif0-N1N2Uq@B$YJc>EGs}A;e{?ZCJ*6y!R zc9ICr3F4->urtD^ZAJ% ztK#nN@KjZ7IiQ6DwMK2Z>;<|!pF@WQIAVSkwDl0b$n7lFsVuS6+6R?4ueAPkzaC`% z{RvxVGyC(u=P`r18A8vT~Y5N+9jmqiOAbNHE<+A334AlnV+){IR zzC7=lB*LpdmpkB!R~M?&(hEFs0g-oA^;n0-aQ{53FAS@P!_uP*B^(GVo@yMo7Qr@9 zhkbk%E_C6LbWi&jR#0@qrPR`laV2=)pMblZtTnu-AFmp};2lcj#gA|sqSbt{xlO>j z$EC~YT-)eP+F3=iOmg$5b~H~>RB5csl|vtNE!_j`*&p-XF92e%9i!qKFdT}5&?S2G zPW6a6>YG81lrYcEa3#{yS}3(9*~?1n)Q5pzlQ zIosV|rRGAx*paSkiGUpKrckP@!j-CbiWFDp^4%yhd z4ZdjG@1!Nu`d(D6Rn~Q;mBm~r#g-unFK}D!)cD$1TmIV0cc1IT5%5OMp^Xeq)Kdyd zhZa%A>!|75UYjIrwbYPKQzW+Gc}p`fr^~ z(r&G|Vulh%wys57>Ea{TcF)BQ`TQ*OE>WkufWvy{G*}OJkJ@R%01c+~9t^K9iQ%3y zk|_tjAQt2j9Pi6d$sO^kWFAJd?KJ=FHlEwS>soe@XJ_xE>Wi#*zDoQb4w2`voIW0} zQJ77fNjG9Nq5Muow^nT2dyx8LM&>1pYC^tHBh)!>wZhOlW2v!;x9jd@F%B(i)mauy ze;cV}HU*Bua?crM5NMb5ZQP2~y^Y&ZKpe6PhSY0N%-HA~BZ%LT@Rrf@P|v9E`&<=& z4nq%FA4T@Ze3PoA}7*3z_?%BGIy3-*A)w+vfZzE63I^)V@5-M=<%TSF7 z`O9_m`ivC}m?P)kecyRY`DU_ZPQsTuiJi`-KvZAs}`BmlY z#rUfl+N8thvw_hE!97jq;Dxk$kIay7kC(vFvH&~kRGo}PELzux_Ea`DfHc1RHM|^b z2amjEc=BuLw_1z*M@ed9PgWrcq!u!4J-0$k3k6Q~UJB^nCo= z$*aOHAi0UQOfnq&bx+=>XIG&HkNsP$;vd%k|B=ft4+ZP~gZTg76TSGLk^)h=&pm5K z{8+BY_n+-GH*3Lb&nVu&qG`KT`<32J6j(TVPqfdZh;O;NC57)zqX4`*C@T)bM30wq zS&HuXc}a{~vHTE9v!B7gqtf*Ja8D+nzD4Ri*JVx$KLuG2xtun$=xy&mP3Y<>y>w8? z=j=Nl$N6~!&j0+$*?a!!)B0qxslv_|J`z|Ppm=5nZeHv7f_PNr*c)fu@iHR$MKYEk z)-Rcmj#N8rKPA4#(ye3XJh_ZK(tT>R8X98n)%-kbQ+4mdOd6xtt=Ti=$jT^BOEQH* zgcG0SN5(7LxT!QSRDP#ato~7ayKQGtxfXRdgHxv zJ2tjm4X-G{aGeDo6)7+T5d|$;x!(@mrq!dpe4ySB*s~=>#h=WrF^x& zf+R21Wt}}O57Xh@muA#=2x|2?yAz|@YLVUg{&jvS>qz>Of%t-X{i3EjtdohKO!!65 zdm^){TAw!yD;V=3i5%ZywwsC8{{E-M8jIMJdDKQ4N1}Fm{i*&A_4s(yE$mh&J42?h1xn>%7@D>Jy_q~c3%(0e(U&JZwn$ZHp+X$?ha*OsULK1{VLcQ7OG18W(y663w12+-no`(~4|BzY=~q2-zR+jjU6RX=C9-VybhnpzbadAE=o55J9m2bE$+9W?qNM%`OTj6 zQOU3jKL~z5ztmSle=s$hD_&oi@r?tbR{sb;g?85DN0lf*(aWhoYFMLNL(QI6vzz1o za-`gSxo*gt!yp@cwTD!E*7&CT`)I>^6P&MUqSA4A{s2#{x;nG0;=6xmApbl{bH8-@ zTk5-3$`#_BQQ`{qR~za~aCc4|VBlL+AQs_7PzdM7K4-JH5K8z;i zQ(t>%V*R=5*n(V&+Z3Z(jUO%@SLiK9Xl1b??KZt0S0<9OyojhF`^o&^(8!D_v~U%k zW#he|%M1xR9$q>AZg7y6-Dl6VrNkTXho@a~o%Yv{VdySy*Du@W^!_P0IBL2HflgGOKFia_Om~DWPjR{ z&x%&rw5N4_A}yFj5lP(UP5s!AtDqfs&^IW?xm4T7?>tEFHAr*{L{}Hr3)Q{oWjqpe zOOveI=>Lpfs`hnxm&#oKR+$k1L_x8#`k71NNd+6Yi+c2`N;~B15vHR-TKW6%%l8R{ z*`BLA8q%zjR{zv26F<*RI3I615_{Hxl>`|$zu-*M|@G)Kl7o9^Y^XoV=HAp6!uY}lJl$96O6;bec(28Zyn^*GeE zv1i<^%3h;0E-J#pw(Q(fgnKSs6Un=%Fw1~-Z zx$dye2FVMd`n$kfUQ5qzI-GOtYx!;&r;XYwS-0KdK{<+OFm!pMS;FqAtGs)ev8;dzKgi zi1wO%&E4y#(cS6g>F*OFrB1shN=-^({;>*r6IghaB#c20H)YRrFC#0_HjLb9RLJ`7 z%)dUGws@IAG4A%kg>wIL$U?%;qi1S#YvXVVjU82?N zulvE8#JU7i;Q8@ZEF`$x z@jEz+<99QjKJ?*ja>C~ye|jXH#Tl5O!{!Kk1v9a&p`}R4{$fk-h)Oo@ z--h48Y1!d6_$&KxV}8;#n|t!+@$O`bUHH6Zz8=7WxVso!%_V0(G7j(Qvcgjwb{dYR zhn&-z?v9|!lXlD8>bcd7jZ%7R@#EDs0|I~=_lLh*w@{4$X@+hP|%6ftz+dRtJrhfi$(oP0Oh=@?#INFO=;?!+u;l_6d3mOTd8?f-G@uV`Md_}7$*>$3sSHax z$);}#hVO&we!DS|>B!lXFc$k44|E35{&xLYoGR6`Cn4JAOUjye=nwu(@bI!vWdifk z=j~idfemn%QD_I9b~{#5v9EGAV?+ZW;5t>8-7EJzAi-St!-CZVIq&i2XTnT?T1*Yi z5?hi@=X(ol%v^vnR^UQvy6W93$IUdM>SMl*l@NWElQxZDR;8MEy01tFNtK_R^xKVL zr5GG{>S*R{YIG><*uQmdo=?-17W^*I{apOYHc3Zg>H_VU+1i<&;gy7!NlvMyn=H|7;XB)R<$CkDK(nkH>=sggjLM{qG?2)Hr5bD*Z%6Ol}gp$`fqu@OgrPnuZfAD-jJAp_r@YuE>~=! zNjA;!BNVFA#;uvieJgKx6>h&TW%f-N+^=6TWY&@~+GN%>EZ^|p@mRZ?smI0U;P2jh zXyHy53iuN>4u_72l)Qg!yPr=l33Zi{zv1GEB@OjJKm;StTs#<9NBO|pi=Q{)TiRCf z;-DOrRlI5Zz1YnLjpbJAoeQCCc@{OL$32hrwo5bat}{VrZLM=DgpO7oWceK~?%v`T zmh0u2t=n)CiZeOsJ=W6+f7o*QeN@3%xIC^1wa5uW{KffnNWix zJH=EPc!(@8H|n+ob{VlPw|3v5Ki#@3>(ZGr$E;{~7r=e;fd0@mr61t7UDSGml3}~d zzLa(=CW8qx5MTVLq$;ijSvU*GX(HkNJ3-W2vIev3P%r$vUbTsS-sLW};q$C60vUNt z0*ztK{FAV52$c5312;}9X}MW@&sEm2x8I&aH%@OAGXEW6@~^`|{=W&66i6NN8P`G= zD3KUi;?s|QO60S;5?KCW_9l(~a(&Y8%jy&qCLLVBoossj9uH>|8#um|rY#jFONUc_ zFb}^n>-9fTVUnY~?W3aKP#@#W^(=g%Es=rDf8^19HVx|nOZBAc zps=QByZ?#Rh0-JY-D|V9XtSs0XDlt$l4)V|R`HuSrRLJ$Ss*LpnhAevp000xJFAlI zjGj{#KXM7H^KC;!bDAx@lJl5Mu;*cHtcULjr@Zp6o*&Lsz1S3B-f7o~YU}ZkNs|QJ z`gSVGe#Y#`iYv~yjy8UFXWPW2m>yLQ7c!A^<@eslNV!MV-ZuF=ufCjF1_@p2dZv~7 zUhR|_&Qd?Vb+miq$z5i5;Jbeu$;PrAi_q?6-#W+ho$U)KMVfT?kv2hUKSe8v)B>rr z@Yl8sbr&UFteU+<_`g=Yvc9VgcxrHQT~E zBlNSjC)>Q+-J`v{0P zexrK>Tb}7LB3y=T-?Nuj=fP^9GdCyua#~waVtWNqv3EV&eX+s%J-on|CCLZ9-2$m_ zO}nMUP(SZ^^P=HTk=bAR)RtQ=$m<lLm7IVM%tT|eD2kcRdNVo>{PGA*zJldb$Tgk3WlGDauG3%eiq~!Vm5XBxP zn)97TjR~n-jb`DuB)I>D}Or*SLAd;BVo<0&St4U(1RbhZC1~I8jm_Zqo&Myk;#-Z zCw?$YOH0HLbH-SkurP?aAmNC?Z0Y`F6T7UPsn~AG@Ls@sqf7fNbLOjfA{oU zU(=J-nsx|*uAd5KlMT}wb#0F>LC5Y%k(yuCq+^uL>Qh&As?}0NtwrpBK--|5jKQq9qrCO2YZR&{TL4(iDI<7jl((v1@7LQ zYta=$e_=Pc7$o_8z>K|0$Tj!XxaZjtd;JS%5|5A-+ZL7r25LRmG2#kDbKY#N4)YHj zTmTYlEmnWy&#^^aS<^_{melEvpcW{tRv*k6VBoejU{AwomXC$aGJu3=etw-N?g|OH zXM5lh4=e1iXN!*yrmN{tYjDTnFW_jSJ&}R(*4ReyPpGe>fRt0xOSsCEWVHDX5?6OKD|yuq_@62PVw@$aYt;ejc4mm&IEtM$cJ@- z$S@hr?9ZKiDEDd^=T2vLtC=MPdma&$<^~me)s_OFXSR_~=GJ)HzU^-KkBwLUDSX{f z7pkzKveXg(F-@eXqLjaIC1F|q6}gnh?pqKG_u8lVIG9j??M+IT6uNp>Dj9^=sOz_j zwt2WTXGx!4|B(Ol5;CN|=f!GMi7|NG?Y5x4S|SnDux`?=E0&)xjf#aHws2IrD_}&qB9gtlhNl>#EROUdtTDH+omp+JmY$U)B($ z22O*R{HD$Q^ew!p)iHExr}gJIQ3AJn1H9?GOlhYrgxv;byxyJs4fC$uB3Yo2-rLb) z)Oo~fx{Oo$=ye9oF^&0|oKjq=&aa%GPgv~HV*E2*UgQ1fF5-)p-MZ zub)443y<+mP{8()rKOH`E}HMzRqT>W@_y|R+vV|#)@h`=Dtzx?%msRM2U0K3h@F6FW_dWEZvYsIV>fx!M z{t@hYXB_^q+o@4j_sIGtH-GZ>eF=s>;tB1CR0g$ow|17Cp8EK~e}0%-+aZUmFCa`) zjV1K4WfP{?mV$3z_V$k`$xk2>H6mU ziIkpus*yr1vn6uF%iKDQIliOMAQM8GX) zQ_Es@5KJi@`|5XR);~Pjwd?kEbDXdKv@O<&W~-&z{QRcMrrn7AbH2OnUJTd2U$xHm zx0{u|XnYfu&Gn1zpNwbGhwb`|-krYDW}a|VBIZY1d1P|!3e+lUT<87W=Xvp|sp)JsU3;~m(v?Q@gb?MHuCX>gerDnG!gxu2uGa$U5l z(fPR;|HM@}pHA6ZSafj6-r2ri#SLQDPu>M}cP>~9gDX?mn-h#s&v-G4`j7P)!LU-L zGi$)zI}1Wp&eK1_?d`B1ZcWZdmK zV5UeqOUBfh24z9x&~$a#K?v%I;z;VEp)7yfwosxH~GnctD-ysN+ZTt(iFKi<3S z${Xc~v|6qbEE*^vtJ33^-oUNt@X6t=>5Q$Oc~ne-&wMW+`!t0{YsjAgq)?(EwdPI&dRG&+ z_QTP!d?(>|*f}HVDP!;#@}gq-4Wmp`AqO*uLqg}>*rYB}`2}S*2=;m<3%7akOnP)z z*bxEs1|=ZVN;YGXs{Y!|KI{U)6Lq2$V{hc7y}Vo1p1=)CO{@JiH}3OnmR!PiR=Ikm zA4#;46G)Xqn`U*pk{5a$wE^C)X0!^-rs~4a8(BO0J4*Y^!**e@O`Q9yUKGn~&xB`l z#9B{RqLfp|cMGp8>2|A)QUEzvigmxNj@V2g_oZq0O#Z4rRM;I&LtNOUl~}8m#6Hkr zcFOna*_d^_=)nzJjqaPf0uM|*{^Vhu@qF;!^VoC?TUZbDEc^;#GIW4cq zsglP~J~6%OYj)ks@0xrB7S1=gpk$GkkLS3Jz02rzcQrrx=zo)TX4{H#TM~X2J48i6 zP(Z;B5d<4lumKcNP*e~Bx9?x`uezzKac<5yd6gGAc8)Z0v1ZJOFV1wizEX9Z10+kNLi|%(~F;3TxH_BTHr?cfbvB7K(4lMQfr4wMEHP;&0&WF~$ zI{qv@ZDarBk&63L9{cjUg`K{@QAXvCosdmd2l;X{`$*6nHTj}xeI0dw7eAO-$wvpL z9y`s*36_MPruk7W{Tc}K{uy!g#~f_D*s%Pm2T%TndQ*P^arq3V(|3t?FR#ht_BJA% zJ@e0eKM@N3>qe|$qe+5R=LdB3S(bY2=%%?gnft9N%gWUd%Um^rYp47}-rwT~dx+um=vcnJN^(mf#n4uBtE@XZnl|l zdY22E5%XyjemyaLw*;Cv9R{7f@zH+K9RD^END9NH@N!D1CTs3fIMw@=L{_oW#ikK- zX_e{JSPz(c6!;rER7(5}l(Z|*^XGq_9scKE$n9CH@)azU{P5N6@phXW-MuXMW} zuFK^d1R0wP%C!HLNbRsloocg>6ynZ`*%9_Obo_W6?0P>6Q4cU{~-pXqXgVkj$IgpSmw$=^ak0VFOFKt_0 zK2OiC_W;x!DQFV8$9)`sJpFp5&88Eh65jhw0l)({{xm$$hxWKyic%_bX_4Q#!y=IG zPE&-q3d7QGneB{8n&CJ~Xwfy=%}o7u z+U=)Rvolf}IH6&F6`m|4=`q{%=Us%%<0R^H!AaWmcbwF zLXazNna$OJeKq)C8ftd9br16nsU``ff*~`V>D-s zeMe;u1au)qb;CdHwc`4v9@=YfExsX+%Y|pm9o9X{Ze+3ev7Rik-Cn!v%+VrUE|1TA z(9!oD+MGj_DJ^ZRV`+Tu&ZV7lesR*P)ZwF#d1bg1-#yuPw(#y}Pn%_jj**UAJGHfF zlwVx7%cJK*aB=9yL)sSR3wJ=u{?eVS-8xevhkv)}6A$nLTR(~(NmDIvA^*~XnQM8E z5;+EVj~D1@_tne?z*;a!_H0y}Js;8db7fR0;r+2qtPaEbU|NNbDEokw_whNmD;|g! zs$rA{-p&uc#<|-HTT7}RMN5^=jVC?eFHwPd*2?Afuuhv!Uw=<|39=81Xz%v|CB8Ac}B5 zSjrID+qr?2!#Mq~?Kj6s=M}B>_CP;NL5?me`2}O0+RwxBv|X&a(JU%`7<<);N~eXc zeQ(;DEi@rn-)J19p5?#Mk8gb`{-WO^{!UzK#}rT(<2Dx5`I>gzM)%B` zF6Q31GR+`q)~n>~F@$WpwoC~6X4VEP-sT;mMnkII->Nlh!7FthR`RcTp^ZDO&hh#H z9uhe-4Va+bz>m@7-C!8!kEh379bymyi@RoZ1}`YLzOF(#CChWudXZ9Hf1#JY8gEUZ6 zSt&~fU~)!4lY-j_Lg8b(DgQH zTuKlIqy$*1UQqM?o3(#FU82_9tfl0!%&ax;H?I3I9O>p{y>bJ9l#oM+n55WCys~@AKzHve!2|Y$*hu)iGo#h5TAj1!YDt@eT{pu!fFX-90~2 zFy~Xw*T5lA-p${Gvg2Cw`qrL3Ii#IWD@E{!y;%TF?Si8FYSmk~a=zB2#C^FnerSR7 z-pe7Zuf-(JVGr1^V^*L_NG1T)ch*d{=$%iRow~OPeP7t5RPr;F3C8 z{F9{_h&IU7r>$_dCRC~K=`2&IkGBy5+MRPiu*uuwcM?`7_fCZN{KC8idTIt1OVGNU zM^ro*PG>~s)~nhw030+y?xgztTixe+Ipr_ZPi@e}Khru_pu{?p)0dFV#-D|C`M9!v zg;C4!=4a>fZDpXT+wZqa5A!Awi1oF+DzETZ#OGR+3@I`s^omJ3zve|%2C%eDd5)9NF3RZRo`!}d%Lvts@~WF zp8%avFYh}{zZ4vZgy@Rmt(9h{t zVoqHj0L}Q8xWAhQ{L9VecEdXOiY+|GbDKDrm0F{ir6o784xhrrP)EP^DQL(!$(b1; zvf_N*zw0dRb=ck;?NCF%|5dAmOMOfhy(V3m%`YTt!%xYn4L)M)xGu(x_oT@a1ym?R z>Oy3m<`|Y|`Pop`S^!T*FR*cqVlo&M{?hAe!iv?@0eR5h659}uhA;bhdEQ}ugRcw- z;hoopFGf>PkL>coF}E%LZhsrsJB8dg&sAMpZOfmU)J^r0TGarASgtiL%tq`xtMfM2 zRx~24H!iOfqtQ_HW-_f%>*y9R!va@1k!$;rJ2u+0hjVs^IG%hvgB`w^>PgEu%sxyd z6xUAW_Th);XB~6*HIx~YKahUIxf&S~IdpV_cO_BnSAv^y+A(Bnr$qRYvI-F*2P0#s zVhXwFFMX1&RyLa*O%b!dE)rE7M86JFDMf#iH<;reY}oQdq5WzVgqW_k=V$JULn!gE z9cDst(ySK=V1Y~det^43*b7=jP-QW)d_k^LZCQ~w$#>%4A3-;Ij7^KZRc5jfa2fta zgRv-Se>weHEvd zqit;Sv%>($w5siCoxad6=OWUOAwvI_ z78SedU(lWwp$`lCERpd#DTd8oR9ua!`9eLKV&?|qaNN35%o}_;cbW9mh+XQsvg+QX zi;K+`g*7X(jgnZrd^fANiDLV`mMC+9Q57T&lU{;Ed%cXZpE$1Ru=$V z={tZQ?g#s-Z4T7!cJvr>nR+vt|Lvb&9te<`$S>^4fbpuCH`vpoviLA~_+REg7c zP!qM@U`tMPm9&RfI@&t@Por9ECzyQ`JI$P@CZ{8tL%fE%tq5MFX&f?2tDHpdZi3C` zoKwotA5u{+IN6r=B6>+u6h_O9R#HE^=elCob3$Dch7%-*m-C-bH~-$|B@QjL<=hsf zO&3Lzd57_}>rc;Bh;HktAD*tr=~RDQIVv*UYsAN)cE9>DGJGG2F1{b5}a7Pa#nyxJ%F`>c>T;{1v2rd=D6 zGirs*!4m4U(WVgQ`zhuEEGrKeX#kWJmL1~b7R5(h1Cf*8Tz{j@v#%g>pB?+9lqyE~ zKW(scrgzj_X%~`Z$K*qHX5b+&F^grse-}V8D1s516Ig0` z`oeiIOc9P%5iaaFDuint5YJ!TZ&XaKN?20o%!!o@w%nI-O&pD)5)By*HQ0d zkA7jvPxw-0#H=_tP~OUhX(6!xECzs_x2IyGzuVuX?DVSx8|?vs^p zycN*)uT%JxHJozFfH$_<1_{Sa`?xM(#p6l)B`ulT=qAA?(Gqb6$leB$B%jRVnlLo9s&f0Hz8W=4>F@Xt*g+2or5k>W@|BRR zn@`5@>;O_5RRd=-KNipaLH_NCU+ca74O zkmfP35%~LbPYMQ>685~4e_z&x>Y)1B#E)t-u>4qq#fps|{dJf2{+i~&Wg%^II$2RS z-*8`{)6TXhJA0+NTv_u`V{^`X<%hj40a^|$IsTAL^dbMJ-e8KHfiOvSPL5v&Uj5dq zK1l*nog~sKyL0_2fsZ z{wwhLDBAHQjd)fL4z1BE4|e#rvPRJPM?S0q6cTvzTlr23t|4(O)zLSCcs0HAyCvRy ztO$Sk-!%8ngf&GgyKs6QRvF6~CqdGwsk2=N?n$9kc{UsCFan%(Q=L~&Ku%1EYV{bN z!`8n29()(!ufQTa8img7G#Zo!jqB?@8^Gdq?i?Z!U^mqxJ!_63UCg3c$8ANBBt5M( zOt;I4#626qvQb$?%{#5N^ejIs%nC^l>9sLz-;%F=Oh^msuZBIp^z**ijl8dQUIx$b z-pTRC;XW_pYP#+Pb+tx{Tm7%1?YgdWTD^@kjae5^pWdIeI@TsnesW3hCU|aJZ<4oz z;Tg&U(CzkMZOg*x55(#`nTyvQ4yn^s`mw5BU$x@<9gP{-}}M-EW7%^daXGloO3hUU8-c+dbY0 zR8|IK;nta=gRaj<`)tk$#rCyv9aiU?N*7|<3rG|^lsgp+Mv}5aja--uwn=@*1HE%V zmsqY`dLRu06FxX{7*!R1kKsk3DGx2H6As<@sMk72BPSgS*oGgrQS;Co_=lf}fhMtt zGbA0)dZ$$G2Mb`3-HXhg^eO{u+wL~+e~y!m){HX$JKVOSQ1KZPB$Y;?=+XQDr6kx( z*wbDmJzfj1ylE~P+8}TFUg-FrkL=|rvszG;eoAfFCaDG1kvQ7CK0#z@u{X18GyiS@ zTo~}&PHnt-T8ur$;M5q_8&!5tub;189#=eT^Ki_L1!<3lP0RL9u1~i3GPn?{{su<+ z+_5q!y@fxtvSwd)j}jx+MNK$qI2iB%TUL>yQL>%H=NZo0{pk!tJtyj#h2R z6|dHvsZrwdk?Hqly~7V@m3UXAC#Ut|F6L>5VHA&={J!nCce3-Be2Fp9CoP?K=hAbV zSL-;Vm%G>IH01`ss4ogPbK2&62j36-zv+76^_9z&5aUC;0qLt(>1E2?fpzS^TecA_{apx4Dd92u2wSF9#_E0pToq}(iHdbQ|3{a2XV z=KVYsZ$0PUfEA`QHFtaMMc1yM;a{<_WTRhyQF;p7ZN`GDkREkzu~$ACS8xQ?u9~&H za>RRi^w2XWUZ?qA+0mNK-{T&zhu5V`=O+jR3!MY?CD(dWr^^F!w{hU@p<(j8vd7y; zS?hdvnLU|MH38g-1+@8bB`(PRxm>eERgcj5$Z$knPDlF1$J>aj;zpH@7 znmokYm;Nq8zS*S5@ZmakQJhn5dJ(t{Yw9ZuRtbi@78SvL^BcIpwbW(SuFz?()>BRj zvl>r=&S@_eKCD3|lcq1TI~S?QHYt@d^K!J_CM59QGmqS0SCdGcc3m)Lm0KMrImXyBwVHXK zb&0Z!P=sQZ_j;xGNqe(*0yb%9)qVe%v+r7Ye>CevjU`e+x97qk`A&US5uML>T&ooZ zttg@)UwynQF!t3U`PO~r6Auw|ZFs9xp8m=%b#ncwC>Cbt#Y-rE8k6Ex3)KVHO~NGW z5J$MWt@d(NY16sUGerQ?S-u4&1)ovs)^_Rxb90|U=z2KpPTYdSefstCu0of&pt4N5 znSTw3cK;VVf4rmw%5bNmE|Kko{l&!ovd-j%d~XE-as~3e&dZDE==b@_t>0pPP@P&w zixPLmQN`Vv8xsaoJk(z=9kf`7Vm5)ZYVqpAX;>6swj{-Kw?iJRSkL`uzgK)X0A8BuoB@LbqQkrE7Y> zDpG&*=zc!m;nO9|ldhb-@nhb;hSl}77lcCAKwJ_XSxifqs>lF!Z4b-vZ_((Eb%t=5 zjj{ctB0a2iyR*t*vR@r2ixZyxi6J*Lb8Jn#3o$>j8N=<@C7yly^TtP`;~;JO3cqo$8bOZ!wtf8%|!|JbK=*dNDs!&9%j z_1XDLQm%Mc8tS4NRW%vkO$OdqC^Xuy>PUH4l=8uRmQOL`zoj0v?6>Cx%V?2IG{5I2 zsYroPA1fogwJ3I90@hqrm5lCVt@A|!QBes#)1tG{MYgMq-dBhXoCgfs?A-g?e%3Uk z{t2j-Z_By(@}wraVgFRzr zp^WSKBTPyQrJ`p@K3ONIGaTAnpeb+2X1HOTz@S_0J4 zwv*A(d-|g8-u6?|2f1QC%Fe4^IIV-n2>K!Zs$}Q95<9mXofRR9vY4MpU^2G5M5AtS z$DKJCak)c0sXwQum%5B08U9`lVDq47y&Ezt@ZLrE-1_=CKT1W~NaH z8(sr~Gx^tkmM;FTSa9N9TmV4K;7Vw)W&d_NaM{0GlgiV+RP@;hdI-|r)q0&wzP!DW z<#M=*_El|t-ZUGb_SdsJIV23RiWY!VO73NC3Sim-alOa%6zjX0hTY~+xPl4J!o5}p zvRHeEw+gNrf9VWUtdGXsE9YKqRX<$j9n3NIZ_~5ZhdmLbL9Ej1XEK%dFUy7&h=Z0d z+s=&OQ3rDyn#`DT_j8)A_t)X?UDX=q5W0%nmJ)vA; zJOEDqxwG4;v9;%KK&hLhXx5cm3w$srCk!z-KiOJ@&KB3wU$p&`;zcLdZSV`9>{oez zDMaMh>Nhk8$5`pt?_y@PayRESDsHXl5|-Ik%NSK(v(9vrdd7Lty6MpFl?|AZEmG^_ zZPDEK8Ra`hg)1)Yzm%oa+?r@8_XAS`&;z9oMu`G4gWM}_uaVDv+{KWU=9fIYf493O z=b3wGt1uv$C8=ER+3;B+y7D&{qNv7kZ26TnO;28@V{KzJ8FP9}E7(l^W=!s(YnOC_ zteotWQ>u0t%7bn16IYS3=N?0Kk_+o*J*?Q^o(r9yHx$a>QG8OZJ<&fIhW!*-i7{vK zVW}Qd>Gh}W^-1$Xga55F=cin0^6;OXQ(RDSvGV%-VuO13r~PU3Q@NI|#o|H})L9q$ z<%*^1)9{=*EO?`)r{fkpLoxS1&rl6$&WG9H_K^SDtzv^1*k?ag=mn5 zBdf;q?+f(1?R}fafzZ?{WdnBkx_*=X+>nb}>-f*y|(Yri6Z>#JWj3kjNu6+EEtc&$bN9kepZfvA~%z}epU~7k+xqsXRAa@UU zOlSUD|E7OZG$W>Lw@fnhAm;*RTs}I}b7}N}u>bLH^Z>&>7>4oEiM{=2VW^XhW{k>A z({VvmDuc;?G$=41yQGL%<=p~}qSf_da^*sXi{|$x!lKgb3Q4q_=92vF#SZpI?})GC zmdcFsL>0Cj0B=_(F%*JAI?(V9L|Yf<>x23WM%wh-i?FURV($e1>*&2V<9@Ah_ zNs1Mn;{N#7LH)OtHs^<06N=st!M(LV$F~z+eq0Y)4@K8RzvqqlE#+N-+0DWW)F$2X zjbu_GxTyVUu5jUc?GAmtB5ddSlhpS6Tij<%oi6n7cgIh5#mjj6Szy8Lh7+HNtoDk# zv1lq?D}IBR_bW^?wAtg_1=GD2pkDCn8>@@>=jKxPIhsy8%RLG66@X`7Na9LV*Pv9# zJz>uH?+P<`JDJRed7$1OT!_XnbdG%|_+#5lJLTTz-3UF7EDu)Mz2Elf%VtPqq(u**cU4y-PXdL(0J^aZX8nhPB^t2t7{6zqi zVtEr1h4VT}hlAzzHf{%Xy(T-8&HlcLk83SZKlQ@g9%P03Us*XB<-Ij~6m zkfYI*I-kxaA^Vr(?o&q0YA4&9*l;ZwWvy}^v=Hl@!a#{$mX$Y2c;$}UtY4zP>XLu6 zPlOoVFo^y!u;e7j)|{kMOtc(&@*2D~t$mTY@uwGuG&W54 z!D&9O(&o3Y8r^~Ozu5G!=i%Y-6o1lvsN=A_U^f#JfPZsexAGd>VdbNY`#hDbeKl>Gzjbb!)Tg6|S{$#x1AoBA%kv6m`QvYLIUw)-wKMh# z!)tPTUL0;2O{tDB>#SrOr+GJ;OYLm(7!|jThCr^ptrg9Wx%})a^dKdFP4)}jhY8Emui=~SOZI@ksSohM}vlKjd?DXc%ncO`76{i<%aG&=4;iK<)i(#&? z*0`rO|Etkc*f7t#gA2+HGj?_;7f)lk`rdFk>x1OjOSiQ8fb7v@ChLk~;_sIlY8C+N zyUGvJ%A=FDX+0asaC8JGP4##wf95;?)ld#u_1-q;i(dR%C0X5S4apwX8Eu(*C*h!h7YT{c!{qL@gFyCu!DRT+7-SoLG%tunhXmibWa@$a0xy~5zK zlr-`ivBb_T5~a^VWDCm%dL(qXE}xCAg_Un#!g=_Aj2SHNoo-PcAXS&{#-nAASg!36 zKy}r*_KFgyh!`IYhFJRE>Y(hVYiH-aGRFHcg)yp%$9Eq7#T9EGJjY~t)wf*f4_TEj z_I4_#XVCA1OHvMwo*LuhA5<_k$J>pZpVQyo1m!WCZ@0tQ zCD|VvYGvNPBA;imyg!eFtk_Xs1Vrdgu@6FOv)KPlpu5RgcBQ!*AwfIJi{xZ<*_|oB zf3Icr^Q*kPVl#RPH5m97nfA^F6?Q7lq8LApV#|Y%{Ut--;VeT_KyzUJ(NhvUYzfp0 zuc_d!P0xx@2)(@J-M5vZM0G~Hx>lwcuG3RftJRCr^Q%bJiCzxHf0&RY96;c!-P-7w zueOH>Eo6r|uRf7scRAdu#}oHOcG}LrwoM)XRhPqhoixH1Om2H>g|>Oo{uq{UUx?{x z)&oSiA_7k=z|#b#(VCC+EC66DJ~u&%SlLZgL$hDFzmi%m!j;1B(*SvHK$*{i6zNlTBK7(n{KGl-Ex}#Tw z$31S0K6yUDHvR20X!jn1G49N#CGGISS{(C`=d}K7MYhkJHQ;w2(vHwTp>lS1w+nqJ z#FK1!{#}1N*AmujQwJz5eKkx|)sORl-!FF^TPIJk>#w1fwk3|C@SmD8mGQ9!|mE-EN(5PNI;bC~A4llT4^t@nNOR(24?4T)xg=X9VP#T7}* zV>4&=!16U0L)HFTomaBcfP_qqQ3N)zvh9y?rCFMrM%)h%bnv%JUtgfX&X(p75etq9EirVd3;P+3xcpLmPogVMnDpO-AIU`+E$v$7~hPA(Gh&6On}kG1NsLpWxqN;Pzf_ zgOIhoHhx}he`;Oxc+|FQNV|MZ?KOO&p54^iE33pQAl=w8U0$`0gWUl5NySPbqN40D z1@z}-+S-}Z3poldvUB~QWgIH7$~$(zhX@~?GNOdslJr^QY;!~51UIAdRQgA{nLZJq z=KX~skGS=#qu&zB@14yo6El!(8?9)?6ij0Y-F>sC^w+U+-Lm-l8jAdGq23$cE^S}j z%N#W$uiCuxG+U|EVDqQ4wW&~==*-4NSXg4!^CCCddL8;R&p&%|DAM%BGa4!V+YQJ( z{Ajy)>rcnzsbaLozj*X4x3IOh7?U&A0blGWz9p4y*(b7I3DN`q2{=#d$&Mu_g5FU>CC(J~+Ixfsxr}i3%yAv0R~> zGCk|{kVT!>6J}9qkD_w}G`TxEJP6+#yM2abHC^xIhWl}>Ggmuf9cN*@ z3|rQh-`?1H0?0lZ6giTjoa4TC6|Ss9Y;|x|Lj@!usJ(fQMt1A{TAI1 zk&P=d*1Tt$x$Ibj8apw%ZZTeg9dJSAit1qq9D_fk@Ijh5M0CIO$bU4@=~cw_!T-FX zH@|7Krp&^S1NAo_e=PM>%+|lXqOCKP>_1D@ioK|Y<`nH>(Sga1y_r$k+f8ip*JZN^ zFoB|_6^ilvNna4WbkSQuhI-X7a>rMD3})t7eA?#~!zrcrE8Oew`Ru;w9WTBH^V1hz zxyu4RY=YC$@&;pEJ}(jbcI}bUIV&!NS_+)Bzj>#5ZD3}{8CKr4lY#2koQw|blTT~M*SoRmN$MhL%b!CbuEmR4blkgl0&AD-uXbx6FE?hc z(z3W(KktKs*4vDCYx-%^VYGbp_~y6FuOo3QlY_$5IUD z=6XvjuSWH*Hjp|F7k9UC>Wei;AUeRPezPxHUx9JpIk0!nxZ(#V*a2u$<-D411mmRa ztEI!zt(+=Zt2F^Qk`$rr=OI;ueZSvYUV;?fuoHUEFZ;RC@k-h<7ex6^vqe7#MI5gt zn=Kp;bFUu6PQ@>x9U}Lkn%R8J;oNfR^}r)`x9vLlJunwyvk<|grm!EXio40Fq^oO2 z?(|ZVT0GQi!tNcBM;KhBLD@4)BR_}(fSl4if))H3NQM>|qk6RpMB3##?^qg}-Sr}v78Rz)wy;l9rNF3{1U z+q>vfwtyNFzK5mWrtE1%W8lFtKdbK^fsaBZ+oQ~`4iTlaZsnkS8oe01+E>=~IU>3P zue0X{t=TO6%94y{h`Fw19w1f-vj;iBua|I(?V#QEZF#(n-ClW=l-9mQSi`uDEj z($ip*$=P?_OTvCVyGUDXj3{%5H;d1Be7x7n!4A+Sf^MVjrr%+a`DZhp{Jn0G7xzpv z$|d7P)y?wsKF8JyeJnwyq_?ApDG0yherw0vgiZ71?JnSoGhfv^7ON3XK z;bPjI#}W8ac16N%zZ}(MmK}~JuWIWxXkC#;n;$<#7wLZzX*UEzSoc$$cGVRz4swOG zEap?b+!(alRpsGQPhS?w)7SGrY#*l{{;YzZv#|<)*98V|x*Gi|^$aX0RqNw(3DGwR z1lx8GpE`ElZe!j_{JD5pD4ur#liHL_LCXBh>R&k3OJl0;8y|1O6phB9yl$4pLu5Wj zg49sZLth>RwlY{aQVk3?eJgmXouER~)7HAPIQWHqcljNl=w(X#=Y!o1zUM;jF+mWZ zx-41tLnI~Wqb~0TB}^gC!PD>l<)Wc-*;d72O_=URn-tDz45BkT1@JucVG|$0fj7c7 z4U|B3Tt_v%bjh;-#%He6c{+yx=w4Lx;qJ|;h_uhGTML4N##LK=*DGYJ-9KkMb&_w- z1*BH6zPV%7zspFWF7NYfK&ZA}o*F>4$LV`(JuC}yX*{NHxt{fP3(*f!TE_yLclaor z^h(+!**=u@2U7KQB{!P)D|Ny-b{Tc=>leJaf^g$~yz=?Se8J zeYE6oG_PkSgG98wE6rh(Fn)>EzbRl-yWqB%iTT^pIQW;5+K$W`Fbf!uU_!nYnUdo-Zx022HPa-vd@n_j^K8vDAuofneM z$Rfa!VqvJlO8L21bdWf|!@f@c5E>XSH0l%RK)X$iZinTWj=;B|)Ie~sHxd`t%ki9z zAA)G~H>L=XZxM0!a58KzX$CGod6EWiKHZhWXeJwuJ z0dtC1o00}h)s`_o)~mKpa? zE+B96Q;uLqlnqBugb{Yc4I#sG?v%r^>_q&Isw^*~0E%}I@knxgcju)eNVw+27NSba zZSf5+xNif?>hp17A~ug^*BCqp}bz}GyNXI{^po!L)%CbnuX9EG9-wPnT)G-YV?u8-=O3H z8L3=r*Y1+Uisoq~&hPoG??%Jrakh(+^Lt)Qb?G3Ya;fmQ=60tIZ!2!0z0mC6_V!%b ztNmY7CXrsKtQa?M_1C7erwA}S^yLD@`k9=I?HX z?VIqOc6MjjcL;6&l#kJLgbE$};#;HRb2pi+7n#;q3V*>?+O_B+x?6%ta?^jnY*J%h z!#Mh@sqdI!4qBsKphcAS62ksE`zd^*b9(&HD+ioia=D4auefH?)!Q3hF#@Wfu+{~~_89xd-!0v(L`2%c~ ziJk%TAb|SG`M}2&cJjJk!j!rb~U}F+NEb60G2= z^O`b8V=Y?dhbZ=+&2T7$m#3FB|4@ZW-F-O3KIh{d2!B5}XvwD~R1B`HT63GQEX%Qnr=MqOJD$H3 zS6S0q2n0kK%61lLx;|CT`VMR^@QvVLBZPF6Ad@tBqvHcRNcNj8=?MHjFB_;tV z1Fnq9>15RA!^UY`Zzw_b=s(fnYp6*~-5-a2*(g?V01s}u-Xva>%CG&Wti{nxViYn8 z!hHKz4ORTKg&Bv5F~x=;gVb$zT1b`oT}fyu-)H4PA9V2YWB)h}zL(|I$oAr{y&JP@ zC19ugx%o$}`!lW7oQ}%PA(4$|$Ajebwv}3-%viVc(u-yOK`(D{Jy=6ksc=Mla_MmF zDFz=;C%X11<-Xe`JA({6QsKSQEbhI2t3p^nPow8Sm3c~J($Q%ymc|>pl=Z4~wvyc6 zyPw<&*Pt(jVOFkA>a)#;{WBAK>I^#~Zv486J2@$uq|z<6JQ7a=usGr#BS6^Y7T&A7 z_2HK5Zv=N6BdCQh9lL&gya`@e*S5lI`K|JANTCI%-0#q@gBR&`f0`NQ{(UHYHZ88t z?&p(ZTAR)MwA8ZPVC{pR&8v@#nwN+>y{sK4FY>XQReRWMn<7xJLCwYq8FCm^`$XA ze-0Y-B6C&!4b{QkN6!Nsd{<7dQFk(=*2-u`FD4Um1`TrFr6*ZqNW_dTpXqoq5v~vi z%e0%eBM$QtEV!8MPN;{cM(?X$CYOP*K2{~D3fo8nt_s1iiB`O0Uy|)_{Y~}n?_^8+oEV`<%fvm4e|@c#hdoiFIbLHes(GmBW7e zfZmo#1!+3rs8<$e-c9>!*f!mK?u{?t2^E&y{KD^|EkX$!m~r zn$+x8i#mqu!v+<0>rm(}5qNi$t@4wZ>0})P5S!d)j=}uC`570}IEIg!OXxVn;FU;7 zi0N-SeTYdLs8%b+dc4iS>p(5Ho7Amu7WV9r2Uz5$$Ry3aVCA7E^2bh4k^d8Mw=NX_)cS2-fZ7lqzF)bSaNGFk+Q|PQ9b{P*`M-gJj_=ah zdJxcK5a#B$bG7=EY+&KyE-IJ3uR0;H;8!boLjW4j~Tfs{bg@n{^OPa?p2l;+6!whhIb2 z&rtxcGZU*;YVsA8FSOd~G&884n*hiiZL4jKp5OM2wQbPqSeOS}e?w*d8Wv@5i;qfgA_klT#Fl@zRp8V9oxLz6_zRh8`iO4ZT4{ zDr{tP7@#C&TY=kt+m8*LgF@Y8Fx_njw}RS=50Z%IuODaIsu;u4Cpv}4$uKa(Pib}?HczIJ$%TO(w|PXulc9>(ptA~OsJ|ZE+8Vxo#(LK z^5NhipI}x3V8!(dANuUN;@bMrj$2Q5GHqBDp|seJtQhG}H-k%MO08i1n6vDGotrE# z^=E^N3Hwxd*90Sf^xqwloHg_DApag!YD@dNg~q@Xz-_f>;?E*+vQkB2RhVtgvQk3x z_gUdB(wB^?{vGsQohq@WvU}H0hn%!fiC!aY5EaDV{VMABmxpek(i$1P)= zh|V6_b)^IUvXTyb=&suqqBZ= z0oAlAL5KT@qHf*5FtlfXyOi1T3(~FVUalMdoS?idece{vAsh(ye{YnVX5F(s@PvjkdpXI!|P3W0Fz- zm~Y>ko2EDPVOI<_prkQ&c2cgrmB6M~jN=dVGkLB`HjMkD%MK2+iLrXsyRFN5y^8KT zX>(QAKyO~(>mQ%5&l|adHLwbs8<;5Y;D>|dq!I28Q^oG9uL_Z`Jg|bFWon;^>Gi*m z&pO0^AfGjf|5qzqaE0jdXO=5L&Dxi15PSoAlO>959~j&haC#tL3y1Xr92(8qs&D+r z?Q3d>Nrg{+qqCp>S7+a~v?>s-dB2}xo5e^@BA|i+6~wG4qKFs(16p-zo_^Mzi!_Rv-%5R`fN6&#I-)ic^BsHulLf5-mFH{M(m%XdwlEOm3gPINr2Pu z)9&T+DDa>1Ex@Hy>C^rO1N11TR#7)|RDICo-f|P|RIttYYOrT(SkE4%3aufKOE#|I zrS#gKUUIJq2QUO>McSvE#PDsA_p*r8Z-A`iQwb}^MIc_iqne*R>fOnR-wYoQbCR7g z(RIDuJp{t5>RcEh|rQ%#qKti-%e#Mtf<#DIFEx!0|Hf2I%=(a-Z@QxR-r zU~$8S{LBvz>wCg$cPv}!-PK92mYLbld_BK{?{Xy@>>Fh5_z;t>)nF40wis8>j0f1Q z;d_7IG{mQ-4j2z9PXZu59I!$94gyT3n5-_8u2zD>g=|ZnWRrql@Q~96V94#uC|Ya7 zE$K8zomx$;joW!78l|M-TR96OFSSjq3Q0XyUT$)>&u1e=>@wvs84~E- zDbcHDQ>gW+pZZGCNJmxS_7tf-?<6PmxmqRJ&0{d~SREP8M-?R_6Ahx&3@eREayDzH zua(L`s=f7MT;Vq9g+oY*vylLx{<|I$ePu@AR4l(POgGBIHeCFBLi9BH0 z!d2G>He^0<)Jyecvsx6gI+qrL6IFd2go0d=jDmI?&1=f6#MXwP;@`TlHEee5Ii0soE`5B7 zrA6It%!al8tr&_Y9TOPW0pCccII=+&z`7Kjd?|Q8T8b%cE@gB4S`_K@CLhio@9qK6 z5!$oedKsI{lrw}GIAUiiB#`-Uo%*#F2*X7{ZT`($(F ztQyrRo!M#mZJ(u?yuY|{YI-5ljJCnQj?*u$!kuFev_v=pw&@ zgj->e9OtZLWoTql{$^D~DP%-<_F~RqFdW9JC_Q z*)Ryu_cg7M zT?Vt9I8)`F)`?#TS$$YfiO7dZNvY^Xq&=$faSB6NhkcD^+ElxaLR;5(~TI zC}Ye1wv{qwQ>oH~qwA42s0No)V70gL&h6*OBbhpz5-r(>c-H*YSoJ$oVn{nNS&AT@+W}N_+YUCA8a*&gIx6%<%%m7ozJGDYW8zg}8 zKYyaNYvriyj<1z8e(r8@ud)xGKL3`;8dutw$f)Xp07^O%)k~NUhl?^j5xD0nz(5k{ zeuyxBY(1#3m)GPtHGz|ztF*@7fek16zdjB?WT4LfB-RwB+!udJIq;6 zO=f14UBBS;qM!Kyn63hTHeqi^x@7@-261VAZQHZjJ19O9vC=y?vPU($>?jFH_p*uZ z2E(6pG|v#3G^tmQev|IlwIy%#&-K#_NYuzRG058zXLt)bXy(Z3qWN$;>z7`7u6VnF zm2#89Wt&U2yr+pahw>%yN|g@#tbcZv9p2WP$h0r24IUp0`a@Mke>)1~_8b{tJr*D_ zm0q}6zO*1Xs-3w#J;-tKrX??ZM>k;2_Azo%lbfJKyjI>tMoPGG`XbrMMJKs_B^D24 z$*ZH5wKz<>3Z$UX*Q|e$(AyQ1L*Utu(R{-T@EVy)0zD|f6SP?$X*ZTq; zsEV24E8RX8R^T|ZgaqF9w%RqW_lm^Uabq**7|9)J2JFGi{m93yOq;CqRL#VPWIkjmXcdI+|jrxZ%m%LCPyVDHqmTU@E0 z4Y=(7nvn)mf-9|ar%84(1Pi_6{>-hPl}A~oo&)10zCtN{;!5GRT^}vtphl%8NZF;& zFpn2H9df<8uH*zef%{NpppQ!WSgc6IicQK_%wwt!bhv9V7fca(owN( z(E*qdUoMTb59Y*;d9`)x#R#@Rhwn8}iIzyuwB`6wt9xP8Oct^EwQcgaZmw!s zP$0eb?y*y$(0h+>Ro3UjrI!cFQg~u-APnP`(Hz1qU1snWjBdw$&R?5ZwjpN4)D6wt zhUzfgeRMO;;|>&_@BN$P?(x!gTRhrt=HVoX`Y>-zc-CDXT2(oqax?jG9|S&fb_)6e ze2lb4^Ne3X^aYt(?!fGbi>D5X82`M8x5ccU$ZMZ0J=+dj4IYp+Oy}7Dm z%#Eut9i7dszEo1RWlKpgSy9u^7c6sVoYi!Q7)m3-(6;X1SHooyb?El!15~=HoQMiJ z6*=XTW8!c%syQL$i>Z9{oZc;#Zhv&U_0;Dxp+@h(s*pbj$z{pLtZnctZ{t#JK^hIP z1Cw$FPVOU}wrr3f_r2lzL`(+3&0CJCDS3aH=yKmv)|iRjz3e*6kKdyFD%poZ*}p9; zMxgenqD>k>D;10?ShBfp`29o^hQY8%ESt7BI(ig=ngBgHb$e!^0a{$jQrWgmi0_107qyYo%W zaw?syL{G1~Y>YcilhvRMu3!5f=7)d)8pJsPnc(xYx;iI%co$`A4@Ej@&s^l(w zZQQ}uj;-vF|0&IY{r{&KaE`?PD@teoaprZ$k?PqQ5YujOP8)6I!HP;CqUzNT7qQFX zV}Q_yrNYTn$_EhHyRedx^Ne1i{^r_{z{Q4OSPV80!T=O5EonI!Vl12p2Fn$9e6NFR zax<*1A=$T4e7W!@dp%P-maGDq(u_cuFNZVWzjFEs)MHT>eO$!|)QBH%upAXnF)LVT zq*+X?o4GT!pbxp1iRG1McH}B7yKTyaQ5X*>KcoGw9k9MoD@~tp_aytzco;~a>pXJ8XNh1wl$@E7kAhsl` z>+;LvdQ~xw14u-vU@tk#B-69^MgLe7j5snJvsp#3D#@@Zw@M&tTvqzd*l>4%GWa;VCuR1r5jh&;krDF z$@_IUdxypOWIo7dtWR&A)@MQkEaJ}j=Z%iruSh*K4L98c6e=Rm9lSx!4xsMuVQ&B4 z(4tjt^SHKF!MjC`d0M_yNO5QtK@>Y%++(v&+34y(a*Ax@tX-Dx)QQd+a(h4RJG*2) zI;`sb-ga+J+vubbB{n4T?mC73kjt-4`Mz`7`FX;N-PH(vDxcl5si#n~uQkCI&8P!b z?lva!Zrx9uts}Z?+O=}ANt8Tn_u%VVW^aNm$P4^pbTNL7CSv!2tTNrsY=eKU+1@)N zKc^&+R>VeV(;}H^SU^7Ug+BPgFJ>K)+%Ag1Luyq|qh zEU9I$qkV#DBC3c~n;VEu-i@cT;r7%%O6@{-i2$3e@J#US^)3I_z3|gqL*XPA1N$g#5^C-5b^)g^N%26L3`@DaM8R=|j+y zN;(yUQdCrCsVUteV|!T8>mB1*dpQ@zBHd1FlX>X7>1=$3j~2W8QBc2E-aoKTbB1Zc z!#iM;&1>fzHi?PB$bMknaf+_;6<_w z>7_@o)tb~@7`0kEMN6%l^d6yMSK|Qi%%ESn_8ZGT!kWYGR~zk~oA-3e1Kj_%3eFHW zyqoQekjaQ(slyENUAI<}{a6lte9jIn*A#9J@YuRwVOgo{*GhE7R8ootLALwg%Lw3y zjpp%qWY5DhfzUJcgt3VV=I?4#K(W*A9HXk!8ZFW|8 zoEUW>4V70)W!mF{b9UHlswh{R`(`RR(KC(-dYG?N=ahK3&(JJoLw@t})XTw+DI&=e zm+obA>Vq^NX>K2Y$>7A1u`~J?zkc%_bpT0C2U#3pJtvCmO;ZTWIFTj^L$!J?jT&h88QpNX;A^NPdc0b#6D;2?_I6?|8KYWmR;3W>maxqm-@6~5M0bdtu9)+lTr&@~ zA&9TcGTU4FOR!X2CZ&w+%!8%>AiOF7WD=fh9faM2AJbC=Z13hMkPnb3Z#v!C>vG2K z>z-FX3t}zyK<&1;=d18?iT=gx0w+z?)5-0lXrlgb-w`ZcZ1r|QN5~Y%k9zv(rL}z% zWY^AB2i(FFB3(MvUwHack4mI_X}9d7L9L3jOlib{>(N%NoY#8!rF4R0XG}ACdNE!J zh0$8(_Y{N3N_O8kTZL$xMb`W>2M4#S@KCbE9Qto9vUOr=qW?1$_uIsYxc@15E zz?a%B8T=w@Aie^qc*hlU>=FQd_-af;4!LD`rr1zc1K0z$R~B3n_6W?WsbfB5 zuk1@%?q4e=5uM~LLIRpN3;JvCOt;TjU>6#KWks_5&m3WqHOuZ5M)Ss!$~x&a{gvF* z23F}@Rwt)0o=;bJTzWRE{QG`Odk>cK6a1;ACY13ew^v=|U3)d1PmQ5uhXnc2(#~0x zK&tC?0DVkqVV2*bTe~!#@^Mzj-r2ppKj6;IZ0t5|P!+&yk#~9msm5uyHD74AP|aR3 zm{VUBXj|IWfB`Q|uU&o7m-IR1QXFEPc{j7k)>?9KxpSIoMpm~}Nd`xt(4+ET!ewKpBD=>ez+ zC0^wQ5}X=0$VEF>p`%;XZFCK(!k|{c1DHJr=|Xp48A6fXd7X#BjT@+qLN^+0`p(l` z$expdmstnlFqdX_B2z=6xLY2 zCCt`oP!+Smbbhg+&(EPFYaLR}8ns$*s6@LBAJx|ER$6NvanHpk!!U?fw;!8sVlmM5 zs?lxZ+vh$_th2)ASwZl5y5|V(09zKya&ZWfCc8-fP{Q>I*#a{S+8CgBf@+KiHqzKa zdL?JDZ@6UgKC+9es+SU@vaPB75UZ^KmDcH8M_&49yputZjA!#@ZWn;?W&3Q*`F3ajAFb$}_c!{_7>)vR|0_md;6eEnqwwErLC*H49ANps zV;D;Ql*0h{{8Nske#=1{e#)`G?Ep^)u!Q*Ub;faq_=%CsPkRZD`))5o5ae%+V!rFb zP$>2rqv-E@a3q6$_a8@cKmEc{#82Benq|MP#b_Fo*#CRmAO?Vu?-)xGKjm2NyB!!y z)96?KLG0&QVk}4;f7gX&neX4j3H+!3AVx7Dg7;sa5tKslul~acipIX`jB_~1Dt%oG zxp(ZVpK%WMjXyC8s{P-uB}jt%E=Q0g#(lMoASsA`UrS>g`4yuG4Ew$ZMG)W5ji5+| z`u;tNWQnhSAt<=-S34j^;Xn7KX#8u8LpkRAU4R%z{nU>Jp3HYUXq@>iN76s{rK#_C z1|A!WeT_eeaqQ2v3`YINz^(YZE(}Tj)CGq9Z;bhV&k2g*$e(^;QSLVel)>-&vgG%> zN>D7t{yYyD_rJf#{l+-#_jzzI!F=EM--PfJgT&@fjHSN&iiWr9yDl_LP2bO#MhWzL ztkEb*|6WUD-*tf*;%EHPC`>S4^`l|ve%G01>7OwUMs+{Wl7VObJ%(6}#lQQWWw4)X zITBvVAk0pSkHS4+r4f7CoVqDIpNRK-J zP0DqTAgu&ea{tc~+drAVV-IvT-eBtwx|NC$3{~iCSjzOsQ@4v<9-#?Sf|MLDV@}l@Be(zuJ|Fi6q PC@cr;pG-zI+JF8xg*(#n literal 0 HcmV?d00001 diff --git a/09_vector_retrieval/PurplePulse/results/exact_fused/million_compare.csv b/09_vector_retrieval/PurplePulse/results/exact_fused/million_compare.csv new file mode 100644 index 00000000..5470c7bd --- /dev/null +++ b/09_vector_retrieval/PurplePulse/results/exact_fused/million_compare.csv @@ -0,0 +1,7 @@ +top_k,mode,average_ms,qps,run_p50_ms,run_p99_ms,gpu_buffer_mib,initialization_ms,distance_or_fused_ms,final_topk_ms,maximum_score_error +10,matrix,563.347,1775.1,563.383,563.491,734.335,209.622,538.646,23.8401,0.0 +10,fused,179.736,5563.72,178.205,184.355,488.437,198.913,178.791,0.18313,0.0 +50,matrix,1396.89,715.874,1396.78,1397.28,732.49,180.337,538.588,857.022,0.0 +50,fused,195.186,5123.32,193.357,199.844,488.935,169.291,193.217,0.80857,0.0 +100,matrix,2253.36,443.781,2253.79,2254.14,732.526,175.024,538.601,1712.97,0.0 +100,fused,212.73,4700.8,210.322,217.367,489.558,172.003,209.586,1.80797,0.0 diff --git a/09_vector_retrieval/PurplePulse/results/figures/architecture.svg b/09_vector_retrieval/PurplePulse/results/figures/architecture.svg new file mode 100644 index 00000000..505a3ce3 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/results/figures/architecture.svg @@ -0,0 +1,84 @@ + + WarpFusion system architecture + Versioned inputs flow through host-side validation and index construction into resident GPU Exact and IVF pipelines, followed by strict correctness checks and reproducible artifacts. + + + WarpFusion: from versioned inputs to auditable results + + + Inputs + + Vector database + FP32 / FP16 + metric + + Queries + parameters + K, batch, nprobe, layout + + Public provenance + checksum + ground truth + + Versioned binary I/O + magic / shape / dtype + fail-closed validation + + + Host orchestration + + CPU Exact reference + stable score + ID ordering + + IVF index construction + deterministic k-means + continuous bucket layout + + Reproduction driver + build → test → datasets + benchmarks → plots → PDF + checkpoint + resume + + + Resident GPU engine + + Exact fused scan + query tile → warp heap + block merge; no score matrix + + IVF-Flat search + center selection → bucket tasks + warp scan → local heap + query-level k-way merge + + Persistent buffers + only queries/results transfer per batch + + + Evidence + + Correctness + ID + score + recall fallback + + Performance + QPS / P50 / P99 + GPU buffers + + Artifacts + CSV + JSON + SVG + PDF + + + → + + → + + → + + Every optimization is accepted only after reproducible A/B and correctness checks + diff --git a/09_vector_retrieval/PurplePulse/results/figures/batch-qps.svg b/09_vector_retrieval/PurplePulse/results/figures/batch-qps.svg new file mode 100644 index 00000000..0aa35880 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/results/figures/batch-qps.svg @@ -0,0 +1,53 @@ + +Batch size vs throughput +Benchmark curves generated from the checked-in raw CSV data. + + +Batch size vs throughput + +4049 + +4834 + +5619 + +6404 + +7190 + +8 + +16 + +32 + +64 + +128 + + +Batch size (queries) +Throughput (QPS) + +K=10 + + + + + + + +K=50 + + + + + + +K=100 + + + + + + diff --git a/09_vector_retrieval/PurplePulse/results/figures/recall-latency.svg b/09_vector_retrieval/PurplePulse/results/figures/recall-latency.svg new file mode 100644 index 00000000..2ee25924 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/results/figures/recall-latency.svg @@ -0,0 +1,139 @@ + +Recall–latency frontier +Benchmark curves generated from the checked-in raw CSV data. + + +Recall–latency frontier + +WarpFusion + +FAISS GPU +K = 10 + +97 + +159 + +221 + +283 + +345 + +0.87 + +0.90 + +0.93 + +0.97 + +1.00 + + +Recall@K +1000-query latency (ms) + + +128 + +160 + +192 + +224 + + +128 + +160 + +192 + +224 +K = 50 + +78 + +123 + +168 + +214 + +259 + +0.85 + +0.89 + +0.92 + +0.96 + +1.00 + + +Recall@K + + +128 + +160 + +192 + +224 + + +128 + +160 + +192 + +224 +K = 100 + +88 + +136 + +184 + +232 + +280 + +0.84 + +0.88 + +0.92 + +0.96 + +0.99 + + +Recall@K + + +128 + +160 + +192 + +224 + + +128 + +160 + +192 + +224 +Matched batch=64 · point labels: nprobe + diff --git a/09_vector_retrieval/PurplePulse/results/figures/recall-qps.svg b/09_vector_retrieval/PurplePulse/results/figures/recall-qps.svg new file mode 100644 index 00000000..93bc2dbc --- /dev/null +++ b/09_vector_retrieval/PurplePulse/results/figures/recall-qps.svg @@ -0,0 +1,139 @@ + +Recall–throughput frontier +Benchmark curves generated from the checked-in raw CSV data. + + +Recall–throughput frontier + +WarpFusion + +FAISS GPU +K = 10 + +2595 + +4243 + +5892 + +7541 + +9189 + +0.87 + +0.90 + +0.93 + +0.97 + +1.00 + + +Recall@K +Throughput (QPS) + + +128 + +160 + +192 + +224 + + +128 + +160 + +192 + +224 +K = 50 + +3496 + +5529 + +7561 + +9594 + +11626 + +0.85 + +0.89 + +0.92 + +0.96 + +1.00 + + +Recall@K + + +128 + +160 + +192 + +224 + + +128 + +160 + +192 + +224 +K = 100 + +3260 + +5036 + +6811 + +8586 + +10362 + +0.84 + +0.88 + +0.92 + +0.96 + +0.99 + + +Recall@K + + +128 + +160 + +192 + +224 + + +128 + +160 + +192 + +224 +Matched batch=64 · point labels: nprobe + diff --git a/09_vector_retrieval/PurplePulse/results/figures/report-architecture.svg b/09_vector_retrieval/PurplePulse/results/figures/report-architecture.svg new file mode 100644 index 00000000..59c06608 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/results/figures/report-architecture.svg @@ -0,0 +1,47 @@ + +WarpFusion 系统流程 +图中数据由仓库内原始 CSV 自动生成。 + + +WarpFusion 向量检索系统的组成与数据流 + +输入 + +向量库与查询 + +检索参数 + +公开数据校验和 + +CPU 端准备 + +格式与参数检查 + +精确结果参考 + +构建并保存 IVF 索引 + +GPU 常驻检索 + +Exact：边算距离边选 Top-K + +IVF：选桶、扫描、合并 + +每批只传查询与结果 + +输出证据 + +ID 与分数 + +召回率与延迟 + +CSV、图表、报告 + +→ + +→ + +→ + +每一次优化都要同时通过正确性检查和同输入 A/B 对照 + diff --git a/09_vector_retrieval/PurplePulse/results/figures/report-exact-breakdown.svg b/09_vector_retrieval/PurplePulse/results/figures/report-exact-breakdown.svg new file mode 100644 index 00000000..1f3f0ef7 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/results/figures/report-exact-breakdown.svg @@ -0,0 +1,48 @@ + +Exact 耗时分解 +图中数据由仓库内原始 CSV 自动生成。 + + +Exact 融合路径的 GPU 执行时间分解 + +距离与向量读取 + +局部 Top-K + +最终合并 +百万条 128 维合成数据 + + + +K=10 +98.3% + + + +K=50 +89.0% + + + +K=100 +80.7% + +67.7 万条 384 维文本向量 + + + +K=10 +98.7% + + + +K=50 +95.1% + + + +K=100 +89.6% + +柱顶数字表示“距离计算与向量读取”占 GPU 核函数流水的比例 + diff --git a/09_vector_retrieval/PurplePulse/results/figures/report-exact-fused.svg b/09_vector_retrieval/PurplePulse/results/figures/report-exact-fused.svg new file mode 100644 index 00000000..cec47e1c --- /dev/null +++ b/09_vector_retrieval/PurplePulse/results/figures/report-exact-fused.svg @@ -0,0 +1,44 @@ + +Exact 融合路径吞吐 +图中数据由仓库内原始 CSV 自动生成。 + + +Exact 融合路径与距离矩阵基线的吞吐对比 + +0 + +1,000 + +2,000 + +3,000 + +4,000 + +5,000 + +6,000 + + +1,775 + +5,564 +K=10 +加速比 3.13× + +716 + +5,123 +K=50 +加速比 7.16× + +444 + +4,701 +K=100 +加速比 10.59× + +先写距离矩阵 + +边计算边选 Top-K + diff --git a/09_vector_retrieval/PurplePulse/results/figures/report-k100-partitioned.svg b/09_vector_retrieval/PurplePulse/results/figures/report-k100-partitioned.svg new file mode 100644 index 00000000..e4b4e320 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/results/figures/report-k100-partitioned.svg @@ -0,0 +1,51 @@ + +K=100 分区候选 +图中数据由仓库内原始 CSV 自动生成。 + + +K=100 条件下分区候选结构的性能影响 + +0 + +10,000 + +20,000 + +30,000 + +40,000 + +50,000 + +32 + +64 + +128 + +160 + +192 + +224 + + + + + + + + +完整大候选表 + + + + + + + + +4 个小候选表 +Recall@100 相同；加速比 1.38× +搜索桶数 nprobe + diff --git a/09_vector_retrieval/PurplePulse/results/figures/report-k50-scheduling.svg b/09_vector_retrieval/PurplePulse/results/figures/report-k50-scheduling.svg new file mode 100644 index 00000000..28a9568f --- /dev/null +++ b/09_vector_retrieval/PurplePulse/results/figures/report-k50-scheduling.svg @@ -0,0 +1,55 @@ + +K=50 任务调度 +图中数据由仓库内原始 CSV 自动生成。 + + +K=50 条件下按桶任务调度的性能影响 + +0 + +5,000 + +10,000 + +15,000 + +20,000 + +25,000 + +32 + +64 + +96 + +128 + +160 + +192 + +224 + + + + + + + + + +按查询顺序 + + + + + + + + + +按桶重排 ++24.8% +搜索桶数 nprobe + diff --git a/09_vector_retrieval/PurplePulse/results/figures/report-real-comparison.svg b/09_vector_retrieval/PurplePulse/results/figures/report-real-comparison.svg new file mode 100644 index 00000000..95e64b6c --- /dev/null +++ b/09_vector_retrieval/PurplePulse/results/figures/report-real-comparison.svg @@ -0,0 +1,46 @@ + +公开数据对比 +图中数据由仓库内原始 CSV 自动生成。 + + +公开数据集上的端到端吞吐对比（nprobe=160） +SIFT1M（图像特征） + + +1.81× +K=10 +召回 0.9999/0.9991 + + +1.43× +K=50 +召回 0.9997/0.9995 + + +1.33× +K=100 +召回 0.9996/0.9995 + +Yahoo MiniLM（文本向量） + + +2.53× +K=10 +召回 0.9939/0.9953 + + +2.43× +K=50 +召回 0.9915/0.9913 + + +2.34× +K=100 +召回 0.9892/0.9891 + + +WarpFusion + +FAISS GPU +召回数字依次为 WarpFusion / FAISS；两者各自训练聚类中心 + diff --git a/09_vector_retrieval/PurplePulse/results/figures/report-recall-qps.svg b/09_vector_retrieval/PurplePulse/results/figures/report-recall-qps.svg new file mode 100644 index 00000000..3d7b23ca --- /dev/null +++ b/09_vector_retrieval/PurplePulse/results/figures/report-recall-qps.svg @@ -0,0 +1,91 @@ + +召回率与吞吐权衡 +图中数据由仓库内原始 CSV 自动生成。 + + +公开数据集上的召回率与吞吐权衡（K=10） +SIFT1M（图像特征） + +97.5% + +0 + +98.1% + +19,613 + +98.7% + +39,225 + +99.4% + +58,838 + +100.0% + +78,450 + + + + + + + + + + + + + + +nprobe=32 +nprobe=224 + + +Yahoo MiniLM（文本向量） + +95.7% + +0 + +96.7% + +10,792 + +97.8% + +21,585 + +98.8% + +32,377 + +99.8% + +43,169 + + + + + + + + + + + + + + +nprobe=32 +nprobe=224 + + + +WarpFusion + +FAISS GPU +横轴为 Recall@10，纵轴为 QPS；每条曲线包含 nprobe=32、64、128、160、192、224 +越靠近右上方表示在相同召回率附近具有更高吞吐 + diff --git a/09_vector_retrieval/PurplePulse/results/formal_100k15/exact_gpu.csv b/09_vector_retrieval/PurplePulse/results/formal_100k15/exact_gpu.csv new file mode 100644 index 00000000..7b73c030 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/results/formal_100k15/exact_gpu.csv @@ -0,0 +1,4 @@ +implementation,backend,metric,num_vectors,num_queries,dim,dtype,top_k,batch_size,warmup,run_samples,average_query_ms,qps,run_p50_ms,run_p99_ms,batch_latency_samples,batch_p50_ms,batch_p99_ms,gpu_initialization_ms,database_h2d_ms,gpu_buffer_mib,query_h2d_ms,fused_kernel_ms,final_topk_ms,result_d2h_ms +purplepulse,gpu,inner_product,1000000,1000,128,fp32,10,64,5,100,149.755,6677.57,150.025,151.143,1600,9.53296,9.64477,200.701,48.602,488.437,0.22932,148.772,0.187956,0.363535 +purplepulse,gpu,inner_product,1000000,1000,128,fp32,50,64,5,100,167.387,5974.19,167.671,169.027,1600,10.6347,10.8679,173.011,49.114,488.935,0.232645,165.594,0.844275,0.411974 +purplepulse,gpu,inner_product,1000000,1000,128,fp32,100,64,5,100,184.966,5406.4,185.035,187.947,1600,11.7263,12.1334,172.278,48.5765,489.558,0.232269,181.896,1.88726,0.492971 diff --git a/09_vector_retrieval/PurplePulse/results/formal_100k15/faiss_cpu_flat.csv b/09_vector_retrieval/PurplePulse/results/formal_100k15/faiss_cpu_flat.csv new file mode 100644 index 00000000..5f039ec3 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/results/formal_100k15/faiss_cpu_flat.csv @@ -0,0 +1,4 @@ +implementation,backend,index_type,metric,num_vectors,num_queries,dim,dtype,cpu_threads,top_k,batch_size,nlist,nprobe,train_samples,build_ms,load_ms,gpu_transfer_ms,gpu_index_mib,gpu_temp_mib,average_query_ms,qps,batch_p50_ms,batch_p99_ms,recall_at_k,min_query_recall,training_iterations,training_seed,run_samples,batch_latency_samples,mean_absolute_score_error,max_absolute_score_error +faiss-1.14.1,cpu,flat,inner_product,1000000,1000,128,fp32,1,10,64,0,0,0,0.0,325.0962197780609,0.0,0.0,0.0,13079.959641397,76.45283528513971,834.8349779844284,846.3362380862236,1.0,1.0,0,0,20,320,2.16458e-06,1.15e-05 +faiss-1.14.1,cpu,flat,inner_product,1000000,1000,128,fp32,1,50,64,0,0,0,0.0,325.0962197780609,0.0,0.0,0.0,13080.880285799503,76.44745446417637,835.1001441478729,847.1395075321198,1.0,1.0,0,0,20,320,1.95558e-06,1.33e-05 +faiss-1.14.1,cpu,flat,inner_product,1000000,1000,128,fp32,1,100,64,0,0,0,0.0,325.0962197780609,0.0,0.0,0.0,13596.499814093113,73.54834065186945,867.8400814533234,881.017416715622,1.0,1.0,0,0,20,320,1.886558e-06,1.33e-05 diff --git a/09_vector_retrieval/PurplePulse/results/formal_100k15/faiss_gpu.csv b/09_vector_retrieval/PurplePulse/results/formal_100k15/faiss_gpu.csv new file mode 100644 index 00000000..e1985ca6 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/results/formal_100k15/faiss_gpu.csv @@ -0,0 +1,16 @@ +implementation,backend,index_type,metric,num_vectors,num_queries,dim,dtype,cpu_threads,top_k,batch_size,nlist,nprobe,train_samples,build_ms,load_ms,gpu_transfer_ms,gpu_index_mib,gpu_temp_mib,average_query_ms,qps,batch_p50_ms,batch_p99_ms,recall_at_k,min_query_recall,training_iterations,training_seed,run_samples,batch_latency_samples,mean_absolute_score_error,max_absolute_score_error +faiss-1.14.1,gpu,flat,inner_product,1000000,1000,128,fp32,1,10,64,0,0,0,261.927954852581,0.0,338.9444574713707,492.095947265625,1536.0,47.11136355996132,21226.301351419028,2.9594004154205322,2.972230315208435,1.0,1.0,0,0,100,1600,2.15207999999e-06,1.14999999994e-05 +faiss-1.14.1,gpu,flat,inner_product,1000000,1000,128,fp32,1,50,64,0,0,0,261.927954852581,0.0,338.9444574713707,492.095947265625,1536.0,48.20252262055874,20745.802203586176,3.023482859134674,3.048732876777649,1.0,1.0,0,0,100,1600,1.956738e-06,1.14999999994e-05 +faiss-1.14.1,gpu,flat,inner_product,1000000,1000,128,fp32,1,100,64,0,0,0,261.927954852581,0.0,338.9444574713707,492.095947265625,1536.0,49.90370690822601,20038.591558719705,3.1260475516319275,3.1578615307807922,1.0,1.0,0,0,100,1600,1.88528e-06,1.14999999994e-05 +faiss-1.14.1,gpu,ivf_flat,inner_product,1000000,1000,128,fp32,1,10,64,256,128,100000,6923.940047621727,0.0,784.832239151001,497.984375,1536.0,145.7934482395649,6859.01878359321,9.360350668430328,10.512225329875946,0.8795,0.5,15,2026,100,1600,0.09018460273,3.191515 +faiss-1.14.1,gpu,ivf_flat,inner_product,1000000,1000,128,fp32,1,50,64,256,128,100000,6923.940047621727,0.0,784.832239151001,497.984375,1536.0,90.35561479628086,11067.380840190588,5.7607293128967285,5.909278988838196,0.85972,0.68,15,2026,100,1600,0.11686476591,3.191515 +faiss-1.14.1,gpu,ivf_flat,inner_product,1000000,1000,128,fp32,1,100,64,256,128,100000,6923.940047621727,0.0,784.832239151001,497.984375,1536.0,100.42996548116207,9957.187530723317,6.3972026109695435,6.478957831859589,0.8491,0.7,15,2026,100,1600,0.132494336279,3.191515 +faiss-1.14.1,gpu,ivf_flat,inner_product,1000000,1000,128,fp32,1,10,64,256,160,100000,6923.940047621727,0.0,784.832239151001,497.984375,1536.0,209.93292428553104,4763.42623913481,13.52139562368393,14.626868069171906,0.9369,0.7,15,2026,100,1600,0.04626592506,1.8539982 +faiss-1.14.1,gpu,ivf_flat,inner_product,1000000,1000,128,fp32,1,50,64,256,160,100000,6923.940047621727,0.0,784.832239151001,497.984375,1536.0,113.9417802542448,8776.411933959984,7.0451050996780396,11.37205958366394,0.92312,0.8,15,2026,100,1600,0.060067846784,1.8539982 +faiss-1.14.1,gpu,ivf_flat,inner_product,1000000,1000,128,fp32,1,100,64,256,160,100000,6923.940047621727,0.0,784.832239151001,497.984375,1536.0,126.59424863755703,7899.253013168305,8.060820400714874,8.196607232093811,0.91634,0.8,15,2026,100,1600,0.069078025599,1.8539982 +faiss-1.14.1,gpu,ivf_flat,inner_product,1000000,1000,128,fp32,1,10,64,256,192,100000,6923.940047621727,0.0,784.832239151001,497.984375,1536.0,268.18819120526314,3728.724950587515,17.25468784570694,19.591324031352997,0.9722,0.7,15,2026,100,1600,0.0204540375,1.8539982 +faiss-1.14.1,gpu,ivf_flat,inner_product,1000000,1000,128,fp32,1,50,64,256,192,100000,6923.940047621727,0.0,784.832239151001,497.984375,1536.0,134.80234675109386,7418.268480492054,8.60326737165451,8.92588496208191,0.96408,0.84,15,2026,100,1600,0.026476802958,1.8539982 +faiss-1.14.1,gpu,ivf_flat,inner_product,1000000,1000,128,fp32,1,100,64,256,192,100000,6923.940047621727,0.0,784.832239151001,497.984375,1536.0,152.11867079138756,6573.815001127506,9.70180332660675,9.853780269622803,0.96127,0.88,15,2026,100,1600,0.030801589658,1.8539982 +faiss-1.14.1,gpu,ivf_flat,inner_product,1000000,1000,128,fp32,1,10,64,256,224,100000,6923.940047621727,0.0,784.832239151001,497.984375,1536.0,327.7342140674591,3051.252988173414,21.058037877082825,22.79273420572281,0.993,0.8,15,2026,100,1600,0.00526308172,0.8752346 +faiss-1.14.1,gpu,ivf_flat,inner_product,1000000,1000,128,fp32,1,50,64,256,224,100000,6923.940047621727,0.0,784.832239151001,497.984375,1536.0,152.58803740143776,6553.593696006064,9.723648428916931,9.950794279575348,0.98988,0.92,15,2026,100,1600,0.006967170764,0.8752346 +faiss-1.14.1,gpu,ivf_flat,inner_product,1000000,1000,128,fp32,1,100,64,256,224,100000,6923.940047621727,0.0,784.832239151001,497.984375,1536.0,177.92695373296738,5620.283936860961,11.344268918037415,11.505231261253357,0.98879,0.95,15,2026,100,1600,0.008430840391,0.8752346 diff --git a/09_vector_retrieval/PurplePulse/results/formal_100k15/ivf_k10.csv b/09_vector_retrieval/PurplePulse/results/formal_100k15/ivf_k10.csv new file mode 100644 index 00000000..a029865f --- /dev/null +++ b/09_vector_retrieval/PurplePulse/results/formal_100k15/ivf_k10.csv @@ -0,0 +1,5 @@ +nlist,nprobe,training_samples,training_iterations,training_seed,top_k,average_ms,qps,run_p50_ms,run_p99_ms,run_samples,batch_p50_ms,batch_p99_ms,batch_latency_samples,recall_at_k,min_query_recall,mean_absolute_score_error,max_absolute_score_error,index_load_ms,gpu_initialization_ms,index_h2d_ms,gpu_buffer_mib,center_ms,scan_ms,merge_ms +256,128,100000,15,0,10,142.972,6994.35,143.046,144.389,100,8.91104,8.99946,1600,0.8876,0.4,0.0854709033,3.3332901,362.111,249.23843,50.082817,497.10743,1.4063,136.9,0.177871 +256,160,100000,15,0,10,175.542,5696.66,175.499,176.839,100,10.9579,11.1168,1600,0.9409,0.6,0.042425712,3.1734657,349.508,187.561331,52.624416,497.349617,1.71734,169.143,0.183409 +256,192,100000,15,0,10,213.302,4688.19,213.26,214.714,100,13.3933,13.5356,1600,0.9734,0.7,0.0190803799,2.0794105,338.971,178.590357,50.026463,497.591805,2.00364,206.648,0.187812 +256,224,100000,15,0,10,250.392,3993.74,250.245,252.798,100,15.7794,15.9556,1600,0.9931,0.8,0.00587754848,2.0794105,348.109,179.173165,50.73613,497.833992,2.28967,243.249,0.190981 diff --git a/09_vector_retrieval/PurplePulse/results/formal_100k15/ivf_k100.csv b/09_vector_retrieval/PurplePulse/results/formal_100k15/ivf_k100.csv new file mode 100644 index 00000000..6cf20b1a --- /dev/null +++ b/09_vector_retrieval/PurplePulse/results/formal_100k15/ivf_k100.csv @@ -0,0 +1,5 @@ +nlist,nprobe,training_samples,training_iterations,training_seed,top_k,average_ms,qps,run_p50_ms,run_p99_ms,batch_p50_ms,batch_p99_ms,recall_at_k,min_query_recall,mean_absolute_score_error,max_absolute_score_error,run_samples,batch_latency_samples,index_load_ms,gpu_initialization_ms,index_h2d_ms,gpu_buffer_mib,center_ms,scan_ms,merge_ms +256,128,100000,15,0,100,149.901,6671.05,149.694,154.968,9.26218,9.39283,0.85191,0.7,0.129033640485,3.3332900999999993,100,1600,289.568,202.359561,48.876545,505.610847,1.36324,141.761,1.26084 +256,160,100000,15,0,100,189.384,5280.28,189.335,190.917,11.738,11.9248,0.9175800000000001,0.83,0.067447758554,3.1734657000000013,100,1600,321.707,176.778803,49.860607,507.96241,1.63587,181.098,1.30515 +256,192,100000,15,0,100,228.495,4376.47,228.377,229.989,14.3074,14.4852,0.96228,0.9,0.02965746239,2.0794105000000016,100,1600,321.716,176.690704,49.861534,510.313972,1.90256,219.933,1.32067 +256,224,100000,15,0,100,273.984,3649.85,273.844,275.722,17.1825,17.2907,0.9892000000000001,0.93,0.008260867503000001,2.0794105000000016,100,1600,316.19,174.228553,48.925697,512.665535,2.16944,265.061,1.36051 diff --git a/09_vector_retrieval/PurplePulse/results/formal_100k15/ivf_k50.csv b/09_vector_retrieval/PurplePulse/results/formal_100k15/ivf_k50.csv new file mode 100644 index 00000000..983b84aa --- /dev/null +++ b/09_vector_retrieval/PurplePulse/results/formal_100k15/ivf_k50.csv @@ -0,0 +1,5 @@ +nlist,nprobe,training_samples,training_iterations,training_seed,top_k,average_ms,qps,run_p50_ms,run_p99_ms,run_samples,batch_p50_ms,batch_p99_ms,batch_latency_samples,recall_at_k,min_query_recall,mean_absolute_score_error,max_absolute_score_error,index_load_ms,gpu_initialization_ms,index_h2d_ms,gpu_buffer_mib,center_ms,scan_ms,merge_ms +256,128,100000,15,0,50,137.089,7294.52,137.215,139.576,100,8.49632,8.70291,1600,0.86248,0.7,0.112961036,3.3332901,349.347,219.744959,51.002369,500.886726,1.43383,129.996,0.649493 +256,160,100000,15,0,50,178.399,5605.41,178.397,180.654,100,11.18,11.4261,1600,0.92482,0.8,0.0581850285,3.1734657,328.392,175.771099,48.182274,502.066414,1.73579,171.117,0.716433 +256,192,100000,15,0,50,223.565,4472.96,223.664,226.713,100,14.0604,14.5124,1600,0.9658,0.82,0.0254461265,2.0794105,310.692,176.079546,48.196609,503.246101,2.00779,215.991,0.727207 +256,224,100000,15,0,50,273.458,3656.87,273.591,276.441,100,17.2885,17.7533,1600,0.99054,0.92,0.00710551607,2.0794105,313.004,177.229013,49.465343,504.425789,2.27946,265.604,0.743718 diff --git a/09_vector_retrieval/PurplePulse/results/formal_4090d_20260908/bucket_major_formal.csv b/09_vector_retrieval/PurplePulse/results/formal_4090d_20260908/bucket_major_formal.csv new file mode 100644 index 00000000..96ce1d3d --- /dev/null +++ b/09_vector_retrieval/PurplePulse/results/formal_4090d_20260908/bucket_major_formal.csv @@ -0,0 +1,5 @@ +top_k,nprobe,batch_size,mode,average_ms,qps,run_p50_ms,run_p99_ms,batch_p50_ms,batch_p99_ms,scan_ms,merge_ms,recall_at_k,run_samples,speedup_percent +50,160,40,warp_compact,166.411,6009.22,166.384,169.040,6.49440,6.62154,157.329,1.02553,0.92482,100,0.0000 +50,160,40,bucket_major,143.592,6964.16,143.500,145.534,5.56659,5.62608,134.757,1.04728,0.92482,100,15.8912 +100,160,64,warp1,189.329,5281.82,189.296,190.881,11.72820,11.91420,180.957,1.30519,0.91758,100,0.0000 +100,160,64,bucket_major,185.624,5387.23,185.513,187.381,11.49960,11.54870,177.212,1.30388,0.91758,100,1.9957 diff --git a/09_vector_retrieval/PurplePulse/results/formal_4090d_20260908/build_threads.csv b/09_vector_retrieval/PurplePulse/results/formal_4090d_20260908/build_threads.csv new file mode 100644 index 00000000..ae47a2c6 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/results/formal_4090d_20260908/build_threads.csv @@ -0,0 +1,6 @@ +threads,build_ms,speedup_vs_single,bucket_min,bucket_average,bucket_max +1,67528.30,1.000,3734,3906.25,4091 +8,9179.44,7.356,3734,3906.25,4091 +16,6476.62,10.426,3734,3906.25,4091 +32,7472.99,9.036,3734,3906.25,4091 +64,10018.90,6.740,3734,3906.25,4091 diff --git a/09_vector_retrieval/PurplePulse/results/formal_4090d_20260908/faiss_gpu_batch40.csv b/09_vector_retrieval/PurplePulse/results/formal_4090d_20260908/faiss_gpu_batch40.csv new file mode 100644 index 00000000..0e258a18 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/results/formal_4090d_20260908/faiss_gpu_batch40.csv @@ -0,0 +1,2 @@ +implementation,backend,index_type,metric,num_vectors,num_queries,dim,dtype,cpu_threads,top_k,batch_size,nlist,nprobe,train_samples,training_iterations,training_seed,build_ms,load_ms,gpu_transfer_ms,gpu_index_mib,gpu_temp_mib,average_query_ms,qps,run_p50_ms,run_p99_ms,run_samples,batch_p50_ms,batch_p99_ms,batch_latency_samples,recall_at_k,min_query_recall,mean_absolute_score_error,max_absolute_score_error +faiss-1.14.1,gpu,ivf_flat,inner_product,1000000,1000,128,fp32,1,50,40,256,160,100000,15,2026,0.0,311.1584857106209,417.3387736082077,497.984375,1536.0,120.07527381181717,8328.109262255002,119.6761205792427,126.95340067148209,100,4.623956978321075,5.8260709047317505,2500,0.92312,0.8,0.0600678468132019,1.8539981842041016 diff --git a/09_vector_retrieval/PurplePulse/results/formal_4090d_20260908/k100_formal.csv b/09_vector_retrieval/PurplePulse/results/formal_4090d_20260908/k100_formal.csv new file mode 100644 index 00000000..6cf20b1a --- /dev/null +++ b/09_vector_retrieval/PurplePulse/results/formal_4090d_20260908/k100_formal.csv @@ -0,0 +1,5 @@ +nlist,nprobe,training_samples,training_iterations,training_seed,top_k,average_ms,qps,run_p50_ms,run_p99_ms,batch_p50_ms,batch_p99_ms,recall_at_k,min_query_recall,mean_absolute_score_error,max_absolute_score_error,run_samples,batch_latency_samples,index_load_ms,gpu_initialization_ms,index_h2d_ms,gpu_buffer_mib,center_ms,scan_ms,merge_ms +256,128,100000,15,0,100,149.901,6671.05,149.694,154.968,9.26218,9.39283,0.85191,0.7,0.129033640485,3.3332900999999993,100,1600,289.568,202.359561,48.876545,505.610847,1.36324,141.761,1.26084 +256,160,100000,15,0,100,189.384,5280.28,189.335,190.917,11.738,11.9248,0.9175800000000001,0.83,0.067447758554,3.1734657000000013,100,1600,321.707,176.778803,49.860607,507.96241,1.63587,181.098,1.30515 +256,192,100000,15,0,100,228.495,4376.47,228.377,229.989,14.3074,14.4852,0.96228,0.9,0.02965746239,2.0794105000000016,100,1600,321.716,176.690704,49.861534,510.313972,1.90256,219.933,1.32067 +256,224,100000,15,0,100,273.984,3649.85,273.844,275.722,17.1825,17.2907,0.9892000000000001,0.93,0.008260867503000001,2.0794105000000016,100,1600,316.19,174.228553,48.925697,512.665535,2.16944,265.061,1.36051 diff --git a/09_vector_retrieval/PurplePulse/results/formal_4090d_20260908/k100_heap_formal.csv b/09_vector_retrieval/PurplePulse/results/formal_4090d_20260908/k100_heap_formal.csv new file mode 100644 index 00000000..0f7f5d3c --- /dev/null +++ b/09_vector_retrieval/PurplePulse/results/formal_4090d_20260908/k100_heap_formal.csv @@ -0,0 +1,6 @@ +top_k,nprobe,batch_size,mode,average_ms,qps,run_p50_ms,run_p99_ms,batch_p50_ms,batch_p99_ms,scan_ms,merge_ms,recall_at_k,run_samples,speedup_percent,output_equal +100,160,64,warp1_pre_heap,188.737,5298.37,188.738,190.409,11.6936,11.8875,180.438,1.30079,0.91758,100,0.0000,true +100,160,64,warp1_heap_hole,184.311,5425.61,184.179,185.741,11.3594,11.6603,175.945,1.30443,0.91758,100,2.4015,true +100,160,64,bucket_major_heap_hole,182.091,5491.76,181.988,184.321,11.2720,11.3244,173.743,1.30183,0.91758,100,3.6500,true +50,160,40,bucket_major_pre_heap,143.912,6948.69,143.683,145.979,5.57686,5.64160,135.087,1.04701,0.92482,100,0.0000,true +50,160,40,bucket_major_heap_hole,144.077,6940.71,143.905,146.137,5.58560,5.63120,135.202,1.04671,0.92482,100,-0.1148,true diff --git a/09_vector_retrieval/PurplePulse/results/formal_4090d_20260908/k100_selection_negative.csv b/09_vector_retrieval/PurplePulse/results/formal_4090d_20260908/k100_selection_negative.csv new file mode 100644 index 00000000..281fb5c5 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/results/formal_4090d_20260908/k100_selection_negative.csv @@ -0,0 +1,4 @@ +top_k,nprobe,batch_size,variant,baseline_qps,variant_qps,speedup_percent,baseline_average_ms,variant_average_ms,baseline_run_p99_ms,variant_run_p99_ms,baseline_scan_ms,variant_scan_ms,repeat,output_equal,accepted +100,160,64,distributed_register_topk,5483.49,5507.23,0.4329,182.366,181.579,183.618,183.087,174.026,173.294,100,true,false +100,160,64,candidate_buffer_8,5508.19,4258.85,-22.6815,181.548,234.805,182.597,239.191,173.114,225.342,10,true,false +100,160,64,warp_bitonic_sort_128,5496.78,4467.83,-18.7191,181.925,223.822,182.873,229.924,173.398,214.114,10,true,false diff --git a/09_vector_retrieval/PurplePulse/results/formal_4090d_20260908/k100_warp_ab.csv b/09_vector_retrieval/PurplePulse/results/formal_4090d_20260908/k100_warp_ab.csv new file mode 100644 index 00000000..4ae4c393 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/results/formal_4090d_20260908/k100_warp_ab.csv @@ -0,0 +1,5 @@ +nprobe,recall_at_100,warp2_qps,warp1_qps,speedup +128,0.85191,6083.83,6671.05,1.0965 +160,0.91758,4732.60,5280.28,1.1157 +192,0.96228,3802.43,4376.47,1.1510 +224,0.98920,3172.27,3649.85,1.1505 diff --git a/09_vector_retrieval/PurplePulse/results/formal_4090d_20260908/k50_batch_sweep.csv b/09_vector_retrieval/PurplePulse/results/formal_4090d_20260908/k50_batch_sweep.csv new file mode 100644 index 00000000..9b975843 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/results/formal_4090d_20260908/k50_batch_sweep.csv @@ -0,0 +1,14 @@ +phase,batch_size,distance_mode,nprobe,repeat,average_ms,qps,run_p50_ms,run_p99_ms,batch_p50_ms,batch_p99_ms,recall_at_50 +screen,8,warp_compact,160,10,221.035,4524.18,220.442,222.343,1.71549,1.82170,0.92482 +screen,32,warp_compact,160,10,176.682,5659.88,176.371,179.702,5.48048,5.68102,0.92482 +screen,36,warp_compact,160,10,165.588,6039.07,164.590,170.808,5.76765,6.05037,0.92482 +screen,40,warp_compact,160,10,164.127,6092.85,162.782,169.588,6.34922,6.58454,0.92482 +screen,44,warp_compact,160,10,179.752,5563.22,179.204,183.228,7.68848,7.96790,0.92482 +screen,48,warp_compact,160,10,169.576,5897.08,169.234,173.093,7.90368,8.23034,0.92482 +screen,56,warp_compact,160,10,170.855,5852.92,169.855,174.363,9.25526,9.65008,0.92482 +screen,64,warp_compact,160,10,172.609,5793.46,171.877,176.778,10.75470,11.28760,0.92482 +screen,80,warp_compact,160,10,179.003,5586.49,178.177,182.870,13.94200,14.62970,0.92482 +screen,96,warp_compact,160,10,186.759,5354.51,186.730,190.213,17.51390,18.33810,0.92482 +screen,128,warp_compact,160,10,204.907,4880.26,204.788,205.879,25.84200,26.50500,0.92482 +formal,64,warp_compact,160,100,174.301,5737.20,174.550,176.348,10.91210,11.21160,0.92482 +formal,40,warp_compact,160,100,166.116,6019.91,166.303,168.097,6.49126,6.62845,0.92482 diff --git a/09_vector_retrieval/PurplePulse/results/formal_4090d_20260908/k50_bucket_major_sweep.csv b/09_vector_retrieval/PurplePulse/results/formal_4090d_20260908/k50_bucket_major_sweep.csv new file mode 100644 index 00000000..0916ec76 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/results/formal_4090d_20260908/k50_bucket_major_sweep.csv @@ -0,0 +1,8 @@ +nprobe,recall_at_50,probe_major_qps,bucket_major_qps,speedup_percent,probe_major_scan_ms,bucket_major_scan_ms +32,0.43754,21263.50,21242.00,-0.1011,40.2078,40.2667 +64,0.64044,11394.10,12146.90,6.6069,80.2197,74.9906 +96,0.77072,9869.36,10646.10,7.8702,93.6169,86.2451 +128,0.86248,7075.94,7946.55,12.3038,133.1150,117.7040 +160,0.92482,6073.52,6968.55,14.7366,155.7970,134.7540 +192,0.96580,4834.77,5834.75,20.6831,197.5560,162.0160 +224,0.99054,4089.10,5103.30,24.8025,234.5500,186.3430 diff --git a/09_vector_retrieval/PurplePulse/results/formal_4090d_20260908/k50_warp_sweep.csv b/09_vector_retrieval/PurplePulse/results/formal_4090d_20260908/k50_warp_sweep.csv new file mode 100644 index 00000000..2c21e8a8 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/results/formal_4090d_20260908/k50_warp_sweep.csv @@ -0,0 +1,5 @@ +distance_mode,batch_size,nprobe,repeat,average_ms,qps,run_p50_ms,run_p99_ms,batch_p50_ms,batch_p99_ms,recall_at_50 +warp1,64,160,10,173.287,5770.77,173.122,174.171,10.71210,10.92040,0.92482 +warp2,64,160,10,172.888,5784.10,171.745,176.958,10.75100,11.21010,0.92482 +warp4,64,160,10,200.599,4985.06,199.023,204.347,12.51780,13.21600,0.92482 +warp8,64,160,10,249.540,4007.38,249.300,251.634,15.67670,16.00280,0.92482 diff --git a/09_vector_retrieval/PurplePulse/results/formal_4090d_20260908/mnist_ivf.csv b/09_vector_retrieval/PurplePulse/results/formal_4090d_20260908/mnist_ivf.csv new file mode 100644 index 00000000..9b73583d --- /dev/null +++ b/09_vector_retrieval/PurplePulse/results/formal_4090d_20260908/mnist_ivf.csv @@ -0,0 +1,7 @@ +nlist,nprobe,training_samples,training_iterations,training_seed,top_k,average_ms,qps,run_p50_ms,run_p99_ms,batch_p50_ms,batch_p99_ms,recall_at_k,min_query_recall,mean_absolute_score_error,max_absolute_score_error,run_samples,batch_latency_samples,index_load_ms,gpu_initialization_ms,index_h2d_ms,gpu_buffer_mib,center_ms,scan_ms,merge_ms +256,8,60000,15,0,10,50.5922,197659.0,50.5448,50.842,0.62016,0.686976,0.9784200000000001,0.2,0.070545293826,15.823303199999998,10,790,122.125,136.370596,17.868799,181.312263,8.89748,33.4003,0.838154 +256,16,60000,15,0,10,76.2134,131211.0,76.1832,76.4513,0.960512,1.04634,0.99471,0.5,0.016609592507,15.019317599999997,10,790,122.665,138.350996,17.955839,181.433357,9.23802,58.6378,0.842787 +256,32,60000,15,0,10,130.492,76632.9,130.419,130.974,1.64867,1.84045,0.9990899999999999,0.7,0.002338636801,5.184761099999999,10,790,117.358,134.273648,18.180096,181.675545,9.97351,111.809,0.855194 +256,64,60000,15,0,10,244.534,40894.1,243.641,252.355,3.10214,3.38707,0.9999100000000001,0.9,0.0002657260519999999,4.437892900000001,10,790,116.904,168.861066,17.852415,182.15992,11.3344,223.302,0.884618 +256,96,60000,15,0,10,383.836,26052.8,383.126,385.85,4.88115,5.24064,0.9999899999999999,0.9,2.0008849999999966e-06,0.20008849999999967,10,790,121.686,136.029167,17.821695,182.644295,12.7204,360.285,0.913581 +256,128,60000,15,0,10,561.745,17801.7,560.289,566.901,7.14714,7.54618,0.9999899999999999,0.9,2.0008849999999966e-06,0.20008849999999967,10,790,108.656,135.780264,17.857536,183.12867,14.1201,536.037,0.939718 diff --git a/09_vector_retrieval/PurplePulse/results/memory_100k/source1_compare.csv b/09_vector_retrieval/PurplePulse/results/memory_100k/source1_compare.csv new file mode 100644 index 00000000..219fcd27 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/results/memory_100k/source1_compare.csv @@ -0,0 +1,3 @@ +mode,nlist,nprobe,rerank_factor,session_id,source_type,average_ms,qps,batch_p50_ms,batch_p99_ms,scan_ms,merge_ms,rerank_ms,gpu_buffer_mib,recall_vs_fused,min_query_recall_vs_fused +fused,64,64,1,any,1,20.8149,48042.6,1.28106,1.29066,18.6603,0.164685,0.0,52.068733,1.0,1.0 +rerank,64,64,10,any,1,46.3539,21573.2,2.88906,2.96419,42.4594,1.10371,0.840845,56.353401,0.507,0.1 diff --git a/09_vector_retrieval/PurplePulse/results/million/adaptive_formal_k10.csv b/09_vector_retrieval/PurplePulse/results/million/adaptive_formal_k10.csv new file mode 100644 index 00000000..c110e883 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/results/million/adaptive_formal_k10.csv @@ -0,0 +1,2 @@ +policy,execution,target_mass,temperature,min_nprobe,max_nprobe,step,actual_nprobe_min,actual_nprobe_average,actual_nprobe_max,average_ms,qps,batch_p50_ms,batch_p99_ms,recall_at_k,min_query_recall,center_ms,policy_ms,scan_ms,merge_ms,gpu_buffer_mib +score_mass,masked,0.93,0.2,64,224,16,128.0,151.632,176.0,138.24,7233.8,8.59325,8.94669,0.9201,0.5,2.73393,0.0,130.854,0.180659,497.888924 diff --git a/09_vector_retrieval/PurplePulse/results/million/adaptive_formal_k100.csv b/09_vector_retrieval/PurplePulse/results/million/adaptive_formal_k100.csv new file mode 100644 index 00000000..14a3c907 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/results/million/adaptive_formal_k100.csv @@ -0,0 +1,2 @@ +policy,execution,target_mass,temperature,min_nprobe,max_nprobe,step,actual_nprobe_min,actual_nprobe_average,actual_nprobe_max,average_ms,qps,batch_p50_ms,batch_p99_ms,recall_at_k,min_query_recall,center_ms,policy_ms,scan_ms,merge_ms,gpu_buffer_mib +score_mass,masked,0.93,0.2,64,224,16,128.0,151.632,176.0,183.852,5439.17,11.4511,11.9546,0.8999299999999999,0.77,2.68297,0.0,174.929,1.30663,512.720467 diff --git a/09_vector_retrieval/PurplePulse/results/million/adaptive_formal_k50.csv b/09_vector_retrieval/PurplePulse/results/million/adaptive_formal_k50.csv new file mode 100644 index 00000000..27a67f0a --- /dev/null +++ b/09_vector_retrieval/PurplePulse/results/million/adaptive_formal_k50.csv @@ -0,0 +1,2 @@ +policy,execution,target_mass,temperature,min_nprobe,max_nprobe,step,actual_nprobe_min,actual_nprobe_average,actual_nprobe_max,average_ms,qps,batch_p50_ms,batch_p99_ms,recall_at_k,min_query_recall,center_ms,policy_ms,scan_ms,merge_ms,gpu_buffer_mib +score_mass,masked,0.93,0.2,64,224,16,128.0,151.632,176.0,147.644,6773.04,4.57968,4.7751,0.90742,0.74,5.34852,0.0,135.713,1.29049,500.259163 diff --git a/09_vector_retrieval/PurplePulse/results/million/adaptive_grouped_formal_k100.csv b/09_vector_retrieval/PurplePulse/results/million/adaptive_grouped_formal_k100.csv new file mode 100644 index 00000000..01cc40c1 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/results/million/adaptive_grouped_formal_k100.csv @@ -0,0 +1,2 @@ +policy,execution,target_mass,temperature,min_nprobe,max_nprobe,step,actual_nprobe_min,actual_nprobe_average,actual_nprobe_max,average_ms,qps,batch_p50_ms,batch_p99_ms,recall_at_k,min_query_recall,center_ms,policy_ms,scan_ms,merge_ms,gpu_buffer_mib +score_mass,grouped,0.93,0.2,64,224,16,128.0,151.632,176.0,204.68,4885.68,12.3324,13.4839,0.8999299999999999,0.77,2.64728,0.155221,195.867,1.29021,512.720711 diff --git a/09_vector_retrieval/PurplePulse/results/million/adaptive_k100_t02.csv b/09_vector_retrieval/PurplePulse/results/million/adaptive_k100_t02.csv new file mode 100644 index 00000000..8194d8f2 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/results/million/adaptive_k100_t02.csv @@ -0,0 +1,6 @@ +policy,target_mass,temperature,min_nprobe,max_nprobe,step,actual_nprobe_min,actual_nprobe_average,actual_nprobe_max,average_ms,qps,batch_p50_ms,batch_p99_ms,recall_at_k,min_query_recall,center_ms,scan_ms,merge_ms,gpu_buffer_mib +score_mass,0.9,0.2,64,224,16,112.0,133.648,160.0,165.603,6038.55,10.3261,10.5847,0.86194,0.72,2.61127,156.707,1.27728,512.720467 +score_mass,0.93,0.2,64,224,16,128.0,151.632,176.0,184.14,5430.65,11.4747,11.8041,0.8999299999999999,0.77,2.65982,175.136,1.30286,512.720467 +score_mass,0.95,0.2,64,224,16,144.0,166.736,192.0,201.136,4971.77,12.5503,13.6421,0.9265399999999999,0.83,2.69391,192.094,1.3114,512.720467 +score_mass,0.97,0.2,64,224,16,160.0,186.096,208.0,225.922,4426.3,14.1753,14.9012,0.95353,0.87,2.73035,216.811,1.32234,512.720467 +score_mass,0.99,0.2,64,224,16,208.0,210.128,224.0,247.336,4043.08,15.4013,16.8479,0.97889,0.92,2.83173,238.087,1.37412,512.720467 diff --git a/09_vector_retrieval/PurplePulse/results/million/adaptive_k10_t02.csv b/09_vector_retrieval/PurplePulse/results/million/adaptive_k10_t02.csv new file mode 100644 index 00000000..ac167a5b --- /dev/null +++ b/09_vector_retrieval/PurplePulse/results/million/adaptive_k10_t02.csv @@ -0,0 +1,8 @@ +policy,target_mass,temperature,min_nprobe,max_nprobe,step,actual_nprobe_min,actual_nprobe_average,actual_nprobe_max,average_ms,qps,batch_p50_ms,batch_p99_ms,recall_at_k,min_query_recall,center_ms,scan_ms,merge_ms,gpu_buffer_mib +score_mass,0.8,0.2,64,224,16,80.0,92.72,112.0,85.1168,11748.6,5.18259,5.38298,0.7753,0.3,2.61823,77.8633,0.178816,497.888924 +score_mass,0.85,0.2,64,224,16,96.0,110.544,144.0,101.222,9879.26,6.22883,6.37872,0.8275,0.3,2.62646,93.9646,0.178214,497.888924 +score_mass,0.9,0.2,64,224,16,112.0,133.648,160.0,121.367,8239.49,7.51754,7.72566,0.8837,0.4,2.69535,114.027,0.179782,497.888924 +score_mass,0.93,0.2,64,224,16,128.0,151.632,176.0,137.663,7264.12,8.53437,8.85264,0.9201,0.5,2.73062,130.283,0.182208,497.888924 +score_mass,0.95,0.2,64,224,16,144.0,166.736,192.0,150.562,6641.78,9.35613,9.99891,0.9416,0.6,2.77469,143.094,0.182662,497.888924 +score_mass,0.97,0.2,64,224,16,160.0,186.096,208.0,168.812,5923.76,10.529,10.9825,0.9653999999999999,0.7,2.83329,161.33,0.186131,497.888924 +score_mass,0.99,0.2,64,224,16,208.0,210.128,224.0,188.458,5306.22,11.7034,12.3358,0.986,0.8,2.92934,180.494,0.189094,497.888924 diff --git a/09_vector_retrieval/PurplePulse/results/million/adaptive_k50_t02.csv b/09_vector_retrieval/PurplePulse/results/million/adaptive_k50_t02.csv new file mode 100644 index 00000000..7cafd04a --- /dev/null +++ b/09_vector_retrieval/PurplePulse/results/million/adaptive_k50_t02.csv @@ -0,0 +1,6 @@ +policy,target_mass,temperature,min_nprobe,max_nprobe,step,actual_nprobe_min,actual_nprobe_average,actual_nprobe_max,average_ms,qps,batch_p50_ms,batch_p99_ms,recall_at_k,min_query_recall,center_ms,scan_ms,merge_ms,gpu_buffer_mib +score_mass,0.9,0.2,64,224,16,112.0,133.648,160.0,133.11,7512.6,4.09942,4.43338,0.87076,0.68,5.20479,121.363,1.22873,500.259163 +score_mass,0.93,0.2,64,224,16,128.0,151.632,176.0,148.125,6751.06,4.59411,4.68419,0.90742,0.74,5.34125,136.162,1.28753,500.259163 +score_mass,0.95,0.2,64,224,16,144.0,166.736,192.0,172.805,5786.88,5.18554,5.90125,0.93202,0.8,5.34197,160.811,1.32697,500.259163 +score_mass,0.97,0.2,64,224,16,160.0,186.096,208.0,193.919,5156.8,6.06669,6.15334,0.95636,0.84,5.41334,181.8,1.35101,500.259163 +score_mass,0.99,0.2,64,224,16,208.0,210.128,224.0,205.788,4859.38,6.43651,6.56304,0.98078,0.9,5.60418,193.437,1.39775,500.259163 diff --git a/09_vector_retrieval/PurplePulse/results/million/adaptive_vs_fixed_n152.csv b/09_vector_retrieval/PurplePulse/results/million/adaptive_vs_fixed_n152.csv new file mode 100644 index 00000000..886506b1 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/results/million/adaptive_vs_fixed_n152.csv @@ -0,0 +1,7 @@ +top_k,policy,configured_max_nprobe,actual_nprobe_average,average_ms,qps,batch_p99_ms,recall_at_k,min_query_recall,gpu_buffer_mib +10,fixed,152,152.000,136.843,7307.65,8.69318,0.92120,0.50,497.289070 +10,score_mass_masked,224,151.632,138.240,7233.80,8.94669,0.92010,0.50,497.888924 +50,fixed,152,152.000,143.793,6954.42,4.57341,0.90874,0.76,498.904549 +50,score_mass_masked,224,151.632,147.644,6773.04,4.77510,0.90742,0.74,500.259163 +100,fixed,152,152.000,177.226,5642.53,11.5828,0.90179,0.80,507.374519 +100,score_mass_masked,224,151.632,183.852,5439.17,11.9546,0.89993,0.77,512.720467 diff --git a/09_vector_retrieval/PurplePulse/results/million/faiss_cpu_flat.csv b/09_vector_retrieval/PurplePulse/results/million/faiss_cpu_flat.csv new file mode 100644 index 00000000..79bcf46c --- /dev/null +++ b/09_vector_retrieval/PurplePulse/results/million/faiss_cpu_flat.csv @@ -0,0 +1,4 @@ +implementation,backend,index_type,metric,num_vectors,num_queries,dim,dtype,cpu_threads,top_k,batch_size,nlist,nprobe,train_samples,build_ms,load_ms,gpu_transfer_ms,average_query_ms,qps,batch_p50_ms,batch_p99_ms,recall_at_k,min_query_recall +faiss-1.14.1,cpu,flat,inner_product,1000000,1000,128,fp32,1,10,64,0,0,0,0.0,287.047304213047,0.0,13172.113172709942,75.91796296374113,840.2729630470276,852.7840971946716,1.0,1.0 +faiss-1.14.1,cpu,flat,inner_product,1000000,1000,128,fp32,1,50,64,0,0,0,0.0,287.047304213047,0.0,13208.435845871767,75.70919158550822,842.0012220740318,853.2666340470314,1.0,1.0 +faiss-1.14.1,cpu,flat,inner_product,1000000,1000,128,fp32,1,100,64,0,0,0,0.0,287.047304213047,0.0,13687.985599040985,73.05676885502149,873.8687261939049,878.7985518574715,1.0,1.0 diff --git a/09_vector_retrieval/PurplePulse/results/million/faiss_cpu_ivf_rep.csv b/09_vector_retrieval/PurplePulse/results/million/faiss_cpu_ivf_rep.csv new file mode 100644 index 00000000..27133436 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/results/million/faiss_cpu_ivf_rep.csv @@ -0,0 +1,2 @@ +implementation,backend,index_type,metric,num_vectors,num_queries,dim,dtype,cpu_threads,top_k,batch_size,nlist,nprobe,train_samples,build_ms,load_ms,gpu_transfer_ms,average_query_ms,qps,batch_p50_ms,batch_p99_ms,recall_at_k,min_query_recall +faiss-1.14.1,cpu,ivf_flat,inner_product,1000000,1000,128,fp32,1,10,64,256,128,100000,0.0,291.03782773017883,0.0,22949.28689301014,43.574338700022025,1464.3422737717628,1503.8493275642395,0.8795000000000001,0.5 diff --git a/09_vector_retrieval/PurplePulse/results/million/faiss_gpu_formal.csv b/09_vector_retrieval/PurplePulse/results/million/faiss_gpu_formal.csv new file mode 100644 index 00000000..3e191318 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/results/million/faiss_gpu_formal.csv @@ -0,0 +1,16 @@ +implementation,backend,index_type,metric,num_vectors,num_queries,dim,dtype,cpu_threads,top_k,batch_size,nlist,nprobe,train_samples,build_ms,load_ms,gpu_transfer_ms,average_query_ms,qps,batch_p50_ms,batch_p99_ms,recall_at_k,min_query_recall +faiss-1.14.1,gpu,flat,inner_product,1000000,1000,128,fp32,1,10,64,0,0,0,0.0,306.81489408016205,243.20194125175476,48.33947420120239,20687.026835205543,3.070421516895294,3.1209364533424377,1.0,1.0 +faiss-1.14.1,gpu,flat,inner_product,1000000,1000,128,fp32,1,50,64,0,0,0,0.0,306.81489408016205,243.20194125175476,48.48332405090332,20625.648500298495,3.0353665351867676,3.0637606978416443,1.0,1.0 +faiss-1.14.1,gpu,flat,inner_product,1000000,1000,128,fp32,1,100,64,0,0,0,0.0,306.81489408016205,243.20194125175476,50.514741241931915,19796.20157234236,3.131583333015442,4.262790083885193,1.0,1.0 +faiss-1.14.1,gpu,ivf_flat,inner_product,1000000,1000,128,fp32,1,10,64,256,128,100000,0.0,301.6926869750023,408.60651433467865,147.6534053683281,6772.617248518276,9.405627846717834,11.32049411535263,0.8795000000000001,0.5 +faiss-1.14.1,gpu,ivf_flat,inner_product,1000000,1000,128,fp32,1,50,64,256,128,100000,0.0,301.6926869750023,408.60651433467865,91.0657986998558,10981.07098687955,5.769222974777222,7.686242461204529,0.85972,0.68 +faiss-1.14.1,gpu,ivf_flat,inner_product,1000000,1000,128,fp32,1,100,64,256,128,100000,0.0,301.6926869750023,408.60651433467865,102.06692963838577,9797.492719168811,6.41188770532608,11.775985360145569,0.8490999999999999,0.7 +faiss-1.14.1,gpu,ivf_flat,inner_product,1000000,1000,128,fp32,1,10,64,256,160,100000,0.0,301.6926869750023,408.60651433467865,209.44822281599045,4774.449678088433,13.43446969985962,18.21625977754593,0.9369000000000001,0.7 +faiss-1.14.1,gpu,ivf_flat,inner_product,1000000,1000,128,fp32,1,50,64,256,160,100000,0.0,301.6926869750023,408.60651433467865,114.03494030237198,8769.242105519825,7.04357773065567,12.079395353794098,0.92312,0.8 +faiss-1.14.1,gpu,ivf_flat,inner_product,1000000,1000,128,fp32,1,100,64,256,160,100000,0.0,301.6926869750023,408.60651433467865,128.0077800154686,7812.025174400798,8.069895207881927,13.705261051654816,0.9163399999999999,0.8 +faiss-1.14.1,gpu,ivf_flat,inner_product,1000000,1000,128,fp32,1,10,64,256,192,100000,0.0,301.6926869750023,408.60651433467865,265.83249419927597,3761.7673603527574,17.10372418165207,18.838346004486084,0.9722000000000001,0.7 +faiss-1.14.1,gpu,ivf_flat,inner_product,1000000,1000,128,fp32,1,50,64,256,192,100000,0.0,301.6926869750023,408.60651433467865,136.0035389661789,7352.749844610133,8.596211671829224,14.135561883449554,0.96408,0.84 +faiss-1.14.1,gpu,ivf_flat,inner_product,1000000,1000,128,fp32,1,100,64,256,192,100000,0.0,301.6926869750023,408.60651433467865,152.1145612001419,6573.9926020906605,9.69243049621582,9.861893951892853,0.96127,0.88 +faiss-1.14.1,gpu,ivf_flat,inner_product,1000000,1000,128,fp32,1,10,64,256,224,100000,0.0,301.6926869750023,408.60651433467865,327.99044251441956,3048.8693278190162,21.03523164987564,23.185282945632935,0.993,0.8 +faiss-1.14.1,gpu,ivf_flat,inner_product,1000000,1000,128,fp32,1,50,64,256,224,100000,0.0,301.6926869750023,408.60651433467865,152.65787541866302,6550.5955539962015,9.729109704494476,9.92652028799057,0.9898800000000001,0.92 +faiss-1.14.1,gpu,ivf_flat,inner_product,1000000,1000,128,fp32,1,100,64,256,224,100000,0.0,301.6926869750023,408.60651433467865,178.12331765890121,5614.088111220557,11.344507336616516,11.540621519088745,0.9887900000000001,0.95 diff --git a/09_vector_retrieval/PurplePulse/results/million/faiss_gpu_formal_v2.csv b/09_vector_retrieval/PurplePulse/results/million/faiss_gpu_formal_v2.csv new file mode 100644 index 00000000..0e6d4825 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/results/million/faiss_gpu_formal_v2.csv @@ -0,0 +1,16 @@ +implementation,backend,index_type,metric,num_vectors,num_queries,dim,dtype,cpu_threads,top_k,batch_size,nlist,nprobe,train_samples,build_ms,load_ms,gpu_transfer_ms,average_query_ms,qps,batch_p50_ms,batch_p99_ms,recall_at_k,min_query_recall +faiss-1.14.1,gpu,flat,inner_product,1000000,1000,128,fp32,1,10,64,0,0,0,0.0,282.5198844075203,244.08532679080963,48.60575199127197,20573.69671349942,3.0652806162834167,3.078766167163849,1.0,1.0 +faiss-1.14.1,gpu,flat,inner_product,1000000,1000,128,fp32,1,50,64,0,0,0,0.0,282.5198844075203,244.08532679080963,48.42245727777481,20651.574831560338,3.0355453491210938,3.049500286579132,1.0,1.0 +faiss-1.14.1,gpu,flat,inner_product,1000000,1000,128,fp32,1,100,64,0,0,0,0.0,282.5198844075203,244.08532679080963,50.34593939781189,19862.575054930083,3.135010600090027,4.179045557975769,1.0,1.0 +faiss-1.14.1,gpu,ivf_flat,inner_product,1000000,1000,128,fp32,1,10,64,256,128,100000,0.0,297.7650985121727,769.0969333052635,147.0141589641571,6802.065916955698,9.391851723194122,12.21979409456253,0.8795000000000001,0.5 +faiss-1.14.1,gpu,ivf_flat,inner_product,1000000,1000,128,fp32,1,50,64,256,128,100000,0.0,297.7650985121727,769.0969333052635,90.75252562761307,11018.977081732392,5.763053894042969,6.491273641586304,0.85972,0.68 +faiss-1.14.1,gpu,ivf_flat,inner_product,1000000,1000,128,fp32,1,100,64,256,128,100000,0.0,297.7650985121727,769.0969333052635,102.74720638990402,9732.624712006384,6.406337022781372,16.36991649866104,0.8490999999999999,0.7 +faiss-1.14.1,gpu,ivf_flat,inner_product,1000000,1000,128,fp32,1,10,64,256,160,100000,0.0,297.7650985121727,769.0969333052635,210.92544347047806,4741.0117221821265,13.606108725070953,14.795109629631042,0.9369000000000001,0.7 +faiss-1.14.1,gpu,ivf_flat,inner_product,1000000,1000,128,fp32,1,50,64,256,160,100000,0.0,297.7650985121727,769.0969333052635,114.68802243471146,8719.306330085783,7.03873485326767,12.096747756004333,0.92312,0.8 +faiss-1.14.1,gpu,ivf_flat,inner_product,1000000,1000,128,fp32,1,100,64,256,160,100000,0.0,297.7650985121727,769.0969333052635,128.31582576036453,7793.270970858606,8.082374930381775,14.721490442752838,0.9163399999999999,0.8 +faiss-1.14.1,gpu,ivf_flat,inner_product,1000000,1000,128,fp32,1,10,64,256,192,100000,0.0,297.7650985121727,769.0969333052635,266.3765713572502,3754.0839079982475,17.110303044319153,19.420921802520752,0.9722000000000001,0.7 +faiss-1.14.1,gpu,ivf_flat,inner_product,1000000,1000,128,fp32,1,50,64,256,192,100000,0.0,297.7650985121727,769.0969333052635,134.9662497639656,7409.259735295604,8.605390787124634,9.006954729557037,0.96408,0.84 +faiss-1.14.1,gpu,ivf_flat,inner_product,1000000,1000,128,fp32,1,100,64,256,192,100000,0.0,297.7650985121727,769.0969333052635,152.0623043179512,6576.2517836706775,9.698398411273956,9.813271462917328,0.96127,0.88 +faiss-1.14.1,gpu,ivf_flat,inner_product,1000000,1000,128,fp32,1,10,64,256,224,100000,0.0,297.7650985121727,769.0969333052635,327.7781292796135,3050.844185967462,21.011285483837128,23.136727511882782,0.993,0.8 +faiss-1.14.1,gpu,ivf_flat,inner_product,1000000,1000,128,fp32,1,50,64,256,224,100000,0.0,297.7650985121727,769.0969333052635,154.548217356205,6470.472562586635,9.722404181957245,19.09337192773819,0.9898800000000001,0.92 +faiss-1.14.1,gpu,ivf_flat,inner_product,1000000,1000,128,fp32,1,100,64,256,224,100000,0.0,297.7650985121727,769.0969333052635,178.02205979824066,5617.281370260174,11.34926825761795,11.463411152362823,0.9887900000000001,0.95 diff --git a/09_vector_retrieval/PurplePulse/results/million/faiss_gpu_formal_v3.csv b/09_vector_retrieval/PurplePulse/results/million/faiss_gpu_formal_v3.csv new file mode 100644 index 00000000..fb3624fb --- /dev/null +++ b/09_vector_retrieval/PurplePulse/results/million/faiss_gpu_formal_v3.csv @@ -0,0 +1,16 @@ +implementation,backend,index_type,metric,num_vectors,num_queries,dim,dtype,cpu_threads,top_k,batch_size,nlist,nprobe,train_samples,build_ms,load_ms,gpu_transfer_ms,gpu_index_mib,gpu_temp_mib,average_query_ms,qps,batch_p50_ms,batch_p99_ms,recall_at_k,min_query_recall +faiss-1.14.1,gpu,flat,inner_product,1000000,1000,128,fp32,1,10,64,0,0,0,0.0,285.8908995985985,241.17807298898697,492.095947265625,1536.0,48.58236759901047,20583.599553109638,3.072679042816162,3.2324790954589844,1.0,1.0 +faiss-1.14.1,gpu,flat,inner_product,1000000,1000,128,fp32,1,50,64,0,0,0,0.0,285.8908995985985,241.17807298898697,492.095947265625,1536.0,48.51717799901962,20611.25649188019,3.0357763171195984,3.1501129269599915,1.0,1.0 +faiss-1.14.1,gpu,flat,inner_product,1000000,1000,128,fp32,1,100,64,0,0,0,0.0,285.8908995985985,241.17807298898697,492.095947265625,1536.0,50.23513734340668,19906.385308833025,3.1277909874916077,3.754034638404846,1.0,1.0 +faiss-1.14.1,gpu,ivf_flat,inner_product,1000000,1000,128,fp32,1,10,64,256,128,100000,0.0,286.5690514445305,414.5883098244667,497.984375,1536.0,145.5121487379074,6872.278422615924,9.345516562461853,10.649532079696655,0.8795000000000001,0.5 +faiss-1.14.1,gpu,ivf_flat,inner_product,1000000,1000,128,fp32,1,50,64,256,128,100000,0.0,286.5690514445305,414.5883098244667,497.984375,1536.0,90.37265181541443,11065.294421618762,5.753464996814728,5.99844753742218,0.85972,0.68 +faiss-1.14.1,gpu,ivf_flat,inner_product,1000000,1000,128,fp32,1,100,64,256,128,100000,0.0,286.5690514445305,414.5883098244667,497.984375,1536.0,101.29805654287338,9871.857705155084,6.404206156730652,9.461544454097748,0.8490999999999999,0.7 +faiss-1.14.1,gpu,ivf_flat,inner_product,1000000,1000,128,fp32,1,10,64,256,160,100000,0.0,286.5690514445305,414.5883098244667,497.984375,1536.0,210.255828499794,4756.110720616621,13.41235637664795,19.407659769058228,0.9369000000000001,0.7 +faiss-1.14.1,gpu,ivf_flat,inner_product,1000000,1000,128,fp32,1,50,64,256,160,100000,0.0,286.5690514445305,414.5883098244667,497.984375,1536.0,114.66595083475113,8720.98467522528,7.035769522190094,11.133134365081787,0.92312,0.8 +faiss-1.14.1,gpu,ivf_flat,inner_product,1000000,1000,128,fp32,1,100,64,256,160,100000,0.0,286.5690514445305,414.5883098244667,497.984375,1536.0,126.64697766304016,7895.96418684876,8.061528205871582,8.296430110931396,0.9163399999999999,0.8 +faiss-1.14.1,gpu,ivf_flat,inner_product,1000000,1000,128,fp32,1,10,64,256,192,100000,0.0,286.5690514445305,414.5883098244667,497.984375,1536.0,267.04441756010056,3744.69539238708,17.224609851837158,19.959919154644012,0.9722000000000001,0.7 +faiss-1.14.1,gpu,ivf_flat,inner_product,1000000,1000,128,fp32,1,50,64,256,192,100000,0.0,286.5690514445305,414.5883098244667,497.984375,1536.0,134.86263751983643,7414.952120100082,8.609533309936523,8.816443383693695,0.96408,0.84 +faiss-1.14.1,gpu,ivf_flat,inner_product,1000000,1000,128,fp32,1,100,64,256,192,100000,0.0,286.5690514445305,414.5883098244667,497.984375,1536.0,152.62544453144073,6551.987468865329,9.692922234535217,12.389101088047028,0.96127,0.88 +faiss-1.14.1,gpu,ivf_flat,inner_product,1000000,1000,128,fp32,1,10,64,256,224,100000,0.0,286.5690514445305,414.5883098244667,497.984375,1536.0,327.8968617320061,3049.7394659949864,20.965807139873505,22.999733686447144,0.993,0.8 +faiss-1.14.1,gpu,ivf_flat,inner_product,1000000,1000,128,fp32,1,50,64,256,224,100000,0.0,286.5690514445305,414.5883098244667,497.984375,1536.0,152.5898739695549,6553.5148171072115,9.715430438518524,9.946219623088837,0.9898800000000001,0.92 +faiss-1.14.1,gpu,ivf_flat,inner_product,1000000,1000,128,fp32,1,100,64,256,224,100000,0.0,286.5690514445305,414.5883098244667,497.984375,1536.0,177.9081165790558,5620.879020185889,11.339344084262848,11.475272476673126,0.9887900000000001,0.95 diff --git a/09_vector_retrieval/PurplePulse/results/million/faiss_smoke.csv b/09_vector_retrieval/PurplePulse/results/million/faiss_smoke.csv new file mode 100644 index 00000000..4e618673 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/results/million/faiss_smoke.csv @@ -0,0 +1,2 @@ +implementation,backend,index_type,metric,num_vectors,num_queries,dim,dtype,top_k,batch_size,nlist,nprobe,train_samples,build_ms,load_ms,gpu_transfer_ms,average_query_ms,qps,batch_p50_ms,batch_p99_ms,recall_at_k,min_query_recall +faiss-1.14.1,gpu,flat,inner_product,1000000,1000,128,fp32,10,64,0,0,0,274.1727828979492,0.0,251.55793875455856,49.115635454654694,20360.115281888913,3.0778199434280396,3.120236098766327,1.0,1.0 diff --git a/09_vector_retrieval/PurplePulse/results/million/fixed_k100_matched.csv b/09_vector_retrieval/PurplePulse/results/million/fixed_k100_matched.csv new file mode 100644 index 00000000..dc15ccc9 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/results/million/fixed_k100_matched.csv @@ -0,0 +1,6 @@ +nlist,nprobe,top_k,average_ms,qps,run_p50_ms,run_p99_ms,batch_p50_ms,batch_p99_ms,recall_at_k,min_query_recall,index_load_ms,gpu_initialization_ms,index_h2d_ms,gpu_buffer_mib,center_ms,scan_ms,merge_ms +256,134,100,158.268,6318.4,158.068,158.738,9.8616,10.0134,0.86378,0.74,344.942,208.431089,48.417759,506.051765,1.41524,150.679,1.26942 +256,152,100,176.19,5675.69,176.228,176.272,10.988,11.2054,0.90179,0.8,344.017,177.436624,50.082817,507.374519,1.5794,168.389,1.29674 +256,167,100,189.8,5268.7,190.014,190.136,11.8462,12.0964,0.92743,0.84,344.508,187.914744,50.082817,508.476814,1.71052,181.834,1.30818 +256,186,100,217.75,4592.41,217.776,218.191,13.6669,13.929,0.9538099999999999,0.88,336.696,181.442069,49.500095,509.873055,1.8641,209.622,1.32333 +256,210,100,241.183,4146.22,239.757,244.452,15.0537,15.7287,0.97908,0.92,344.602,178.693661,50.06739,511.636726,2.09754,232.831,1.3568 diff --git a/09_vector_retrieval/PurplePulse/results/million/fixed_k10_matched.csv b/09_vector_retrieval/PurplePulse/results/million/fixed_k10_matched.csv new file mode 100644 index 00000000..a85b50ab --- /dev/null +++ b/09_vector_retrieval/PurplePulse/results/million/fixed_k10_matched.csv @@ -0,0 +1,8 @@ +nlist,nprobe,top_k,average_ms,qps,run_p50_ms,run_p99_ms,batch_p50_ms,batch_p99_ms,recall_at_k,min_query_recall,index_load_ms,gpu_initialization_ms,index_h2d_ms,gpu_buffer_mib,center_ms,scan_ms,merge_ms +256,93,10,83.1175,12031.2,83.2045,83.3808,5.05821,5.22131,0.7783,0.3,312.033,216.239709,50.641918,496.842537,1.10461,77.4891,0.172749 +256,111,10,100.201,9979.94,100.283,100.514,6.16221,6.3735,0.8274,0.4,307.664,213.145584,49.911808,496.978767,1.25535,94.4129,0.175782 +256,134,10,120.456,8301.77,120.33,120.927,7.4625,7.63642,0.8841,0.4,309.589,178.642062,50.307072,497.15284,1.45938,114.445,0.176813 +256,152,10,137.41,7277.47,137.476,137.653,8.54922,8.78115,0.9212,0.5,306.14,178.288343,50.642944,497.28907,1.60839,131.26,0.178214 +256,167,10,149.536,6687.33,149.459,150.496,9.30378,9.62016,0.9424,0.5,309.172,178.600067,50.193409,497.402596,1.75187,143.246,0.182848 +256,186,10,168.075,5949.74,167.111,171.99,10.4939,11.012,0.9652999999999999,0.7,306.003,182.767851,49.24416,497.546394,1.92886,161.6,0.18425 +256,210,10,187.992,5319.39,187.114,192.062,11.6859,12.2507,0.9863,0.8,308.577,182.787988,50.119743,497.728035,2.1658,180.956,0.187187 diff --git a/09_vector_retrieval/PurplePulse/results/million/fixed_k50_matched.csv b/09_vector_retrieval/PurplePulse/results/million/fixed_k50_matched.csv new file mode 100644 index 00000000..f05dc6f4 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/results/million/fixed_k50_matched.csv @@ -0,0 +1,6 @@ +nlist,nprobe,top_k,average_ms,qps,run_p50_ms,run_p99_ms,batch_p50_ms,batch_p99_ms,recall_at_k,min_query_recall,index_load_ms,gpu_initialization_ms,index_h2d_ms,gpu_buffer_mib,center_ms,scan_ms,merge_ms +256,134,50,131.16,7624.27,129.903,133.805,4.01434,4.20758,0.87142,0.72,322.803,204.905654,48.527359,498.572762,2.90965,121.589,1.24021 +256,152,50,143.03,6991.56,143.109,143.371,4.4273,4.52742,0.90874,0.76,341.521,175.386222,48.854015,498.904549,3.19151,133.51,1.25125 +256,167,50,156.838,6376.02,156.572,157.382,4.88458,4.98432,0.9325800000000001,0.8,338.704,175.705726,48.726082,499.181038,3.46049,146.904,1.36697 +256,186,50,189.449,5278.46,189.428,189.518,5.92461,6.01203,0.95666,0.86,341.27,178.056469,49.153023,499.531258,3.70864,179.291,1.36316 +256,210,50,202.336,4942.26,202.292,202.526,6.32573,6.4479,0.9809399999999999,0.9,339.97,175.408315,48.654335,499.97364,4.17606,191.626,1.38848 diff --git a/09_vector_retrieval/PurplePulse/results/million/optimized_v1_batch_sweep.csv b/09_vector_retrieval/PurplePulse/results/million/optimized_v1_batch_sweep.csv new file mode 100644 index 00000000..9b5b0ff3 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/results/million/optimized_v1_batch_sweep.csv @@ -0,0 +1,14 @@ +top_k,nprobe,batch_size,average_ms,qps,batch_p50_ms,batch_p99_ms +10,160,8,163.117,6130.58,1.26016,1.28992 +10,160,16,150.854,6628.93,2.33667,2.38550 +10,160,32,146.206,6839.68,4.53709,4.65718 +10,160,64,143.411,6972.97,8.89664,9.39501 +10,160,128,151.117,6617.41,18.7522,20.2491 +50,160,8,208.800,4789.27,1.61629,1.71613 +50,160,32,150.702,6635.63,4.66973,4.87680 +50,160,64,156.270,6399.20,9.66397,11.6425 +50,160,128,186.526,5361.18,23.4392,24.0252 +100,160,8,234.454,4265.24,1.80531,2.00845 +100,160,32,188.907,5293.60,5.83405,6.11971 +100,160,64,186.597,5359.15,11.5481,12.1629 +100,160,128,187.932,5321.08,23.2395,24.5809 diff --git a/09_vector_retrieval/PurplePulse/results/million/optimized_v1_k10.csv b/09_vector_retrieval/PurplePulse/results/million/optimized_v1_k10.csv new file mode 100644 index 00000000..9d7c9b84 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/results/million/optimized_v1_k10.csv @@ -0,0 +1,5 @@ +nlist,nprobe,top_k,average_ms,qps,run_p50_ms,run_p99_ms,batch_p50_ms,batch_p99_ms,recall_at_k,min_query_recall,index_load_ms,gpu_initialization_ms,index_h2d_ms,gpu_buffer_mib,center_ms,scan_ms,merge_ms +256,128,10,114.492,8734.26,114.611,114.852,7.06685,7.2783,0.8716,0.4,313.585,201.295587,49.244289,497.10743,1.40868,108.51,0.177389 +256,160,10,142.813,7002.15,142.858,142.944,8.85901,9.36838,0.9332,0.5,342.995,182.53147,50.336769,497.349617,1.67885,136.636,0.178406 +256,192,10,172.767,5788.13,171.995,177.516,10.7552,11.4112,0.9708,0.7,339.737,177.759528,49.464321,497.591805,1.9772,166.288,0.182464 +256,224,10,199.941,5001.48,197.428,204.982,12.4952,13.2546,0.9929,0.8,349.578,187.077321,52.121601,497.833992,2.284,192.958,0.186739 diff --git a/09_vector_retrieval/PurplePulse/results/million/optimized_v1_k100.csv b/09_vector_retrieval/PurplePulse/results/million/optimized_v1_k100.csv new file mode 100644 index 00000000..b52ff9de --- /dev/null +++ b/09_vector_retrieval/PurplePulse/results/million/optimized_v1_k100.csv @@ -0,0 +1,5 @@ +nlist,nprobe,top_k,average_ms,qps,run_p50_ms,run_p99_ms,batch_p50_ms,batch_p99_ms,recall_at_k,min_query_recall,index_load_ms,gpu_initialization_ms,index_h2d_ms,gpu_buffer_mib,center_ms,scan_ms,merge_ms +256,128,100,145.736,6861.72,145.666,146.163,9.03696,9.2657,0.84957,0.73,312.923,177.967054,50.384895,505.610847,1.38316,138.097,1.2993 +256,160,100,186.157,5371.81,184.604,191.066,11.5385,12.1275,0.9162899999999999,0.83,320.683,194.827955,65.00045,507.96241,1.67551,178.317,1.34738 +256,192,100,226.505,4414.92,225.136,230.609,14.1054,14.7119,0.96109,0.9,347.076,186.380337,48.910336,510.313972,1.95592,218.348,1.36909 +256,224,100,266.671,3749.94,265.185,271.297,16.6592,17.4817,0.98901,0.94,338.758,179.058335,50.358273,512.665535,2.23116,258.1,1.40443 diff --git a/09_vector_retrieval/PurplePulse/results/million/optimized_v1_k50.csv b/09_vector_retrieval/PurplePulse/results/million/optimized_v1_k50.csv new file mode 100644 index 00000000..b32c6728 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/results/million/optimized_v1_k50.csv @@ -0,0 +1,5 @@ +nlist,nprobe,top_k,average_ms,qps,run_p50_ms,run_p99_ms,batch_p50_ms,batch_p99_ms,recall_at_k,min_query_recall,index_load_ms,gpu_initialization_ms,index_h2d_ms,gpu_buffer_mib,center_ms,scan_ms,merge_ms +256,128,50,124.367,8040.72,124.425,124.588,7.71194,7.91891,0.85764,0.68,321.321,174.418505,49.756161,500.886726,1.39822,117.718,0.63879 +256,160,50,155.774,6419.56,155.123,158.704,9.6673,10.2908,0.9224399999999999,0.8,316.876,181.233274,48.408577,502.066414,1.68376,148.768,0.707846 +256,192,50,198.084,5048.36,196.159,202.343,12.3866,13.1108,0.96362,0.86,309.553,172.6877,49.701889,503.246101,1.95297,190.765,0.710074 +256,224,50,246.481,4057.12,247.034,249.947,15.5024,15.9968,0.99036,0.92,308.338,177.079955,50.042881,504.425789,2.20839,238.829,0.722426 diff --git a/09_vector_retrieval/PurplePulse/results/million/optimized_v1_k50_b32.csv b/09_vector_retrieval/PurplePulse/results/million/optimized_v1_k50_b32.csv new file mode 100644 index 00000000..5e2e53c0 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/results/million/optimized_v1_k50_b32.csv @@ -0,0 +1,5 @@ +nlist,nprobe,top_k,average_ms,qps,run_p50_ms,run_p99_ms,batch_p50_ms,batch_p99_ms,recall_at_k,min_query_recall,index_load_ms,gpu_initialization_ms,index_h2d_ms,gpu_buffer_mib,center_ms,scan_ms,merge_ms +256,128,50,124.917,8005.3,124.63,126.174,3.83552,4.02794,0.85764,0.68,313.973,198.869566,48.350239,498.462166,2.72706,115.871,1.23165 +256,160,50,149.803,6675.43,149.619,151.027,4.64115,4.85293,0.9224399999999999,0.8,314.481,185.112281,53.036034,499.05201,3.31795,140.0,1.36364 +256,192,50,192.869,5184.87,191.486,196.276,5.99981,6.26022,0.96362,0.86,306.071,172.908014,49.908737,499.641853,3.85896,182.466,1.3929 +256,224,50,215.971,4630.24,213.406,220.59,6.712,7.03693,0.99036,0.92,304.8,180.173094,49.196033,500.231697,4.48155,204.91,1.44743 diff --git a/09_vector_retrieval/PurplePulse/results/million/purplepulse_faiss_compare_k10.csv b/09_vector_retrieval/PurplePulse/results/million/purplepulse_faiss_compare_k10.csv new file mode 100644 index 00000000..5133f5a7 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/results/million/purplepulse_faiss_compare_k10.csv @@ -0,0 +1,5 @@ +nlist,nprobe,top_k,average_ms,qps,recall_at_k,min_query_recall,gpu_buffer_mib,center_ms,scan_ms,merge_ms +256,128,10,163.146,6129.47,0.8716,0.4,497.10743,1.38557,157.214,0.174054 +256,160,10,208.906,4786.85,0.9332,0.5,497.349617,1.65597,202.667,0.176768 +256,192,10,254.128,3935.02,0.9708,0.7,497.591805,1.94223,247.62,0.181888 +256,224,10,302.84,3302.07,0.9929,0.8,497.833992,2.19123,296.028,0.182029 diff --git a/09_vector_retrieval/PurplePulse/results/million/purplepulse_faiss_compare_k100.csv b/09_vector_retrieval/PurplePulse/results/million/purplepulse_faiss_compare_k100.csv new file mode 100644 index 00000000..12f04801 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/results/million/purplepulse_faiss_compare_k100.csv @@ -0,0 +1,5 @@ +nlist,nprobe,top_k,average_ms,qps,recall_at_k,min_query_recall,gpu_buffer_mib,center_ms,scan_ms,merge_ms +256,128,100,256.457,3899.29,0.84957,0.73,505.610847,1.4057,248.785,1.31703 +256,160,100,321.382,3111.56,0.9162899999999999,0.83,507.96241,1.67763,313.296,1.34572 +256,192,100,390.036,2563.86,0.96109,0.9,510.313972,1.95847,381.672,1.36621 +256,224,100,456.162,2192.2,0.98901,0.94,512.665535,2.23994,447.389,1.40833 diff --git a/09_vector_retrieval/PurplePulse/results/million/purplepulse_faiss_compare_k100_v2.csv b/09_vector_retrieval/PurplePulse/results/million/purplepulse_faiss_compare_k100_v2.csv new file mode 100644 index 00000000..46886e81 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/results/million/purplepulse_faiss_compare_k100_v2.csv @@ -0,0 +1,5 @@ +nlist,nprobe,top_k,average_ms,qps,run_p50_ms,run_p99_ms,batch_p50_ms,batch_p99_ms,recall_at_k,min_query_recall,index_load_ms,gpu_initialization_ms,index_h2d_ms,gpu_buffer_mib,center_ms,scan_ms,merge_ms +256,128,100,255.924,3907.4,255.738,259.037,16.0383,16.4815,0.84957,0.73,333.052,180.812547,54.655998,505.610847,1.40155,248.179,1.31416 +256,160,100,321.41,3111.29,320.81,324.92,20.2298,20.6631,0.9162899999999999,0.83,336.804,180.254384,49.406975,507.96241,1.68063,313.259,1.34637 +256,192,100,389.844,2565.13,391.768,392.87,24.5921,25.0963,0.96109,0.9,331.437,175.987579,47.669247,510.313972,1.96227,381.331,1.37202 +256,224,100,456.191,2192.07,455.981,459.528,28.8079,30.7119,0.98901,0.94,331.743,175.22775,48.033791,512.665535,2.23484,447.087,1.40794 diff --git a/09_vector_retrieval/PurplePulse/results/million/purplepulse_faiss_compare_k10_v2.csv b/09_vector_retrieval/PurplePulse/results/million/purplepulse_faiss_compare_k10_v2.csv new file mode 100644 index 00000000..962acdd5 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/results/million/purplepulse_faiss_compare_k10_v2.csv @@ -0,0 +1,5 @@ +nlist,nprobe,top_k,average_ms,qps,run_p50_ms,run_p99_ms,batch_p50_ms,batch_p99_ms,recall_at_k,min_query_recall,index_load_ms,gpu_initialization_ms,index_h2d_ms,gpu_buffer_mib,center_ms,scan_ms,merge_ms +256,128,10,162.302,6161.37,161.765,165.397,10.1036,10.8613,0.8716,0.4,332.034,203.129712,48.421825,497.10743,1.37898,156.38,0.173862 +256,160,10,209.596,4771.09,206.716,215.386,13.0472,14.4953,0.9332,0.5,330.143,174.002704,48.356354,497.349617,1.65799,203.376,0.176749 +256,192,10,253.267,3948.4,252.499,260.228,15.8918,17.18,0.9708,0.7,328.531,175.200326,49.212418,497.591805,1.9384,246.704,0.179731 +256,224,10,301.739,3314.12,300.145,307.802,19.0054,19.9611,0.9929,0.8,341.581,179.865042,49.264641,497.833992,2.19372,294.926,0.181549 diff --git a/09_vector_retrieval/PurplePulse/results/million/purplepulse_faiss_compare_k50.csv b/09_vector_retrieval/PurplePulse/results/million/purplepulse_faiss_compare_k50.csv new file mode 100644 index 00000000..726fc8c0 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/results/million/purplepulse_faiss_compare_k50.csv @@ -0,0 +1,5 @@ +nlist,nprobe,top_k,average_ms,qps,recall_at_k,min_query_recall,gpu_buffer_mib,center_ms,scan_ms,merge_ms +256,128,50,204.712,4884.91,0.85764,0.68,500.886726,1.39819,198.059,0.638522 +256,160,50,258.077,3874.82,0.9224399999999999,0.8,502.066414,1.67948,250.966,0.702528 +256,192,50,315.27,3171.89,0.96362,0.86,503.246101,1.94605,307.915,0.705555 +256,224,50,378.974,2638.7,0.99036,0.92,504.425789,2.21338,371.29,0.725606 diff --git a/09_vector_retrieval/PurplePulse/results/million/purplepulse_faiss_compare_k50_v2.csv b/09_vector_retrieval/PurplePulse/results/million/purplepulse_faiss_compare_k50_v2.csv new file mode 100644 index 00000000..5ec01024 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/results/million/purplepulse_faiss_compare_k50_v2.csv @@ -0,0 +1,5 @@ +nlist,nprobe,top_k,average_ms,qps,run_p50_ms,run_p99_ms,batch_p50_ms,batch_p99_ms,recall_at_k,min_query_recall,index_load_ms,gpu_initialization_ms,index_h2d_ms,gpu_buffer_mib,center_ms,scan_ms,merge_ms +256,128,50,204.374,4892.99,202.363,208.238,12.7027,13.2948,0.85764,0.68,330.555,173.378368,47.857822,500.886726,1.39581,197.712,0.635014 +256,160,50,257.772,3879.4,256.8,261.436,16.1263,17.0315,0.9224399999999999,0.8,334.215,177.807709,49.944576,502.066414,1.67325,250.726,0.701837 +256,192,50,315.139,3173.2,313.446,319.739,19.7917,21.9328,0.96362,0.86,331.813,175.237001,48.2304,503.246101,1.94225,307.562,0.706618 +256,224,50,377.72,2647.47,377.203,379.404,23.8011,24.621,0.99036,0.92,334.021,177.086034,49.295296,504.425789,2.21521,370.053,0.726291 diff --git a/09_vector_retrieval/PurplePulse/results/million/sweep_smoke.csv b/09_vector_retrieval/PurplePulse/results/million/sweep_smoke.csv new file mode 100644 index 00000000..f6868835 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/results/million/sweep_smoke.csv @@ -0,0 +1,3 @@ +nlist,nprobe,top_k,average_ms,qps,recall_at_k,min_query_recall,gpu_buffer_mib,center_ms,scan_ms,merge_ms +256,64,10,94.8201,10546.3,0.6613,0.2,496.183968,3.40621,84.5987,0.9104 +256,128,10,159.119,6284.6,0.8716,0.4,496.305061,5.37437,146.86,0.964864 diff --git a/09_vector_retrieval/PurplePulse/results/real/sift1m/build.csv b/09_vector_retrieval/PurplePulse/results/real/sift1m/build.csv new file mode 100644 index 00000000..703131ab --- /dev/null +++ b/09_vector_retrieval/PurplePulse/results/real/sift1m/build.csv @@ -0,0 +1,2 @@ +dataset,nlist,training_iterations,training_samples,threads,build_ms,save_ms,minimum_bucket,average_bucket,maximum_bucket,empty_buckets,index_bytes +sift1m,1024,15,100000,16,25467.3,286.4,188,976.562,3684,0,520532520 diff --git a/09_vector_retrieval/PurplePulse/results/real/sift1m/dataset_manifest.json b/09_vector_retrieval/PurplePulse/results/real/sift1m/dataset_manifest.json new file mode 100644 index 00000000..8131a38b --- /dev/null +++ b/09_vector_retrieval/PurplePulse/results/real/sift1m/dataset_manifest.json @@ -0,0 +1,58 @@ +{ + "dataset": "SIFT1M", + "source_url": "ftp://ftp.irisa.fr/local/texmex/corpus/sift.tar.gz", + "metric": "l2", + "database_rows": 1000000, + "query_rows": 10000, + "dim": 128, + "groundtruth_k": 100, + "inputs": { + "base": { + "path": "/data/real-datasets/raw/sift1m/extracted/sift/sift_base.fvecs", + "kind": "fvecs", + "rows": 1000000, + "dim": 128, + "bytes": 516000000, + "sha256": "21f66e2975057b5728ba56de1c825bac4f4d89d596609ae985741c6242631816" + }, + "queries": { + "path": "/data/real-datasets/raw/sift1m/extracted/sift/sift_query.fvecs", + "kind": "fvecs", + "rows": 10000, + "dim": 128, + "bytes": 5160000, + "sha256": "f7fc9be140accdfd64116c2fa2365ecdb69b8f084970c6b0532db5ff79ac8fdc" + }, + "groundtruth": { + "path": "/data/real-datasets/raw/sift1m/extracted/sift/sift_groundtruth.ivecs", + "kind": "ivecs", + "rows": 10000, + "dim": 100, + "bytes": 4040000, + "sha256": "2b71de0a8d5a83e6a84eec3e23fb8b611d8801dd9b3a6cd62f070ab65ea65f4f" + } + }, + "outputs": { + "database": { + "path": "data/real/sift1m/database.bin", + "bytes": 512000028, + "sha256": "c8e127794ab289f0e45c28440be05085b8961ca52907676dd313638d6fd4f861" + }, + "queries": { + "path": "data/real/sift1m/queries.bin", + "bytes": 5120024, + "sha256": "d75631c98e58a0f3e103db6490fd110be7a0881ad738ef979a4c512aebe2f032" + }, + "groundtruth": { + "path": "data/real/sift1m/groundtruth.ivecs", + "bytes": 4040000, + "sha256": "2b71de0a8d5a83e6a84eec3e23fb8b611d8801dd9b3a6cd62f070ab65ea65f4f" + } + }, + "archive": { + "path": "/data/real-datasets/raw/sift1m/sift.tar.gz", + "bytes": 168280445, + "md5": "b23d1b3b2ee8469d819b61ca900ef0ed", + "sha256": "92f1270c5e3a0cb46b89983e72b0511e4df065c31a9fa0276d8c9b1fca5bc81a" + } +} diff --git a/09_vector_retrieval/PurplePulse/results/real/sift1m/environment.json b/09_vector_retrieval/PurplePulse/results/real/sift1m/environment.json new file mode 100644 index 00000000..7025574d --- /dev/null +++ b/09_vector_retrieval/PurplePulse/results/real/sift1m/environment.json @@ -0,0 +1,26 @@ +{ + "dataset": "sift1m", + "timestamp_utc": "2026-09-10T07:55:09.404971+00:00", + "git_commit": "fatal: not a git repository (or any parent up to mount point /)\nStopping at filesystem boundary (GIT_DISCOVERY_ACROSS_FILESYSTEM not set).", + "gpu": "NVIDIA GeForce RTX 4090 D, 570.124.06, 24564 MiB", + "nvcc": "nvcc: NVIDIA (R) Cuda compiler driver\nCopyright (c) 2005-2025 NVIDIA Corporation\nBuilt on Wed_Jan_15_19:20:09_PST_2025\nCuda compilation tools, release 12.8, V12.8.61\nBuild cuda_12.8.r12.8/compiler.35404655_0", + "python": "3.12.3 (main, Nov 6 2024, 18:32:19) [GCC 13.2.0]", + "parameters": { + "dataset": "sift1m", + "project_root": "/data/purplepulse-work/PurplePulse", + "data_root": "data/real", + "results_root": "results/real", + "build_dir": "build", + "nlist": 1024, + "nprobes": "32,64,128,160,192,224", + "training_samples": 100000, + "iterations": 15, + "threads": 16, + "batch_size": 64, + "warmup": 1, + "repeat": 5, + "skip_index_build": false, + "skip_faiss": false, + "maximum_partition_recall_loss": 0.0001 + } +} diff --git a/09_vector_retrieval/PurplePulse/results/real/sift1m/exact_gpu.csv b/09_vector_retrieval/PurplePulse/results/real/sift1m/exact_gpu.csv new file mode 100644 index 00000000..cb1d8c7c --- /dev/null +++ b/09_vector_retrieval/PurplePulse/results/real/sift1m/exact_gpu.csv @@ -0,0 +1,4 @@ +implementation,top_k,batch_size,average_query_ms,qps,run_p50_ms,run_p99_ms,run_samples,batch_p50_ms,batch_p99_ms,batch_latency_samples,gpu_initialization_ms,database_h2d_ms,gpu_buffer_mib,query_h2d_ms,distance_and_local_topk_ms,final_topk_ms,result_d2h_ms,result_path +WarpFusion Exact fused,10,64,1420.77,7038.42,1420.44,1423.89,5,9.06992,9.17027,785,205.322,48.4529,488.437,3.58849,1409.55,1.7796,3.25878,/data/purplepulse-work/PurplePulse/results/real/sift1m/exact_outputs/exact_k10.txt +WarpFusion Exact fused,50,64,1578.08,6336.81,1577.67,1580.61,5,10.068,10.1572,785,172.465,47.9695,488.935,2.7779,1558.6,7.83877,3.75766,/data/purplepulse-work/PurplePulse/results/real/sift1m/exact_outputs/exact_k50.txt +WarpFusion Exact fused,100,64,1778.64,5622.28,1778.71,1780.42,5,11.3194,11.6274,785,177.354,48.4283,489.558,2.85445,1743.23,17.8673,5.33884,/data/purplepulse-work/PurplePulse/results/real/sift1m/exact_outputs/exact_k100.txt diff --git a/09_vector_retrieval/PurplePulse/results/real/sift1m/faiss_gpu.csv b/09_vector_retrieval/PurplePulse/results/real/sift1m/faiss_gpu.csv new file mode 100644 index 00000000..2cc8ada0 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/results/real/sift1m/faiss_gpu.csv @@ -0,0 +1,22 @@ +implementation,backend,index_type,metric,num_vectors,num_queries,dim,dtype,cpu_threads,top_k,batch_size,nlist,nprobe,train_samples,training_iterations,training_seed,build_ms,load_ms,gpu_transfer_ms,gpu_index_mib,gpu_temp_mib,average_query_ms,qps,run_p50_ms,run_p99_ms,run_samples,batch_p50_ms,batch_p99_ms,batch_latency_samples,recall_at_k,min_query_recall,mean_absolute_score_error,max_absolute_score_error +faiss-1.14.1,gpu,flat,l2,1000000,10000,128,fp32,1,10,64,0,0,0,0,0,246.00856751203537,0.0,240.90760201215744,492.095947265625,1536.0,482.69662857055664,20716.94602386949,482.70057141780853,482.7524647116661,5,3.072179853916168,3.0867978930473328,785,0.99955,0.9,0.0,0.0 +faiss-1.14.1,gpu,flat,l2,1000000,10000,128,fp32,1,50,64,0,0,0,0,0,246.00856751203537,0.0,240.90760201215744,492.095947265625,1536.0,493.0823802947998,20280.586773393297,492.9155260324478,494.4283291697502,5,3.1305477023124695,3.1549036502838135,785,0.999776,0.98,0.0,0.0 +faiss-1.14.1,gpu,flat,l2,1000000,10000,128,fp32,1,100,64,0,0,0,0,0,246.00856751203537,0.0,240.90760201215744,492.095947265625,1536.0,507.0580542087555,19721.607648269415,506.7726969718933,508.07879865169525,5,3.217831254005432,3.245815634727478,785,0.999877,0.99,0.0,0.0 +faiss-1.14.1,gpu,ivf_flat,l2,1000000,10000,128,fp32,1,10,64,1024,32,100000,15,2026,31694.970950484276,0.0,340.7115638256073,504.30078125,1536.0,145.0256809592247,68953.30491715872,145.078606903553,145.1914831995964,5,0.9266659617424011,1.011282205581665,785,0.97762,0.5,83.89023,35511.0 +faiss-1.14.1,gpu,ivf_flat,l2,1000000,10000,128,fp32,1,50,64,1024,32,100000,15,2026,31694.970950484276,0.0,340.7115638256073,504.30078125,1536.0,151.96136981248856,65806.19806428051,151.97588503360748,152.06657350063324,5,0.9674057364463806,1.064293086528778,785,0.96454,0.54,151.144254,35511.0 +faiss-1.14.1,gpu,ivf_flat,l2,1000000,10000,128,fp32,1,100,64,1024,32,100000,15,2026,31694.970950484276,0.0,340.7115638256073,504.30078125,1536.0,164.1660898923874,60913.91959542379,162.75090724229813,166.51777923107147,5,1.034475862979889,1.2471377849578857,785,0.955319,0.56,204.169815,35511.0 +faiss-1.14.1,gpu,ivf_flat,l2,1000000,10000,128,fp32,1,10,64,1024,64,100000,15,2026,31694.970950484276,0.0,340.7115638256073,504.30078125,1536.0,285.0419953465462,35082.549811098106,285.0293591618538,285.17842292785645,5,1.823335886001587,1.9802674651145935,785,0.99511,0.7,16.27643,9002.0 +faiss-1.14.1,gpu,ivf_flat,l2,1000000,10000,128,fp32,1,50,64,1024,64,100000,15,2026,31694.970950484276,0.0,340.7115638256073,504.30078125,1536.0,292.5386443734169,34183.51794655647,292.5541028380394,292.60001331567764,5,1.8706843256950378,2.030685544013977,785,0.992236,0.76,29.796736,9002.0 +faiss-1.14.1,gpu,ivf_flat,l2,1000000,10000,128,fp32,1,100,64,1024,64,100000,15,2026,31694.970950484276,0.0,340.7115638256073,504.30078125,1536.0,305.8733895421028,32693.26571680575,305.97837269306183,306.770958006382,5,1.952625811100006,2.138972282409668,785,0.989512,0.78,42.427,9002.0 +faiss-1.14.1,gpu,ivf_flat,l2,1000000,10000,128,fp32,1,10,64,1024,128,100000,15,2026,31694.970950484276,0.0,340.7115638256073,504.30078125,1536.0,579.1918680071831,17265.435777623836,579.1034325957298,579.5566886663437,5,3.7212222814559937,4.00301069021225,785,0.9989299999999999,0.8,1.32864,5480.0 +faiss-1.14.1,gpu,ivf_flat,l2,1000000,10000,128,fp32,1,50,64,1024,128,100000,15,2026,31694.970950484276,0.0,340.7115638256073,504.30078125,1536.0,588.677728176117,16987.22326557641,588.556744158268,590.2841538190842,5,3.773339092731476,4.068478941917419,785,0.9990460000000001,0.88,2.733834,5480.0 +faiss-1.14.1,gpu,ivf_flat,l2,1000000,10000,128,fp32,1,100,64,1024,128,100000,15,2026,31694.970950484276,0.0,340.7115638256073,504.30078125,1536.0,605.4432585835457,16516.824422812682,605.4337322711945,606.0454100370407,5,3.88181209564209,4.145830869674683,785,0.998775,0.91,4.259166,5480.0 +faiss-1.14.1,gpu,ivf_flat,l2,1000000,10000,128,fp32,1,10,64,1024,160,100000,15,2026,31694.970950484276,0.0,340.7115638256073,504.30078125,1536.0,734.3974649906158,13616.604736139414,734.5470413565636,734.5698103308678,5,4.706636071205139,4.988819360733032,785,0.9991200000000001,0.9,0.70007,5251.0 +faiss-1.14.1,gpu,ivf_flat,l2,1000000,10000,128,fp32,1,50,64,1024,160,100000,15,2026,31694.970950484276,0.0,340.7115638256073,504.30078125,1536.0,743.8984677195549,13442.694714314075,743.5071393847466,745.405986905098,5,4.7682300209999084,5.052067339420319,785,0.999516,0.94,1.140368,5251.0 +faiss-1.14.1,gpu,ivf_flat,l2,1000000,10000,128,fp32,1,100,64,1024,160,100000,15,2026,31694.970950484276,0.0,340.7115638256073,504.30078125,1536.0,763.2167413830757,13102.437954752324,763.1471678614616,763.8469263911247,5,4.884809255599976,5.159102380275726,785,0.9994569999999999,0.95,1.665954,5251.0 +faiss-1.14.1,gpu,ivf_flat,l2,1000000,10000,128,fp32,1,10,64,1024,192,100000,15,2026,31694.970950484276,0.0,340.7115638256073,504.30078125,1536.0,894.5333555340767,11179.012988318977,894.5060148835182,895.1161950826645,5,5.7231709361076355,6.028883159160614,785,0.9992,0.9,0.32828,5251.0 +faiss-1.14.1,gpu,ivf_flat,l2,1000000,10000,128,fp32,1,50,64,1024,192,100000,15,2026,31694.970950484276,0.0,340.7115638256073,504.30078125,1536.0,904.1486114263535,11060.128693030172,903.9210975170135,905.0416499376297,5,5.783669650554657,6.090313196182251,785,0.999682,0.96,0.469276,5251.0 +faiss-1.14.1,gpu,ivf_flat,l2,1000000,10000,128,fp32,1,100,64,1024,192,100000,15,2026,31694.970950484276,0.0,340.7115638256073,504.30078125,1536.0,928.1674966216087,10773.917462525362,927.5803938508034,932.3575720191002,5,5.919240415096283,6.227999925613403,785,0.9997090000000001,0.95,0.710559,5251.0 +faiss-1.14.1,gpu,ivf_flat,l2,1000000,10000,128,fp32,1,10,64,1024,224,100000,15,2026,31694.970950484276,0.0,340.7115638256073,504.30078125,1536.0,1057.640977203846,9455.004311990302,1057.6919615268707,1058.4268048405647,5,6.769798696041107,7.08375871181488,785,0.9992700000000001,0.9,0.01325,578.0 +faiss-1.14.1,gpu,ivf_flat,l2,1000000,10000,128,fp32,1,50,64,1024,224,100000,15,2026,31694.970950484276,0.0,340.7115638256073,504.30078125,1536.0,1068.3847710490227,9359.923756851409,1068.4531033039093,1068.631336092949,5,6.834805011749268,7.147148251533508,785,0.9997540000000001,0.96,0.133192,1078.0 +faiss-1.14.1,gpu,ivf_flat,l2,1000000,10000,128,fp32,1,100,64,1024,224,100000,15,2026,31694.970950484276,0.0,340.7115638256073,504.30078125,1536.0,1096.5277925133705,9119.695887578759,1096.0408672690392,1098.961740732193,5,7.007323205471039,7.347196340560913,785,0.999814,0.96,0.246457,1078.0 diff --git a/09_vector_retrieval/PurplePulse/results/real/sift1m/figures/recall-latency.svg b/09_vector_retrieval/PurplePulse/results/real/sift1m/figures/recall-latency.svg new file mode 100644 index 00000000..b63ca015 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/results/real/sift1m/figures/recall-latency.svg @@ -0,0 +1,163 @@ + +Recall–latency frontier +Benchmark curves generated from the checked-in raw CSV data. + + +Recall–latency frontier + +WarpFusion + +FAISS GPU +K = 10 + +64 + +331 + +598 + +864 + +1131 + +0.98 + +0.98 + +0.99 + +0.99 + +1.00 + + +Recall@K +1000-query latency (ms) + + +32 + +64 + +128 + +160 + +192 + +224 + + +32 + +64 + +128 + +160 + +192 + +224 +K = 50 + +79 + +344 + +610 + +876 + +1142 + +0.96 + +0.97 + +0.98 + +0.99 + +1.00 + + +Recall@K + + +32 + +64 + +128 + +160 + +192 + +224 + + +32 + +64 + +128 + +160 + +192 + +224 +K = 100 + +90 + +360 + +630 + +901 + +1171 + +0.95 + +0.97 + +0.98 + +0.99 + +1.00 + + +Recall@K + + +32 + +64 + +128 + +160 + +192 + +224 + + +32 + +64 + +128 + +160 + +192 + +224 +Matched batch=64 · point labels: nprobe + diff --git a/09_vector_retrieval/PurplePulse/results/real/sift1m/figures/recall-qps.svg b/09_vector_retrieval/PurplePulse/results/real/sift1m/figures/recall-qps.svg new file mode 100644 index 00000000..10ab1f61 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/results/real/sift1m/figures/recall-qps.svg @@ -0,0 +1,163 @@ + +Recall–throughput frontier +Benchmark curves generated from the checked-in raw CSV data. + + +Recall–throughput frontier + +WarpFusion + +FAISS GPU +K = 10 + +4400 + +22724 + +41047 + +59370 + +77694 + +0.98 + +0.98 + +0.99 + +0.99 + +1.00 + + +Recall@K +Throughput (QPS) + + +32 + +64 + +128 + +160 + +192 + +224 + + +32 + +64 + +128 + +160 + +192 + +224 +K = 50 + +4844 + +21214 + +37583 + +53952 + +70322 + +0.96 + +0.97 + +0.98 + +0.99 + +1.00 + + +Recall@K + + +32 + +64 + +128 + +160 + +192 + +224 + + +32 + +64 + +128 + +160 + +192 + +224 +K = 100 + +4976 + +19996 + +35017 + +50037 + +65057 + +0.95 + +0.97 + +0.98 + +0.99 + +1.00 + + +Recall@K + + +32 + +64 + +128 + +160 + +192 + +224 + + +32 + +64 + +128 + +160 + +192 + +224 +Matched batch=64 · point labels: nprobe + diff --git a/09_vector_retrieval/PurplePulse/results/real/sift1m/groundtruth_score_validation.jsonl b/09_vector_retrieval/PurplePulse/results/real/sift1m/groundtruth_score_validation.jsonl new file mode 100644 index 00000000..30a29462 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/results/real/sift1m/groundtruth_score_validation.jsonl @@ -0,0 +1,3 @@ +{"exact_query_fraction": 0.9944, "max_rank_score_error": 0.0, "max_reported_score_error": 0.0, "mean_rank_score_error": 0.0, "mean_reported_score_error": 0.0, "metric": "l2", "min_query_recall": 0.9, "queries": 10000, "recall_at_k": 0.9994400000000001, "score_atol": 0.0001, "score_equivalent_query_fraction": 1.0, "score_rtol": 1e-05, "top_k": 10} +{"exact_query_fraction": 0.9898, "max_rank_score_error": 0.0, "max_reported_score_error": 0.0, "mean_rank_score_error": 0.0, "mean_reported_score_error": 0.0, "metric": "l2", "min_query_recall": 0.98, "queries": 10000, "recall_at_k": 0.9997960000000001, "score_atol": 0.0001, "score_equivalent_query_fraction": 1.0, "score_rtol": 1e-05, "top_k": 50} +{"exact_query_fraction": 0.9889, "max_rank_score_error": 0.0, "max_reported_score_error": 0.0, "mean_rank_score_error": 0.0, "mean_reported_score_error": 0.0, "metric": "l2", "min_query_recall": 0.99, "queries": 10000, "recall_at_k": 0.9998890000000001, "score_atol": 0.0001, "score_equivalent_query_fraction": 1.0, "score_rtol": 1e-05, "top_k": 100} diff --git a/09_vector_retrieval/PurplePulse/results/real/sift1m/groundtruth_validation.json b/09_vector_retrieval/PurplePulse/results/real/sift1m/groundtruth_validation.json new file mode 100644 index 00000000..07a1d3b9 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/results/real/sift1m/groundtruth_validation.json @@ -0,0 +1,23 @@ +[ + { + "exact_query_fraction": 0.9944, + "min_query_recall": 0.9, + "queries": 10000, + "recall_at_k": 0.9994400000000001, + "top_k": 10 + }, + { + "exact_query_fraction": 0.9898, + "min_query_recall": 0.98, + "queries": 10000, + "recall_at_k": 0.9997960000000001, + "top_k": 50 + }, + { + "exact_query_fraction": 0.9889, + "min_query_recall": 0.99, + "queries": 10000, + "recall_at_k": 0.9998890000000001, + "top_k": 100 + } +] diff --git a/09_vector_retrieval/PurplePulse/results/real/sift1m/ivf_k10.csv b/09_vector_retrieval/PurplePulse/results/real/sift1m/ivf_k10.csv new file mode 100644 index 00000000..32a957a3 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/results/real/sift1m/ivf_k10.csv @@ -0,0 +1,7 @@ +implementation,backend,num_vectors,num_queries,dim,metric,batch_size,distance_mode,nlist,nprobe,training_samples,training_iterations,training_seed,top_k,average_ms,qps,run_p50_ms,run_p99_ms,batch_p50_ms,batch_p99_ms,recall_at_k,min_query_recall,mean_absolute_score_error,max_absolute_score_error,run_samples,batch_latency_samples,index_load_ms,gpu_initialization_ms,index_h2d_ms,gpu_buffer_mib,center_ms,scan_ms,merge_ms +WarpFusion,gpu,1000000,10000,128,l2,64,bucket_major,1024,32,100000,15,2026,10,137.667,72639.1,136.666,141.631,0.844768,0.919616,0.97893,0.2,79.97344,33777.0,5,785,323.204,193.740457,49.689697,496.972664,10.2928,114.537,1.64833 +WarpFusion,gpu,1000000,10000,128,l2,64,bucket_major,1024,64,100000,15,2026,10,199.05,50238.7,199.113,199.474,1.22992,1.30381,0.9957799999999999,0.6,14.64322,22403.0,5,785,332.983,175.210298,48.152576,497.230476,13.4793,172.77,1.67068 +WarpFusion,gpu,1000000,10000,128,l2,64,bucket_major,1024,128,100000,15,2026,10,338.589,29534.3,338.737,340.325,2.11696,2.23069,0.9996700000000001,0.8,0.78821,3055.0,5,785,320.283,176.065727,48.905216,497.746101,20.0255,304.608,1.71103 +WarpFusion,gpu,1000000,10000,128,l2,64,bucket_major,1024,160,100000,15,2026,10,406.849,24579.1,406.96,407.761,2.5561,2.67661,0.99987,0.9,0.23446,3055.0,5,785,322.505,184.066255,49.703007,498.003914,23.2551,370.073,1.7466 +WarpFusion,gpu,1000000,10000,128,l2,64,bucket_major,1024,192,100000,15,2026,10,474.668,21067.4,474.156,476.9,2.98656,3.12528,0.99995,0.9,0.07875,1525.0,5,785,321.781,179.427563,49.660927,498.261726,26.5432,433.943,1.77025 +WarpFusion,gpu,1000000,10000,128,l2,64,bucket_major,1024,224,100000,15,2026,10,542.219,18442.7,542.007,544.065,3.41907,3.56627,1.0,1.0,0.0,0.0,5,785,316.701,177.705606,49.788929,498.519539,29.7752,498.496,1.80524 diff --git a/09_vector_retrieval/PurplePulse/results/real/sift1m/ivf_k100.csv b/09_vector_retrieval/PurplePulse/results/real/sift1m/ivf_k100.csv new file mode 100644 index 00000000..9aa67e03 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/results/real/sift1m/ivf_k100.csv @@ -0,0 +1,7 @@ +implementation,backend,num_vectors,num_queries,dim,metric,batch_size,distance_mode,nlist,nprobe,training_samples,training_iterations,training_seed,top_k,average_ms,qps,run_p50_ms,run_p99_ms,batch_p50_ms,batch_p99_ms,recall_at_k,min_query_recall,mean_absolute_score_error,max_absolute_score_error,run_samples,batch_latency_samples,index_load_ms,gpu_initialization_ms,index_h2d_ms,gpu_buffer_mib,center_ms,scan_ms,merge_ms +WarpFusion,gpu,1000000,10000,128,l2,64,bucket_major_partitioned,1024,32,100000,15,2026,100,197.7,50581.8,196.9,201.022,1.17728,1.27478,0.955644,0.57,202.235993,33777.0,5,785,318.913,199.390626,48.195457,499.147957,10.2146,155.665,11.3115 +WarpFusion,gpu,1000000,10000,128,l2,64,bucket_major_partitioned,1024,64,100000,15,2026,100,287.526,34779.5,285.984,292.142,1.73152,1.84867,0.9896149999999999,0.72,42.083326,22403.0,5,785,318.984,188.132779,48.891903,501.515144,13.4691,239.71,11.4772 +WarpFusion,gpu,1000000,10000,128,l2,64,bucket_major_partitioned,1024,128,100000,15,2026,100,478.162,20913.4,477.592,481.989,2.95587,3.07408,0.9988469999999999,0.87,4.317628,3201.0,5,785,319.94,188.279765,48.712704,506.249519,19.9136,423.282,12.6424 +WarpFusion,gpu,1000000,10000,128,l2,64,bucket_major_partitioned,1024,160,100000,15,2026,100,575.966,17362.2,575.245,579.948,3.57984,3.6977,0.9995649999999999,0.94,1.670581,3201.0,5,785,319.493,188.226338,48.700382,508.616707,23.2414,516.87,13.0008 +WarpFusion,gpu,1000000,10000,128,l2,64,bucket_major_partitioned,1024,192,100000,15,2026,100,672.357,14873.1,671.714,675.204,4.19066,4.33942,0.9998309999999999,0.96,0.666237,3201.0,5,785,323.998,201.666764,57.35117,510.983894,26.508,608.886,13.2047 +WarpFusion,gpu,1000000,10000,128,l2,64,bucket_major_partitioned,1024,224,100000,15,2026,100,775.442,12895.9,770.922,788.185,4.80515,4.93936,0.9999379999999999,0.96,0.256532,1754.0,5,785,315.571,187.56733,48.707584,513.351082,29.7979,700.814,13.5694 diff --git a/09_vector_retrieval/PurplePulse/results/real/sift1m/ivf_k100_full_heap.csv b/09_vector_retrieval/PurplePulse/results/real/sift1m/ivf_k100_full_heap.csv new file mode 100644 index 00000000..dab2d21e --- /dev/null +++ b/09_vector_retrieval/PurplePulse/results/real/sift1m/ivf_k100_full_heap.csv @@ -0,0 +1,7 @@ +implementation,backend,num_vectors,num_queries,dim,metric,batch_size,distance_mode,nlist,nprobe,training_samples,training_iterations,training_seed,top_k,average_ms,qps,run_p50_ms,run_p99_ms,batch_p50_ms,batch_p99_ms,recall_at_k,min_query_recall,mean_absolute_score_error,max_absolute_score_error,run_samples,batch_latency_samples,index_load_ms,gpu_initialization_ms,index_h2d_ms,gpu_buffer_mib,center_ms,scan_ms,merge_ms +WarpFusion,gpu,1000000,10000,128,l2,64,bucket_major,1024,32,100000,15,2026,100,436.985,22884.1,436.115,440.615,2.71795,2.91997,0.955644,0.57,202.235993,33777.0,5,785,322.883,213.866939,58.734592,499.147957,10.1781,393.527,11.2142 +WarpFusion,gpu,1000000,10000,128,l2,64,bucket_major,1024,64,100000,15,2026,100,545.796,18321.9,545.788,549.061,3.4225,3.68883,0.9896149999999999,0.72,42.083326,22403.0,5,785,320.381,193.838597,49.660801,501.515144,13.4119,497.891,11.4252 +WarpFusion,gpu,1000000,10000,128,l2,64,bucket_major,1024,128,100000,15,2026,100,709.398,14096.5,709.035,712.222,4.43107,4.71782,0.9988469999999999,0.87,4.317628,3201.0,5,785,318.368,182.89103,48.803841,506.249519,19.9153,654.032,12.5987 +WarpFusion,gpu,1000000,10000,128,l2,64,bucket_major,1024,160,100000,15,2026,100,794.208,12591.2,793.008,798.834,4.96826,5.28915,0.9995649999999999,0.94,1.670581,3201.0,5,785,321.814,204.20748,50.63168,508.616707,23.1448,735.464,12.9091 +WarpFusion,gpu,1000000,10000,128,l2,64,bucket_major,1024,192,100000,15,2026,100,876.632,11407.3,877.072,880.335,5.48989,5.83808,0.9998309999999999,0.96,0.666237,3201.0,5,785,322.317,191.25412,49.747967,510.983894,26.3964,813.507,13.1192 +WarpFusion,gpu,1000000,10000,128,l2,64,bucket_major,1024,224,100000,15,2026,100,957.07,10448.6,955.921,962.226,5.99789,6.38269,0.9999379999999999,0.96,0.256532,1754.0,5,785,329.588,199.467905,53.533695,513.351082,29.6314,889.532,13.4925 diff --git a/09_vector_retrieval/PurplePulse/results/real/sift1m/ivf_k100_partitioned.csv b/09_vector_retrieval/PurplePulse/results/real/sift1m/ivf_k100_partitioned.csv new file mode 100644 index 00000000..9aa67e03 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/results/real/sift1m/ivf_k100_partitioned.csv @@ -0,0 +1,7 @@ +implementation,backend,num_vectors,num_queries,dim,metric,batch_size,distance_mode,nlist,nprobe,training_samples,training_iterations,training_seed,top_k,average_ms,qps,run_p50_ms,run_p99_ms,batch_p50_ms,batch_p99_ms,recall_at_k,min_query_recall,mean_absolute_score_error,max_absolute_score_error,run_samples,batch_latency_samples,index_load_ms,gpu_initialization_ms,index_h2d_ms,gpu_buffer_mib,center_ms,scan_ms,merge_ms +WarpFusion,gpu,1000000,10000,128,l2,64,bucket_major_partitioned,1024,32,100000,15,2026,100,197.7,50581.8,196.9,201.022,1.17728,1.27478,0.955644,0.57,202.235993,33777.0,5,785,318.913,199.390626,48.195457,499.147957,10.2146,155.665,11.3115 +WarpFusion,gpu,1000000,10000,128,l2,64,bucket_major_partitioned,1024,64,100000,15,2026,100,287.526,34779.5,285.984,292.142,1.73152,1.84867,0.9896149999999999,0.72,42.083326,22403.0,5,785,318.984,188.132779,48.891903,501.515144,13.4691,239.71,11.4772 +WarpFusion,gpu,1000000,10000,128,l2,64,bucket_major_partitioned,1024,128,100000,15,2026,100,478.162,20913.4,477.592,481.989,2.95587,3.07408,0.9988469999999999,0.87,4.317628,3201.0,5,785,319.94,188.279765,48.712704,506.249519,19.9136,423.282,12.6424 +WarpFusion,gpu,1000000,10000,128,l2,64,bucket_major_partitioned,1024,160,100000,15,2026,100,575.966,17362.2,575.245,579.948,3.57984,3.6977,0.9995649999999999,0.94,1.670581,3201.0,5,785,319.493,188.226338,48.700382,508.616707,23.2414,516.87,13.0008 +WarpFusion,gpu,1000000,10000,128,l2,64,bucket_major_partitioned,1024,192,100000,15,2026,100,672.357,14873.1,671.714,675.204,4.19066,4.33942,0.9998309999999999,0.96,0.666237,3201.0,5,785,323.998,201.666764,57.35117,510.983894,26.508,608.886,13.2047 +WarpFusion,gpu,1000000,10000,128,l2,64,bucket_major_partitioned,1024,224,100000,15,2026,100,775.442,12895.9,770.922,788.185,4.80515,4.93936,0.9999379999999999,0.96,0.256532,1754.0,5,785,315.571,187.56733,48.707584,513.351082,29.7979,700.814,13.5694 diff --git a/09_vector_retrieval/PurplePulse/results/real/sift1m/ivf_k50.csv b/09_vector_retrieval/PurplePulse/results/real/sift1m/ivf_k50.csv new file mode 100644 index 00000000..95eba584 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/results/real/sift1m/ivf_k50.csv @@ -0,0 +1,7 @@ +implementation,backend,num_vectors,num_queries,dim,metric,batch_size,distance_mode,nlist,nprobe,training_samples,training_iterations,training_seed,top_k,average_ms,qps,run_p50_ms,run_p99_ms,batch_p50_ms,batch_p99_ms,recall_at_k,min_query_recall,mean_absolute_score_error,max_absolute_score_error,run_samples,batch_latency_samples,index_load_ms,gpu_initialization_ms,index_h2d_ms,gpu_buffer_mib,center_ms,scan_ms,merge_ms +WarpFusion,gpu,1000000,10000,128,l2,64,bucket_major,1024,32,100000,15,2026,50,229.621,43549.9,229.04,231.593,1.42186,1.5537,0.965052,0.48,148.988358,33777.0,5,785,321.818,220.079691,49.472607,497.939461,10.1987,198.142,5.8998 +WarpFusion,gpu,1000000,10000,128,l2,64,bucket_major,1024,64,100000,15,2026,50,309.041,32358.1,309.69,310.375,1.91482,2.07626,0.9924219999999999,0.78,29.384078,22403.0,5,785,321.078,177.157338,49.529858,499.134773,13.5669,272.944,6.03525 +WarpFusion,gpu,1000000,10000,128,l2,64,bucket_major,1024,128,100000,15,2026,50,449.871,22228.6,449.435,452.469,2.79856,2.98934,0.999224,0.9,2.673216,3201.0,5,785,321.842,179.021586,49.711105,501.525398,19.984,406.742,6.18315 +WarpFusion,gpu,1000000,10000,128,l2,64,bucket_major,1024,160,100000,15,2026,50,519.838,19236.8,520.489,520.999,3.24864,3.43622,0.999706,0.92,1.05729,3201.0,5,785,318.684,175.394173,48.728191,502.720711,23.2292,473.323,6.74316 +WarpFusion,gpu,1000000,10000,128,l2,64,bucket_major,1024,192,100000,15,2026,50,602.184,16606.2,601.594,604.995,3.75677,3.90189,0.999898,0.96,0.459158,3201.0,5,785,317.694,179.918136,48.764927,503.916023,26.5244,548.342,6.86728 +WarpFusion,gpu,1000000,10000,128,l2,64,bucket_major,1024,224,100000,15,2026,50,677.719,14755.4,677.844,678.803,4.26278,4.40144,0.9999620000000001,0.96,0.172746,1754.0,5,785,321.349,181.939438,49.854465,505.111336,29.7551,623.967,7.04164 diff --git a/09_vector_retrieval/PurplePulse/results/real/sift1m/ivf_k50_full_heap.csv b/09_vector_retrieval/PurplePulse/results/real/sift1m/ivf_k50_full_heap.csv new file mode 100644 index 00000000..95eba584 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/results/real/sift1m/ivf_k50_full_heap.csv @@ -0,0 +1,7 @@ +implementation,backend,num_vectors,num_queries,dim,metric,batch_size,distance_mode,nlist,nprobe,training_samples,training_iterations,training_seed,top_k,average_ms,qps,run_p50_ms,run_p99_ms,batch_p50_ms,batch_p99_ms,recall_at_k,min_query_recall,mean_absolute_score_error,max_absolute_score_error,run_samples,batch_latency_samples,index_load_ms,gpu_initialization_ms,index_h2d_ms,gpu_buffer_mib,center_ms,scan_ms,merge_ms +WarpFusion,gpu,1000000,10000,128,l2,64,bucket_major,1024,32,100000,15,2026,50,229.621,43549.9,229.04,231.593,1.42186,1.5537,0.965052,0.48,148.988358,33777.0,5,785,321.818,220.079691,49.472607,497.939461,10.1987,198.142,5.8998 +WarpFusion,gpu,1000000,10000,128,l2,64,bucket_major,1024,64,100000,15,2026,50,309.041,32358.1,309.69,310.375,1.91482,2.07626,0.9924219999999999,0.78,29.384078,22403.0,5,785,321.078,177.157338,49.529858,499.134773,13.5669,272.944,6.03525 +WarpFusion,gpu,1000000,10000,128,l2,64,bucket_major,1024,128,100000,15,2026,50,449.871,22228.6,449.435,452.469,2.79856,2.98934,0.999224,0.9,2.673216,3201.0,5,785,321.842,179.021586,49.711105,501.525398,19.984,406.742,6.18315 +WarpFusion,gpu,1000000,10000,128,l2,64,bucket_major,1024,160,100000,15,2026,50,519.838,19236.8,520.489,520.999,3.24864,3.43622,0.999706,0.92,1.05729,3201.0,5,785,318.684,175.394173,48.728191,502.720711,23.2292,473.323,6.74316 +WarpFusion,gpu,1000000,10000,128,l2,64,bucket_major,1024,192,100000,15,2026,50,602.184,16606.2,601.594,604.995,3.75677,3.90189,0.999898,0.96,0.459158,3201.0,5,785,317.694,179.918136,48.764927,503.916023,26.5244,548.342,6.86728 +WarpFusion,gpu,1000000,10000,128,l2,64,bucket_major,1024,224,100000,15,2026,50,677.719,14755.4,677.844,678.803,4.26278,4.40144,0.9999620000000001,0.96,0.172746,1754.0,5,785,321.349,181.939438,49.854465,505.111336,29.7551,623.967,7.04164 diff --git a/09_vector_retrieval/PurplePulse/results/real/sift1m/ivf_k50_partitioned.csv b/09_vector_retrieval/PurplePulse/results/real/sift1m/ivf_k50_partitioned.csv new file mode 100644 index 00000000..b4ded49b --- /dev/null +++ b/09_vector_retrieval/PurplePulse/results/real/sift1m/ivf_k50_partitioned.csv @@ -0,0 +1,7 @@ +implementation,backend,num_vectors,num_queries,dim,metric,batch_size,distance_mode,nlist,nprobe,training_samples,training_iterations,training_seed,top_k,average_ms,qps,run_p50_ms,run_p99_ms,batch_p50_ms,batch_p99_ms,recall_at_k,min_query_recall,mean_absolute_score_error,max_absolute_score_error,run_samples,batch_latency_samples,index_load_ms,gpu_initialization_ms,index_h2d_ms,gpu_buffer_mib,center_ms,scan_ms,merge_ms +WarpFusion,gpu,1000000,10000,128,l2,64,bucket_major_partitioned4x16,1024,32,100000,15,2026,50,163.699,61087.6,163.231,166.021,1.00186,1.06045,0.964726,0.48,149.200156,33777.0,5,785,323.638,178.231923,49.849342,497.939461,10.2109,132.098,5.94147 +WarpFusion,gpu,1000000,10000,128,l2,64,bucket_major_partitioned4x16,1024,64,100000,15,2026,50,221.181,45211.8,220.699,223.219,1.35101,1.45536,0.9920959999999999,0.78,29.595786,22403.0,5,785,322.103,185.831212,50.363392,499.134773,13.4626,186.244,6.04764 +WarpFusion,gpu,1000000,10000,128,l2,64,bucket_major_partitioned4x16,1024,128,100000,15,2026,50,350.25,28551.0,349.978,352.581,2.1705,2.27488,0.998898,0.9,2.884924,3201.0,5,785,319.932,176.105079,48.959488,501.525398,19.9973,307.617,6.20828 +WarpFusion,gpu,1000000,10000,128,l2,64,bucket_major_partitioned4x16,1024,160,100000,15,2026,50,419.173,23856.5,418.995,421.675,2.6041,2.72541,0.9993799999999999,0.9,1.268998,3201.0,5,785,323.14,183.377534,49.666943,502.720711,23.2758,371.624,6.76509 +WarpFusion,gpu,1000000,10000,128,l2,64,bucket_major_partitioned4x16,1024,192,100000,15,2026,50,485.037,20617.0,484.752,487.521,3.02928,3.15424,0.9995719999999999,0.9,0.670866,3201.0,5,785,319.942,181.11161,49.281025,503.916023,26.5112,434.734,6.87158 +WarpFusion,gpu,1000000,10000,128,l2,64,bucket_major_partitioned4x16,1024,224,100000,15,2026,50,551.673,18126.7,551.729,552.486,3.45821,3.58438,0.9996360000000001,0.9,0.384454,2193.0,5,785,320.227,217.122509,50.38176,505.111336,29.7846,498.491,7.05448 diff --git a/09_vector_retrieval/PurplePulse/results/real/sift1m/partitioned_topk_validation.json b/09_vector_retrieval/PurplePulse/results/real/sift1m/partitioned_topk_validation.json new file mode 100644 index 00000000..73040506 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/results/real/sift1m/partitioned_topk_validation.json @@ -0,0 +1,110 @@ +{ + "50": { + "selected_mode": "full_heap", + "maximum_allowed_recall_loss": 0.0001, + "comparisons": [ + { + "nprobe": 32, + "partitioned_recall": 0.964726, + "full_heap_recall": 0.965052, + "partitioned_qps": 61087.6, + "full_heap_qps": 43549.9, + "recall_loss": 0.0003260000000000485 + }, + { + "nprobe": 64, + "partitioned_recall": 0.9920959999999999, + "full_heap_recall": 0.9924219999999999, + "partitioned_qps": 45211.8, + "full_heap_qps": 32358.1, + "recall_loss": 0.0003260000000000485 + }, + { + "nprobe": 128, + "partitioned_recall": 0.998898, + "full_heap_recall": 0.999224, + "partitioned_qps": 28551.0, + "full_heap_qps": 22228.6, + "recall_loss": 0.0003260000000000485 + }, + { + "nprobe": 160, + "partitioned_recall": 0.9993799999999999, + "full_heap_recall": 0.999706, + "partitioned_qps": 23856.5, + "full_heap_qps": 19236.8, + "recall_loss": 0.0003260000000000485 + }, + { + "nprobe": 192, + "partitioned_recall": 0.9995719999999999, + "full_heap_recall": 0.999898, + "partitioned_qps": 20617.0, + "full_heap_qps": 16606.2, + "recall_loss": 0.0003260000000000485 + }, + { + "nprobe": 224, + "partitioned_recall": 0.9996360000000001, + "full_heap_recall": 0.9999620000000001, + "partitioned_qps": 18126.7, + "full_heap_qps": 14755.4, + "recall_loss": 0.0003260000000000485 + } + ] + }, + "100": { + "selected_mode": "partitioned", + "maximum_allowed_recall_loss": 0.0001, + "comparisons": [ + { + "nprobe": 32, + "partitioned_recall": 0.955644, + "full_heap_recall": 0.955644, + "partitioned_qps": 50581.8, + "full_heap_qps": 22884.1, + "recall_loss": 0.0 + }, + { + "nprobe": 64, + "partitioned_recall": 0.9896149999999999, + "full_heap_recall": 0.9896149999999999, + "partitioned_qps": 34779.5, + "full_heap_qps": 18321.9, + "recall_loss": 0.0 + }, + { + "nprobe": 128, + "partitioned_recall": 0.9988469999999999, + "full_heap_recall": 0.9988469999999999, + "partitioned_qps": 20913.4, + "full_heap_qps": 14096.5, + "recall_loss": 0.0 + }, + { + "nprobe": 160, + "partitioned_recall": 0.9995649999999999, + "full_heap_recall": 0.9995649999999999, + "partitioned_qps": 17362.2, + "full_heap_qps": 12591.2, + "recall_loss": 0.0 + }, + { + "nprobe": 192, + "partitioned_recall": 0.9998309999999999, + "full_heap_recall": 0.9998309999999999, + "partitioned_qps": 14873.1, + "full_heap_qps": 11407.3, + "recall_loss": 0.0 + }, + { + "nprobe": 224, + "partitioned_recall": 0.9999379999999999, + "full_heap_recall": 0.9999379999999999, + "partitioned_qps": 12895.9, + "full_heap_qps": 10448.6, + "recall_loss": 0.0 + } + ] + } +} diff --git a/09_vector_retrieval/PurplePulse/results/real/yahoo-minilm-384/build.csv b/09_vector_retrieval/PurplePulse/results/real/yahoo-minilm-384/build.csv new file mode 100644 index 00000000..64805273 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/results/real/yahoo-minilm-384/build.csv @@ -0,0 +1,2 @@ +dataset,nlist,training_iterations,training_samples,threads,build_ms,save_ms,minimum_bucket,average_bucket,maximum_bucket,empty_buckets,index_bytes +yahoo-minilm-384,1024,15,100000,16,87542.6,543.356,55,661.431,1878,0,1047340016 diff --git a/09_vector_retrieval/PurplePulse/results/real/yahoo-minilm-384/dataset_manifest.json b/09_vector_retrieval/PurplePulse/results/real/yahoo-minilm-384/dataset_manifest.json new file mode 100644 index 00000000..a6ea091a --- /dev/null +++ b/09_vector_retrieval/PurplePulse/results/real/yahoo-minilm-384/dataset_manifest.json @@ -0,0 +1,30 @@ +{ + "dataset": "yahoo-minilm-384-normalized", + "source_url": "https://huggingface.co/datasets/vector-index-bench/vibe/resolve/main/yahoo-minilm-384-normalized.hdf5", + "source_path": "/data/real-datasets/raw/vibe/yahoo-minilm-384-normalized.hdf5", + "source_bytes": 1043492672, + "source_sha256": "8b0519850249d7ea4aea258eb43ae837dca1bb2f7a85ee0a5145afdb699c4136", + "distance": "normalized", + "warpfusion_metric": "cosine", + "database_rows": 677305, + "query_rows": 1000, + "dim": 384, + "groundtruth_k": 100, + "outputs": { + "database": { + "path": "data/real/yahoo-minilm-384-normalized/database.bin", + "bytes": 1040340508, + "sha256": "867ab60b87ab69cb837825dc69b1d7c8b4d7c3a49e80e4a378292424f14a24d4" + }, + "queries": { + "path": "data/real/yahoo-minilm-384-normalized/queries.bin", + "bytes": 1536024, + "sha256": "aa822e4c811a8ac119a100cd6f252ec4d2d5e1a2b1eb6d2cff9fb3de2d1c26cf" + }, + "groundtruth": { + "path": "data/real/yahoo-minilm-384-normalized/groundtruth.ivecs", + "bytes": 404000, + "sha256": "cc9b1496e1bb9d1da67128bbe1e2ef4c70e9bce6fb654290a3be0af780511324" + } + } +} diff --git a/09_vector_retrieval/PurplePulse/results/real/yahoo-minilm-384/environment.json b/09_vector_retrieval/PurplePulse/results/real/yahoo-minilm-384/environment.json new file mode 100644 index 00000000..5e72e50f --- /dev/null +++ b/09_vector_retrieval/PurplePulse/results/real/yahoo-minilm-384/environment.json @@ -0,0 +1,35 @@ +{ + "dataset": "yahoo-minilm-384", + "timestamp_utc": "2026-09-10T11:55:25.088905+00:00", + "git_commit": "fatal: not a git repository (or any parent up to mount point /)\nStopping at filesystem boundary (GIT_DISCOVERY_ACROSS_FILESYSTEM not set).", + "source_sha256": { + "src/gpu_ivf_flat.cu": "1a53f5471c1ba2d9edc9f2752fc1dab43152db3d9cae511715f287fea217be91", + "src/gpu_search.cu": "52fc4696718ca42a34210c5c708d3356c3d2cb4b43bc0deadccef4aac0b49bfe", + "src/ivf_flat.cpp": "ea7854fc9597e66ff5d910f4a4ed5e91abf77d1457eea86d5406ca617bea79ea", + "scripts/run_real_benchmarks.py": "c4bf1a520f7392e48759ac64a388fdc25416ff28df92f577bbfe104b13996837", + "scripts/benchmark_exact.py": "a9f1e5754ecde98bd66cc0a333106c052c4488ea20076596a72f11d28f1d552a", + "scripts/benchmark_ivf_sweep.py": "bb454aee7a1276462f2ecef105ad23d9180a454e39e3043fae00884156f948da", + "scripts/benchmark_faiss.py": "0cd80ffcaacd191a7e8b68c40508d917ece3d3bbf2dbb58952f04fa2d0eaa969" + }, + "gpu": "NVIDIA GeForce RTX 4090 D, 570.124.06, 24564 MiB", + "nvcc": "nvcc: NVIDIA (R) Cuda compiler driver\nCopyright (c) 2005-2025 NVIDIA Corporation\nBuilt on Wed_Jan_15_19:20:09_PST_2025\nCuda compilation tools, release 12.8, V12.8.61\nBuild cuda_12.8.r12.8/compiler.35404655_0", + "python": "3.12.3 (main, Nov 6 2024, 18:32:19) [GCC 13.2.0]", + "parameters": { + "dataset": "yahoo-minilm-384", + "project_root": "/data/purplepulse-work/PurplePulse", + "data_root": "data/real", + "results_root": "results/real", + "build_dir": "build", + "nlist": 1024, + "nprobes": "32,64,128,160,192,224", + "training_samples": 100000, + "iterations": 15, + "threads": 16, + "batch_size": 64, + "warmup": 1, + "repeat": 5, + "skip_index_build": false, + "skip_faiss": false, + "maximum_partition_recall_loss": 0.0001 + } +} diff --git a/09_vector_retrieval/PurplePulse/results/real/yahoo-minilm-384/exact_gpu.csv b/09_vector_retrieval/PurplePulse/results/real/yahoo-minilm-384/exact_gpu.csv new file mode 100644 index 00000000..9fadda9d --- /dev/null +++ b/09_vector_retrieval/PurplePulse/results/real/yahoo-minilm-384/exact_gpu.csv @@ -0,0 +1,4 @@ +implementation,top_k,batch_size,average_query_ms,qps,run_p50_ms,run_p99_ms,run_samples,batch_p50_ms,batch_p99_ms,batch_latency_samples,gpu_initialization_ms,database_h2d_ms,gpu_buffer_mib,query_h2d_ms,distance_and_local_topk_ms,final_topk_ms,result_d2h_ms,result_path +WarpFusion Exact fused,10,64,276.992,3610.22,276.461,282.893,5,17.6113,18.5666,80,256.683,103.47,992.364,0.358611,275.887,0.188109,0.330854,/data/purplepulse-work/PurplePulse/results/real/yahoo-minilm-384/exact_outputs/exact_k10.txt +WarpFusion Exact fused,50,64,288.082,3471.23,284.597,294.777,5,18.2826,19.5238,80,250.861,98.1649,992.862,0.349184,286.057,0.854976,0.423923,/data/purplepulse-work/PurplePulse/results/real/yahoo-minilm-384/exact_outputs/exact_k50.txt +WarpFusion Exact fused,100,64,304.328,3285.92,301.115,312.348,5,19.2952,20.0785,80,231.485,102.26,993.485,0.357984,300.941,1.92299,0.499981,/data/purplepulse-work/PurplePulse/results/real/yahoo-minilm-384/exact_outputs/exact_k100.txt diff --git a/09_vector_retrieval/PurplePulse/results/real/yahoo-minilm-384/faiss_gpu.csv b/09_vector_retrieval/PurplePulse/results/real/yahoo-minilm-384/faiss_gpu.csv new file mode 100644 index 00000000..6ff91590 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/results/real/yahoo-minilm-384/faiss_gpu.csv @@ -0,0 +1,22 @@ +implementation,backend,index_type,metric,num_vectors,num_queries,dim,dtype,cpu_threads,top_k,batch_size,nlist,nprobe,train_samples,training_iterations,training_seed,build_ms,load_ms,gpu_transfer_ms,gpu_index_mib,gpu_temp_mib,average_query_ms,qps,run_p50_ms,run_p99_ms,run_samples,batch_p50_ms,batch_p99_ms,batch_latency_samples,recall_at_k,min_query_recall,mean_absolute_score_error,max_absolute_score_error +faiss-1.14.1,gpu,flat,cosine,677305,1000,384,fp32,1,10,64,0,0,0,0,0,450.8100524544716,0.0,299.5612174272537,994.729736328125,1536.0,47.129690647125244,21218.04718573931,47.55561053752899,47.75427281856537,5,2.9764994978904724,2.999044954776764,80,1.0,1.0,1.4573633670806886e-07,8.344650268554688e-07 +faiss-1.14.1,gpu,flat,cosine,677305,1000,384,fp32,1,50,64,0,0,0,0,0,450.8100524544716,0.0,299.5612174272537,994.729736328125,1536.0,47.176842391490936,21196.840426530227,47.17285931110382,47.21493273973465,5,2.949640154838562,2.9679685831069946,80,1.0,1.0,1.2792527675628663e-07,8.344650268554688e-07 +faiss-1.14.1,gpu,flat,cosine,677305,1000,384,fp32,1,100,64,0,0,0,0,0,450.8100524544716,0.0,299.5612174272537,994.729736328125,1536.0,48.6848309636116,20540.278772815866,48.51270467042923,49.375757575035095,5,3.0298978090286255,3.2334327697753906,80,1.0,1.0,1.2031018733978273e-07,8.344650268554688e-07 +faiss-1.14.1,gpu,ivf_flat,cosine,677305,1000,384,fp32,1,10,64,1024,32,100000,15,2026,77886.4715769887,0.0,682.7266216278076,1022.198486328125,1536.0,30.19937574863434,33113.2672517319,30.18796443939209,30.236780643463135,5,1.9363686442375183,2.031967043876648,80,0.9599,0.2,0.0015490862041711807,0.13176971673965454 +faiss-1.14.1,gpu,ivf_flat,cosine,677305,1000,384,fp32,1,50,64,1024,32,100000,15,2026,77886.4715769887,0.0,682.7266216278076,1022.198486328125,1536.0,31.084707379341125,32170.159680016783,31.085841357707977,31.138867139816284,5,1.9916370511054993,2.074703574180603,80,0.9444000000000001,0.34,0.002021763826608658,0.13176971673965454 +faiss-1.14.1,gpu,ivf_flat,cosine,677305,1000,384,fp32,1,100,64,1024,32,100000,15,2026,77886.4715769887,0.0,682.7266216278076,1022.198486328125,1536.0,32.6606884598732,30617.848157995697,32.656192779541016,32.72579610347748,5,2.067975699901581,2.176031470298767,80,0.93456,0.31,0.0023760333928465843,0.13176971673965454 +faiss-1.14.1,gpu,ivf_flat,cosine,677305,1000,384,fp32,1,10,64,1024,64,100000,15,2026,77886.4715769887,0.0,682.7266216278076,1022.198486328125,1536.0,61.67723387479782,16213.43787936336,61.669930815696716,61.73589825630188,5,3.944084048271179,4.0833353996276855,80,0.9822000000000001,0.3,0.0006998934537172318,0.11930292844772339 +faiss-1.14.1,gpu,ivf_flat,cosine,677305,1000,384,fp32,1,50,64,1024,64,100000,15,2026,77886.4715769887,0.0,682.7266216278076,1022.198486328125,1536.0,62.69751042127609,15949.5966152534,62.69342452287674,62.74265795946121,5,4.007898271083832,4.138417541980743,80,0.97208,0.4,0.0009457416796684265,0.11930292844772339 +faiss-1.14.1,gpu,ivf_flat,cosine,677305,1000,384,fp32,1,100,64,1024,64,100000,15,2026,77886.4715769887,0.0,682.7266216278076,1022.198486328125,1536.0,64.59192037582397,15481.81249576671,64.58926200866699,64.62705135345459,5,4.12413477897644,4.209011793136597,80,0.9666200000000001,0.4,0.0011352579164505005,0.11930292844772339 +faiss-1.14.1,gpu,ivf_flat,cosine,677305,1000,384,fp32,1,10,64,1024,128,100000,15,2026,77886.4715769887,0.0,682.7266216278076,1022.198486328125,1536.0,127.10106819868088,7867.754489968783,127.10943818092346,127.14478373527527,5,8.125759661197662,8.281677961349487,80,0.9933,0.6,0.00025944703221321105,0.1106138825416565 +faiss-1.14.1,gpu,ivf_flat,cosine,677305,1000,384,fp32,1,50,64,1024,128,100000,15,2026,77886.4715769887,0.0,682.7266216278076,1022.198486328125,1536.0,128.5264492034912,7780.499704124998,128.52223217487335,128.5785660147667,5,8.207350969314575,8.37651640176773,80,0.98806,0.5,0.0003864972972869873,0.1106138825416565 +faiss-1.14.1,gpu,ivf_flat,cosine,677305,1000,384,fp32,1,100,64,1024,128,100000,15,2026,77886.4715769887,0.0,682.7266216278076,1022.198486328125,1536.0,131.11717849969864,7626.765702575718,131.09293580055237,131.2248781323433,5,8.35118442773819,8.524872362613678,80,0.9850300000000001,0.53,0.00047720189005136487,0.1106138825416565 +faiss-1.14.1,gpu,ivf_flat,cosine,677305,1000,384,fp32,1,10,64,1024,160,100000,15,2026,77886.4715769887,0.0,682.7266216278076,1022.198486328125,1536.0,160.87044924497604,6216.182056389886,160.87330132722855,160.94182431697845,5,10.255873203277588,10.470017790794373,80,0.9953,0.6,0.00018345237970352174,0.07067233324050903 +faiss-1.14.1,gpu,ivf_flat,cosine,677305,1000,384,fp32,1,50,64,1024,160,100000,15,2026,77886.4715769887,0.0,682.7266216278076,1022.198486328125,1536.0,162.41146624088287,6157.2007392435935,162.38412261009216,162.4923124909401,5,10.346010327339172,10.57722419500351,80,0.9912799999999999,0.52,0.00027075362503528594,0.07067233324050903 +faiss-1.14.1,gpu,ivf_flat,cosine,677305,1000,384,fp32,1,100,64,1024,160,100000,15,2026,77886.4715769887,0.0,682.7266216278076,1022.198486328125,1536.0,165.0216579437256,6059.810648254499,165.03334790468216,165.03560543060303,5,10.53435355424881,10.722257196903229,80,0.98908,0.57,0.0003443580350279808,0.07067233324050903 +faiss-1.14.1,gpu,ivf_flat,cosine,677305,1000,384,fp32,1,10,64,1024,192,100000,15,2026,77886.4715769887,0.0,682.7266216278076,1022.198486328125,1536.0,194.4973424077034,5141.458426222657,194.54151391983032,194.56788152456284,5,12.407995760440826,12.624137103557587,80,0.9966,0.6,0.0001404004067182541,0.07067233324050903 +faiss-1.14.1,gpu,ivf_flat,cosine,677305,1000,384,fp32,1,50,64,1024,192,100000,15,2026,77886.4715769887,0.0,682.7266216278076,1022.198486328125,1536.0,196.2209790945053,5096.295027242592,196.22181355953217,196.2629333138466,5,12.520894408226013,12.721635401248932,80,0.9933600000000001,0.52,0.00020388494014739991,0.07067233324050903 +faiss-1.14.1,gpu,ivf_flat,cosine,677305,1000,384,fp32,1,100,64,1024,192,100000,15,2026,77886.4715769887,0.0,682.7266216278076,1022.198486328125,1536.0,199.3123009800911,5017.251795712739,199.30288940668106,199.33469593524933,5,12.710973620414734,12.910954654216766,80,0.9917,0.58,0.00026132633090019227,0.07067233324050903 +faiss-1.14.1,gpu,ivf_flat,cosine,677305,1000,384,fp32,1,10,64,1024,224,100000,15,2026,77886.4715769887,0.0,682.7266216278076,1022.198486328125,1536.0,228.4289449453354,4377.728926775488,228.42742502689362,228.47234457731247,5,14.590464532375336,14.774948358535767,80,0.9972000000000001,0.6,0.00012997233867645264,0.07067233324050903 +faiss-1.14.1,gpu,ivf_flat,cosine,677305,1000,384,fp32,1,50,64,1024,224,100000,15,2026,77886.4715769887,0.0,682.7266216278076,1022.198486328125,1536.0,230.3800955414772,4340.652770586085,230.39449006319046,230.4425686597824,5,14.698363840579987,14.881014823913574,80,0.99466,0.54,0.00016983119666576387,0.07067233324050903 +faiss-1.14.1,gpu,ivf_flat,cosine,677305,1000,384,fp32,1,100,64,1024,224,100000,15,2026,77886.4715769887,0.0,682.7266216278076,1022.198486328125,1536.0,234.14455652236938,4270.86589093718,234.07414555549622,234.4672828912735,5,14.94162529706955,15.218183398246765,80,0.9933599999999999,0.61,0.0002106537815928459,0.07067233324050903 diff --git a/09_vector_retrieval/PurplePulse/results/real/yahoo-minilm-384/figures/recall-latency.svg b/09_vector_retrieval/PurplePulse/results/real/yahoo-minilm-384/figures/recall-latency.svg new file mode 100644 index 00000000..050c3feb --- /dev/null +++ b/09_vector_retrieval/PurplePulse/results/real/yahoo-minilm-384/figures/recall-latency.svg @@ -0,0 +1,163 @@ + +Recall–latency frontier +Benchmark curves generated from the checked-in raw CSV data. + + +Recall–latency frontier + +WarpFusion + +FAISS GPU +K = 10 + +9 + +68 + +127 + +186 + +245 + +0.96 + +0.97 + +0.98 + +0.99 + +1.00 + + +Recall@K +1000-query latency (ms) + + +32 + +64 + +128 + +160 + +192 + +224 + + +32 + +64 + +128 + +160 + +192 + +224 +K = 50 + +15 + +73 + +131 + +189 + +246 + +0.94 + +0.96 + +0.97 + +0.98 + +1.00 + + +Recall@K + + +32 + +64 + +128 + +160 + +192 + +224 + + +32 + +64 + +128 + +160 + +192 + +224 +K = 100 + +11 + +71 + +131 + +191 + +251 + +0.93 + +0.95 + +0.96 + +0.98 + +1.00 + + +Recall@K + + +32 + +64 + +128 + +160 + +192 + +224 + + +32 + +64 + +128 + +160 + +192 + +224 +Matched batch=64 · point labels: nprobe + diff --git a/09_vector_retrieval/PurplePulse/results/real/yahoo-minilm-384/figures/recall-qps.svg b/09_vector_retrieval/PurplePulse/results/real/yahoo-minilm-384/figures/recall-qps.svg new file mode 100644 index 00000000..7ff37c8f --- /dev/null +++ b/09_vector_retrieval/PurplePulse/results/real/yahoo-minilm-384/figures/recall-qps.svg @@ -0,0 +1,163 @@ + +Recall–throughput frontier +Benchmark curves generated from the checked-in raw CSV data. + + +Recall–throughput frontier + +WarpFusion + +FAISS GPU +K = 10 + +1530 + +11852 + +22175 + +32497 + +42819 + +0.96 + +0.97 + +0.98 + +0.99 + +1.00 + + +Recall@K +Throughput (QPS) + + +32 + +64 + +128 + +160 + +192 + +224 + + +32 + +64 + +128 + +160 + +192 + +224 +K = 50 + +2114 + +10185 + +18255 + +26326 + +34397 + +0.94 + +0.96 + +0.97 + +0.98 + +1.00 + + +Recall@K + + +32 + +64 + +128 + +160 + +192 + +224 + + +32 + +64 + +128 + +160 + +192 + +224 +K = 100 + +1708 + +10997 + +20286 + +29575 + +38863 + +0.93 + +0.95 + +0.96 + +0.98 + +1.00 + + +Recall@K + + +32 + +64 + +128 + +160 + +192 + +224 + + +32 + +64 + +128 + +160 + +192 + +224 +Matched batch=64 · point labels: nprobe + diff --git a/09_vector_retrieval/PurplePulse/results/real/yahoo-minilm-384/groundtruth_validation.json b/09_vector_retrieval/PurplePulse/results/real/yahoo-minilm-384/groundtruth_validation.json new file mode 100644 index 00000000..9e640d86 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/results/real/yahoo-minilm-384/groundtruth_validation.json @@ -0,0 +1,47 @@ +[ + { + "exact_query_fraction": 0.998, + "max_rank_score_error": 0.0, + "max_reported_score_error": 2.980232238769531e-07, + "mean_rank_score_error": 0.0, + "mean_reported_score_error": 5.479156970977783e-08, + "metric": "cosine", + "min_query_recall": 0.9, + "queries": 1000, + "recall_at_k": 0.9997999999999999, + "score_atol": 0.0001, + "score_equivalent_query_fraction": 1.0, + "score_rtol": 1e-05, + "top_k": 10 + }, + { + "exact_query_fraction": 1.0, + "max_rank_score_error": 0.0, + "max_reported_score_error": 2.980232238769531e-07, + "mean_rank_score_error": 0.0, + "mean_reported_score_error": 5.029141902923584e-08, + "metric": "cosine", + "min_query_recall": 1.0, + "queries": 1000, + "recall_at_k": 1.0, + "score_atol": 0.0001, + "score_equivalent_query_fraction": 1.0, + "score_rtol": 1e-05, + "top_k": 50 + }, + { + "exact_query_fraction": 1.0, + "max_rank_score_error": 0.0, + "max_reported_score_error": 2.980232238769531e-07, + "mean_rank_score_error": 0.0, + "mean_reported_score_error": 4.790693521499634e-08, + "metric": "cosine", + "min_query_recall": 1.0, + "queries": 1000, + "recall_at_k": 1.0, + "score_atol": 0.0001, + "score_equivalent_query_fraction": 1.0, + "score_rtol": 1e-05, + "top_k": 100 + } +] diff --git a/09_vector_retrieval/PurplePulse/results/real/yahoo-minilm-384/ivf_k10.csv b/09_vector_retrieval/PurplePulse/results/real/yahoo-minilm-384/ivf_k10.csv new file mode 100644 index 00000000..1abab854 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/results/real/yahoo-minilm-384/ivf_k10.csv @@ -0,0 +1,7 @@ +implementation,backend,num_vectors,num_queries,dim,metric,batch_size,distance_mode,nlist,nprobe,training_samples,training_iterations,training_seed,top_k,average_ms,qps,run_p50_ms,run_p99_ms,batch_p50_ms,batch_p99_ms,recall_at_k,min_query_recall,mean_absolute_score_error,max_absolute_score_error,run_samples,batch_latency_samples,index_load_ms,gpu_initialization_ms,index_h2d_ms,gpu_buffer_mib,center_ms,scan_ms,merge_ms +WarpFusion,gpu,677305,1000,384,cosine,64,bucket_major,1024,32,100000,15,2026,10,25.0177,39971.7,24.9527,25.2257,1.40381,1.45821,0.96,0.2,0.0015903574357999997,0.16803198999999996,5,80,615.32,253.841354,105.347069,999.437943,2.41437,19.017,0.175494 +WarpFusion,gpu,677305,1000,384,cosine,64,bucket_major,1024,64,100000,15,2026,10,33.1919,30127.8,33.187,33.255,1.93594,1.95795,0.9795,0.4,0.0007539571803000002,0.11731612700000005,5,80,598.462,221.988553,99.067902,999.695755,2.59712,27.1201,0.169581 +WarpFusion,gpu,677305,1000,384,cosine,64,bucket_major,1024,128,100000,15,2026,10,53.2645,18774.2,53.2607,53.3304,3.21763,3.25683,0.9907,0.5,0.00034863495470000005,0.074507355,5,80,583.333,228.371225,103.077888,1000.21138,3.2778,46.466,0.175968 +WarpFusion,gpu,677305,1000,384,cosine,64,bucket_major,1024,160,100000,15,2026,10,63.5599,15733.2,63.5563,63.6589,3.8767,3.93107,0.9939,0.5,0.0002204378745000001,0.074507355,5,80,576.442,221.602178,99.047424,1000.469193,3.61023,56.4528,0.178182 +WarpFusion,gpu,677305,1000,384,cosine,64,bucket_major,1024,192,100000,15,2026,10,74.1052,13494.3,74.0714,74.2573,4.55146,4.61754,0.9956,0.5,0.0001645866181000001,0.074507355,5,80,581.862,226.575909,103.015427,1000.727005,3.97519,66.6217,0.183008 +WarpFusion,gpu,677305,1000,384,cosine,64,bucket_major,1024,224,100000,15,2026,10,84.5358,11829.3,84.4814,84.7255,5.2119,5.28419,0.9967,0.5,0.00013676807620000017,0.074507355,5,80,582.47,221.918123,98.994179,1000.984818,4.31267,76.7173,0.185715 diff --git a/09_vector_retrieval/PurplePulse/results/real/yahoo-minilm-384/ivf_k100.csv b/09_vector_retrieval/PurplePulse/results/real/yahoo-minilm-384/ivf_k100.csv new file mode 100644 index 00000000..4dd1b25e --- /dev/null +++ b/09_vector_retrieval/PurplePulse/results/real/yahoo-minilm-384/ivf_k100.csv @@ -0,0 +1,7 @@ +implementation,backend,num_vectors,num_queries,dim,metric,batch_size,distance_mode,nlist,nprobe,training_samples,training_iterations,training_seed,top_k,average_ms,qps,run_p50_ms,run_p99_ms,batch_p50_ms,batch_p99_ms,recall_at_k,min_query_recall,mean_absolute_score_error,max_absolute_score_error,run_samples,batch_latency_samples,index_load_ms,gpu_initialization_ms,index_h2d_ms,gpu_buffer_mib,center_ms,scan_ms,merge_ms +WarpFusion,gpu,677305,1000,384,cosine,64,bucket_major_partitioned,1024,32,100000,15,2026,100,27.5476,36300.8,27.4732,27.9229,1.51718,1.58736,0.937,0.35,0.00226213649931,0.16803198999999996,5,80,632.016,219.81129,96.653313,1001.613235,2.26611,19.967,1.14769 +WarpFusion,gpu,677305,1000,384,cosine,64,bucket_major_partitioned,1024,64,100000,15,2026,100,36.2878,27557.5,36.2143,36.6426,2.07827,2.16842,0.9675,0.44,0.00111137691699,0.11731612700000005,5,80,615.445,219.880745,97.542145,1003.980423,2.59802,28.3386,1.16737 +WarpFusion,gpu,677305,1000,384,cosine,64,bucket_major_partitioned,1024,128,100000,15,2026,100,59.1108,16917.4,59.0373,59.3614,3.54618,3.59792,0.9851599999999999,0.58,0.0004964717999000001,0.074507355,5,80,622.461,223.660037,97.46637,1008.714798,3.29774,50.2849,1.31774 +WarpFusion,gpu,677305,1000,384,cosine,64,bucket_major_partitioned,1024,160,100000,15,2026,100,70.4416,14196.2,70.4452,70.9362,4.27254,4.34474,0.98923,0.67,0.00034501102042000016,0.074507355,5,80,621.258,222.388821,96.962563,1011.081985,3.65519,61.2786,1.33953 +WarpFusion,gpu,677305,1000,384,cosine,64,bucket_major_partitioned,1024,192,100000,15,2026,100,82.0801,12183.2,82.0289,82.457,5.01482,5.08573,0.99185,0.68,0.0002647012558900001,0.074507355,5,80,634.804,221.601371,98.573311,1013.449173,4.01476,72.4373,1.37266 +WarpFusion,gpu,677305,1000,384,cosine,64,bucket_major_partitioned,1024,224,100000,15,2026,100,93.9092,10648.6,93.8809,94.1201,5.77248,5.8792,0.9936799999999999,0.75,0.0002082749046100001,0.074507355,5,80,616.427,220.378178,97.820671,1015.81636,4.37754,83.9071,1.41418 diff --git a/09_vector_retrieval/PurplePulse/results/real/yahoo-minilm-384/ivf_k100_full_heap.csv b/09_vector_retrieval/PurplePulse/results/real/yahoo-minilm-384/ivf_k100_full_heap.csv new file mode 100644 index 00000000..b6a4388b --- /dev/null +++ b/09_vector_retrieval/PurplePulse/results/real/yahoo-minilm-384/ivf_k100_full_heap.csv @@ -0,0 +1,7 @@ +implementation,backend,num_vectors,num_queries,dim,metric,batch_size,distance_mode,nlist,nprobe,training_samples,training_iterations,training_seed,top_k,average_ms,qps,run_p50_ms,run_p99_ms,batch_p50_ms,batch_p99_ms,recall_at_k,min_query_recall,mean_absolute_score_error,max_absolute_score_error,run_samples,batch_latency_samples,index_load_ms,gpu_initialization_ms,index_h2d_ms,gpu_buffer_mib,center_ms,scan_ms,merge_ms +WarpFusion,gpu,677305,1000,384,cosine,64,bucket_major,1024,32,100000,15,2026,100,69.145,14462.4,69.0678,69.5509,4.09882,4.34435,0.937,0.35,0.00226213649931,0.16803198999999996,5,80,624.728,224.893472,97.354752,1001.613235,2.26195,61.5402,1.1458 +WarpFusion,gpu,677305,1000,384,cosine,64,bucket_major,1024,64,100000,15,2026,100,75.3244,13275.9,75.2989,75.787,4.49338,4.65229,0.9675,0.44,0.00111137691699,0.11731612700000005,5,80,626.195,229.177001,98.456573,1003.980423,2.59912,67.3547,1.16408 +WarpFusion,gpu,677305,1000,384,cosine,64,bucket_major,1024,128,100000,15,2026,100,104.647,9555.98,104.555,104.995,6.41245,6.49898,0.9851599999999999,0.58,0.0004964717999000001,0.074507355,5,80,622.307,221.062987,97.443871,1008.714798,3.26175,95.8832,1.30049 +WarpFusion,gpu,677305,1000,384,cosine,64,bucket_major,1024,160,100000,15,2026,100,121.055,8260.7,121.057,121.314,7.46528,7.55472,0.98923,0.67,0.00034501102042000016,0.074507355,5,80,633.156,226.854412,102.834175,1011.081985,3.59485,111.886,1.31443 +WarpFusion,gpu,677305,1000,384,cosine,64,bucket_major,1024,192,100000,15,2026,100,136.881,7305.64,136.765,137.326,8.47526,8.57216,0.99185,0.68,0.0002647012558900001,0.074507355,5,80,623.851,230.293752,106.638306,1013.449173,3.92856,127.4,1.34267 +WarpFusion,gpu,677305,1000,384,cosine,64,bucket_major,1024,224,100000,15,2026,100,153.724,6505.14,153.634,154.178,9.56003,9.63667,0.9936799999999999,0.75,0.0002082749046100001,0.074507355,5,80,630.281,223.004934,98.985985,1015.81636,4.26166,143.838,1.37277 diff --git a/09_vector_retrieval/PurplePulse/results/real/yahoo-minilm-384/ivf_k100_partitioned.csv b/09_vector_retrieval/PurplePulse/results/real/yahoo-minilm-384/ivf_k100_partitioned.csv new file mode 100644 index 00000000..4dd1b25e --- /dev/null +++ b/09_vector_retrieval/PurplePulse/results/real/yahoo-minilm-384/ivf_k100_partitioned.csv @@ -0,0 +1,7 @@ +implementation,backend,num_vectors,num_queries,dim,metric,batch_size,distance_mode,nlist,nprobe,training_samples,training_iterations,training_seed,top_k,average_ms,qps,run_p50_ms,run_p99_ms,batch_p50_ms,batch_p99_ms,recall_at_k,min_query_recall,mean_absolute_score_error,max_absolute_score_error,run_samples,batch_latency_samples,index_load_ms,gpu_initialization_ms,index_h2d_ms,gpu_buffer_mib,center_ms,scan_ms,merge_ms +WarpFusion,gpu,677305,1000,384,cosine,64,bucket_major_partitioned,1024,32,100000,15,2026,100,27.5476,36300.8,27.4732,27.9229,1.51718,1.58736,0.937,0.35,0.00226213649931,0.16803198999999996,5,80,632.016,219.81129,96.653313,1001.613235,2.26611,19.967,1.14769 +WarpFusion,gpu,677305,1000,384,cosine,64,bucket_major_partitioned,1024,64,100000,15,2026,100,36.2878,27557.5,36.2143,36.6426,2.07827,2.16842,0.9675,0.44,0.00111137691699,0.11731612700000005,5,80,615.445,219.880745,97.542145,1003.980423,2.59802,28.3386,1.16737 +WarpFusion,gpu,677305,1000,384,cosine,64,bucket_major_partitioned,1024,128,100000,15,2026,100,59.1108,16917.4,59.0373,59.3614,3.54618,3.59792,0.9851599999999999,0.58,0.0004964717999000001,0.074507355,5,80,622.461,223.660037,97.46637,1008.714798,3.29774,50.2849,1.31774 +WarpFusion,gpu,677305,1000,384,cosine,64,bucket_major_partitioned,1024,160,100000,15,2026,100,70.4416,14196.2,70.4452,70.9362,4.27254,4.34474,0.98923,0.67,0.00034501102042000016,0.074507355,5,80,621.258,222.388821,96.962563,1011.081985,3.65519,61.2786,1.33953 +WarpFusion,gpu,677305,1000,384,cosine,64,bucket_major_partitioned,1024,192,100000,15,2026,100,82.0801,12183.2,82.0289,82.457,5.01482,5.08573,0.99185,0.68,0.0002647012558900001,0.074507355,5,80,634.804,221.601371,98.573311,1013.449173,4.01476,72.4373,1.37266 +WarpFusion,gpu,677305,1000,384,cosine,64,bucket_major_partitioned,1024,224,100000,15,2026,100,93.9092,10648.6,93.8809,94.1201,5.77248,5.8792,0.9936799999999999,0.75,0.0002082749046100001,0.074507355,5,80,616.427,220.378178,97.820671,1015.81636,4.37754,83.9071,1.41418 diff --git a/09_vector_retrieval/PurplePulse/results/real/yahoo-minilm-384/ivf_k50.csv b/09_vector_retrieval/PurplePulse/results/real/yahoo-minilm-384/ivf_k50.csv new file mode 100644 index 00000000..6d42ef91 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/results/real/yahoo-minilm-384/ivf_k50.csv @@ -0,0 +1,7 @@ +implementation,backend,num_vectors,num_queries,dim,metric,batch_size,distance_mode,nlist,nprobe,training_samples,training_iterations,training_seed,top_k,average_ms,qps,run_p50_ms,run_p99_ms,batch_p50_ms,batch_p99_ms,recall_at_k,min_query_recall,mean_absolute_score_error,max_absolute_score_error,run_samples,batch_latency_samples,index_load_ms,gpu_initialization_ms,index_h2d_ms,gpu_buffer_mib,center_ms,scan_ms,merge_ms +WarpFusion,gpu,677305,1000,384,cosine,64,bucket_major,1024,32,100000,15,2026,50,32.1664,31088.4,32.0637,32.3548,1.82246,1.90678,0.9469399999999999,0.32,0.00192249446772,0.16803198999999996,5,80,622.945,219.127451,96.250877,1000.404739,2.26944,25.5575,0.596019 +WarpFusion,gpu,677305,1000,384,cosine,64,bucket_major,1024,64,100000,15,2026,50,38.3504,26075.4,38.2919,38.5666,2.2232,2.31315,0.97312,0.4,0.0009359371467000001,0.11731612700000005,5,80,623.44,221.35587,97.981346,1001.600052,2.59305,31.4994,0.612141 +WarpFusion,gpu,677305,1000,384,cosine,64,bucket_major,1024,128,100000,15,2026,50,56.6138,17663.5,56.5905,56.8182,3.40432,3.53843,0.9881599999999999,0.54,0.0004134946768800001,0.074507355,5,80,621.404,218.141578,96.24678,1003.990677,3.27071,49.0523,0.630266 +WarpFusion,gpu,677305,1000,384,cosine,64,bucket_major,1024,160,100000,15,2026,50,66.8911,14949.7,66.9044,67.0506,4.05846,4.18694,0.99152,0.66,0.00027749032166000015,0.074507355,5,80,626.545,227.419422,98.321411,1005.185989,3.62828,58.881,0.692832 +WarpFusion,gpu,677305,1000,384,cosine,64,bucket_major,1024,192,100000,15,2026,50,77.1075,12968.9,77.1068,77.3744,4.71923,4.81955,0.99336,0.66,0.00021764280564000017,0.074507355,5,80,621.796,223.496225,96.375809,1006.381302,3.9801,68.7534,0.7072 +WarpFusion,gpu,677305,1000,384,cosine,64,bucket_major,1024,224,100000,15,2026,50,87.4793,11431.3,87.4535,87.779,5.37523,5.5209,0.99482,0.76,0.00017535253728000015,0.074507355,5,80,628.405,224.773498,98.170883,1007.576614,4.34845,78.7322,0.729683 diff --git a/09_vector_retrieval/PurplePulse/results/real/yahoo-minilm-384/ivf_k50_full_heap.csv b/09_vector_retrieval/PurplePulse/results/real/yahoo-minilm-384/ivf_k50_full_heap.csv new file mode 100644 index 00000000..6d42ef91 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/results/real/yahoo-minilm-384/ivf_k50_full_heap.csv @@ -0,0 +1,7 @@ +implementation,backend,num_vectors,num_queries,dim,metric,batch_size,distance_mode,nlist,nprobe,training_samples,training_iterations,training_seed,top_k,average_ms,qps,run_p50_ms,run_p99_ms,batch_p50_ms,batch_p99_ms,recall_at_k,min_query_recall,mean_absolute_score_error,max_absolute_score_error,run_samples,batch_latency_samples,index_load_ms,gpu_initialization_ms,index_h2d_ms,gpu_buffer_mib,center_ms,scan_ms,merge_ms +WarpFusion,gpu,677305,1000,384,cosine,64,bucket_major,1024,32,100000,15,2026,50,32.1664,31088.4,32.0637,32.3548,1.82246,1.90678,0.9469399999999999,0.32,0.00192249446772,0.16803198999999996,5,80,622.945,219.127451,96.250877,1000.404739,2.26944,25.5575,0.596019 +WarpFusion,gpu,677305,1000,384,cosine,64,bucket_major,1024,64,100000,15,2026,50,38.3504,26075.4,38.2919,38.5666,2.2232,2.31315,0.97312,0.4,0.0009359371467000001,0.11731612700000005,5,80,623.44,221.35587,97.981346,1001.600052,2.59305,31.4994,0.612141 +WarpFusion,gpu,677305,1000,384,cosine,64,bucket_major,1024,128,100000,15,2026,50,56.6138,17663.5,56.5905,56.8182,3.40432,3.53843,0.9881599999999999,0.54,0.0004134946768800001,0.074507355,5,80,621.404,218.141578,96.24678,1003.990677,3.27071,49.0523,0.630266 +WarpFusion,gpu,677305,1000,384,cosine,64,bucket_major,1024,160,100000,15,2026,50,66.8911,14949.7,66.9044,67.0506,4.05846,4.18694,0.99152,0.66,0.00027749032166000015,0.074507355,5,80,626.545,227.419422,98.321411,1005.185989,3.62828,58.881,0.692832 +WarpFusion,gpu,677305,1000,384,cosine,64,bucket_major,1024,192,100000,15,2026,50,77.1075,12968.9,77.1068,77.3744,4.71923,4.81955,0.99336,0.66,0.00021764280564000017,0.074507355,5,80,621.796,223.496225,96.375809,1006.381302,3.9801,68.7534,0.7072 +WarpFusion,gpu,677305,1000,384,cosine,64,bucket_major,1024,224,100000,15,2026,50,87.4793,11431.3,87.4535,87.779,5.37523,5.5209,0.99482,0.76,0.00017535253728000015,0.074507355,5,80,628.405,224.773498,98.170883,1007.576614,4.34845,78.7322,0.729683 diff --git a/09_vector_retrieval/PurplePulse/results/real/yahoo-minilm-384/ivf_k50_partitioned.csv b/09_vector_retrieval/PurplePulse/results/real/yahoo-minilm-384/ivf_k50_partitioned.csv new file mode 100644 index 00000000..dcaa1174 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/results/real/yahoo-minilm-384/ivf_k50_partitioned.csv @@ -0,0 +1,7 @@ +implementation,backend,num_vectors,num_queries,dim,metric,batch_size,distance_mode,nlist,nprobe,training_samples,training_iterations,training_seed,top_k,average_ms,qps,run_p50_ms,run_p99_ms,batch_p50_ms,batch_p99_ms,recall_at_k,min_query_recall,mean_absolute_score_error,max_absolute_score_error,run_samples,batch_latency_samples,index_load_ms,gpu_initialization_ms,index_h2d_ms,gpu_buffer_mib,center_ms,scan_ms,merge_ms +WarpFusion,gpu,677305,1000,384,cosine,64,bucket_major_partitioned4x16,1024,32,100000,15,2026,50,29.3989,34014.9,29.2894,29.9744,1.64589,1.76723,0.94596,0.32,0.00193058893402,0.16803198999999996,5,80,592.455,228.222858,103.40966,1000.404739,2.29152,22.7084,0.603386 +WarpFusion,gpu,677305,1000,384,cosine,64,bucket_major_partitioned4x16,1024,64,100000,15,2026,50,37.2287,26861.0,37.1704,37.4512,2.16864,2.20826,0.97214,0.4,0.0009436211343000001,0.11731612700000005,5,80,650.679,228.574469,99.001343,1001.600052,2.59432,30.3215,0.611136 +WarpFusion,gpu,677305,1000,384,cosine,64,bucket_major_partitioned4x16,1024,128,100000,15,2026,50,57.793,17303.1,57.7763,57.9775,3.4871,3.52182,0.98718,0.54,0.00042093429500000015,0.074507355,5,80,624.675,224.919206,97.750015,1003.990677,3.26433,50.1918,0.629805 +WarpFusion,gpu,677305,1000,384,cosine,64,bucket_major_partitioned4x16,1024,160,100000,15,2026,50,67.8663,14734.8,67.8445,67.9957,4.12883,4.15885,0.99054,0.66,0.00028492993978000014,0.074507355,5,80,628.667,226.838536,99.302399,1005.185989,3.5943,59.8969,0.687155 +WarpFusion,gpu,677305,1000,384,cosine,64,bucket_major_partitioned4x16,1024,192,100000,15,2026,50,78.2938,12772.4,78.2749,78.4451,4.79248,4.85242,0.99238,0.66,0.00022508242376000016,0.074507355,5,80,625.165,219.83818,97.390755,1006.381302,3.92787,70.003,0.695942 +WarpFusion,gpu,677305,1000,384,cosine,64,bucket_major_partitioned4x16,1024,224,100000,15,2026,50,89.3739,11189.0,89.1604,90.7402,5.47133,5.56445,0.9938400000000001,0.76,0.00018279215540000014,0.074507355,5,80,629.193,222.462564,99.461121,1007.576614,4.27971,80.129,0.717869 diff --git a/09_vector_retrieval/PurplePulse/results/real/yahoo-minilm-384/partitioned_topk_validation.json b/09_vector_retrieval/PurplePulse/results/real/yahoo-minilm-384/partitioned_topk_validation.json new file mode 100644 index 00000000..769e3860 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/results/real/yahoo-minilm-384/partitioned_topk_validation.json @@ -0,0 +1,110 @@ +{ + "50": { + "selected_mode": "full_heap", + "maximum_allowed_recall_loss": 0.0001, + "comparisons": [ + { + "nprobe": 32, + "partitioned_recall": 0.94596, + "full_heap_recall": 0.9469399999999999, + "partitioned_qps": 34014.9, + "full_heap_qps": 31088.4, + "recall_loss": 0.0009799999999998699 + }, + { + "nprobe": 64, + "partitioned_recall": 0.97214, + "full_heap_recall": 0.97312, + "partitioned_qps": 26861.0, + "full_heap_qps": 26075.4, + "recall_loss": 0.0009799999999999809 + }, + { + "nprobe": 128, + "partitioned_recall": 0.98718, + "full_heap_recall": 0.9881599999999999, + "partitioned_qps": 17303.1, + "full_heap_qps": 17663.5, + "recall_loss": 0.0009799999999999809 + }, + { + "nprobe": 160, + "partitioned_recall": 0.99054, + "full_heap_recall": 0.99152, + "partitioned_qps": 14734.8, + "full_heap_qps": 14949.7, + "recall_loss": 0.0009799999999999809 + }, + { + "nprobe": 192, + "partitioned_recall": 0.99238, + "full_heap_recall": 0.99336, + "partitioned_qps": 12772.4, + "full_heap_qps": 12968.9, + "recall_loss": 0.0009799999999999809 + }, + { + "nprobe": 224, + "partitioned_recall": 0.9938400000000001, + "full_heap_recall": 0.99482, + "partitioned_qps": 11189.0, + "full_heap_qps": 11431.3, + "recall_loss": 0.0009799999999999809 + } + ] + }, + "100": { + "selected_mode": "partitioned", + "maximum_allowed_recall_loss": 0.0001, + "comparisons": [ + { + "nprobe": 32, + "partitioned_recall": 0.937, + "full_heap_recall": 0.937, + "partitioned_qps": 36300.8, + "full_heap_qps": 14462.4, + "recall_loss": 0.0 + }, + { + "nprobe": 64, + "partitioned_recall": 0.9675, + "full_heap_recall": 0.9675, + "partitioned_qps": 27557.5, + "full_heap_qps": 13275.9, + "recall_loss": 0.0 + }, + { + "nprobe": 128, + "partitioned_recall": 0.9851599999999999, + "full_heap_recall": 0.9851599999999999, + "partitioned_qps": 16917.4, + "full_heap_qps": 9555.98, + "recall_loss": 0.0 + }, + { + "nprobe": 160, + "partitioned_recall": 0.98923, + "full_heap_recall": 0.98923, + "partitioned_qps": 14196.2, + "full_heap_qps": 8260.7, + "recall_loss": 0.0 + }, + { + "nprobe": 192, + "partitioned_recall": 0.99185, + "full_heap_recall": 0.99185, + "partitioned_qps": 12183.2, + "full_heap_qps": 7305.64, + "recall_loss": 0.0 + }, + { + "nprobe": 224, + "partitioned_recall": 0.9936799999999999, + "full_heap_recall": 0.9936799999999999, + "partitioned_qps": 10648.6, + "full_heap_qps": 6505.14, + "recall_loss": 0.0 + } + ] + } +} diff --git a/09_vector_retrieval/PurplePulse/results/stage7_exact_block_tile/smoke/synthetic/results.csv b/09_vector_retrieval/PurplePulse/results/stage7_exact_block_tile/smoke/synthetic/results.csv new file mode 100644 index 00000000..f8835128 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/results/stage7_exact_block_tile/smoke/synthetic/results.csv @@ -0,0 +1,8 @@ +variant,num_vectors,dim,num_queries,top_k,batch_size,exact_query_tile,exact_fused_layout,exact_fused_blocks,average_query_ms,qps,run_p50_ms,run_p99_ms,gpu_buffer_mib,fused_kernel_ms,final_merge_ms,result_path +baseline,1000000,128,1000,10,64,1,warp_query,0,144.318,6929.15,144.353,144.475,488.437,143.308,0.179403,results/stage7_exact_block_tile/smoke/synthetic/outputs/baseline/exact_k10.txt +warp_best,1000000,128,1000,10,64,4,warp_query,64,107.937,9264.65,107.945,107.978,488.789,106.992,0.193227,results/stage7_exact_block_tile/smoke/synthetic/outputs/warp_best/exact_k10.txt +block64,1000000,128,1000,10,64,8,block_tile,64,127.981,7813.64,127.983,128.028,488.789,126.986,0.19456,results/stage7_exact_block_tile/smoke/synthetic/outputs/block64/exact_k10.txt +block96,1000000,128,1000,10,64,8,block_tile,96,128.289,7794.89,128.276,128.338,489.023,127.271,0.209867,results/stage7_exact_block_tile/smoke/synthetic/outputs/block96/exact_k10.txt +block128,1000000,128,1000,10,64,8,block_tile,128,110.501,9049.68,110.525,110.587,489.257,109.478,0.222496,results/stage7_exact_block_tile/smoke/synthetic/outputs/block128/exact_k10.txt +block160,1000000,128,1000,10,64,8,block_tile,160,114.417,8739.97,114.366,114.544,489.492,113.375,0.247509,results/stage7_exact_block_tile/smoke/synthetic/outputs/block160/exact_k10.txt +block192,1000000,128,1000,10,64,8,block_tile,192,112.468,8891.39,112.471,112.568,489.726,111.306,0.261931,results/stage7_exact_block_tile/smoke/synthetic/outputs/block192/exact_k10.txt diff --git a/09_vector_retrieval/PurplePulse/results/stage7_exact_block_tile/smoke/yahoo/results.csv b/09_vector_retrieval/PurplePulse/results/stage7_exact_block_tile/smoke/yahoo/results.csv new file mode 100644 index 00000000..f3e4c174 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/results/stage7_exact_block_tile/smoke/yahoo/results.csv @@ -0,0 +1,8 @@ +variant,num_vectors,dim,num_queries,top_k,batch_size,exact_query_tile,exact_fused_layout,exact_fused_blocks,average_query_ms,qps,run_p50_ms,run_p99_ms,gpu_buffer_mib,fused_kernel_ms,final_merge_ms,result_path +baseline,677305,384,1000,10,64,1,warp_query,0,276.259,3619.8,273.933,282.329,992.364,275.211,0.187712,results/stage7_exact_block_tile/smoke/yahoo/outputs/baseline/exact_k10.txt +warp_best,677305,384,1000,10,64,8,warp_query,160,143.091,6988.56,143.01,143.254,993.419,142.032,0.248512,results/stage7_exact_block_tile/smoke/yahoo/outputs/warp_best/exact_k10.txt +block64,677305,384,1000,10,64,8,block_tile,64,267.849,3733.45,268.055,268.136,992.716,266.743,0.193952,results/stage7_exact_block_tile/smoke/yahoo/outputs/block64/exact_k10.txt +block96,677305,384,1000,10,64,8,block_tile,96,274.073,3648.66,274.36,274.45,992.95,272.968,0.209461,results/stage7_exact_block_tile/smoke/yahoo/outputs/block96/exact_k10.txt +block128,677305,384,1000,10,64,8,block_tile,128,208.578,4794.38,208.393,209.233,993.185,207.503,0.221515,results/stage7_exact_block_tile/smoke/yahoo/outputs/block128/exact_k10.txt +block160,677305,384,1000,10,64,8,block_tile,160,225.601,4432.61,225.549,225.791,993.419,224.466,0.248672,results/stage7_exact_block_tile/smoke/yahoo/outputs/block160/exact_k10.txt +block192,677305,384,1000,10,64,8,block_tile,192,233.273,4286.82,233.268,233.288,993.653,232.133,0.261152,results/stage7_exact_block_tile/smoke/yahoo/outputs/block192/exact_k10.txt diff --git a/09_vector_retrieval/PurplePulse/results/stage7_exact_profile/cosine_norm_ab/synthetic1m_baseline/breakdown.csv b/09_vector_retrieval/PurplePulse/results/stage7_exact_profile/cosine_norm_ab/synthetic1m_baseline/breakdown.csv new file mode 100644 index 00000000..f7cfd3c9 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/results/stage7_exact_profile/cosine_norm_ab/synthetic1m_baseline/breakdown.csv @@ -0,0 +1,4 @@ +num_vectors,dim,num_queries,top_k,batch_size,average_query_ms,qps,run_p50_ms,run_p99_ms,gpu_buffer_mib,distance_only_ms,local_topk_estimated_ms,fused_kernel_ms,final_merge_ms,kernel_pipeline_ms,distance_share,local_topk_share,final_merge_share,result_path +1000000,128,1000,10,64,148.343,6741.14,148.321,150.597,488.437,145.312,2.13207,147.444,0.18549,147.62948999999998,0.9843019846508989,0.01444203322791402,0.001256456281194225,/data/purplepulse-work/PurplePulse/results/stage7_exact_profile/cosine_norm_ab/synthetic1m_baseline/outputs/exact_profile_k10.txt +1000000,128,1000,50,64,164.833,6066.75,165.297,166.813,488.935,146.071,16.8916,162.962,0.828016,163.79001599999998,0.8918187052378089,0.10312960711842169,0.005055350870714855,/data/purplepulse-work/PurplePulse/results/stage7_exact_profile/cosine_norm_ab/synthetic1m_baseline/outputs/exact_profile_k50.txt +1000000,128,1000,100,64,182.551,5477.94,182.487,185.795,489.558,146.649,32.9103,179.559,1.86196,181.42096,0.808335486704513,0.1814029646850066,0.010263202223160984,/data/purplepulse-work/PurplePulse/results/stage7_exact_profile/cosine_norm_ab/synthetic1m_baseline/outputs/exact_profile_k100.txt diff --git a/09_vector_retrieval/PurplePulse/results/stage7_exact_profile/cosine_norm_ab/synthetic1m_optimized/breakdown.csv b/09_vector_retrieval/PurplePulse/results/stage7_exact_profile/cosine_norm_ab/synthetic1m_optimized/breakdown.csv new file mode 100644 index 00000000..94fb45be --- /dev/null +++ b/09_vector_retrieval/PurplePulse/results/stage7_exact_profile/cosine_norm_ab/synthetic1m_optimized/breakdown.csv @@ -0,0 +1,4 @@ +num_vectors,dim,num_queries,top_k,batch_size,average_query_ms,qps,run_p50_ms,run_p99_ms,gpu_buffer_mib,distance_only_ms,local_topk_estimated_ms,fused_kernel_ms,final_merge_ms,kernel_pipeline_ms,distance_share,local_topk_share,final_merge_share,result_path +1000000,128,1000,10,64,150.884,6627.6,150.712,153.343,488.437,147.996,1.97462,149.97,0.187618,150.15761799999999,0.9856043401008134,0.013150315157503365,0.0012494737363241872,/data/purplepulse-work/PurplePulse/results/stage7_exact_profile/cosine_norm_ab/synthetic1m_optimized/outputs/exact_profile_k10.txt +1000000,128,1000,50,64,166.862,5992.98,167.2,169.552,488.935,147.771,17.2596,165.03,0.838797,165.868797,0.890890888899375,0.10405573749956117,0.005056990917948238,/data/purplepulse-work/PurplePulse/results/stage7_exact_profile/cosine_norm_ab/synthetic1m_optimized/outputs/exact_profile_k50.txt +1000000,128,1000,100,64,184.321,5425.31,184.607,187.837,489.558,147.835,33.4004,181.236,1.87907,183.11507,0.8073338802753919,0.18240115354787564,0.01026168954854453,/data/purplepulse-work/PurplePulse/results/stage7_exact_profile/cosine_norm_ab/synthetic1m_optimized/outputs/exact_profile_k100.txt diff --git a/09_vector_retrieval/PurplePulse/results/stage7_exact_profile/cosine_norm_ab/yahoo_baseline/breakdown.csv b/09_vector_retrieval/PurplePulse/results/stage7_exact_profile/cosine_norm_ab/yahoo_baseline/breakdown.csv new file mode 100644 index 00000000..6c250dcc --- /dev/null +++ b/09_vector_retrieval/PurplePulse/results/stage7_exact_profile/cosine_norm_ab/yahoo_baseline/breakdown.csv @@ -0,0 +1,4 @@ +num_vectors,dim,num_queries,top_k,batch_size,average_query_ms,qps,run_p50_ms,run_p99_ms,gpu_buffer_mib,distance_only_ms,local_topk_estimated_ms,fused_kernel_ms,final_merge_ms,kernel_pipeline_ms,distance_share,local_topk_share,final_merge_share,result_path +677305,384,1000,10,64,285.991,3496.61,286.103,289.843,992.364,282.23,2.68989,284.919,0.195202,285.114202,0.9898840465337466,0.009434430067429613,0.0006846449550064854,/data/purplepulse-work/PurplePulse/results/stage7_exact_profile/cosine_norm_ab/yahoo_baseline/outputs/exact_profile_k10.txt +677305,384,1000,50,64,299.294,3341.2,299.837,305.487,992.862,284.014,12.8595,296.874,0.890322,297.76432200000005,0.9538214588381746,0.04318683955695672,0.0029900224245133027,/data/purplepulse-work/PurplePulse/results/stage7_exact_profile/cosine_norm_ab/yahoo_baseline/outputs/exact_profile_k50.txt +677305,384,1000,100,64,316.414,3160.41,316.024,321.712,993.485,282.913,30.1593,313.072,2.01341,315.08541,0.8978930506493461,0.09571785631076983,0.006390045162675098,/data/purplepulse-work/PurplePulse/results/stage7_exact_profile/cosine_norm_ab/yahoo_baseline/outputs/exact_profile_k100.txt diff --git a/09_vector_retrieval/PurplePulse/results/stage7_exact_profile/cosine_norm_ab/yahoo_optimized/breakdown.csv b/09_vector_retrieval/PurplePulse/results/stage7_exact_profile/cosine_norm_ab/yahoo_optimized/breakdown.csv new file mode 100644 index 00000000..ffd5621d --- /dev/null +++ b/09_vector_retrieval/PurplePulse/results/stage7_exact_profile/cosine_norm_ab/yahoo_optimized/breakdown.csv @@ -0,0 +1,4 @@ +num_vectors,dim,num_queries,top_k,batch_size,average_query_ms,qps,run_p50_ms,run_p99_ms,gpu_buffer_mib,distance_only_ms,local_topk_estimated_ms,fused_kernel_ms,final_merge_ms,kernel_pipeline_ms,distance_share,local_topk_share,final_merge_share,result_path +677305,384,1000,10,64,293.485,3407.33,292.798,299.461,994.948,274.714,17.7212,292.435,0.185157,292.620157,0.9388075066886113,0.060560421338301716,0.0006327554530018245,/data/purplepulse-work/PurplePulse/results/stage7_exact_profile/cosine_norm_ab/yahoo_optimized/outputs/exact_profile_k10.txt +677305,384,1000,50,64,300.28,3330.23,300.525,305.539,995.446,273.776,24.6115,298.388,0.846878,299.234878,0.9149200849507925,0.08224809943445162,0.002830144686542857,/data/purplepulse-work/PurplePulse/results/stage7_exact_profile/cosine_norm_ab/yahoo_optimized/outputs/exact_profile_k50.txt +677305,384,1000,100,64,308.686,3239.53,308.944,313.71,996.069,274.059,31.3277,305.387,1.93324,307.32024,0.8917700962357703,0.10193829082002538,0.006290636763787507,/data/purplepulse-work/PurplePulse/results/stage7_exact_profile/cosine_norm_ab/yahoo_optimized/outputs/exact_profile_k100.txt diff --git a/09_vector_retrieval/PurplePulse/results/stage7_exact_profile/synthetic1m/breakdown.csv b/09_vector_retrieval/PurplePulse/results/stage7_exact_profile/synthetic1m/breakdown.csv new file mode 100644 index 00000000..88e49ac8 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/results/stage7_exact_profile/synthetic1m/breakdown.csv @@ -0,0 +1,4 @@ +num_vectors,dim,num_queries,top_k,batch_size,average_query_ms,qps,run_p50_ms,run_p99_ms,gpu_buffer_mib,distance_only_ms,local_topk_estimated_ms,fused_kernel_ms,final_merge_ms,kernel_pipeline_ms,distance_share,local_topk_share,final_merge_share,result_path +1000000,128,1000,10,64,148.312,6742.53,149.409,150.703,488.437,144.937,2.37689,147.314,0.185445,147.49944499999998,0.9826274261574343,0.016114569109056648,0.0012572589679913713,results/stage7_exact_profile/synthetic1m/outputs/exact_profile_k10.txt +1000000,128,1000,50,64,164.39,6083.1,164.069,166.782,488.935,145.529,17.1322,162.661,0.826717,163.487717,0.8901524999581467,0.10479197039616132,0.005056752979185586,results/stage7_exact_profile/synthetic1m/outputs/exact_profile_k50.txt +1000000,128,1000,100,64,182.36,5483.66,182.359,185.6,489.558,146.286,33.0835,179.37,1.8608,181.23080000000002,0.8071806778980173,0.18254899277606235,0.01026757041297616,results/stage7_exact_profile/synthetic1m/outputs/exact_profile_k100.txt diff --git a/09_vector_retrieval/PurplePulse/results/stage7_exact_profile/yahoo-minilm-384/breakdown.csv b/09_vector_retrieval/PurplePulse/results/stage7_exact_profile/yahoo-minilm-384/breakdown.csv new file mode 100644 index 00000000..f46787a3 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/results/stage7_exact_profile/yahoo-minilm-384/breakdown.csv @@ -0,0 +1,4 @@ +num_vectors,dim,num_queries,top_k,batch_size,average_query_ms,qps,run_p50_ms,run_p99_ms,gpu_buffer_mib,distance_only_ms,local_topk_estimated_ms,fused_kernel_ms,final_merge_ms,kernel_pipeline_ms,distance_share,local_topk_share,final_merge_share,result_path +677305,384,1000,10,64,283.703,3524.81,283.749,287.772,992.364,279.28,3.3966,282.676,0.193875,282.869875,0.9873090939782824,0.012007641322710663,0.0006853858156511011,results/stage7_exact_profile/yahoo-minilm-384/outputs/exact_profile_k10.txt +677305,384,1000,50,64,297.128,3365.55,298.132,301.107,992.862,280.778,13.6133,294.391,0.883347,295.27434700000003,0.9509054980654991,0.04610390349961556,0.0029916144391642662,results/stage7_exact_profile/yahoo-minilm-384/outputs/exact_profile_k50.txt +677305,384,1000,100,64,315.293,3171.65,314.946,320.001,993.485,281.066,30.7045,311.77,2.00405,313.77405,0.8957592254681354,0.09785544725575618,0.0063869207794589765,results/stage7_exact_profile/yahoo-minilm-384/outputs/exact_profile_k100.txt diff --git a/09_vector_retrieval/PurplePulse/results/stage7_exact_query_tile/formal/sift_k10/results.csv b/09_vector_retrieval/PurplePulse/results/stage7_exact_query_tile/formal/sift_k10/results.csv new file mode 100644 index 00000000..c08230db --- /dev/null +++ b/09_vector_retrieval/PurplePulse/results/stage7_exact_query_tile/formal/sift_k10/results.csv @@ -0,0 +1,4 @@ +variant,num_vectors,dim,num_queries,top_k,batch_size,exact_query_tile,average_query_ms,qps,run_p50_ms,run_p99_ms,gpu_buffer_mib,fused_kernel_ms,final_merge_ms,result_path +baseline,1000000,128,10000,10,64,1,1440.91,6940.08,1442.63,1445.25,488.437,1431.54,1.80152,results/stage7_exact_query_tile/formal/sift_k10/outputs/baseline/exact_k10.txt +tile4,1000000,128,10000,10,64,4,1083.7,9227.65,1083.44,1085.15,488.789,1074.32,1.94307,results/stage7_exact_query_tile/formal/sift_k10/outputs/tile4/exact_k10.txt +tile8,1000000,128,10000,10,64,8,1249.66,8002.19,1249.55,1251.36,489.257,1240.05,2.22865,results/stage7_exact_query_tile/formal/sift_k10/outputs/tile8/exact_k10.txt diff --git a/09_vector_retrieval/PurplePulse/results/stage7_exact_query_tile/formal/sift_k50/results.csv b/09_vector_retrieval/PurplePulse/results/stage7_exact_query_tile/formal/sift_k50/results.csv new file mode 100644 index 00000000..e4c449f4 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/results/stage7_exact_query_tile/formal/sift_k50/results.csv @@ -0,0 +1,4 @@ +variant,num_vectors,dim,num_queries,top_k,batch_size,exact_query_tile,average_query_ms,qps,run_p50_ms,run_p99_ms,gpu_buffer_mib,fused_kernel_ms,final_merge_ms,result_path +baseline,1000000,128,10000,50,64,1,1614.64,6193.34,1613.93,1618.6,488.935,1596.65,8.04563,results/stage7_exact_query_tile/formal/sift_k50/outputs/baseline/exact_k50.txt +tile2,1000000,128,10000,50,64,2,1678.86,5956.41,1678.08,1682.58,489.521,1659.58,9.09063,results/stage7_exact_query_tile/formal/sift_k50/outputs/tile2/exact_k50.txt +tile4,1000000,128,10000,50,64,4,1815.17,5509.14,1815.15,1817.72,490.693,1792.9,12.0159,results/stage7_exact_query_tile/formal/sift_k50/outputs/tile4/exact_k50.txt diff --git a/09_vector_retrieval/PurplePulse/results/stage7_exact_query_tile/formal/synthetic_k10/results.csv b/09_vector_retrieval/PurplePulse/results/stage7_exact_query_tile/formal/synthetic_k10/results.csv new file mode 100644 index 00000000..07b7d5d9 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/results/stage7_exact_query_tile/formal/synthetic_k10/results.csv @@ -0,0 +1,4 @@ +variant,num_vectors,dim,num_queries,top_k,batch_size,exact_query_tile,average_query_ms,qps,run_p50_ms,run_p99_ms,gpu_buffer_mib,fused_kernel_ms,final_merge_ms,result_path +baseline,1000000,128,1000,10,64,1,148.4,6738.55,148.281,150.72,488.437,147.509,0.185906,results/stage7_exact_query_tile/formal/synthetic_k10/outputs/baseline/exact_k10.txt +tile4,1000000,128,1000,10,64,4,108.113,9249.56,108.062,108.902,488.789,107.201,0.193621,results/stage7_exact_query_tile/formal/synthetic_k10/outputs/tile4/exact_k10.txt +tile8,1000000,128,1000,10,64,8,124.16,8054.13,124.14,124.527,489.257,123.258,0.222749,results/stage7_exact_query_tile/formal/synthetic_k10/outputs/tile8/exact_k10.txt diff --git a/09_vector_retrieval/PurplePulse/results/stage7_exact_query_tile/formal/synthetic_k50/results.csv b/09_vector_retrieval/PurplePulse/results/stage7_exact_query_tile/formal/synthetic_k50/results.csv new file mode 100644 index 00000000..02d07a40 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/results/stage7_exact_query_tile/formal/synthetic_k50/results.csv @@ -0,0 +1,4 @@ +variant,num_vectors,dim,num_queries,top_k,batch_size,exact_query_tile,average_query_ms,qps,run_p50_ms,run_p99_ms,gpu_buffer_mib,fused_kernel_ms,final_merge_ms,result_path +baseline,1000000,128,1000,50,64,1,165.038,6059.22,165.614,167.005,488.935,163.268,0.829941,results/stage7_exact_query_tile/formal/synthetic_k50/outputs/baseline/exact_k50.txt +tile2,1000000,128,1000,50,64,2,167.406,5973.5,167.385,167.861,489.521,165.643,0.91159,results/stage7_exact_query_tile/formal/synthetic_k50/outputs/tile2/exact_k50.txt +tile4,1000000,128,1000,50,64,4,179.957,5556.89,179.917,180.472,490.693,177.9,1.21166,results/stage7_exact_query_tile/formal/synthetic_k50/outputs/tile4/exact_k50.txt diff --git a/09_vector_retrieval/PurplePulse/results/stage7_exact_query_tile/formal/yahoo_k10/results.csv b/09_vector_retrieval/PurplePulse/results/stage7_exact_query_tile/formal/yahoo_k10/results.csv new file mode 100644 index 00000000..608235d0 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/results/stage7_exact_query_tile/formal/yahoo_k10/results.csv @@ -0,0 +1,4 @@ +variant,num_vectors,dim,num_queries,top_k,batch_size,exact_query_tile,average_query_ms,qps,run_p50_ms,run_p99_ms,gpu_buffer_mib,fused_kernel_ms,final_merge_ms,result_path +baseline,677305,384,1000,10,64,1,282.932,3534.41,283.266,287.692,992.364,281.68,0.193827,results/stage7_exact_query_tile/formal/yahoo_k10/outputs/baseline/exact_k10.txt +tile4,677305,384,1000,10,64,4,172.99,5780.69,172.903,173.936,992.716,171.953,0.195746,results/stage7_exact_query_tile/formal/yahoo_k10/outputs/tile4/exact_k10.txt +tile8,677305,384,1000,10,64,8,165.936,6026.44,165.689,167.77,993.185,164.89,0.227648,results/stage7_exact_query_tile/formal/yahoo_k10/outputs/tile8/exact_k10.txt diff --git a/09_vector_retrieval/PurplePulse/results/stage7_exact_query_tile/formal/yahoo_k10_grid/results.csv b/09_vector_retrieval/PurplePulse/results/stage7_exact_query_tile/formal/yahoo_k10_grid/results.csv new file mode 100644 index 00000000..0ac279ff --- /dev/null +++ b/09_vector_retrieval/PurplePulse/results/stage7_exact_query_tile/formal/yahoo_k10_grid/results.csv @@ -0,0 +1,5 @@ +variant,num_vectors,dim,num_queries,top_k,batch_size,exact_query_tile,average_query_ms,qps,run_p50_ms,run_p99_ms,gpu_buffer_mib,fused_kernel_ms,final_merge_ms,result_path +baseline,677305,384,1000,10,64,1,279.583,3576.76,279.976,282.94,992.364,278.604,0.189806,results/stage7_exact_query_tile/formal/yahoo_k10_grid/outputs/baseline/exact_k10.txt +tile8b96,677305,384,1000,10,64,8,155.005,6451.4,154.98,155.297,992.95,153.982,0.209971,results/stage7_exact_query_tile/formal/yahoo_k10_grid/outputs/tile8b96/exact_k10.txt +tile8b160,677305,384,1000,10,64,8,148.22,6746.74,148.701,150.207,993.419,147.163,0.258874,results/stage7_exact_query_tile/formal/yahoo_k10_grid/outputs/tile8b160/exact_k10.txt +tile8b192,677305,384,1000,10,64,8,161.791,6180.83,161.839,165.113,993.653,160.523,0.273869,results/stage7_exact_query_tile/formal/yahoo_k10_grid/outputs/tile8b192/exact_k10.txt diff --git a/09_vector_retrieval/PurplePulse/results/stage7_exact_query_tile/formal/yahoo_k50/results.csv b/09_vector_retrieval/PurplePulse/results/stage7_exact_query_tile/formal/yahoo_k50/results.csv new file mode 100644 index 00000000..28f87b5f --- /dev/null +++ b/09_vector_retrieval/PurplePulse/results/stage7_exact_query_tile/formal/yahoo_k50/results.csv @@ -0,0 +1,4 @@ +variant,num_vectors,dim,num_queries,top_k,batch_size,exact_query_tile,average_query_ms,qps,run_p50_ms,run_p99_ms,gpu_buffer_mib,fused_kernel_ms,final_merge_ms,result_path +baseline,677305,384,1000,50,64,1,297.321,3363.36,298.175,300.758,992.862,295.338,0.884942,results/stage7_exact_query_tile/formal/yahoo_k50/outputs/baseline/exact_k50.txt +tile2,677305,384,1000,50,64,2,282.258,3542.85,282.201,282.68,993.448,280.25,0.91304,results/stage7_exact_query_tile/formal/yahoo_k50/outputs/tile2/exact_k50.txt +tile4,677305,384,1000,50,64,4,245.484,4073.59,245.404,246.268,994.62,243.265,1.21372,results/stage7_exact_query_tile/formal/yahoo_k50/outputs/tile4/exact_k50.txt diff --git a/09_vector_retrieval/PurplePulse/results/stage7_exact_query_tile/grid_sweep_smoke/sift/results.csv b/09_vector_retrieval/PurplePulse/results/stage7_exact_query_tile/grid_sweep_smoke/sift/results.csv new file mode 100644 index 00000000..971f32ac --- /dev/null +++ b/09_vector_retrieval/PurplePulse/results/stage7_exact_query_tile/grid_sweep_smoke/sift/results.csv @@ -0,0 +1,7 @@ +variant,num_vectors,dim,num_queries,top_k,batch_size,exact_query_tile,average_query_ms,qps,run_p50_ms,run_p99_ms,gpu_buffer_mib,fused_kernel_ms,final_merge_ms,result_path +baseline,1000000,128,10000,10,64,1,1419.46,7044.92,1420.71,1420.97,488.437,1409.63,1.77203,results/stage7_exact_query_tile/grid_sweep_smoke/sift/outputs/baseline/exact_k10.txt +b48,1000000,128,10000,10,64,4,1401.5,7135.21,1400.22,1404.32,488.671,1391.46,1.83397,results/stage7_exact_query_tile/grid_sweep_smoke/sift/outputs/b48/exact_k10.txt +b64,1000000,128,10000,10,64,4,1081.66,9245.07,1082.62,1082.77,488.789,1069.38,1.93764,results/stage7_exact_query_tile/grid_sweep_smoke/sift/outputs/b64/exact_k10.txt +b80,1000000,128,10000,10,64,4,1165.8,8577.83,1165.78,1166.72,488.906,1156.29,2.0629,results/stage7_exact_query_tile/grid_sweep_smoke/sift/outputs/b80/exact_k10.txt +b96,1000000,128,10000,10,64,4,1167.56,8564.9,1167.57,1168.89,489.023,1157.08,2.09634,results/stage7_exact_query_tile/grid_sweep_smoke/sift/outputs/b96/exact_k10.txt +b128,1000000,128,10000,10,64,4,1108.76,9019.09,1108.76,1109.19,489.257,1098.89,2.2765,results/stage7_exact_query_tile/grid_sweep_smoke/sift/outputs/b128/exact_k10.txt diff --git a/09_vector_retrieval/PurplePulse/results/stage7_exact_query_tile/grid_sweep_smoke/synthetic/results.csv b/09_vector_retrieval/PurplePulse/results/stage7_exact_query_tile/grid_sweep_smoke/synthetic/results.csv new file mode 100644 index 00000000..5f13e51a --- /dev/null +++ b/09_vector_retrieval/PurplePulse/results/stage7_exact_query_tile/grid_sweep_smoke/synthetic/results.csv @@ -0,0 +1,7 @@ +variant,num_vectors,dim,num_queries,top_k,batch_size,exact_query_tile,average_query_ms,qps,run_p50_ms,run_p99_ms,gpu_buffer_mib,fused_kernel_ms,final_merge_ms,result_path +baseline,1000000,128,1000,10,64,1,144.327,6928.73,144.353,144.507,488.437,143.409,0.180853,results/stage7_exact_query_tile/grid_sweep_smoke/synthetic/outputs/baseline/exact_k10.txt +b48,1000000,128,1000,10,64,4,137.91,7251.11,137.898,137.98,488.671,136.996,0.182005,results/stage7_exact_query_tile/grid_sweep_smoke/synthetic/outputs/b48/exact_k10.txt +b64,1000000,128,1000,10,64,4,108.217,9240.66,108.29,108.299,488.789,107.135,0.193525,results/stage7_exact_query_tile/grid_sweep_smoke/synthetic/outputs/b64/exact_k10.txt +b80,1000000,128,1000,10,64,4,117.234,8529.92,117.126,117.468,488.906,116.217,0.209515,results/stage7_exact_query_tile/grid_sweep_smoke/synthetic/outputs/b80/exact_k10.txt +b96,1000000,128,1000,10,64,4,115.928,8626.03,115.844,116.125,489.023,114.927,0.209259,results/stage7_exact_query_tile/grid_sweep_smoke/synthetic/outputs/b96/exact_k10.txt +b128,1000000,128,1000,10,64,4,109.236,9154.46,109.204,109.376,489.257,108.28,0.227925,results/stage7_exact_query_tile/grid_sweep_smoke/synthetic/outputs/b128/exact_k10.txt diff --git a/09_vector_retrieval/PurplePulse/results/stage7_exact_query_tile/grid_sweep_smoke/yahoo/results.csv b/09_vector_retrieval/PurplePulse/results/stage7_exact_query_tile/grid_sweep_smoke/yahoo/results.csv new file mode 100644 index 00000000..a9c90c11 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/results/stage7_exact_query_tile/grid_sweep_smoke/yahoo/results.csv @@ -0,0 +1,7 @@ +variant,num_vectors,dim,num_queries,top_k,batch_size,exact_query_tile,average_query_ms,qps,run_p50_ms,run_p99_ms,gpu_buffer_mib,fused_kernel_ms,final_merge_ms,result_path +baseline,677305,384,1000,10,64,1,281.815,3548.43,279.12,289.263,992.364,280.821,0.192853,results/stage7_exact_query_tile/grid_sweep_smoke/yahoo/outputs/baseline/exact_k10.txt +tile4b64,677305,384,1000,10,64,4,172.696,5790.53,172.792,172.878,992.716,171.699,0.195808,results/stage7_exact_query_tile/grid_sweep_smoke/yahoo/outputs/tile4b64/exact_k10.txt +tile8b96,677305,384,1000,10,64,8,154.405,6476.45,154.43,154.607,992.95,153.403,0.205579,results/stage7_exact_query_tile/grid_sweep_smoke/yahoo/outputs/tile8b96/exact_k10.txt +tile8b128,677305,384,1000,10,64,8,164.453,6080.77,164.162,165.051,993.185,163.424,0.224533,results/stage7_exact_query_tile/grid_sweep_smoke/yahoo/outputs/tile8b128/exact_k10.txt +tile8b160,677305,384,1000,10,64,8,145.189,6887.55,145.205,145.243,993.419,144.154,0.254208,results/stage7_exact_query_tile/grid_sweep_smoke/yahoo/outputs/tile8b160/exact_k10.txt +tile8b192,677305,384,1000,10,64,8,158.742,6299.54,158.723,158.9,993.653,157.687,0.268971,results/stage7_exact_query_tile/grid_sweep_smoke/yahoo/outputs/tile8b192/exact_k10.txt diff --git a/09_vector_retrieval/PurplePulse/results/stage7_exact_query_tile/scaled_grid_smoke/synthetic1m/results.csv b/09_vector_retrieval/PurplePulse/results/stage7_exact_query_tile/scaled_grid_smoke/synthetic1m/results.csv new file mode 100644 index 00000000..eb66cd5e --- /dev/null +++ b/09_vector_retrieval/PurplePulse/results/stage7_exact_query_tile/scaled_grid_smoke/synthetic1m/results.csv @@ -0,0 +1,10 @@ +variant,num_vectors,dim,num_queries,top_k,batch_size,exact_query_tile,average_query_ms,qps,run_p50_ms,run_p99_ms,gpu_buffer_mib,fused_kernel_ms,final_merge_ms,result_path +baseline,1000000,128,1000,10,64,1,144.322,6928.96,144.326,144.484,488.437,143.405,0.181024,results/stage7_exact_query_tile/scaled_grid_smoke/synthetic1m/outputs/baseline/exact_k10.txt +baseline,1000000,128,1000,50,64,1,159.416,6272.88,159.487,159.83,488.935,157.621,0.799499,results/stage7_exact_query_tile/scaled_grid_smoke/synthetic1m/outputs/baseline/exact_k50.txt +baseline,1000000,128,1000,100,64,1,176.94,5651.64,177.059,177.241,489.558,173.815,1.80062,results/stage7_exact_query_tile/scaled_grid_smoke/synthetic1m/outputs/baseline/exact_k100.txt +tile2,1000000,128,1000,10,64,2,155.53,6429.62,155.525,155.568,488.554,154.618,0.179424,results/stage7_exact_query_tile/scaled_grid_smoke/synthetic1m/outputs/tile2/exact_k10.txt +tile2,1000000,128,1000,50,64,2,167.2,5980.88,167.148,167.354,489.521,165.295,0.909653,results/stage7_exact_query_tile/scaled_grid_smoke/synthetic1m/outputs/tile2/exact_k50.txt +tile2,1000000,128,1000,100,64,2,258.83,3863.53,258.864,258.93,490.729,255.229,2.29613,results/stage7_exact_query_tile/scaled_grid_smoke/synthetic1m/outputs/tile2/exact_k100.txt +tile4,1000000,128,1000,10,64,4,108.063,9253.86,108.035,108.12,488.789,107.147,0.193248,results/stage7_exact_query_tile/scaled_grid_smoke/synthetic1m/outputs/tile4/exact_k10.txt +tile4,1000000,128,1000,50,64,4,179.836,5560.61,179.889,179.969,490.693,177.511,1.20859,results/stage7_exact_query_tile/scaled_grid_smoke/synthetic1m/outputs/tile4/exact_k50.txt +tile4,1000000,128,1000,100,64,4,311.977,3205.37,311.897,312.184,493.073,303.926,6.71577,results/stage7_exact_query_tile/scaled_grid_smoke/synthetic1m/outputs/tile4/exact_k100.txt diff --git a/09_vector_retrieval/PurplePulse/results/stage7_exact_query_tile/scaled_grid_smoke/yahoo/results.csv b/09_vector_retrieval/PurplePulse/results/stage7_exact_query_tile/scaled_grid_smoke/yahoo/results.csv new file mode 100644 index 00000000..48f8a757 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/results/stage7_exact_query_tile/scaled_grid_smoke/yahoo/results.csv @@ -0,0 +1,10 @@ +variant,num_vectors,dim,num_queries,top_k,batch_size,exact_query_tile,average_query_ms,qps,run_p50_ms,run_p99_ms,gpu_buffer_mib,fused_kernel_ms,final_merge_ms,result_path +baseline,677305,384,1000,10,64,1,276.786,3612.9,273.801,283.9,992.364,275.793,0.186955,results/stage7_exact_query_tile/scaled_grid_smoke/yahoo/outputs/baseline/exact_k10.txt +baseline,677305,384,1000,50,64,1,287.502,3478.24,282.853,296.864,992.862,285.579,0.851019,results/stage7_exact_query_tile/scaled_grid_smoke/yahoo/outputs/baseline/exact_k50.txt +baseline,677305,384,1000,100,64,1,306.175,3266.11,303.427,314.989,993.485,302.749,1.93477,results/stage7_exact_query_tile/scaled_grid_smoke/yahoo/outputs/baseline/exact_k100.txt +tile2,677305,384,1000,10,64,2,273.55,3655.63,273.601,273.631,992.481,272.532,0.179893,results/stage7_exact_query_tile/scaled_grid_smoke/yahoo/outputs/tile2/exact_k10.txt +tile2,677305,384,1000,50,64,2,281.422,3553.38,281.261,281.774,993.448,279.484,0.907637,results/stage7_exact_query_tile/scaled_grid_smoke/yahoo/outputs/tile2/exact_k50.txt +tile2,677305,384,1000,100,64,2,430.109,2324.99,430.126,430.181,994.657,425.991,2.29615,results/stage7_exact_query_tile/scaled_grid_smoke/yahoo/outputs/tile2/exact_k100.txt +tile4,677305,384,1000,10,64,4,172.223,5806.42,172.072,172.564,992.716,171.208,0.195893,results/stage7_exact_query_tile/scaled_grid_smoke/yahoo/outputs/tile4/exact_k10.txt +tile4,677305,384,1000,50,64,4,244.817,4084.68,244.819,244.932,994.62,242.611,1.20675,results/stage7_exact_query_tile/scaled_grid_smoke/yahoo/outputs/tile4/exact_k50.txt +tile4,677305,384,1000,100,64,4,438.85,2278.68,438.846,439.097,997.0,430.492,6.69938,results/stage7_exact_query_tile/scaled_grid_smoke/yahoo/outputs/tile4/exact_k100.txt diff --git a/09_vector_retrieval/PurplePulse/results/stage7_exact_query_tile/smoke/synthetic1m/results.csv b/09_vector_retrieval/PurplePulse/results/stage7_exact_query_tile/smoke/synthetic1m/results.csv new file mode 100644 index 00000000..4fbace69 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/results/stage7_exact_query_tile/smoke/synthetic1m/results.csv @@ -0,0 +1,10 @@ +variant,num_vectors,dim,num_queries,top_k,batch_size,exact_query_tile,average_query_ms,qps,run_p50_ms,run_p99_ms,gpu_buffer_mib,fused_kernel_ms,final_merge_ms,result_path +baseline,1000000,128,1000,10,64,1,144.492,6920.78,144.478,144.568,488.437,143.571,0.179381,results/stage7_exact_query_tile/smoke/synthetic1m/outputs/baseline/exact_k10.txt +baseline,1000000,128,1000,50,64,1,159.407,6273.26,159.396,159.543,488.935,157.604,0.797781,results/stage7_exact_query_tile/smoke/synthetic1m/outputs/baseline/exact_k50.txt +baseline,1000000,128,1000,100,64,1,176.951,5651.27,176.837,177.386,489.558,173.766,1.80062,results/stage7_exact_query_tile/smoke/synthetic1m/outputs/baseline/exact_k100.txt +tile2,1000000,128,1000,10,64,2,157.074,6366.42,157.068,157.177,488.437,156.117,0.166123,results/stage7_exact_query_tile/smoke/synthetic1m/outputs/tile2/exact_k10.txt +tile2,1000000,128,1000,50,64,2,162.361,6159.13,162.333,162.425,488.935,160.603,0.750944,results/stage7_exact_query_tile/smoke/synthetic1m/outputs/tile2/exact_k50.txt +tile2,1000000,128,1000,100,64,2,317.018,3154.4,316.998,317.379,489.558,313.943,1.72026,results/stage7_exact_query_tile/smoke/synthetic1m/outputs/tile2/exact_k100.txt +tile4,1000000,128,1000,10,64,4,183.473,5450.4,183.454,183.512,488.437,182.569,0.169333,results/stage7_exact_query_tile/smoke/synthetic1m/outputs/tile4/exact_k10.txt +tile4,1000000,128,1000,50,64,4,190.591,5246.83,190.522,190.759,488.935,188.841,0.751019,results/stage7_exact_query_tile/smoke/synthetic1m/outputs/tile4/exact_k50.txt +tile4,1000000,128,1000,100,64,4,388.451,2574.33,388.913,389.133,489.558,385.257,1.71516,results/stage7_exact_query_tile/smoke/synthetic1m/outputs/tile4/exact_k100.txt diff --git a/09_vector_retrieval/PurplePulse/results/stage7_exact_query_tile/smoke/yahoo/results.csv b/09_vector_retrieval/PurplePulse/results/stage7_exact_query_tile/smoke/yahoo/results.csv new file mode 100644 index 00000000..72658165 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/results/stage7_exact_query_tile/smoke/yahoo/results.csv @@ -0,0 +1,10 @@ +variant,num_vectors,dim,num_queries,top_k,batch_size,exact_query_tile,average_query_ms,qps,run_p50_ms,run_p99_ms,gpu_buffer_mib,fused_kernel_ms,final_merge_ms,result_path +baseline,677305,384,1000,10,64,1,276.616,3615.12,273.955,283.176,992.364,275.631,0.187733,results/stage7_exact_query_tile/smoke/yahoo/outputs/baseline/exact_k10.txt +baseline,677305,384,1000,50,64,1,287.947,3472.86,283.954,296.233,992.862,285.89,0.856,results/stage7_exact_query_tile/smoke/yahoo/outputs/baseline/exact_k50.txt +baseline,677305,384,1000,100,64,1,305.241,3276.09,301.596,314.124,993.485,301.766,1.92579,results/stage7_exact_query_tile/smoke/yahoo/outputs/baseline/exact_k100.txt +tile2,677305,384,1000,10,64,2,278.579,3589.64,278.699,278.702,992.364,277.589,0.167648,results/stage7_exact_query_tile/smoke/yahoo/outputs/tile2/exact_k10.txt +tile2,677305,384,1000,50,64,2,282.455,3540.39,282.456,282.583,992.862,280.639,0.755019,results/stage7_exact_query_tile/smoke/yahoo/outputs/tile2/exact_k50.txt +tile2,677305,384,1000,100,64,2,548.787,1822.2,548.469,550.376,993.485,544.814,1.72404,results/stage7_exact_query_tile/smoke/yahoo/outputs/tile2/exact_k100.txt +tile4,677305,384,1000,10,64,4,297.312,3363.47,297.352,297.41,992.364,296.289,0.169515,results/stage7_exact_query_tile/smoke/yahoo/outputs/tile4/exact_k10.txt +tile4,677305,384,1000,50,64,4,302.735,3303.22,302.704,303.015,992.862,300.718,0.755563,results/stage7_exact_query_tile/smoke/yahoo/outputs/tile4/exact_k50.txt +tile4,677305,384,1000,100,64,4,608.46,1643.49,608.515,608.763,993.485,604.992,1.72365,results/stage7_exact_query_tile/smoke/yahoo/outputs/tile4/exact_k100.txt diff --git a/09_vector_retrieval/PurplePulse/results/stage7_exact_query_tile/tile8_smoke/synthetic1m/results.csv b/09_vector_retrieval/PurplePulse/results/stage7_exact_query_tile/tile8_smoke/synthetic1m/results.csv new file mode 100644 index 00000000..764b6241 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/results/stage7_exact_query_tile/tile8_smoke/synthetic1m/results.csv @@ -0,0 +1,4 @@ +variant,num_vectors,dim,num_queries,top_k,batch_size,exact_query_tile,average_query_ms,qps,run_p50_ms,run_p99_ms,gpu_buffer_mib,fused_kernel_ms,final_merge_ms,result_path +baseline,1000000,128,1000,10,64,1,144.391,6925.65,144.49,144.516,488.437,143.458,0.180992,results/stage7_exact_query_tile/tile8_smoke/synthetic1m/outputs/baseline/exact_k10.txt +tile4,1000000,128,1000,10,64,4,108.02,9257.58,108.021,108.08,488.789,107.096,0.194528,results/stage7_exact_query_tile/tile8_smoke/synthetic1m/outputs/tile4/exact_k10.txt +tile8,1000000,128,1000,10,64,8,124.133,8055.87,124.209,124.216,489.257,123.157,0.222187,results/stage7_exact_query_tile/tile8_smoke/synthetic1m/outputs/tile8/exact_k10.txt diff --git a/09_vector_retrieval/PurplePulse/results/stage7_exact_query_tile/tile8_smoke/yahoo/results.csv b/09_vector_retrieval/PurplePulse/results/stage7_exact_query_tile/tile8_smoke/yahoo/results.csv new file mode 100644 index 00000000..8f3fc037 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/results/stage7_exact_query_tile/tile8_smoke/yahoo/results.csv @@ -0,0 +1,4 @@ +variant,num_vectors,dim,num_queries,top_k,batch_size,exact_query_tile,average_query_ms,qps,run_p50_ms,run_p99_ms,gpu_buffer_mib,fused_kernel_ms,final_merge_ms,result_path +baseline,677305,384,1000,10,64,1,274.924,3637.37,272.243,281.643,992.364,273.935,0.186016,results/stage7_exact_query_tile/tile8_smoke/yahoo/outputs/baseline/exact_k10.txt +tile4,677305,384,1000,10,64,4,171.876,5818.16,171.869,172.108,992.716,170.885,0.193248,results/stage7_exact_query_tile/tile8_smoke/yahoo/outputs/tile4/exact_k10.txt +tile8,677305,384,1000,10,64,8,163.674,6109.69,163.62,163.812,993.185,162.632,0.22528,results/stage7_exact_query_tile/tile8_smoke/yahoo/outputs/tile8/exact_k10.txt diff --git a/09_vector_retrieval/PurplePulse/results/stage8_stability/coverage.csv b/09_vector_retrieval/PurplePulse/results/stage8_stability/coverage.csv new file mode 100644 index 00000000..4c739814 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/results/stage8_stability/coverage.csv @@ -0,0 +1,10 @@ +suite,cases,status,details +exact_cross_product,576,passed,"dim=64/128/384/768; batch=1/8/32/40/64/128; K=1/10/50/100; FP32/FP16; L2/IP/cosine" +exact_full_batch_boundary,6,passed,"dim=65; query_count=batch+1; batch=1/8/32/40/64/128" +exact_non_warp_dimension,6,passed,"dim=65; K=100; FP32/FP16; L2/IP/cosine" +exact_recommended_paths,3,passed,"128/K10/tile4/64; 384/K10/tile8/160; 384/K50/tile4/64" +exact_safe_fallback,2,passed,"FP32 dim768 block-tile rejected then warp-query passed; K50/tile8 rejected then tile1 passed" +ivf_empty_skewed_buckets,48,passed,"dim=65; 4 empty buckets; largest bucket=125/128; K=1/10/50/100; FP32/FP16; L2/IP/cosine; warp/bucket-major" +ctest,14,passed,"all project tests" +python_unittest,29,passed,"2 skipped because local h5py is unavailable" +compute_sanitizer,1,passed,"memcheck ERROR SUMMARY: 0 errors" diff --git a/09_vector_retrieval/PurplePulse/scripts/benchmark_adaptive_nprobe.py b/09_vector_retrieval/PurplePulse/scripts/benchmark_adaptive_nprobe.py new file mode 100644 index 00000000..62d19056 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/scripts/benchmark_adaptive_nprobe.py @@ -0,0 +1,167 @@ +#!/usr/bin/env python3 +"""扫描 score-mass 自适应 nprobe 参数并输出质量、性能和实际 probe CSV。""" + +import argparse +import csv +import re +import subprocess +from pathlib import Path + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--search-binary", required=True) + parser.add_argument("--index", required=True) + parser.add_argument("--queries", required=True) + parser.add_argument("--params", required=True) + parser.add_argument("--exact", required=True) + parser.add_argument("--output-dir", required=True) + parser.add_argument("--csv", required=True) + parser.add_argument("--max-nprobe", type=int, required=True) + parser.add_argument("--min-nprobe", type=int, required=True) + parser.add_argument("--step", type=int, default=16) + parser.add_argument("--temperature", type=float, default=0.2) + parser.add_argument("--execution", choices=("masked", "grouped"), + default="masked") + parser.add_argument("--target-masses", required=True, + help="逗号分隔，例如 0.70,0.80,0.90") + parser.add_argument("--warmup", type=int, default=1) + parser.add_argument("--repeat", type=int, default=5) + return parser.parse_args() + + +def read_ids(path: Path) -> list[list[int]]: + rows: dict[int, list[int]] = {} + for line_number, line in enumerate(path.read_text().splitlines(), 1): + fields = line.split() + if len(fields) != 3: + raise ValueError(f"{path}:{line_number}: 结果行必须有三列") + query_id, vector_id = int(fields[0]), int(fields[1]) + rows.setdefault(query_id, []).append(vector_id) + return [rows[query_id] for query_id in sorted(rows)] + + +def recall(exact: list[list[int]], approximate: list[list[int]]) -> tuple[float, float]: + if len(exact) != len(approximate) or not exact: + raise ValueError("exact 与 approximate 的 query 数不一致或为空") + recalls = [] + for expected, actual in zip(exact, approximate): + if len(expected) != len(actual) or not expected: + raise ValueError("exact 与 approximate 的 K 不一致或为空") + recalls.append(len(set(expected) & set(actual)) / len(expected)) + return sum(recalls) / len(recalls), min(recalls) + + +def extract(text: str, pattern: str) -> float: + match = re.search(pattern, text) + if match is None: + raise ValueError(f"无法从 ivf_search 输出解析字段: {pattern}") + return float(match.group(1)) + + +def extract_pair(text: str, pattern: str) -> tuple[float, float]: + match = re.search(pattern, text) + if match is None: + raise ValueError(f"无法从 ivf_search 输出解析字段: {pattern}") + return float(match.group(1)), float(match.group(2)) + + +def extract_triple(text: str, pattern: str) -> tuple[float, float, float]: + match = re.search(pattern, text) + if match is None: + raise ValueError(f"无法从 ivf_search 输出解析字段: {pattern}") + return float(match.group(1)), float(match.group(2)), float(match.group(3)) + + +def main() -> None: + args = parse_args() + masses = [float(value) for value in args.target_masses.split(",")] + if (not masses or any(value <= 0.0 or value > 1.0 for value in masses) or + args.min_nprobe <= 0 or args.min_nprobe > args.max_nprobe or + args.step <= 0 or args.temperature <= 0.0): + raise ValueError("自适应 nprobe 参数不合法") + output_dir = Path(args.output_dir) + output_dir.mkdir(parents=True, exist_ok=True) + csv_path = Path(args.csv) + csv_path.parent.mkdir(parents=True, exist_ok=True) + exact_ids = read_ids(Path(args.exact)) + rows = [] + for target_mass in masses: + label = f"{target_mass:.4f}".rstrip("0").rstrip(".").replace(".", "p") + result_path = output_dir / f"mass_{label}.txt" + counts_path = output_dir / f"mass_{label}_nprobes.txt" + command = [ + args.search_binary, + "--index", args.index, + "--queries", args.queries, + "--params", args.params, + "--nprobe", str(args.max_nprobe), + "--nprobe-policy", "score_mass", + "--adaptive-execution", args.execution, + "--adaptive-nprobe-min", str(args.min_nprobe), + "--adaptive-nprobe-step", str(args.step), + "--adaptive-target-mass", str(target_mass), + "--adaptive-temperature", str(args.temperature), + "--backend", "gpu", + "--warmup", str(args.warmup), + "--repeat", str(args.repeat), + "--output", str(result_path), + "--probe-counts-output", str(counts_path), + ] + completed = subprocess.run(command, check=True, text=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT) + print(completed.stdout, end="") + approximate_ids = read_ids(result_path) + recall_at_k, minimum_recall = recall(exact_ids, approximate_ids) + actual_min, actual_average, actual_max = extract_triple( + completed.stdout, + r"实际 nprobe min/avg/max: ([0-9.eE+-]+)/([0-9.eE+-]+)/([0-9.eE+-]+)", + ) + batch_p50_ms, batch_p99_ms = extract_pair( + completed.stdout, r"batch P50/P99: ([0-9.eE+-]+)/([0-9.eE+-]+) ms" + ) + row = { + "policy": "score_mass", + "execution": args.execution, + "target_mass": target_mass, + "temperature": args.temperature, + "min_nprobe": args.min_nprobe, + "max_nprobe": args.max_nprobe, + "step": args.step, + "actual_nprobe_min": actual_min, + "actual_nprobe_average": actual_average, + "actual_nprobe_max": actual_max, + "average_ms": extract(completed.stdout, r"平均查询: ([0-9.eE+-]+) ms"), + "qps": extract(completed.stdout, r"QPS: ([0-9.eE+-]+)"), + "batch_p50_ms": batch_p50_ms, + "batch_p99_ms": batch_p99_ms, + "recall_at_k": recall_at_k, + "min_query_recall": minimum_recall, + "center_ms": extract( + completed.stdout, r"平均 GPU 中心选择: ([0-9.eE+-]+) ms" + ), + "policy_ms": extract( + completed.stdout, r"平均自适应分层开销: ([0-9.eE+-]+) ms" + ), + "scan_ms": extract( + completed.stdout, r"平均 GPU 桶扫描\+局部 Top-K: ([0-9.eE+-]+) ms" + ), + "merge_ms": extract( + completed.stdout, r"平均 GPU 最终 Top-K 归并: ([0-9.eE+-]+) ms" + ), + "gpu_buffer_mib": extract( + completed.stdout, r"GPU 缓冲区: ([0-9.eE+-]+) MiB" + ), + } + rows.append(row) + with csv_path.open("w", newline="") as csv_file: + writer = csv.DictWriter(csv_file, fieldnames=list(rows[0])) + writer.writeheader() + writer.writerows(rows) + print(f"recall={recall_at_k:.6f}, actual_avg_nprobe={actual_average:.3f}") + print(f"CSV: {csv_path}") + + +if __name__ == "__main__": + main() diff --git a/09_vector_retrieval/PurplePulse/scripts/benchmark_exact.py b/09_vector_retrieval/PurplePulse/scripts/benchmark_exact.py new file mode 100644 index 00000000..a077a9d4 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/scripts/benchmark_exact.py @@ -0,0 +1,128 @@ +#!/usr/bin/env python3 +"""Benchmark WarpFusion Exact for several K values and write one stable CSV.""" + +from __future__ import annotations + +import argparse +import csv +import re +import subprocess +from pathlib import Path + + +def extract(text: str, pattern: str) -> float: + match = re.search(pattern, text) + if match is None: + raise ValueError(f"cannot parse vector_search output: {pattern}") + return float(match.group(1)) + + +def parse_configs(text: str) -> list[tuple[int, Path]]: + result = [] + for item in text.split(","): + fields = item.split(":", 1) + if len(fields) != 2: + raise ValueError("configs must use K:path,K:path syntax") + top_k, path = int(fields[0]), Path(fields[1]) + if top_k <= 0 or not path.is_file(): + raise ValueError(f"invalid Exact config: {item}") + result.append((top_k, path)) + if not result: + raise ValueError("at least one Exact config is required") + return result + + +def write_csv(path: Path, rows: list[dict[str, object]]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("w", newline="") as output: + writer = csv.DictWriter(output, fieldnames=list(rows[0])) + writer.writeheader() + writer.writerows(rows) + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--search-binary", required=True) + parser.add_argument("--database", required=True) + parser.add_argument("--queries", required=True) + parser.add_argument("--configs", required=True, help="K:path,K:path") + parser.add_argument("--output-dir", type=Path, required=True) + parser.add_argument("--csv", type=Path, required=True) + parser.add_argument("--warmup", type=int, default=1) + parser.add_argument("--repeat", type=int, default=5) + args = parser.parse_args() + if args.warmup < 0 or args.repeat <= 0: + raise ValueError("warmup must be non-negative and repeat must be positive") + + args.output_dir.mkdir(parents=True, exist_ok=True) + rows: list[dict[str, object]] = [] + for expected_k, config in parse_configs(args.configs): + result_path = args.output_dir / f"exact_k{expected_k}.txt" + command = [ + args.search_binary, + "--database", args.database, + "--queries", args.queries, + "--params", str(config), + "--backend", "gpu", + "--warmup", str(args.warmup), + "--repeat", str(args.repeat), + "--output", str(result_path), + ] + completed = subprocess.run( + command, + check=True, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + ) + print(completed.stdout, end="") + top_k = int(extract(completed.stdout, r"top_k: (\d+)")) + if top_k != expected_k: + raise ValueError(f"config {config} produced K={top_k}, expected {expected_k}") + row: dict[str, object] = { + "implementation": "WarpFusion Exact fused", + "top_k": top_k, + "batch_size": int(extract(completed.stdout, r"batch_size: (\d+)")), + "average_query_ms": extract( + completed.stdout, r"平均常驻查询时间: ([0-9.eE+-]+) ms" + ), + "qps": extract(completed.stdout, r"常驻 QPS: ([0-9.eE+-]+)"), + "run_p50_ms": extract(completed.stdout, r"run P50: ([0-9.eE+-]+) ms"), + "run_p99_ms": extract(completed.stdout, r"run P99: ([0-9.eE+-]+) ms"), + "run_samples": int(extract(completed.stdout, r"run samples: (\d+)")), + "batch_p50_ms": extract(completed.stdout, r"batch P50: ([0-9.eE+-]+) ms"), + "batch_p99_ms": extract(completed.stdout, r"batch P99: ([0-9.eE+-]+) ms"), + "batch_latency_samples": int( + extract(completed.stdout, r"batch samples: (\d+)") + ), + "gpu_initialization_ms": extract( + completed.stdout, r"GPU 一次性初始化总时间: ([0-9.eE+-]+) ms" + ), + "database_h2d_ms": extract( + completed.stdout, r"数据库一次性 H2D: ([0-9.eE+-]+) ms" + ), + "gpu_buffer_mib": extract( + completed.stdout, r"GPU 缓冲区: ([0-9.eE+-]+) MiB" + ), + "query_h2d_ms": extract( + completed.stdout, r"平均查询 H2D: ([0-9.eE+-]+) ms" + ), + "distance_and_local_topk_ms": extract( + completed.stdout, + r"平均融合距离\+局部 Top-K kernel: ([0-9.eE+-]+) ms", + ), + "final_topk_ms": extract( + completed.stdout, r"平均最终 Top-K kernel: ([0-9.eE+-]+) ms" + ), + "result_d2h_ms": extract( + completed.stdout, r"平均结果 D2H: ([0-9.eE+-]+) ms" + ), + "result_path": str(result_path), + } + rows.append(row) + write_csv(args.csv, rows) + print(f"CSV: {args.csv}") + + +if __name__ == "__main__": + main() diff --git a/09_vector_retrieval/PurplePulse/scripts/benchmark_exact_breakdown.py b/09_vector_retrieval/PurplePulse/scripts/benchmark_exact_breakdown.py new file mode 100644 index 00000000..5ad672f9 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/scripts/benchmark_exact_breakdown.py @@ -0,0 +1,174 @@ +#!/usr/bin/env python3 +"""Profile fused Exact distance, local selection, and final merge by K.""" + +from __future__ import annotations + +import argparse +import csv +import re +import subprocess +from pathlib import Path + + +def extract(text: str, pattern: str) -> float: + match = re.search(pattern, text) + if match is None: + raise ValueError(f"cannot parse vector_search output: {pattern}") + return float(match.group(1)) + + +def parse_k_paths(text: str) -> list[tuple[int, Path]]: + result = [] + for item in text.split(","): + fields = item.split(":", 1) + if len(fields) != 2: + raise ValueError("paths must use K:path,K:path syntax") + top_k, path = int(fields[0]), Path(fields[1]) + if top_k <= 0 or not path.is_file(): + raise ValueError(f"invalid K:path item: {item}") + result.append((top_k, path)) + if not result: + raise ValueError("at least one K:path item is required") + return result + + +def read_results(path: Path) -> dict[int, list[tuple[int, float]]]: + queries: dict[int, list[tuple[int, float]]] = {} + for line_number, line in enumerate(path.read_text().splitlines(), 1): + fields = line.split() + if len(fields) != 3: + raise ValueError(f"{path}:{line_number}: expected three columns") + query_id, vector_id, score = int(fields[0]), int(fields[1]), float(fields[2]) + queries.setdefault(query_id, []).append((vector_id, score)) + return queries + + +def compare_results(expected_path: Path, actual_path: Path, tolerance: float) -> None: + expected = read_results(expected_path) + actual = read_results(actual_path) + if expected.keys() != actual.keys(): + raise ValueError("profile and reference query sets differ") + for query_id in expected: + if len(expected[query_id]) != len(actual[query_id]): + raise ValueError(f"query {query_id} Top-K lengths differ") + for rank, (expected_item, actual_item) in enumerate( + zip(expected[query_id], actual[query_id]) + ): + expected_id, expected_score = expected_item + actual_id, actual_score = actual_item + if expected_id != actual_id: + raise ValueError( + f"query {query_id} rank {rank} IDs differ: " + f"{expected_id} != {actual_id}" + ) + error = abs(expected_score - actual_score) + if error > tolerance: + raise ValueError( + f"query {query_id} rank {rank} score error {error} " + f"exceeds {tolerance}" + ) + + +def parse_profile_output(text: str) -> dict[str, object]: + fused_ms = extract(text, r"剖析融合 kernel: ([0-9.eE+-]+) ms") + distance_ms = extract(text, r"剖析距离-only kernel: ([0-9.eE+-]+) ms") + local_ms = extract(text, r"剖析局部 Top-K 估算: ([0-9.eE+-]+) ms") + merge_ms = extract(text, r"剖析最终归并 kernel: ([0-9.eE+-]+) ms") + pipeline_ms = fused_ms + merge_ms + return { + "num_vectors": int(extract(text, r"向量库: (\d+) x")), + "dim": int(extract(text, r"向量库: \d+ x (\d+)")), + "num_queries": int(extract(text, r"查询数: (\d+)")), + "top_k": int(extract(text, r"top_k: (\d+)")), + "batch_size": int(extract(text, r"batch_size: (\d+)")), + "average_query_ms": extract( + text, r"平均常驻查询时间: ([0-9.eE+-]+) ms" + ), + "qps": extract(text, r"常驻 QPS: ([0-9.eE+-]+)"), + "run_p50_ms": extract(text, r"run P50: ([0-9.eE+-]+) ms"), + "run_p99_ms": extract(text, r"run P99: ([0-9.eE+-]+) ms"), + "gpu_buffer_mib": extract(text, r"GPU 缓冲区: ([0-9.eE+-]+) MiB"), + "distance_only_ms": distance_ms, + "local_topk_estimated_ms": local_ms, + "fused_kernel_ms": fused_ms, + "final_merge_ms": merge_ms, + "kernel_pipeline_ms": pipeline_ms, + "distance_share": distance_ms / pipeline_ms if pipeline_ms else 0.0, + "local_topk_share": local_ms / pipeline_ms if pipeline_ms else 0.0, + "final_merge_share": merge_ms / pipeline_ms if pipeline_ms else 0.0, + } + + +def write_csv(path: Path, rows: list[dict[str, object]]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("w", newline="") as output: + writer = csv.DictWriter(output, fieldnames=list(rows[0])) + writer.writeheader() + writer.writerows(rows) + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--search-binary", required=True) + parser.add_argument("--database", required=True) + parser.add_argument("--queries", required=True) + parser.add_argument("--configs", required=True, help="K:path,K:path") + parser.add_argument("--reference-results", help="optional K:path,K:path") + parser.add_argument("--output-dir", type=Path, required=True) + parser.add_argument("--csv", type=Path, required=True) + parser.add_argument("--warmup", type=int, default=1) + parser.add_argument("--repeat", type=int, default=5) + parser.add_argument("--profile-repeat", type=int, default=5) + parser.add_argument("--tolerance", type=float, default=5e-5) + args = parser.parse_args() + if args.warmup < 0 or args.repeat <= 0 or args.profile_repeat <= 0: + raise ValueError("warmup must be non-negative and repeats must be positive") + + configs = parse_k_paths(args.configs) + references = ( + dict(parse_k_paths(args.reference_results)) + if args.reference_results + else {} + ) + if references and set(references) != {top_k for top_k, _ in configs}: + raise ValueError("reference-results must cover the same K values as configs") + + args.output_dir.mkdir(parents=True, exist_ok=True) + rows: list[dict[str, object]] = [] + for expected_k, config in configs: + result_path = args.output_dir / f"exact_profile_k{expected_k}.txt" + command = [ + args.search_binary, + "--database", args.database, + "--queries", args.queries, + "--params", str(config), + "--backend", "gpu", + "--output", str(result_path), + "--warmup", str(args.warmup), + "--repeat", str(args.repeat), + "--profile-breakdown", "1", + "--profile-repeat", str(args.profile_repeat), + ] + completed = subprocess.run( + command, + check=True, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + ) + print(completed.stdout, end="") + row = parse_profile_output(completed.stdout) + if row["top_k"] != expected_k: + raise ValueError( + f"config {config} produced K={row['top_k']}, expected {expected_k}" + ) + if references: + compare_results(references[expected_k], result_path, args.tolerance) + row["result_path"] = str(result_path) + rows.append(row) + write_csv(args.csv, rows) + print(f"CSV: {args.csv}") + + +if __name__ == "__main__": + main() diff --git a/09_vector_retrieval/PurplePulse/scripts/benchmark_exact_fused.py b/09_vector_retrieval/PurplePulse/scripts/benchmark_exact_fused.py new file mode 100644 index 00000000..a30e5cf0 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/scripts/benchmark_exact_fused.py @@ -0,0 +1,148 @@ +#!/usr/bin/env python3 +"""Compare matrix-based and fused exact-search modes and write a CSV.""" + +import argparse +import csv +import re +import subprocess +from pathlib import Path + + +def extract(text: str, pattern: str) -> float: + match = re.search(pattern, text) + if match is None: + raise ValueError(f"无法解析字段: {pattern}") + return float(match.group(1)) + + +def read_results(path: Path) -> dict[int, dict[int, float]]: + queries: dict[int, dict[int, float]] = {} + for line_number, line in enumerate(path.read_text().splitlines(), 1): + fields = line.split() + if len(fields) != 3: + raise ValueError(f"{path}:{line_number}: 结果行必须有三列") + query_id, vector_id, score = int(fields[0]), int(fields[1]), float(fields[2]) + queries.setdefault(query_id, {})[vector_id] = score + return queries + + +def compare_results(expected_path: Path, actual_path: Path, + tolerance: float) -> tuple[float, float]: + expected = read_results(expected_path) + actual = read_results(actual_path) + if expected.keys() != actual.keys(): + raise ValueError("baseline 与 fused 的 query 集合不同") + maximum_error = 0.0 + error_sum = 0.0 + error_count = 0 + for query_id in expected: + if expected[query_id].keys() != actual[query_id].keys(): + raise ValueError(f"query {query_id} 的 Top-K 候选集合不同") + for vector_id, score in expected[query_id].items(): + error = abs(score - actual[query_id][vector_id]) + maximum_error = max(maximum_error, error) + error_sum += error + error_count += 1 + if maximum_error > tolerance: + raise ValueError( + f"最大分数误差 {maximum_error} 超过容差 {tolerance}" + ) + return error_sum / error_count if error_count else 0.0, maximum_error + + +def run_search(args: argparse.Namespace, config: str, output: Path) -> tuple[str, dict]: + command = [ + args.search_binary, + "--database", args.database, + "--queries", args.queries, + "--params", config, + "--backend", "gpu", + "--output", str(output), + "--warmup", str(args.warmup), + "--repeat", str(args.repeat), + ] + completed = subprocess.run( + command, check=True, text=True, stdout=subprocess.PIPE, + stderr=subprocess.STDOUT + ) + text = completed.stdout + print(text, end="") + row = { + "top_k": int(extract(text, r"top_k: (\d+)")), + "average_ms": extract(text, r"平均常驻查询时间: ([0-9.eE+-]+) ms"), + "qps": extract(text, r"常驻 QPS: ([0-9.eE+-]+)"), + "run_p50_ms": extract(text, r"run P50: ([0-9.eE+-]+) ms"), + "run_p99_ms": extract(text, r"run P99: ([0-9.eE+-]+) ms"), + "run_samples": int(extract(text, r"run samples: (\d+)")), + "batch_latency_samples": int(extract(text, r"batch samples: (\d+)")), + "gpu_buffer_mib": extract(text, r"GPU 缓冲区: ([0-9.eE+-]+) MiB"), + "initialization_ms": extract( + text, r"GPU 一次性初始化总时间: ([0-9.eE+-]+) ms" + ), + "distance_or_fused_ms": extract( + text, + r"平均(?:距离|融合距离\+局部 Top-K) kernel: ([0-9.eE+-]+) ms", + ), + "final_topk_ms": extract( + text, r"平均最终 Top-K kernel: ([0-9.eE+-]+) ms" + ), + } + return text, row + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--search-binary", required=True) + parser.add_argument("--database", required=True) + parser.add_argument("--queries", required=True) + parser.add_argument("--baseline-configs", required=True) + parser.add_argument("--fused-configs", required=True) + parser.add_argument("--output-dir", required=True) + parser.add_argument("--csv", required=True) + parser.add_argument("--warmup", type=int, default=1) + parser.add_argument("--repeat", type=int, default=5) + parser.add_argument("--tolerance", type=float, default=5e-5) + args = parser.parse_args() + + baseline_configs = args.baseline_configs.split(",") + fused_configs = args.fused_configs.split(",") + if len(baseline_configs) != len(fused_configs) or not baseline_configs: + parser.error("baseline-configs 与 fused-configs 必须是一一对应的非空列表") + + output_dir = Path(args.output_dir) + output_dir.mkdir(parents=True, exist_ok=True) + rows = [] + for baseline_config, fused_config in zip(baseline_configs, fused_configs): + baseline_path = output_dir / f"baseline_{len(rows) // 2}.txt" + _, baseline = run_search(args, baseline_config, baseline_path) + baseline["mode"] = "matrix" + baseline["mean_absolute_score_error"] = 0.0 + baseline["maximum_score_error"] = 0.0 + + fused_path = output_dir / f"fused_{len(rows) // 2}.txt" + _, fused = run_search(args, fused_config, fused_path) + if baseline["top_k"] != fused["top_k"]: + raise ValueError("成对配置的 top_k 不一致") + fused["mode"] = "fused" + fused["mean_absolute_score_error"], fused["maximum_score_error"] = compare_results( + baseline_path, fused_path, args.tolerance + ) + rows.extend((baseline, fused)) + + csv_path = Path(args.csv) + csv_path.parent.mkdir(parents=True, exist_ok=True) + fieldnames = [ + "top_k", "mode", "average_ms", "qps", "run_p50_ms", "run_p99_ms", + "run_samples", "batch_latency_samples", "gpu_buffer_mib", + "initialization_ms", "distance_or_fused_ms", "final_topk_ms", + "mean_absolute_score_error", "maximum_score_error", + ] + with csv_path.open("w", newline="") as csv_file: + writer = csv.DictWriter(csv_file, fieldnames=fieldnames) + writer.writeheader() + writer.writerows(rows) + print(f"CSV: {csv_path}") + + +if __name__ == "__main__": + main() diff --git a/09_vector_retrieval/PurplePulse/scripts/benchmark_exact_query_tile.py b/09_vector_retrieval/PurplePulse/scripts/benchmark_exact_query_tile.py new file mode 100644 index 00000000..bcca5e90 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/scripts/benchmark_exact_query_tile.py @@ -0,0 +1,128 @@ +#!/usr/bin/env python3 +"""Benchmark Exact fused query-tile variants with strict result validation.""" + +from __future__ import annotations + +import argparse +import csv +import re +import subprocess +from pathlib import Path + +from benchmark_exact_breakdown import compare_results, extract, parse_k_paths + + +def parse_variant(text: str) -> tuple[str, list[tuple[int, Path]]]: + fields = text.split("=", 1) + if len(fields) != 2 or not re.fullmatch(r"[A-Za-z0-9_-]+", fields[0]): + raise ValueError("variant must use label=K:path,K:path syntax") + return fields[0], parse_k_paths(fields[1]) + + +def parse_search_output(text: str, label: str) -> dict[str, object]: + layout = re.search(r"exact_fused_layout: ([A-Za-z0-9_-]+)", text) + if layout is None: + raise ValueError("cannot parse exact_fused_layout") + return { + "variant": label, + "num_vectors": int(extract(text, r"向量库: (\d+) x")), + "dim": int(extract(text, r"向量库: \d+ x (\d+)")), + "num_queries": int(extract(text, r"查询数: (\d+)")), + "top_k": int(extract(text, r"top_k: (\d+)")), + "batch_size": int(extract(text, r"batch_size: (\d+)")), + "exact_query_tile": int(extract(text, r"exact_query_tile: (\d+)")), + "exact_fused_layout": layout.group(1), + "exact_fused_blocks": int( + extract(text, r"exact_fused_blocks: (\d+)") + ), + "average_query_ms": extract( + text, r"平均常驻查询时间: ([0-9.eE+-]+) ms" + ), + "qps": extract(text, r"常驻 QPS: ([0-9.eE+-]+)"), + "run_p50_ms": extract(text, r"run P50: ([0-9.eE+-]+) ms"), + "run_p99_ms": extract(text, r"run P99: ([0-9.eE+-]+) ms"), + "gpu_buffer_mib": extract(text, r"GPU 缓冲区: ([0-9.eE+-]+) MiB"), + "fused_kernel_ms": extract( + text, r"平均融合距离\+局部 Top-K kernel: ([0-9.eE+-]+) ms" + ), + "final_merge_ms": extract( + text, r"平均最终 Top-K kernel: ([0-9.eE+-]+) ms" + ), + } + + +def write_csv(path: Path, rows: list[dict[str, object]]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("w", newline="") as output: + writer = csv.DictWriter(output, fieldnames=list(rows[0])) + writer.writeheader() + writer.writerows(rows) + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--search-binary", required=True) + parser.add_argument("--database", required=True) + parser.add_argument("--queries", required=True) + parser.add_argument( + "--variant", action="append", required=True, + help="repeat label=K:path,K:path for each A/B variant", + ) + parser.add_argument("--reference-results", required=True) + parser.add_argument("--output-dir", type=Path, required=True) + parser.add_argument("--csv", type=Path, required=True) + parser.add_argument("--warmup", type=int, default=1) + parser.add_argument("--repeat", type=int, default=5) + parser.add_argument("--tolerance", type=float, default=5e-5) + args = parser.parse_args() + if args.warmup < 0 or args.repeat <= 0: + raise ValueError("warmup must be non-negative and repeat positive") + + references = dict(parse_k_paths(args.reference_results)) + variants = [parse_variant(item) for item in args.variant] + expected_keys = set(references) + if any({top_k for top_k, _ in configs} != expected_keys + for _, configs in variants): + raise ValueError("every variant must cover the reference K values") + + rows: list[dict[str, object]] = [] + for label, configs in variants: + variant_dir = args.output_dir / label + variant_dir.mkdir(parents=True, exist_ok=True) + for expected_k, config in configs: + result_path = variant_dir / f"exact_k{expected_k}.txt" + command = [ + args.search_binary, + "--database", args.database, + "--queries", args.queries, + "--params", str(config), + "--backend", "gpu", + "--output", str(result_path), + "--warmup", str(args.warmup), + "--repeat", str(args.repeat), + ] + completed = subprocess.run( + command, + check=True, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + ) + print(completed.stdout, end="") + row = parse_search_output(completed.stdout, label) + if row["top_k"] != expected_k: + raise ValueError( + f"config {config} produced K={row['top_k']}, " + f"expected {expected_k}" + ) + compare_results( + references[expected_k], result_path, args.tolerance + ) + row["result_path"] = str(result_path) + rows.append(row) + write_csv(args.csv, rows) + print(f"CSV: {args.csv}") + + +if __name__ == "__main__": + main() diff --git a/09_vector_retrieval/PurplePulse/scripts/benchmark_faiss.py b/09_vector_retrieval/PurplePulse/scripts/benchmark_faiss.py new file mode 100644 index 00000000..2a9e818c --- /dev/null +++ b/09_vector_retrieval/PurplePulse/scripts/benchmark_faiss.py @@ -0,0 +1,506 @@ +#!/usr/bin/env python3 +"""在本项目自定义数据上运行可复现的 FAISS Flat/IVF-Flat 基准。""" + +from __future__ import annotations + +import argparse +import csv +import gc +import json +import os +import struct +import time +from dataclasses import asdict, dataclass +from pathlib import Path +from typing import Iterable + +# 固定 CPU 对照的线程口径；用户可在命令前显式覆盖。 +os.environ.setdefault("OMP_NUM_THREADS", "1") + +import numpy as np + + +DTYPES = {1: np.dtype(" np.memmap: + return np.memmap( + self.path, + dtype=DTYPES[self.dtype_code], + mode="r", + offset=self.header_bytes, + shape=(self.rows, self.dim), + ) + + +@dataclass +class BenchmarkRow: + implementation: str + backend: str + index_type: str + metric: str + num_vectors: int + num_queries: int + dim: int + dtype: str + cpu_threads: int + top_k: int + batch_size: int + nlist: int + nprobe: int + train_samples: int + training_iterations: int + training_seed: int + build_ms: float + load_ms: float + gpu_transfer_ms: float + gpu_index_mib: float + gpu_temp_mib: float + average_query_ms: float + qps: float + run_p50_ms: float + run_p99_ms: float + run_samples: int + batch_p50_ms: float + batch_p99_ms: float + batch_latency_samples: int + recall_at_k: float + min_query_recall: float + mean_absolute_score_error: float + max_absolute_score_error: float + + +def _read_header(path: Path, expected_magic: bytes) -> tuple[bytes, int]: + with path.open("rb") as source: + magic = source.read(8) + if magic != expected_magic: + raise ValueError(f"{path}: magic/version 不正确") + return magic, path.stat().st_size + + +def read_database(path: Path) -> VectorFile: + _, actual_size = _read_header(path, b"PPVEC001") + with path.open("rb") as source: + source.seek(8) + rows, dim, dtype_code, metric_code = struct.unpack(" VectorFile: + _, actual_size = _read_header(path, b"PPQRY001") + with path.open("rb") as source: + source.seek(8) + rows, dim, dtype_code = struct.unpack(" list[int]: + try: + values = [int(item.strip()) for item in text.split(",")] + except ValueError as error: + raise ValueError(f"{name} 必须是逗号分隔的正整数") from error + if not values or any(value <= 0 for value in values): + raise ValueError(f"{name} 必须是逗号分隔的正整数") + return values + + +def percentile(values: Iterable[float], fraction: float) -> float: + array = np.asarray(list(values), dtype=np.float64) + if array.size == 0: + return 0.0 + return float(np.percentile(array, fraction * 100.0, method="higher")) + + +def gpu_memory_mib(resource) -> tuple[float, float]: + """返回 FAISS GPU 的索引数据与临时工作区保留量。""" + if not hasattr(resource, "getMemoryInfo"): + return 0.0, 0.0 + index_bytes = 0 + temp_bytes = 0 + for categories in resource.getMemoryInfo().values(): + for name, (_, allocated_bytes) in categories.items(): + if name == "TemporaryMemoryBuffer": + temp_bytes += allocated_bytes + else: + index_bytes += allocated_bytes + scale = 1024.0 * 1024.0 + return index_bytes / scale, temp_bytes / scale + + +def recall(reference: np.ndarray, actual: np.ndarray) -> tuple[float, float]: + if reference.shape != actual.shape or reference.ndim != 2 or reference.shape[1] == 0: + raise ValueError("reference 与 actual 的形状必须相同且 K>0") + per_query = np.asarray( + [len(set(expected) & set(found)) / reference.shape[1] + for expected, found in zip(reference.tolist(), actual.tolist())], + dtype=np.float64, + ) + return float(per_query.mean()), float(per_query.min()) + + +def score_errors(reference: np.ndarray, actual: np.ndarray) -> tuple[float, float]: + """按名次比较 Top-K 分数，量化近似结果相对 exact 的质量损失。""" + if reference.shape != actual.shape or reference.size == 0: + raise ValueError("reference 与 actual 的分数形状必须相同且非空") + errors = np.abs( + np.asarray(reference, dtype=np.float64) + - np.asarray(actual, dtype=np.float64) + ) + return float(errors.mean()), float(errors.max()) + + +def as_float32(values: np.ndarray) -> np.ndarray: + return np.ascontiguousarray(values, dtype=np.float32) + + +def normalize_rows(values: np.ndarray) -> np.ndarray: + values = as_float32(values) + norms = np.linalg.norm(values, axis=1, keepdims=True) + if np.any(norms == 0): + raise ValueError("cosine 数据包含零向量") + return values / norms + + +def import_faiss(): + try: + import faiss # type: ignore + except ImportError as error: + raise RuntimeError( + "未安装 FAISS；请在隔离环境中安装 faiss-cpu 或 CUDA 对应的 " + "faiss-gpu 包" + ) from error + return faiss + + +def make_flat_index(faiss, dim: int, metric: str): + return faiss.IndexFlatL2(dim) if metric == "l2" else faiss.IndexFlatIP(dim) + + +def make_ivf_index(faiss, dim: int, metric: str, nlist: int, iterations: int, seed: int): + metric_id = faiss.METRIC_L2 if metric == "l2" else faiss.METRIC_INNER_PRODUCT + quantizer = make_flat_index(faiss, dim, metric) + index = faiss.IndexIVFFlat(quantizer, dim, nlist, metric_id) + index.cp.niter = iterations + index.cp.seed = seed + return index + + +def prepare_vectors(database: VectorFile, queries: VectorFile) -> tuple[np.ndarray, np.ndarray]: + if database.dim != queries.dim: + raise ValueError("数据库与 query 维度不一致") + database_values = as_float32(database.mmap()) + query_values = as_float32(queries.mmap()) + if database.metric == "cosine": + database_values = normalize_rows(database_values) + query_values = normalize_rows(query_values) + return database_values, query_values + + +def search_batches(index, queries: np.ndarray, top_k: int, batch_size: int): + distances = np.empty((len(queries), top_k), dtype=np.float32) + ids = np.empty((len(queries), top_k), dtype=np.int64) + latencies = [] + start_all = time.perf_counter() + for start in range(0, len(queries), batch_size): + end = min(start + batch_size, len(queries)) + start_batch = time.perf_counter() + batch_distances, batch_ids = index.search(queries[start:end], top_k) + latencies.append((time.perf_counter() - start_batch) * 1000.0) + distances[start:end] = batch_distances + ids[start:end] = batch_ids + total_ms = (time.perf_counter() - start_all) * 1000.0 + return distances, ids, total_ms, latencies + + +def write_results(path: Path, ids: np.ndarray, scores: np.ndarray) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("w") as output: + for query_id, (query_ids, query_scores) in enumerate(zip(ids, scores)): + for vector_id, score in zip(query_ids, query_scores): + output.write(f"{query_id} {int(vector_id)} {float(score):.9g}\n") + + +def write_csv(path: Path, rows: list[BenchmarkRow]) -> None: + with path.open("w", newline="") as output: + writer = csv.DictWriter(output, fieldnames=list(asdict(rows[0]))) + writer.writeheader() + writer.writerows(asdict(row) for row in rows) + + +def read_result_arrays( + path: Path, num_queries: int, minimum_k: int +) -> tuple[np.ndarray, np.ndarray]: + id_rows: list[list[int]] = [[] for _ in range(num_queries)] + score_rows: list[list[float]] = [[] for _ in range(num_queries)] + with path.open() as source: + for line_number, line in enumerate(source, 1): + fields = line.split() + if len(fields) != 3: + raise ValueError(f"{path}:{line_number}: 结果行必须有三列") + query_id, vector_id, score = int(fields[0]), int(fields[1]), float(fields[2]) + if query_id < 0 or query_id >= num_queries: + raise ValueError(f"{path}:{line_number}: query_id 越界") + id_rows[query_id].append(vector_id) + score_rows[query_id].append(score) + if any(len(row) < minimum_k for row in id_rows): + raise ValueError(f"{path}: 每个 query 至少需要 {minimum_k} 个结果") + ids = np.asarray([row[:minimum_k] for row in id_rows], dtype=np.int64) + scores = np.asarray([row[:minimum_k] for row in score_rows], dtype=np.float32) + return ids, scores + + +def read_result_ids(path: Path, num_queries: int, minimum_k: int) -> np.ndarray: + """兼容已有调用者；新代码应使用 read_result_arrays 同时读取分数。""" + return read_result_arrays(path, num_queries, minimum_k)[0] + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--database", type=Path, required=True) + parser.add_argument("--queries", type=Path, required=True) + parser.add_argument("--output-dir", type=Path, required=True) + parser.add_argument("--csv", type=Path, required=True) + parser.add_argument("--index-dir", type=Path, required=True) + parser.add_argument( + "--reference-results", type=Path, + help="可选的本项目 exact 结果，用来交叉验证 FAISS Flat", + ) + parser.add_argument("--backends", default="cpu,gpu", help="cpu,gpu 或二者") + parser.add_argument("--index-types", default="flat,ivf_flat") + parser.add_argument("--top-ks", default="10,50,100") + parser.add_argument("--nprobes", default="128,160,192,224") + parser.add_argument("--nlist", type=int, default=256) + parser.add_argument("--batch-size", type=int, default=64) + parser.add_argument("--train-samples", type=int, default=100000) + parser.add_argument("--iterations", type=int, default=15) + parser.add_argument("--seed", type=int, default=2026) + parser.add_argument("--warmup", type=int, default=1) + parser.add_argument("--repeat", type=int, default=5) + parser.add_argument("--cpu-threads", type=int, default=1) + parser.add_argument("--rebuild", action="store_true") + return parser.parse_args() + + +def validate_args(args: argparse.Namespace) -> tuple[list[str], list[str], list[int], list[int]]: + backends = [item.strip() for item in args.backends.split(",")] + index_types = [item.strip() for item in args.index_types.split(",")] + top_ks = parse_positive_csv(args.top_ks, "top-ks") + nprobes = parse_positive_csv(args.nprobes, "nprobes") + if not backends or any(item not in {"cpu", "gpu"} for item in backends): + raise ValueError("backends 只能包含 cpu,gpu") + if not index_types or any(item not in {"flat", "ivf_flat"} for item in index_types): + raise ValueError("index-types 只能包含 flat,ivf_flat") + for name in ( + "nlist", "batch_size", "train_samples", "iterations", "repeat", + "cpu_threads", + ): + if getattr(args, name) <= 0: + raise ValueError(f"{name} 必须大于 0") + if args.warmup < 0 or any(value > args.nlist for value in nprobes): + raise ValueError("warmup 必须非负，nprobe 必须位于 1..nlist") + return backends, index_types, top_ks, nprobes + + +def main() -> None: + args = parse_args() + backends, index_types, top_ks, nprobes = validate_args(args) + faiss = import_faiss() + faiss.omp_set_num_threads(args.cpu_threads) + database_info = read_database(args.database) + query_info = read_queries(args.queries) + database, queries = prepare_vectors(database_info, query_info) + if max(top_ks) > database_info.rows: + raise ValueError("top_k 不能超过数据库向量数") + + args.output_dir.mkdir(parents=True, exist_ok=True) + args.index_dir.mkdir(parents=True, exist_ok=True) + args.csv.parent.mkdir(parents=True, exist_ok=True) + metric = database_info.metric + assert metric is not None + metric_suffix = "l2" if metric == "l2" else "ip" + + cpu_indexes = {} + build_times = {} + load_times = {} + build_index_types = list(dict.fromkeys(["flat", *index_types])) + for index_type in build_index_types: + training_suffix = "" + if index_type == "ivf_flat": + training_suffix = ( + f"_train{min(args.train_samples, database_info.rows)}" + f"_iter{args.iterations}_seed{args.seed}" + ) + index_path = args.index_dir / ( + f"faiss_{index_type}_{metric_suffix}_{database_info.rows}x" + f"{database_info.dim}_nlist{args.nlist}{training_suffix}.index" + ) + if index_path.exists() and not args.rebuild: + start = time.perf_counter() + cpu_index = faiss.read_index(str(index_path)) + load_times[index_type] = (time.perf_counter() - start) * 1000.0 + build_times[index_type] = 0.0 + else: + start = time.perf_counter() + if index_type == "flat": + cpu_index = make_flat_index(faiss, database_info.dim, metric) + else: + cpu_index = make_ivf_index( + faiss, database_info.dim, metric, args.nlist, + args.iterations, args.seed, + ) + rng = np.random.default_rng(args.seed) + sample_count = min(args.train_samples, database_info.rows) + sample_ids = rng.choice(database_info.rows, sample_count, replace=False) + cpu_index.train(database[sample_ids]) + cpu_index.add(database) + build_times[index_type] = (time.perf_counter() - start) * 1000.0 + load_times[index_type] = 0.0 + faiss.write_index(cpu_index, str(index_path)) + if cpu_index.ntotal != database_info.rows: + raise RuntimeError(f"FAISS {index_type} 索引向量数不正确") + cpu_indexes[index_type] = cpu_index + + max_k = max(top_ks) + faiss_reference_scores, faiss_reference_ids = cpu_indexes["flat"].search( + queries, max_k + ) + if args.reference_results is not None: + reference_ids, reference_scores = read_result_arrays( + args.reference_results, query_info.rows, max_k + ) + else: + reference_ids, reference_scores = faiss_reference_ids, faiss_reference_scores + if args.reference_results is not None: + exact_recall, exact_minimum = recall(reference_ids, faiss_reference_ids) + exact_mean_error, exact_max_error = score_errors( + reference_scores, faiss_reference_scores + ) + print( + f"FAISS Flat vs project exact recall@{max_k}: " + f"{exact_recall:.9f}, min={exact_minimum:.9f}, " + f"mean_score_error={exact_mean_error:.9g}, " + f"max_score_error={exact_max_error:.9g}" + ) + if exact_recall < 1.0: + raise RuntimeError("FAISS Flat 与本项目 exact 的候选 ID 不一致") + rows: list[BenchmarkRow] = [] + for backend in backends: + if backend == "gpu" and ( + not hasattr(faiss, "get_num_gpus") or faiss.get_num_gpus() < 1 + ): + raise RuntimeError("请求了 GPU backend，但 FAISS 没有检测到 GPU") + for index_type in index_types: + gpu_transfer_ms = 0.0 + gpu_index_mib = 0.0 + gpu_temp_mib = 0.0 + if backend == "gpu": + resource = faiss.StandardGpuResources() + start = time.perf_counter() + index = faiss.index_cpu_to_gpu(resource, 0, cpu_indexes[index_type]) + gpu_transfer_ms = (time.perf_counter() - start) * 1000.0 + gpu_index_mib, gpu_temp_mib = gpu_memory_mib(resource) + else: + index = cpu_indexes[index_type] + + probe_values = [0] if index_type == "flat" else nprobes + for nprobe in probe_values: + if index_type == "ivf_flat": + index.nprobe = nprobe + for top_k in top_ks: + for _ in range(args.warmup): + # 与本项目 CLI 一致：预热覆盖完整 query set，避免 + # GPU 时钟爬升和 FAISS 延迟初始化进入正式统计。 + search_batches(index, queries, top_k, args.batch_size) + repeat_ms = [] + all_batch_ms = [] + last_scores = last_ids = None + for _ in range(args.repeat): + last_scores, last_ids, total_ms, batch_ms = search_batches( + index, queries, top_k, args.batch_size + ) + repeat_ms.append(total_ms) + all_batch_ms.extend(batch_ms) + assert last_scores is not None and last_ids is not None + average_ms = float(np.mean(repeat_ms)) + reference = reference_ids[:, :top_k] + recall_at_k, minimum_recall = recall(reference, last_ids) + mean_score_error, max_score_error = score_errors( + reference_scores[:, :top_k], last_scores + ) + row = BenchmarkRow( + implementation=f"faiss-{getattr(faiss, '__version__', 'unknown')}", + backend=backend, + index_type=index_type, + metric=metric, + num_vectors=database_info.rows, + num_queries=query_info.rows, + dim=database_info.dim, + dtype="fp32" if database_info.dtype_code == 1 else "fp16-to-fp32", + cpu_threads=args.cpu_threads, + top_k=top_k, + batch_size=args.batch_size, + nlist=0 if index_type == "flat" else args.nlist, + nprobe=nprobe, + train_samples=0 if index_type == "flat" else min( + args.train_samples, database_info.rows + ), + training_iterations=0 if index_type == "flat" else args.iterations, + training_seed=0 if index_type == "flat" else args.seed, + build_ms=build_times[index_type], + load_ms=load_times[index_type], + gpu_transfer_ms=gpu_transfer_ms, + gpu_index_mib=gpu_index_mib, + gpu_temp_mib=gpu_temp_mib, + average_query_ms=average_ms, + qps=query_info.rows / (average_ms / 1000.0), + run_p50_ms=percentile(repeat_ms, 0.50), + run_p99_ms=percentile(repeat_ms, 0.99), + run_samples=len(repeat_ms), + batch_p50_ms=percentile(all_batch_ms, 0.50), + batch_p99_ms=percentile(all_batch_ms, 0.99), + batch_latency_samples=len(all_batch_ms), + recall_at_k=recall_at_k, + min_query_recall=minimum_recall, + mean_absolute_score_error=mean_score_error, + max_absolute_score_error=max_score_error, + ) + rows.append(row) + # 长矩阵每完成一项立即落盘，远端中断时保留已完成结果。 + write_csv(args.csv, rows) + result_path = args.output_dir / ( + f"faiss_{backend}_{index_type}_k{top_k}_nprobe{nprobe}.txt" + ) + write_results(result_path, last_ids, last_scores) + print(json.dumps(asdict(row), ensure_ascii=False)) + if backend == "gpu": + del index + del resource + gc.collect() + + print(f"CSV: {args.csv}") + + +if __name__ == "__main__": + main() diff --git a/09_vector_retrieval/PurplePulse/scripts/benchmark_ivf_sweep.py b/09_vector_retrieval/PurplePulse/scripts/benchmark_ivf_sweep.py new file mode 100644 index 00000000..520ea9ee --- /dev/null +++ b/09_vector_retrieval/PurplePulse/scripts/benchmark_ivf_sweep.py @@ -0,0 +1,204 @@ +#!/usr/bin/env python3 +"""运行 IVF nprobe 扫描并把性能、显存和 recall 汇总为 CSV。""" + +import argparse +import csv +import re +import subprocess +from pathlib import Path + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--search-binary", required=True) + parser.add_argument("--index", required=True) + parser.add_argument("--queries", required=True) + parser.add_argument("--params", required=True) + parser.add_argument("--exact", required=True) + parser.add_argument("--output-dir", required=True) + parser.add_argument("--csv", required=True) + parser.add_argument("--nlist", type=int, required=True) + parser.add_argument("--training-samples", type=int, default=0) + parser.add_argument("--training-iterations", type=int, default=0) + parser.add_argument("--training-seed", type=int, default=0) + parser.add_argument("--nprobes", required=True, + help="逗号分隔，例如 64,128,192") + parser.add_argument("--warmup", type=int, default=1) + parser.add_argument("--repeat", type=int, default=5) + return parser.parse_args() + + +def read_results(path: Path) -> list[list[tuple[int, float]]]: + rows: dict[int, list[tuple[int, float]]] = {} + for line_number, line in enumerate(path.read_text().splitlines(), 1): + fields = line.split() + if len(fields) != 3: + raise ValueError(f"{path}:{line_number}: 结果行必须有三列") + query_id, vector_id, score = int(fields[0]), int(fields[1]), float(fields[2]) + rows.setdefault(query_id, []).append((vector_id, score)) + return [rows[query_id] for query_id in sorted(rows)] + + +def recall(exact: list[list[int]], approximate: list[list[int]]) -> tuple[float, float]: + if len(exact) != len(approximate) or not exact: + raise ValueError("exact 与 approximate 的 query 数不一致或为空") + recalls = [] + for expected, actual in zip(exact, approximate): + if len(expected) != len(actual) or not expected: + raise ValueError("exact 与 approximate 的 K 不一致或为空") + recalls.append(len(set(expected) & set(actual)) / len(expected)) + return sum(recalls) / len(recalls), min(recalls) + + +def quality_metrics( + exact: list[list[tuple[int, float]]], + approximate: list[list[tuple[int, float]]], +) -> tuple[float, float, float, float]: + if len(exact) != len(approximate) or not exact: + raise ValueError("exact 与 approximate 的 query 数不一致或为空") + exact_ids = [[item[0] for item in query] for query in exact] + approximate_ids = [[item[0] for item in query] for query in approximate] + recall_at_k, minimum_recall = recall(exact_ids, approximate_ids) + errors = [] + for expected, actual in zip(exact, approximate): + if len(expected) != len(actual) or not expected: + raise ValueError("exact 与 approximate 的 K 不一致或为空") + errors.extend( + abs(expected_item[1] - actual_item[1]) + for expected_item, actual_item in zip(expected, actual) + ) + return ( + recall_at_k, + minimum_recall, + sum(errors) / len(errors), + max(errors), + ) + + +def extract(text: str, pattern: str) -> float: + match = re.search(pattern, text) + if match is None: + raise ValueError(f"无法从 ivf_search 输出解析字段: {pattern}") + return float(match.group(1)) + + +def extract_pair(text: str, pattern: str) -> tuple[float, float]: + match = re.search(pattern, text) + if match is None: + raise ValueError(f"无法从 ivf_search 输出解析字段: {pattern}") + return float(match.group(1)), float(match.group(2)) + + +def read_params(path: Path) -> dict[str, str]: + params = {} + for line_number, raw_line in enumerate(path.read_text().splitlines(), 1): + line = raw_line.split("#", 1)[0].strip() + if not line: + continue + if "=" not in line: + raise ValueError(f"{path}:{line_number}: expected key = value") + key, value = (part.strip() for part in line.split("=", 1)) + params[key] = value + return params + + +def main() -> None: + args = parse_args() + nprobes = [int(value) for value in args.nprobes.split(",")] + if not nprobes or any(value <= 0 or value > args.nlist for value in nprobes): + raise ValueError("nprobes 必须位于 1..nlist") + output_dir = Path(args.output_dir) + output_dir.mkdir(parents=True, exist_ok=True) + csv_path = Path(args.csv) + csv_path.parent.mkdir(parents=True, exist_ok=True) + exact_results = read_results(Path(args.exact)) + params = read_params(Path(args.params)) + rows = [] + for nprobe in nprobes: + result_path = output_dir / f"nlist{args.nlist}_nprobe{nprobe}.txt" + command = [ + args.search_binary, + "--index", args.index, + "--queries", args.queries, + "--params", args.params, + "--nlist", str(args.nlist), + "--nprobe", str(nprobe), + "--backend", "gpu", + "--warmup", str(args.warmup), + "--repeat", str(args.repeat), + "--output", str(result_path), + ] + completed = subprocess.run(command, check=True, text=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT) + print(completed.stdout, end="") + approximate_results = read_results(result_path) + recall_at_k, minimum_recall, mean_score_error, max_score_error = ( + quality_metrics(exact_results, approximate_results) + ) + top_k = int(extract(completed.stdout, r"nlist/nprobe/top_k: \d+/\d+/(\d+)")) + run_p50_ms, run_p99_ms = extract_pair( + completed.stdout, r"run P50/P99: ([0-9.eE+-]+)/([0-9.eE+-]+) ms" + ) + batch_p50_ms, batch_p99_ms = extract_pair( + completed.stdout, r"batch P50/P99: ([0-9.eE+-]+)/([0-9.eE+-]+) ms" + ) + row = { + "implementation": "WarpFusion", + "backend": "gpu", + "num_vectors": int(extract(completed.stdout, r"索引: (\d+) x \d+")), + "num_queries": len(approximate_results), + "dim": int(extract(completed.stdout, r"索引: \d+ x (\d+)")), + "metric": re.search( + r"dtype/metric: [^/]+/([^\n]+)", completed.stdout + ).group(1), + "batch_size": int(params["batch_size"]), + "distance_mode": params["distance_mode"], + "nlist": args.nlist, + "nprobe": nprobe, + "training_samples": args.training_samples, + "training_iterations": args.training_iterations, + "training_seed": args.training_seed, + "top_k": top_k, + "average_ms": extract(completed.stdout, r"平均查询: ([0-9.eE+-]+) ms"), + "qps": extract(completed.stdout, r"QPS: ([0-9.eE+-]+)"), + "run_p50_ms": run_p50_ms, + "run_p99_ms": run_p99_ms, + "batch_p50_ms": batch_p50_ms, + "batch_p99_ms": batch_p99_ms, + "recall_at_k": recall_at_k, + "min_query_recall": minimum_recall, + "mean_absolute_score_error": mean_score_error, + "max_absolute_score_error": max_score_error, + "run_samples": int(extract( + completed.stdout, r"run samples: (\d+)" + )), + "batch_latency_samples": int(extract( + completed.stdout, r"batch samples: (\d+)" + )), + "index_load_ms": extract(completed.stdout, r"索引加载: ([0-9.eE+-]+) ms"), + "gpu_initialization_ms": extract( + completed.stdout, r"GPU 初始化: ([0-9.eE+-]+) ms" + ), + "index_h2d_ms": extract(completed.stdout, r"索引 H2D: ([0-9.eE+-]+) ms"), + "gpu_buffer_mib": extract(completed.stdout, r"GPU 缓冲区: ([0-9.eE+-]+) MiB"), + "center_ms": extract(completed.stdout, r"平均 GPU 中心选择: ([0-9.eE+-]+) ms"), + "scan_ms": extract(completed.stdout, r"平均 GPU 桶扫描\+局部 Top-K: ([0-9.eE+-]+) ms"), + "merge_ms": extract(completed.stdout, r"平均 GPU 最终 Top-K 归并: ([0-9.eE+-]+) ms"), + } + rows.append(row) + # 长时间扫描每完成一个 nprobe 就落盘，远端中断时保留已完成结果。 + with csv_path.open("w", newline="") as csv_file: + writer = csv.DictWriter(csv_file, fieldnames=list(rows[0])) + writer.writeheader() + writer.writerows(rows) + print(f"recall@{top_k}={recall_at_k:.6f}, " + f"min_query_recall={minimum_recall:.6f}, " + f"mean_absolute_score_error={mean_score_error:.9g}, " + f"max_absolute_score_error={max_score_error:.9g}") + + print(f"CSV: {csv_path}") + + +if __name__ == "__main__": + main() diff --git a/09_vector_retrieval/PurplePulse/scripts/benchmark_memory_modes.py b/09_vector_retrieval/PurplePulse/scripts/benchmark_memory_modes.py new file mode 100644 index 00000000..a2c89b7c --- /dev/null +++ b/09_vector_retrieval/PurplePulse/scripts/benchmark_memory_modes.py @@ -0,0 +1,150 @@ +#!/usr/bin/env python3 +"""Compare fused Agent-memory scoring with a separate GPU rerank baseline.""" + +from __future__ import annotations + +import argparse +import csv +import re +import subprocess +from pathlib import Path + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--search-binary", required=True) + parser.add_argument("--index", required=True) + parser.add_argument("--queries", required=True) + parser.add_argument("--params", required=True) + parser.add_argument("--memory-metadata", required=True) + parser.add_argument("--output-dir", required=True) + parser.add_argument("--csv", required=True) + parser.add_argument("--nlist", type=int, required=True) + parser.add_argument("--nprobe", type=int, required=True) + parser.add_argument("--rerank-factor", type=int, default=10) + parser.add_argument("--semantic-weight", type=float, default=1.0) + parser.add_argument("--importance-weight", type=float, default=0.15) + parser.add_argument("--recency-weight", type=float, default=0.10) + parser.add_argument("--time-scale", type=float, default=604800.0) + parser.add_argument("--now", type=int, default=1_702_592_000) + parser.add_argument("--min-timestamp", type=int, default=0) + parser.add_argument("--session-id", default="any") + parser.add_argument("--source-type", default="any") + parser.add_argument("--warmup", type=int, default=2) + parser.add_argument("--repeat", type=int, default=5) + return parser.parse_args() + + +def extract(text: str, pattern: str) -> float: + match = re.search(pattern, text) + if match is None: + raise ValueError(f"cannot parse ivf_search output: {pattern}") + return float(match.group(1)) + + +def read_ids(path: Path) -> list[list[int]]: + rows: dict[int, list[int]] = {} + for line_number, line in enumerate(path.read_text().splitlines(), 1): + fields = line.split() + if len(fields) != 3: + raise ValueError(f"{path}:{line_number}: expected three columns") + rows.setdefault(int(fields[0]), []).append(int(fields[1])) + return [rows[query_id] for query_id in sorted(rows)] + + +def recall(reference: list[list[int]], actual: list[list[int]]) -> tuple[float, float]: + if len(reference) != len(actual) or not reference: + raise ValueError("reference and actual query counts differ or are empty") + values = [] + for expected, found in zip(reference, actual): + if not expected: + raise ValueError("reference contains an empty query result") + values.append(len(set(expected) & set(found)) / len(expected)) + return sum(values) / len(values), min(values) + + +def main() -> None: + args = parse_args() + if args.nlist <= 0 or not 0 < args.nprobe <= args.nlist: + raise ValueError("invalid nlist/nprobe") + if args.rerank_factor <= 0 or args.semantic_weight <= 0: + raise ValueError("weights and rerank factor must be positive") + if args.importance_weight < 0 or args.recency_weight < 0 or args.time_scale <= 0: + raise ValueError("invalid memory scoring weights") + + output_dir = Path(args.output_dir) + output_dir.mkdir(parents=True, exist_ok=True) + csv_path = Path(args.csv) + csv_path.parent.mkdir(parents=True, exist_ok=True) + rows: list[dict[str, object]] = [] + reference_ids: list[list[int]] | None = None + for mode in ("fused", "rerank"): + result_path = output_dir / f"{mode}.txt" + command = [ + args.search_binary, + "--index", args.index, + "--queries", args.queries, + "--params", args.params, + "--memory-metadata", args.memory_metadata, + "--memory-mode", mode, + "--memory-semantic-weight", str(args.semantic_weight), + "--memory-importance-weight", str(args.importance_weight), + "--memory-recency-weight", str(args.recency_weight), + "--memory-time-scale", str(args.time_scale), + "--memory-now", str(args.now), + "--filter-min-timestamp", str(args.min_timestamp), + "--filter-session-id", args.session_id, + "--filter-source-type", args.source_type, + "--memory-rerank-factor", str(args.rerank_factor), + "--nlist", str(args.nlist), + "--nprobe", str(args.nprobe), + "--backend", "gpu", + "--warmup", str(args.warmup), + "--repeat", str(args.repeat), + "--output", str(result_path), + ] + completed = subprocess.run( + command, check=True, text=True, stdout=subprocess.PIPE, + stderr=subprocess.STDOUT + ) + print(completed.stdout, end="") + ids = read_ids(result_path) + if reference_ids is None: + reference_ids = ids + recall_at_k, minimum_recall = 1.0, 1.0 + else: + recall_at_k, minimum_recall = recall(reference_ids, ids) + batch_match = re.search( + r"batch P50/P99: ([0-9.eE+-]+)/([0-9.eE+-]+) ms", + completed.stdout, + ) + if batch_match is None: + raise ValueError("cannot parse batch latency") + rows.append({ + "mode": mode, + "nlist": args.nlist, + "nprobe": args.nprobe, + "rerank_factor": 1 if mode == "fused" else args.rerank_factor, + "session_id": args.session_id, + "source_type": args.source_type, + "average_ms": extract(completed.stdout, r"平均查询: ([0-9.eE+-]+) ms"), + "qps": extract(completed.stdout, r"QPS: ([0-9.eE+-]+)"), + "batch_p50_ms": float(batch_match.group(1)), + "batch_p99_ms": float(batch_match.group(2)), + "scan_ms": extract(completed.stdout, r"平均 GPU 桶扫描\+局部 Top-K: ([0-9.eE+-]+) ms"), + "merge_ms": extract(completed.stdout, r"平均 GPU 最终 Top-K 归并: ([0-9.eE+-]+) ms"), + "rerank_ms": extract(completed.stdout, r"平均记忆独立重排: ([0-9.eE+-]+) ms"), + "gpu_buffer_mib": extract(completed.stdout, r"GPU 缓冲区: ([0-9.eE+-]+) MiB"), + "recall_vs_fused": recall_at_k, + "min_query_recall_vs_fused": minimum_recall, + }) + with csv_path.open("w", newline="") as csv_file: + writer = csv.DictWriter(csv_file, fieldnames=list(rows[0])) + writer.writeheader() + writer.writerows(rows) + print(f"{mode}: recall_vs_fused={recall_at_k:.6f}") + print(f"CSV: {csv_path}") + + +if __name__ == "__main__": + main() diff --git a/09_vector_retrieval/PurplePulse/scripts/build_final_report_pdf.py b/09_vector_retrieval/PurplePulse/scripts/build_final_report_pdf.py new file mode 100644 index 00000000..fb7339c2 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/scripts/build_final_report_pdf.py @@ -0,0 +1,739 @@ +#!/usr/bin/env python3 +"""Render docs/FINAL_REPORT.md as a submission-ready A4 technical report.""" + +from __future__ import annotations + +import argparse +import os +import re +import xml.etree.ElementTree as ET +from html import escape +from pathlib import Path + +from reportlab.graphics.shapes import ( + Circle, + Drawing, + Line, + Path as GraphicsPath, + Rect, + String, +) +from reportlab.lib import colors +from reportlab.lib.enums import TA_CENTER, TA_JUSTIFY, TA_LEFT +from reportlab.lib.pagesizes import A4 +from reportlab.lib.styles import ParagraphStyle, getSampleStyleSheet +from reportlab.lib.units import mm +from reportlab.pdfbase import pdfmetrics +from reportlab.pdfbase.ttfonts import TTFont +from reportlab.platypus import ( + BaseDocTemplate, + Flowable, + Frame, + PageTemplate, + Paragraph, + PageBreak, + Spacer, + Table, + TableStyle, +) + + +ROOT = Path(__file__).resolve().parents[1] +SOURCE = ROOT / "docs" / "FINAL_REPORT.md" +OUTPUT = ROOT / "output" / "pdf" / "WarpFusion_Technical_Report.pdf" + +NAVY = colors.HexColor("#142B4A") +BLUE = colors.HexColor("#275D8C") +LIGHT_BLUE = colors.HexColor("#EAF1F7") +LIGHT_GRAY = colors.HexColor("#F4F5F7") +MID_GRAY = colors.HexColor("#B8C0C8") +TEXT = colors.HexColor("#18212B") + + +def choose_font(explicit: str | None, environment_name: str, + candidates: list[str], label: str, + fallback: str | None = None) -> str: + requested = explicit or os.environ.get(environment_name) + if requested: + path = Path(requested).expanduser() + if not path.is_file(): + raise FileNotFoundError(f"{label} font does not exist: {path}") + return str(path) + for candidate in candidates: + if Path(candidate).is_file(): + return candidate + if fallback is not None: + return fallback + raise FileNotFoundError( + f"cannot find a {label} font; pass the corresponding --*-font option" + ) + + +def register_fonts(body_font: str | None = None, + heading_font: str | None = None, + light_font: str | None = None, + mono_font: str | None = None) -> dict[str, str]: + body = choose_font( + body_font, + "PURPLEPULSE_REPORT_BODY_FONT", + [ + "/System/Library/Fonts/Supplemental/Songti.ttc", + "/usr/share/fonts/opentype/noto/NotoSerifCJK-Regular.ttc", + "/usr/share/fonts/opentype/noto/NotoSansCJK-Regular.ttc", + "/usr/share/fonts/truetype/wqy/wqy-zenhei.ttc", + ], + "CJK body", + ) + heading = choose_font( + heading_font, + "PURPLEPULSE_REPORT_HEADING_FONT", + [ + "/System/Library/Fonts/STHeiti Medium.ttc", + "/usr/share/fonts/opentype/noto/NotoSansCJK-Bold.ttc", + "/usr/share/fonts/opentype/noto/NotoSansCJK-Regular.ttc", + ], + "CJK heading", + fallback=body, + ) + light = choose_font( + light_font, + "PURPLEPULSE_REPORT_LIGHT_FONT", + [ + "/System/Library/Fonts/STHeiti Light.ttc", + "/usr/share/fonts/opentype/noto/NotoSansCJK-Regular.ttc", + "/usr/share/fonts/truetype/wqy/wqy-zenhei.ttc", + ], + "CJK light", + fallback=body, + ) + mono = choose_font( + mono_font, + "PURPLEPULSE_REPORT_MONO_FONT", + [ + "/System/Library/Fonts/SFNSMono.ttf", + "/usr/share/fonts/truetype/dejavu/DejaVuSansMono.ttf", + "/usr/share/fonts/truetype/liberation2/LiberationMono-Regular.ttf", + ], + "monospace", + fallback=light, + ) + fonts = {"BodyCN": body, "HeadCN": heading, "LightCN": light, "Mono": mono} + # macOS CJK fonts are TrueType collections. Their first faces are not the + # simplified-Chinese regular faces: Songti index 0 is Black and the Heiti + # collections start with Traditional Chinese. Select the intended faces + # explicitly so simplified glyphs and body weight remain consistent. + collection_faces = { + ("BodyCN", "/System/Library/Fonts/Supplemental/Songti.ttc"): 6, + ("HeadCN", "/System/Library/Fonts/STHeiti Medium.ttc"): 1, + ("LightCN", "/System/Library/Fonts/STHeiti Light.ttc"): 1, + } + for name, path in fonts.items(): + subfont_index = collection_faces.get((name, str(Path(path))), 0) + pdfmetrics.registerFont(TTFont(name, path, subfontIndex=subfont_index)) + return fonts + + +def inline_markup(text: str) -> str: + placeholders: list[str] = [] + + # Keep punctuation portable across PDF renderers and submission systems. + text = text.replace("–", "-").replace("—", "-").replace("‑", "-") + + def save_code(match: re.Match[str]) -> str: + placeholders.append( + f'{escape(match.group(1))}' + ) + return f"@@CODE{len(placeholders) - 1}@@" + + text = re.sub(r"`([^`]+)`", save_code, text) + text = re.sub(r"\[([^\]]+)\]\([^\)]+\)", r"\1", text) + text = escape(text) + text = re.sub(r"\*\*([^*]+)\*\*", r"\1", text) + text = re.sub(r"\*([^*]+)\*", r"\1", text) + for index, value in enumerate(placeholders): + text = text.replace(f"@@CODE{index}@@", value) + return text + + +def equation_text(text: str) -> str: + replacements = { + r"\sum_j": "Σ_j", + r"\cdot": "·", + r"\qquad": " ", + r"\lVert": "||", + r"\rVert": "||", + r"\frac{q\cdot x}{\lVert q\rVert_2\lVert x\rVert_2}": + "(q · x) / (||q||₂ ||x||₂)", + } + for source, target in replacements.items(): + text = text.replace(source, target) + text = text.replace("_{L2}", "_L2").replace("_{IP}", "_IP") + text = text.replace("^2", "²") + text = text.replace("\\", "") + return text.strip() + + +class Rule(Flowable): + def __init__(self, width: float, color: colors.Color = BLUE) -> None: + super().__init__() + self.width = width + self.height = 1.2 + self.color = color + + def draw(self) -> None: + self.canv.setStrokeColor(self.color) + self.canv.setLineWidth(1.2) + self.canv.line(0, 0, self.width, 0) + + +class TechnicalReport(BaseDocTemplate): + def __init__(self, filename: str) -> None: + super().__init__( + filename, + pagesize=A4, + leftMargin=19 * mm, + rightMargin=19 * mm, + topMargin=19 * mm, + bottomMargin=18 * mm, + title="WarpFusion：面向大规模向量检索的 CUDA 并行设计与性能优化", + author="花生 (PurplePulse)", + subject="CUDA GPU 向量检索技术报告", + ) + frame = Frame( + self.leftMargin, + self.bottomMargin, + self.width, + self.height, + id="body", + leftPadding=0, + rightPadding=0, + topPadding=0, + bottomPadding=0, + ) + self.addPageTemplates(PageTemplate(id="main", frames=[frame], onPage=self.header_footer)) + + def header_footer(self, canvas, doc) -> None: # noqa: ANN001 + canvas.saveState() + if doc.page > 1: + canvas.setStrokeColor(MID_GRAY) + canvas.setLineWidth(0.35) + canvas.line(19 * mm, A4[1] - 13 * mm, A4[0] - 19 * mm, A4[1] - 13 * mm) + canvas.setFont("LightCN", 7.4) + canvas.setFillColor(colors.HexColor("#59636E")) + canvas.drawString(19 * mm, A4[1] - 10.5 * mm, "WarpFusion CUDA 向量检索技术报告") + canvas.drawRightString(A4[0] - 19 * mm, A4[1] - 10.5 * mm, "花生 · ID PurplePulse") + canvas.setFont("LightCN", 7.4) + canvas.setFillColor(colors.HexColor("#68727C")) + canvas.drawCentredString(A4[0] / 2, 9 * mm, str(doc.page)) + canvas.restoreState() + + def afterFlowable(self, flowable: Flowable) -> None: + if isinstance(flowable, Paragraph): + level = getattr(flowable, "heading_level", None) + if level in (1, 2): + key = f"section-{self.seq.nextf('section')}" + self.canv.bookmarkPage(key) + self.canv.addOutlineEntry(flowable.getPlainText(), key, level - 1, False) + + +def make_styles() -> dict[str, ParagraphStyle]: + sample = getSampleStyleSheet() + return { + "title": ParagraphStyle( + "ReportTitle", + parent=sample["Title"], + fontName="HeadCN", + fontSize=22, + leading=30, + textColor=NAVY, + alignment=TA_LEFT, + spaceAfter=8 * mm, + ), + "author": ParagraphStyle( + "Author", + fontName="LightCN", + fontSize=10.5, + leading=16, + textColor=colors.HexColor("#3D4955"), + spaceAfter=7 * mm, + ), + "h1": ParagraphStyle( + "Heading1CN", + fontName="HeadCN", + fontSize=14.2, + leading=20, + textColor=NAVY, + spaceBefore=7 * mm, + spaceAfter=3.2 * mm, + keepWithNext=True, + ), + "h2": ParagraphStyle( + "Heading2CN", + fontName="HeadCN", + fontSize=11.2, + leading=16, + textColor=BLUE, + spaceBefore=5 * mm, + spaceAfter=2.2 * mm, + keepWithNext=True, + ), + "references_heading": ParagraphStyle( + "ReferencesHeading", + fontName="HeadCN", + fontSize=12.2, + leading=17, + textColor=NAVY, + spaceBefore=3 * mm, + spaceAfter=1.8 * mm, + keepWithNext=True, + ), + "body": ParagraphStyle( + "BodyCN", + fontName="BodyCN", + fontSize=9.1, + leading=14.2, + textColor=TEXT, + alignment=TA_JUSTIFY, + firstLineIndent=2 * 9.1, + spaceAfter=2.1 * mm, + wordWrap="CJK", + ), + "abstract": ParagraphStyle( + "AbstractCN", + fontName="BodyCN", + fontSize=8.9, + leading=13.8, + textColor=TEXT, + alignment=TA_JUSTIFY, + firstLineIndent=2 * 8.9, + spaceAfter=2 * mm, + leftIndent=5 * mm, + rightIndent=5 * mm, + wordWrap="CJK", + ), + "list": ParagraphStyle( + "ListCN", + fontName="BodyCN", + fontSize=9.1, + leading=14.2, + textColor=TEXT, + alignment=TA_JUSTIFY, + leftIndent=7 * mm, + firstLineIndent=-4.5 * mm, + spaceAfter=1.5 * mm, + wordWrap="CJK", + ), + "references": ParagraphStyle( + "References", + fontName="BodyCN", + fontSize=8.0, + leading=10.8, + textColor=TEXT, + alignment=TA_LEFT, + leftIndent=5 * mm, + firstLineIndent=-3.5 * mm, + spaceAfter=0.8 * mm, + ), + "caption": ParagraphStyle( + "CaptionCN", + fontName="LightCN", + fontSize=7.8, + leading=11, + textColor=colors.HexColor("#4E5965"), + alignment=TA_CENTER, + spaceBefore=1.5 * mm, + spaceAfter=3 * mm, + wordWrap="CJK", + ), + "code": ParagraphStyle( + "CodeCN", + fontName="LightCN", + fontSize=7.7, + leading=11.2, + leftIndent=4 * mm, + rightIndent=4 * mm, + borderColor=MID_GRAY, + borderWidth=0.45, + borderPadding=4, + backColor=LIGHT_GRAY, + spaceBefore=1.5 * mm, + spaceAfter=3 * mm, + wordWrap="CJK", + ), + "equation": ParagraphStyle( + "Equation", + fontName="BodyCN", + fontSize=9.4, + leading=14, + textColor=TEXT, + alignment=TA_CENTER, + spaceBefore=1.5 * mm, + spaceAfter=2.5 * mm, + ), + "table": ParagraphStyle( + "TableCell", + fontName="BodyCN", + fontSize=7.1, + leading=9.7, + textColor=TEXT, + alignment=TA_LEFT, + wordWrap="CJK", + ), + "table_head": ParagraphStyle( + "TableHead", + fontName="HeadCN", + fontSize=7.1, + leading=9.7, + textColor=colors.white, + alignment=TA_CENTER, + wordWrap="CJK", + ), + } + + +def build_table(rows: list[list[str]], width: float, styles: dict[str, ParagraphStyle]) -> Table: + column_count = max(len(row) for row in rows) + normalized = [row + [""] * (column_count - len(row)) for row in rows] + data = [] + for row_index, row in enumerate(normalized): + style = styles["table_head"] if row_index == 0 else styles["table"] + data.append([Paragraph(inline_markup(cell.strip()), style) for cell in row]) + weights = [1.0] * column_count + if column_count >= 3: + weights[-1] = 1.35 + total = sum(weights) + column_widths = [width * value / total for value in weights] + table = Table(data, colWidths=column_widths, repeatRows=1, hAlign="CENTER") + table.setStyle( + TableStyle( + [ + ("BACKGROUND", (0, 0), (-1, 0), NAVY), + ("TEXTCOLOR", (0, 0), (-1, 0), colors.white), + ("GRID", (0, 0), (-1, -1), 0.35, MID_GRAY), + ("VALIGN", (0, 0), (-1, -1), "MIDDLE"), + ("ALIGN", (0, 0), (-1, 0), "CENTER"), + ("LEFTPADDING", (0, 0), (-1, -1), 3.5), + ("RIGHTPADDING", (0, 0), (-1, -1), 3.5), + ("TOPPADDING", (0, 0), (-1, -1), 3.2), + ("BOTTOMPADDING", (0, 0), (-1, -1), 3.2), + ] + + [ + ("BACKGROUND", (0, row), (-1, row), LIGHT_BLUE) + for row in range(2, len(data), 2) + ] + ) + ) + return table + + +def simple_svg_drawing(path: Path) -> Drawing: + """Convert the chart SVG subset used by this report into ReportLab vectors.""" + root = ET.parse(path).getroot() + width = float(root.attrib["width"].removesuffix("px")) + height = float(root.attrib["height"].removesuffix("px")) + drawing = Drawing(width, height) + text_styles = { + "tick": (16, "LightCN", TEXT), + "label": (18, "HeadCN", TEXT), + "panel-title": (21, "HeadCN", TEXT), + "figure-title": (26, "HeadCN", TEXT), + "note": (15, "LightCN", colors.HexColor("#5F6368")), + } + line_styles = { + "grid": (colors.HexColor("#D9DDE3"), 1.0), + "axis": (colors.HexColor("#51565C"), 1.2), + } + + def number(value: str | None, default: float = 0.0) -> float: + return float(value) if value is not None else default + + for element in root: + tag = element.tag.rsplit("}", 1)[-1] + attrs = element.attrib + css_class = attrs.get("class", "") + if tag in {"title", "desc", "style"}: + continue + if tag == "rect": + rect_width = width if attrs.get("width") == "100%" else number(attrs.get("width")) + rect_height = height if attrs.get("height") == "100%" else number(attrs.get("height")) + x = number(attrs.get("x")) + y = height - number(attrs.get("y")) - rect_height + drawing.add( + Rect( + x, + y, + rect_width, + rect_height, + fillColor=colors.HexColor(attrs.get("fill", "#FFFFFF")), + strokeColor=None, + ) + ) + continue + if tag == "line": + stroke, stroke_width = line_styles.get( + css_class, + ( + colors.HexColor(attrs.get("stroke", "#000000")), + number(attrs.get("stroke-width"), 1.0), + ), + ) + drawing.add( + Line( + number(attrs.get("x1")), + height - number(attrs.get("y1")), + number(attrs.get("x2")), + height - number(attrs.get("y2")), + strokeColor=stroke, + strokeWidth=stroke_width, + ) + ) + continue + if tag == "circle": + drawing.add( + Circle( + number(attrs.get("cx")), + height - number(attrs.get("cy")), + number(attrs.get("r")), + fillColor=colors.HexColor(attrs.get("fill", "#000000")), + strokeColor=colors.HexColor(attrs.get("stroke", "#000000")), + strokeWidth=number(attrs.get("stroke-width"), 1.0), + ) + ) + continue + if tag == "path": + tokens = re.findall(r"[ML]|-?\d+(?:\.\d+)?", attrs.get("d", "")) + graphic_path = GraphicsPath() + cursor = 0 + command = "M" + while cursor < len(tokens): + if tokens[cursor] in {"M", "L"}: + command = tokens[cursor] + cursor += 1 + x = float(tokens[cursor]) + y = height - float(tokens[cursor + 1]) + cursor += 2 + if command == "M": + graphic_path.moveTo(x, y) + command = "L" + else: + graphic_path.lineTo(x, y) + graphic_path.fillColor = None + graphic_path.strokeColor = colors.HexColor(attrs.get("stroke", "#000000")) + graphic_path.strokeWidth = number(attrs.get("stroke-width"), 1.0) + drawing.add(graphic_path) + continue + if tag == "text": + font_size, font_name, fill = text_styles.get(css_class, (11, "LightCN", TEXT)) + angle = 90 if "rotate(-90" in attrs.get("transform", "") else 0 + value = "".join(element.itertext()).replace("–", "-").replace("—", "-") + drawing.add( + String( + number(attrs.get("x")), + height - number(attrs.get("y")), + value, + fontName=font_name, + fontSize=font_size, + fillColor=fill, + textAnchor=attrs.get("text-anchor", "start"), + angle=angle, + ) + ) + return drawing + + +def add_svg(story: list[Flowable], path: Path, max_width: float, max_height: float) -> bool: + if path.name.startswith("report-"): + drawing = simple_svg_drawing(path) + else: + try: + from svglib.svglib import svg2rlg + except ImportError: + drawing = simple_svg_drawing(path) + else: + drawing = svg2rlg(str(path)) + if drawing is None: + return False + scale = min(max_width / drawing.width, max_height / drawing.height, 1.0) + drawing.width *= scale + drawing.height *= scale + drawing.scale(scale, scale) + drawing.hAlign = "CENTER" + story.append(Spacer(1, 2 * mm)) + story.append(drawing) + return True + + +def parse_markdown(text: str, doc: TechnicalReport, styles: dict[str, ParagraphStyle]) -> list[Flowable]: + lines = text.splitlines() + story: list[Flowable] = [] + story.append(Spacer(1, 11 * mm)) + story.append(Paragraph(inline_markup(lines[0].removeprefix("# ")), styles["title"])) + story.append(Rule(doc.width)) + story.append(Spacer(1, 5 * mm)) + story.append(Paragraph("作者：花生　　学员 ID：PurplePulse", styles["author"])) + + index = 2 + paragraph_lines: list[str] = [] + in_code = False + code_lines: list[str] = [] + in_equation = False + equation_lines: list[str] = [] + abstract_mode = False + references_mode = False + + def flush_paragraph() -> None: + if not paragraph_lines: + return + value = " ".join(part.strip() for part in paragraph_lines) + style = styles["abstract"] if abstract_mode else styles["body"] + story.append(Paragraph(inline_markup(value), style)) + paragraph_lines.clear() + + while index < len(lines): + line = lines[index] + stripped = line.strip() + + if stripped.startswith("```"): + flush_paragraph() + if in_code: + story.append( + Paragraph("
".join(escape(item) for item in code_lines), styles["code"]) + ) + code_lines.clear() + in_code = False + else: + in_code = True + index += 1 + continue + if in_code: + code_lines.append(line) + index += 1 + continue + + if stripped == "$$": + flush_paragraph() + if in_equation: + rendered = "
".join( + escape(equation_text(item)) for item in equation_lines if item.strip() + ) + story.append(Paragraph(rendered, styles["equation"])) + equation_lines.clear() + in_equation = False + else: + in_equation = True + index += 1 + continue + if in_equation: + equation_lines.append(line) + index += 1 + continue + + if stripped.startswith("!["): + flush_paragraph() + match = re.match(r"!\[([^\]]*)\]\(([^)]+)\)", stripped) + if match: + image_path = (SOURCE.parent / match.group(2)).resolve() + add_svg(story, image_path, doc.width * 0.88, 82 * mm) + index += 1 + continue + + if stripped.startswith("**图"): + flush_paragraph() + story.append(Paragraph(inline_markup(stripped), styles["caption"])) + index += 1 + continue + + if stripped.startswith("## "): + flush_paragraph() + title = stripped[3:].strip() + abstract_mode = title == "摘要" + references_mode = title == "参考文献" + heading_style = styles["references_heading"] if references_mode else styles["h1"] + paragraph = Paragraph(inline_markup(title), heading_style) + paragraph.heading_level = 1 + story.append(paragraph) + index += 1 + continue + + if stripped.startswith("### "): + flush_paragraph() + abstract_mode = False + paragraph = Paragraph(inline_markup(stripped[4:].strip()), styles["h2"]) + paragraph.heading_level = 2 + story.append(paragraph) + index += 1 + continue + + if stripped.startswith("|") and index + 1 < len(lines) and re.match( + r"^\s*\|?\s*:?-+", lines[index + 1] + ): + flush_paragraph() + rows: list[list[str]] = [] + rows.append([cell for cell in stripped.strip("|").split("|")]) + index += 2 + while index < len(lines) and lines[index].strip().startswith("|"): + rows.append([cell for cell in lines[index].strip().strip("|").split("|")]) + index += 1 + story.append(Spacer(1, 1.5 * mm)) + story.append(build_table(rows, doc.width, styles)) + story.append(Spacer(1, 2.8 * mm)) + continue + + list_match = re.match(r"^(\d+)\.\s+(.*)", stripped) + if list_match: + flush_paragraph() + item_parts = [list_match.group(2)] + item_number = list_match.group(1) + index += 1 + while index < len(lines) and lines[index].startswith(" "): + continuation = lines[index].strip() + if continuation: + item_parts.append(continuation) + index += 1 + story.append( + Paragraph( + f"{item_number}.　{inline_markup(' '.join(item_parts))}", + styles["references"] if references_mode else styles["list"], + ) + ) + continue + + if not stripped: + flush_paragraph() + elif index == 2 and stripped.startswith("**作者"): + pass + else: + paragraph_lines.append(stripped) + index += 1 + + flush_paragraph() + return story + + +def main() -> None: + global SOURCE, OUTPUT + parser = argparse.ArgumentParser() + parser.add_argument("--source", type=Path, default=SOURCE) + parser.add_argument("--output", type=Path, default=OUTPUT) + parser.add_argument("--body-font") + parser.add_argument("--heading-font") + parser.add_argument("--light-font") + parser.add_argument("--mono-font") + args = parser.parse_args() + SOURCE = args.source.resolve() + OUTPUT = args.output.resolve() + from build_report_figures import generate_report_figures + + generate_report_figures() + register_fonts(args.body_font, args.heading_font, args.light_font, + args.mono_font) + OUTPUT.parent.mkdir(parents=True, exist_ok=True) + doc = TechnicalReport(str(OUTPUT)) + styles = make_styles() + story = parse_markdown(SOURCE.read_text(encoding="utf-8"), doc, styles) + doc.build(story) + print(OUTPUT) + + +if __name__ == "__main__": + main() diff --git a/09_vector_retrieval/PurplePulse/scripts/build_report_figures.py b/09_vector_retrieval/PurplePulse/scripts/build_report_figures.py new file mode 100644 index 00000000..3042ef65 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/scripts/build_report_figures.py @@ -0,0 +1,389 @@ +#!/usr/bin/env python3 +"""Generate the Chinese figures used by the final technical report.""" + +from __future__ import annotations + +import csv +import html +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] +OUT = ROOT / "results" / "figures" +WIDTH = 1120 +HEIGHT = 520 + +NAVY = "#142B4A" +BLUE = "#2878B5" +ORANGE = "#D95F02" +GREEN = "#2A9D8F" +GOLD = "#E9C46A" +PALE = "#EAF1F7" +GRID = "#D9DDE3" +TEXT = "#18212B" +MUTED = "#65717D" + + +def esc(value: object) -> str: + return html.escape(str(value), quote=True) + + +def attrs(values: dict[str, object]) -> str: + return " ".join( + f'{key.rstrip("_").replace("_", "-")}="{esc(value)}"' + for key, value in values.items() + ) + + +def text(x: float, y: float, value: object, css: str = "label", **extra: object) -> str: + return f'{esc(value)}' + + +def line(x1: float, y1: float, x2: float, y2: float, css: str = "grid", **extra: object) -> str: + return ( + f'' + ) + + +def rect(x: float, y: float, width: float, height: float, fill: str, **extra: object) -> str: + return ( + f'' + ) + + +def circle(x: float, y: float, radius: float, fill: str) -> str: + return ( + f'' + ) + + +def path(points: list[tuple[float, float]], color: str) -> str: + commands = " ".join( + f'{"M" if index == 0 else "L"} {x:.2f} {y:.2f}' + for index, (x, y) in enumerate(points) + ) + return f'' + + +def svg(title: str, body: list[str], width: int = WIDTH, height: int = HEIGHT) -> str: + return "\n".join( + [ + f'', + f'{esc(title)}', + '图中数据由仓库内原始 CSV 自动生成。', + '', + '', + *body, + '', + '', + ] + ) + + +def read_csv(path: Path) -> list[dict[str, str]]: + with path.open(newline="", encoding="utf-8") as source: + rows = list(csv.DictReader(source)) + if not rows: + raise ValueError(f"empty CSV: {path}") + return rows + + +def integer_label(value: float | int) -> str: + """Format chart counts consistently with report tables.""" + return f"{round(float(value)):,}" + + +def write(name: str, contents: str) -> Path: + OUT.mkdir(parents=True, exist_ok=True) + target = OUT / name + target.write_text(contents, encoding="utf-8") + return target + + +def architecture_figure() -> str: + body = [text(40, 42, "WarpFusion 向量检索系统的组成与数据流", "figure-title")] + panels = [ + (36, 210, "输入", ["向量库与查询", "检索参数", "公开数据校验和"]), + (278, 248, "CPU 端准备", ["格式与参数检查", "精确结果参考", "构建并保存 IVF 索引"]), + (558, 300, "GPU 常驻检索", ["Exact：边算距离边选 Top-K", "IVF：选桶、扫描、合并", "每批只传查询与结果"]), + (890, 194, "输出证据", ["ID 与分数", "召回率与延迟", "CSV、图表、报告"]), + ] + for x, width, heading, items in panels: + body.append(rect(x, 78, width, 354, PALE if heading == "GPU 常驻检索" else "#F4F5F7")) + body.append(text(x + 20, 112, heading, "panel-title")) + for index, item in enumerate(items): + y = 142 + index * 86 + body.append(rect(x + 20, y, width - 40, 62, "#FFFFFF")) + body.append(text(x + width / 2, y + 37, item, "label", text_anchor="middle")) + for x1, x2 in ((246, 278), (526, 558), (858, 890)): + body.append(line(x1, 255, x2, 255, "axis", stroke=BLUE, stroke_width=2.4)) + body.append(text((x1 + x2) / 2, 245, "→", "label", text_anchor="middle")) + body.append(line(55, 463, 1065, 463, "axis", stroke=GRID, stroke_width=1.2)) + body.append(text(WIDTH / 2, 494, "每一次优化都要同时通过正确性检查和同输入 A/B 对照", "note", text_anchor="middle")) + return svg("WarpFusion 系统流程", body) + + +def exact_fused_figure() -> str: + rows = read_csv(ROOT / "results/exact_fused/million_compare.csv") + values = {(int(row["top_k"]), row["mode"]): float(row["qps"]) for row in rows} + body = [text(WIDTH / 2, 34, "Exact 融合路径与距离矩阵基线的吞吐对比", "figure-title", text_anchor="middle")] + left, right, top, bottom = 95, 1070, 78, 420 + maximum = 6000.0 + for tick in range(0, 6001, 1000): + y = bottom - tick / maximum * (bottom - top) + body.append(line(left, y, right, y)) + body.append(text(left - 12, y + 4, integer_label(tick), "tick", text_anchor="end")) + body.append(line(left, top, left, bottom, "axis", stroke=MUTED, stroke_width=1.2)) + centers = [260, 560, 860] + bar_width = 82 + for center, top_k in zip(centers, (10, 50, 100)): + matrix = values[(top_k, "matrix")] + fused = values[(top_k, "fused")] + for offset, value, color, label in ((-52, matrix, MUTED, "距离矩阵"), (52, fused, BLUE, "融合路径")): + height = value / maximum * (bottom - top) + body.append(rect(center + offset - bar_width / 2, bottom - height, bar_width, height, color)) + body.append(text(center + offset, bottom - height - 10, integer_label(value), "note", text_anchor="middle")) + body.append(text(center, 452, f"K={top_k}", "label", text_anchor="middle")) + body.append(text(center, 482, f"加速比 {fused / matrix:.2f}×", "note", text_anchor="middle")) + body.extend([ + rect(770, 48, 18, 12, MUTED), text(797, 60, "先写距离矩阵", "note"), + rect(925, 48, 18, 12, BLUE), text(952, 60, "边计算边选 Top-K", "note"), + ]) + return svg("Exact 融合路径吞吐", body) + + +def exact_breakdown_figure() -> str: + datasets = [ + ("百万条 128 维合成数据", ROOT / "results/stage7_exact_profile/synthetic1m/breakdown.csv"), + ("67.7 万条 384 维文本向量", ROOT / "results/stage7_exact_profile/yahoo-minilm-384/breakdown.csv"), + ] + body = [text(WIDTH / 2, 34, "Exact 融合路径的 GPU 执行时间分解", "figure-title", text_anchor="middle")] + colors = [BLUE, GOLD, ORANGE] + labels = ["距离与向量读取", "局部 Top-K", "最终合并"] + for index, (label, _) in enumerate(zip(labels, colors)): + x = 370 + index * 180 + body.append(rect(x, 49, 16, 11, colors[index])) + body.append(text(x + 24, 60, label, "note")) + for panel_index, (dataset_name, source) in enumerate(datasets): + rows = read_csv(source) + x0 = 60 + panel_index * 550 + body.append(text(x0 + 250, 94, dataset_name, "panel-title", text_anchor="middle")) + for bar_index, row in enumerate(rows): + x = x0 + 90 + bar_index * 145 + y = 400 + shares = [ + float(row["distance_share"]), + float(row["local_topk_share"]), + float(row["final_merge_share"]), + ] + for share, color in zip(shares, colors): + height = share * 270 + body.append(rect(x, y - height, 72, height, color)) + y -= height + body.append(text(x + 36, 427, f"K={row['top_k']}", "label", text_anchor="middle")) + body.append(text(x + 36, 118, f"{shares[0] * 100:.1f}%", "note", text_anchor="middle")) + body.append(line(x0 + 50, 400, x0 + 500, 400, "axis", stroke=MUTED, stroke_width=1.2)) + body.append(text(WIDTH / 2, 482, "柱顶数字表示“距离计算与向量读取”占 GPU 核函数流水的比例", "note", text_anchor="middle")) + return svg("Exact 耗时分解", body) + + +def k50_scheduling_figure() -> str: + rows = read_csv(ROOT / "results/formal_4090d_20260908/k50_bucket_major_sweep.csv") + body = [text(WIDTH / 2, 34, "K=50 条件下按桶任务调度的性能影响", "figure-title", text_anchor="middle")] + left, right, top, bottom = 95, 1070, 80, 410 + x_values = [float(row["nprobe"]) for row in rows] + all_qps = [float(row[key]) for row in rows for key in ("probe_major_qps", "bucket_major_qps")] + y_min, y_max = 0.0, max(all_qps) * 1.08 + for tick in range(0, 25001, 5000): + y = bottom - (tick - y_min) / (y_max - y_min) * (bottom - top) + body.append(line(left, y, right, y)) + body.append(text(left - 10, y + 4, integer_label(tick), "tick", text_anchor="end")) + def px(value: float) -> float: + return left + (value - min(x_values)) / (max(x_values) - min(x_values)) * (right - left) + def py(value: float) -> float: + return bottom - (value - y_min) / (y_max - y_min) * (bottom - top) + for value in x_values: + x = px(value) + body.append(line(x, top, x, bottom)) + body.append(text(x, 434, int(value), "tick", text_anchor="middle")) + for key, color, label in (("probe_major_qps", MUTED, "按查询顺序"), ("bucket_major_qps", BLUE, "按桶重排")): + points = [(px(float(row["nprobe"])), py(float(row[key]))) for row in rows] + body.append(path(points, color)) + for x, y in points: + body.append(circle(x, y, 5, color)) + body.append(line(770 if key == "probe_major_qps" else 920, 55, 795 if key == "probe_major_qps" else 945, 55, "axis", stroke=color, stroke_width=3)) + body.append(text(802 if key == "probe_major_qps" else 952, 60, label, "note")) + last = rows[-1] + body.append(text(px(float(last["nprobe"])) - 5, py(float(last["bucket_major_qps"])) - 14, f"+{float(last['speedup_percent']):.1f}%", "label", text_anchor="end")) + body.append(text((left + right) / 2, 474, "搜索桶数 nprobe", "label", text_anchor="middle")) + return svg("K=50 任务调度", body) + + +def k100_partitioned_figure() -> str: + full = read_csv(ROOT / "results/real/sift1m/ivf_k100_full_heap.csv") + partitioned = read_csv(ROOT / "results/real/sift1m/ivf_k100_partitioned.csv") + body = [text(WIDTH / 2, 34, "K=100 条件下分区候选结构的性能影响", "figure-title", text_anchor="middle")] + left, right, top, bottom = 95, 1070, 82, 410 + x_values = [float(row["nprobe"]) for row in full] + maximum = max(float(row["qps"]) for row in partitioned) * 1.08 + for tick in range(0, 50001, 10000): + y = bottom - tick / maximum * (bottom - top) + body.append(line(left, y, right, y)) + body.append(text(left - 10, y + 4, integer_label(tick), "tick", text_anchor="end")) + def px(value: float) -> float: + return left + (value - min(x_values)) / (max(x_values) - min(x_values)) * (right - left) + def py(value: float) -> float: + return bottom - value / maximum * (bottom - top) + for value in x_values: + x = px(value) + body.append(line(x, top, x, bottom)) + body.append(text(x, 434, int(value), "tick", text_anchor="middle")) + for rows, color, label in ((full, MUTED, "完整大候选表"), (partitioned, GREEN, "4 个小候选表")): + points = [(px(float(row["nprobe"])), py(float(row["qps"]))) for row in rows] + body.append(path(points, color)) + for x, y in points: + body.append(circle(x, y, 5, color)) + base_x = 780 if rows is full else 940 + body.append(line(base_x, 56, base_x + 25, 56, "axis", stroke=color, stroke_width=3)) + body.append(text(base_x + 32, 61, label, "note")) + row160_full = next(row for row in full if int(row["nprobe"]) == 160) + row160_part = next(row for row in partitioned if int(row["nprobe"]) == 160) + ratio = float(row160_part["qps"]) / float(row160_full["qps"]) + body.append(text(px(160), py(float(row160_part["qps"])) - 16, f"Recall@100 相同；加速比 {ratio:.2f}×", "label", text_anchor="middle")) + body.append(text((left + right) / 2, 474, "搜索桶数 nprobe", "label", text_anchor="middle")) + return svg("K=100 分区候选", body) + + +def real_comparison_figure() -> str: + datasets = [("SIFT1M（图像特征）", "sift1m"), ("Yahoo MiniLM（文本向量）", "yahoo-minilm-384")] + body = [text(WIDTH / 2, 34, "公开数据集上的端到端吞吐对比（nprobe=160）", "figure-title", text_anchor="middle")] + for panel_index, (title, directory) in enumerate(datasets): + root = ROOT / "results/real" / directory + faiss = read_csv(root / "faiss_gpu.csv") + x0 = 55 + panel_index * 550 + body.append(text(x0 + 250, 80, title, "panel-title", text_anchor="middle")) + warp_values = [] + faiss_values = [] + recalls = [] + for top_k in (10, 50, 100): + warp = next(row for row in read_csv(root / f"ivf_k{top_k}.csv") if int(row["nprobe"]) == 160) + baseline = next( + row for row in faiss + if row["index_type"] == "ivf_flat" and int(row["top_k"]) == top_k and int(row["nprobe"]) == 160 + ) + warp_values.append(float(warp["qps"])) + faiss_values.append(float(baseline["qps"])) + recalls.append((float(warp["recall_at_k"]), float(baseline["recall_at_k"]))) + maximum = max(warp_values + faiss_values) * 1.15 + panel_bottom, panel_top = 405, 112 + for group, top_k in enumerate((10, 50, 100)): + center = x0 + 95 + group * 155 + for offset, value, color in ((-27, warp_values[group], BLUE), (27, faiss_values[group], ORANGE)): + height = value / maximum * (panel_bottom - panel_top) + body.append(rect(center + offset - 22, panel_bottom - height, 44, height, color)) + ratio = warp_values[group] / faiss_values[group] + body.append(text(center, min(panel_bottom - warp_values[group] / maximum * (panel_bottom - panel_top), panel_bottom - faiss_values[group] / maximum * (panel_bottom - panel_top)) - 10, f"{ratio:.2f}×", "note", text_anchor="middle")) + body.append(text(center, 432, f"K={top_k}", "label", text_anchor="middle")) + body.append(text(center, 458, f"召回 {recalls[group][0]:.4f}/{recalls[group][1]:.4f}", "note", text_anchor="middle")) + body.append(line(x0 + 35, panel_bottom, x0 + 505, panel_bottom, "axis", stroke=MUTED, stroke_width=1.2)) + body.extend([ + rect(780, 49, 16, 11, BLUE), text(804, 60, "WarpFusion", "note"), + rect(920, 49, 16, 11, ORANGE), text(944, 60, "FAISS GPU", "note"), + text(WIDTH / 2, 496, "召回数字依次为 WarpFusion / FAISS；两者各自训练聚类中心", "note", text_anchor="middle"), + ]) + return svg("公开数据对比", body) + + +def recall_qps_figure() -> str: + datasets = [ + ("SIFT1M（图像特征）", "sift1m"), + ("Yahoo MiniLM（文本向量）", "yahoo-minilm-384"), + ] + body = [ + text( + WIDTH / 2, + 34, + "公开数据集上的召回率与吞吐权衡（K=10）", + "figure-title", + text_anchor="middle", + ) + ] + panel_top, panel_bottom = 98, 402 + panel_width = 455 + for panel_index, (title, directory) in enumerate(datasets): + root = ROOT / "results" / "real" / directory + warp = read_csv(root / "ivf_k10.csv") + faiss = [ + row + for row in read_csv(root / "faiss_gpu.csv") + if row["index_type"] == "ivf_flat" and int(row["top_k"]) == 10 + ] + series = ((warp, BLUE, "WarpFusion"), (faiss, ORANGE, "FAISS GPU")) + recalls = [float(row["recall_at_k"]) for rows, _, _ in series for row in rows] + qps_values = [float(row["qps"]) for rows, _, _ in series for row in rows] + x_min = max(0.0, min(recalls) - 0.003) + x_max = min(1.0, max(recalls) + 0.001) + y_max = max(qps_values) * 1.08 + left = 78 + panel_index * 550 + right = left + panel_width + + body.append(text((left + right) / 2, 76, title, "panel-title", text_anchor="middle")) + for tick_index in range(5): + fraction = tick_index / 4 + x = left + fraction * panel_width + recall = x_min + fraction * (x_max - x_min) + body.append(line(x, panel_top, x, panel_bottom)) + body.append(text(x, 425, f"{recall * 100:.1f}%", "tick", text_anchor="middle")) + qps_tick = y_max * fraction + y = panel_bottom - fraction * (panel_bottom - panel_top) + body.append(line(left, y, right, y)) + body.append(text(left - 9, y + 4, integer_label(qps_tick), "tick", text_anchor="end")) + + def px(value: float) -> float: + return left + (value - x_min) / (x_max - x_min) * panel_width + + def py(value: float) -> float: + return panel_bottom - value / y_max * (panel_bottom - panel_top) + + for rows, color, _ in series: + points = [(px(float(row["recall_at_k"])), py(float(row["qps"]))) for row in rows] + body.append(path(points, color)) + for x, y in points: + body.append(circle(x, y, 4.5, color)) + body.append(text(left + 8, panel_top + 18, "nprobe=32", "note")) + body.append(text(right - 8, panel_bottom - 10, "nprobe=224", "note", text_anchor="end")) + body.append(line(left, panel_top, left, panel_bottom, "axis", stroke=MUTED, stroke_width=1.2)) + body.append(line(left, panel_bottom, right, panel_bottom, "axis", stroke=MUTED, stroke_width=1.2)) + + body.extend( + [ + line(770, 52, 795, 52, "axis", stroke=BLUE, stroke_width=3), + text(802, 57, "WarpFusion", "note"), + line(920, 52, 945, 52, "axis", stroke=ORANGE, stroke_width=3), + text(952, 57, "FAISS GPU", "note"), + text(WIDTH / 2, 458, "横轴为 Recall@10，纵轴为 QPS；每条曲线包含 nprobe=32、64、128、160、192、224", "note", text_anchor="middle"), + text(WIDTH / 2, 486, "越靠近右上方表示在相同召回率附近具有更高吞吐", "note", text_anchor="middle"), + ] + ) + return svg("召回率与吞吐权衡", body) + + +def generate_report_figures() -> list[Path]: + figures = { + "report-architecture.svg": architecture_figure(), + "report-exact-fused.svg": exact_fused_figure(), + "report-exact-breakdown.svg": exact_breakdown_figure(), + "report-k50-scheduling.svg": k50_scheduling_figure(), + "report-k100-partitioned.svg": k100_partitioned_figure(), + "report-recall-qps.svg": recall_qps_figure(), + "report-real-comparison.svg": real_comparison_figure(), + } + return [write(name, contents) for name, contents in figures.items()] + + +if __name__ == "__main__": + for generated in generate_report_figures(): + print(generated) diff --git a/09_vector_retrieval/PurplePulse/scripts/evaluate_ivecs_groundtruth.py b/09_vector_retrieval/PurplePulse/scripts/evaluate_ivecs_groundtruth.py new file mode 100644 index 00000000..d69b0da7 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/scripts/evaluate_ivecs_groundtruth.py @@ -0,0 +1,241 @@ +#!/usr/bin/env python3 +"""Evaluate WarpFusion text results against TexMex ivecs ground truth.""" + +from __future__ import annotations + +import argparse +import json +import struct +from pathlib import Path + +import numpy as np + +from import_sift1m import inspect_vecs, record_dtype + + +def read_result_rows(path: Path) -> tuple[list[list[int]], list[list[float]]]: + rows: dict[int, list[int]] = {} + scores: dict[int, list[float]] = {} + for line_number, line in enumerate(path.read_text().splitlines(), 1): + parts = line.split() + if len(parts) != 3: + raise ValueError(f"{path}:{line_number}: expected three columns") + query_id = int(parts[0]) + vector_id = int(parts[1]) + score = float(parts[2]) + if query_id < 0 or vector_id < 0: + raise ValueError(f"{path}:{line_number}: IDs must be non-negative") + rows.setdefault(query_id, []).append(vector_id) + scores.setdefault(query_id, []).append(score) + if not rows or sorted(rows) != list(range(len(rows))): + raise ValueError("result query IDs must be contiguous and start at zero") + top_k = len(rows[0]) + if top_k == 0 or any(len(values) != top_k for values in rows.values()): + raise ValueError("all result queries must have the same positive K") + return ( + [rows[index] for index in range(len(rows))], + [scores[index] for index in range(len(rows))], + ) + + +def read_result_ids(path: Path) -> list[list[int]]: + return read_result_rows(path)[0] + + +def read_vectors(path: Path, database: bool): + with path.open("rb") as source: + magic = source.read(8) + expected_magic = b"PPVEC001" if database else b"PPQRY001" + if magic != expected_magic: + raise ValueError(f"{path}: invalid vector file magic") + if database: + rows, dim, dtype_code, metric_code = struct.unpack(" None: + database, metric = read_vectors(database_path, database=True) + queries, _ = read_vectors(query_path, database=False) + if metric is None or len(queries) < len(actual_ids) or database.shape[1] != queries.shape[1]: + raise ValueError("database/query shape or metric mismatch") + equivalent_queries = 0 + rank_errors = [] + reported_errors = [] + descending = metric in {2, 3} + for query_id in range(len(actual_ids)): + expected_scores = vector_scores( + database, queries[query_id], reference[query_id], metric + ) + recomputed_scores = vector_scores( + database, queries[query_id], actual_ids[query_id], metric + ) + expected_scores = np.sort(expected_scores) + recomputed_scores = np.sort(recomputed_scores) + if descending: + expected_scores = expected_scores[::-1] + recomputed_scores = recomputed_scores[::-1] + errors = np.abs( + expected_scores.astype(np.float64) - recomputed_scores.astype(np.float64) + ) + rank_errors.extend(errors.tolist()) + if np.allclose(expected_scores, recomputed_scores, rtol=rtol, atol=atol): + equivalent_queries += 1 + reported_errors.extend( + np.abs( + actual_reported_scores[query_id].astype(np.float64) + - vector_scores( + database, queries[query_id], actual_ids[query_id], metric + ).astype(np.float64) + ).tolist() + ) + result.update( + { + "metric": {1: "l2", 2: "inner_product", 3: "cosine"}[metric], + "score_equivalent_query_fraction": equivalent_queries / len(actual_ids), + "mean_rank_score_error": float(np.mean(rank_errors)), + "max_rank_score_error": float(np.max(rank_errors)), + "mean_reported_score_error": float(np.mean(reported_errors)), + "max_reported_score_error": float(np.max(reported_errors)), + "score_rtol": rtol, + "score_atol": atol, + } + ) + del database + del queries + + +def evaluate( + groundtruth_path: Path, + result_path: Path, + database_path: Path | None = None, + query_path: Path | None = None, + score_rtol: float = 1e-5, + score_atol: float = 1e-4, +) -> dict[str, float | int | str]: + actual, actual_scores = read_result_rows(result_path) + info = inspect_vecs(groundtruth_path, "ivecs") + if len(actual) > info.rows: + raise ValueError("results contain more queries than ground truth") + top_k = len(actual[0]) + if top_k > info.dim: + raise ValueError(f"result K={top_k} exceeds ground truth K={info.dim}") + records = np.memmap( + groundtruth_path, + dtype=record_dtype(info.dim, "ivecs"), + mode="r", + shape=(info.rows,), + ) + reference = np.asarray(records["values"][: len(actual), :top_k]) + actual_array = np.asarray(actual, dtype=np.int64) + per_query = np.asarray( + [ + len(set(expected.tolist()) & set(found.tolist())) / top_k + for expected, found in zip(reference, actual_array) + ], + dtype=np.float64, + ) + result: dict[str, float | int | str] = { + "queries": len(actual), + "top_k": top_k, + "recall_at_k": float(per_query.mean()), + "min_query_recall": float(per_query.min()), + "exact_query_fraction": float(np.mean(per_query == 1.0)), + } + if (database_path is None) != (query_path is None): + raise ValueError("database and queries must be provided together") + if database_path is not None and query_path is not None: + add_score_equivalence( + result, + reference, + actual_array, + np.asarray(actual_scores, dtype=np.float32), + database_path, + query_path, + score_rtol, + score_atol, + ) + del records + return result + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--groundtruth", type=Path, required=True) + parser.add_argument("--results", type=Path, required=True) + parser.add_argument("--database", type=Path) + parser.add_argument("--queries", type=Path) + parser.add_argument("--score-rtol", type=float, default=1e-5) + parser.add_argument("--score-atol", type=float, default=1e-4) + parser.add_argument("--json", action="store_true") + args = parser.parse_args() + result = evaluate( + args.groundtruth, + args.results, + args.database, + args.queries, + args.score_rtol, + args.score_atol, + ) + if args.json: + print(json.dumps(result, sort_keys=True)) + else: + print( + f"recall@{result['top_k']}={result['recall_at_k']:.6f}, " + f"min_query_recall={result['min_query_recall']:.6f}, " + f"exact_query_fraction={result['exact_query_fraction']:.6f}, " + f"queries={result['queries']}" + ) + if "score_equivalent_query_fraction" in result: + print( + "score_equivalent_query_fraction=" + f"{result['score_equivalent_query_fraction']:.6f}, " + f"max_rank_score_error={result['max_rank_score_error']:.9g}, " + f"max_reported_score_error={result['max_reported_score_error']:.9g}" + ) + + +if __name__ == "__main__": + main() diff --git a/09_vector_retrieval/PurplePulse/scripts/evaluate_recall.py b/09_vector_retrieval/PurplePulse/scripts/evaluate_recall.py new file mode 100644 index 00000000..755c1f72 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/scripts/evaluate_recall.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python3 +"""计算近似 Top-K 相对 exact ground truth 的 recall@K 和分数误差。""" + +import argparse +from collections import defaultdict +from pathlib import Path + + +def read_results(path): + results = defaultdict(list) + for line_number, line in enumerate(Path(path).read_text().splitlines(), 1): + parts = line.split() + if len(parts) != 3: + raise ValueError(f"{path}:{line_number} 不是三列") + results[int(parts[0])].append((int(parts[1]), float(parts[2]))) + return results + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--exact", required=True) + parser.add_argument("--approximate", required=True) + args = parser.parse_args() + + exact = read_results(args.exact) + approximate = read_results(args.approximate) + if exact.keys() != approximate.keys(): + raise SystemExit("失败：query ID 集合不同") + + matches = 0 + total = 0 + per_query = [] + for query_id in exact: + if len(exact[query_id]) != len(approximate[query_id]): + raise SystemExit(f"失败：query {query_id} 的 K 不同") + exact_ids = {vector_id for vector_id, _ in exact[query_id]} + approximate_ids = {vector_id for vector_id, _ in approximate[query_id]} + query_matches = len(exact_ids & approximate_ids) + matches += query_matches + total += len(exact[query_id]) + per_query.append(query_matches / len(exact[query_id])) + + score_errors = [ + abs(exact_neighbor[1] - approximate_neighbor[1]) + for query_id in exact + for exact_neighbor, approximate_neighbor in zip( + exact[query_id], approximate[query_id] + ) + ] + + print( + f"recall@K={matches / total:.6f}, " + f"min_query_recall={min(per_query):.6f}, " + f"mean_absolute_score_error={sum(score_errors) / len(score_errors):.9g}, " + f"max_absolute_score_error={max(score_errors):.9g}, " + f"queries={len(per_query)}, K={len(next(iter(exact.values())))}" + ) + + +if __name__ == "__main__": + main() diff --git a/09_vector_retrieval/PurplePulse/scripts/generate_data.py b/09_vector_retrieval/PurplePulse/scripts/generate_data.py new file mode 100644 index 00000000..8e4b4234 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/scripts/generate_data.py @@ -0,0 +1,99 @@ +#!/usr/bin/env python3 +"""生成项目自定义的 FP32/FP16 向量库和查询文件。""" + +import argparse +import random +import struct +from pathlib import Path + +try: + import numpy as np +except ImportError: + np = None + + +METRICS = {"l2": 1, "inner_product": 2, "cosine": 3} +FLOAT32 = 1 +FLOAT16 = 2 +DTYPES = {"fp32": FLOAT32, "fp16": FLOAT16} + + +def write_floats(file, values, dtype, chunk_size=65536): + """分块写入，避免百万级数据时一次创建巨大的 bytes 对象。""" + for start in range(0, len(values), chunk_size): + chunk = values[start : start + chunk_size] + format_code = "f" if dtype == "fp32" else "e" + file.write(struct.pack(f"<{len(chunk)}{format_code}", *chunk)) + + +def make_values(random_generator, count): + return [random_generator.uniform(-1.0, 1.0) for _ in range(count)] + + +def write_random_values(file, random_generator, total_count, dtype): + while total_count: + count = min(total_count, 1_048_576) + if np is None: + write_floats(file, make_values(random_generator, count), dtype) + else: + numpy_dtype = " argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--num-vectors", type=int, required=True) + parser.add_argument("--sessions", type=int, default=16) + parser.add_argument("--source-types", type=int, default=4) + parser.add_argument("--start-timestamp", type=int, default=1_700_000_000) + parser.add_argument("--timestamp-span", type=int, default=2_592_000) + parser.add_argument("--seed", type=int, default=2026) + return parser.parse_args() + + +def main() -> None: + args = parse_args() + if ( + args.num_vectors <= 0 + or args.sessions <= 0 + or args.source_types <= 0 + or args.start_timestamp < 0 + or args.timestamp_span < 0 + ): + raise SystemExit("counts must be positive and timestamps non-negative") + if args.sessions >= 2**32 or args.source_types >= 2**32: + raise SystemExit("sessions and source-types must fit uint32") + + rng = random.Random(args.seed) + timestamps = [ + args.start_timestamp + rng.randrange(args.timestamp_span + 1) + for _ in range(args.num_vectors) + ] + importance = [rng.random() for _ in range(args.num_vectors)] + sessions = [rng.randrange(args.sessions) for _ in range(args.num_vectors)] + source_types = [ + rng.randrange(args.source_types) for _ in range(args.num_vectors) + ] + + args.output.parent.mkdir(parents=True, exist_ok=True) + with args.output.open("wb") as output: + output.write(MAGIC) + output.write(struct.pack(" str: + digest = hashlib.sha256() + with path.open("rb") as source: + while chunk := source.read(chunk_bytes): + digest.update(chunk) + return digest.hexdigest() + + +def normalized_distance(value: object) -> str: + if isinstance(value, bytes): + value = value.decode("utf-8") + distance = str(value).strip().lower() + if distance not in METRIC_CODES: + raise ValueError(f"unsupported ANN-Benchmarks distance: {distance}") + return distance + + +def write_vector_file( + dataset, + destination: Path, + rows: int, + metric_code: int | None, + chunk_rows: int = 16384, +) -> None: + destination.parent.mkdir(parents=True, exist_ok=True) + dim = int(dataset.shape[1]) + with destination.open("wb") as output: + if metric_code is None: + output.write(QUERY_MAGIC) + output.write(struct.pack(" None: + destination.parent.mkdir(parents=True, exist_ok=True) + top_k = int(dataset.shape[1]) + with destination.open("wb") as output: + for start in range(0, rows, chunk_rows): + ids = np.ascontiguousarray( + dataset[start : min(start + chunk_rows, rows)], dtype="= database_rows): + raise ValueError("ground truth contains an out-of-range vector ID") + if np.any(np.diff(np.sort(ids, axis=1), axis=1) == 0): + raise ValueError("ground truth contains duplicate IDs in one query") + records = np.empty((len(ids), top_k + 1), dtype=" None: + for start in range(0, int(dataset.shape[0]), chunk_rows): + values = np.asarray(dataset[start : start + chunk_rows], dtype=np.float32) + norms = np.linalg.norm(values, axis=1) + if not np.all(np.isfinite(norms)) or not np.allclose( + norms, 1.0, rtol=5e-4, atol=5e-4 + ): + maximum_error = float(np.max(np.abs(norms - 1.0))) + raise ValueError( + f"{name} declares normalized vectors but unit-norm validation failed; " + f"maximum absolute norm error={maximum_error}" + ) + + +def convert( + input_path: Path, + database_output: Path, + query_output: Path, + groundtruth_output: Path, + manifest_output: Path, + query_limit: int | None = None, + expected_sha256: str | None = None, + source_url: str | None = None, + dataset_name: str | None = None, + expected_dim: int | None = None, +) -> dict[str, object]: + try: + import h5py + except ImportError as error: + raise RuntimeError("h5py is required to import ANN-Benchmarks HDF5 files") from error + + input_sha256 = sha256_file(input_path) + if expected_sha256 is not None and input_sha256.lower() != expected_sha256.lower(): + raise ValueError( + f"input SHA-256 mismatch: expected {expected_sha256}, got {input_sha256}" + ) + + with h5py.File(input_path, "r") as source: + required = {"train", "test", "neighbors"} + missing = sorted(required - set(source.keys())) + if missing: + raise ValueError(f"HDF5 is missing datasets: {missing}") + train = source["train"] + test = source["test"] + neighbors = source["neighbors"] + if train.ndim != 2 or test.ndim != 2 or neighbors.ndim != 2: + raise ValueError("train, test, and neighbors must be two-dimensional") + if train.shape[1] != test.shape[1]: + raise ValueError("train and test dimensions differ") + if expected_dim is not None and int(train.shape[1]) != expected_dim: + raise ValueError( + f"dataset dimension mismatch: expected {expected_dim}, " + f"got {int(train.shape[1])}" + ) + if test.shape[0] != neighbors.shape[0]: + raise ValueError("test and neighbors row counts differ") + if query_limit is not None and query_limit <= 0: + raise ValueError("query_limit must be positive") + query_rows = int(test.shape[0]) if query_limit is None else min( + query_limit, int(test.shape[0]) + ) + distance = normalized_distance(source.attrs.get("distance", "euclidean")) + if distance == "normalized": + validate_unit_norms(train, "train") + validate_unit_norms(test, "test") + metric_code = METRIC_CODES[distance] + write_vector_file(train, database_output, int(train.shape[0]), metric_code) + write_vector_file(test, query_output, query_rows, None) + write_groundtruth_ivecs( + neighbors, groundtruth_output, query_rows, int(train.shape[0]) + ) + shape = { + "database_rows": int(train.shape[0]), + "query_rows": query_rows, + "dim": int(train.shape[1]), + "groundtruth_k": int(neighbors.shape[1]), + } + + outputs = {} + for name, path in { + "database": database_output, + "queries": query_output, + "groundtruth": groundtruth_output, + }.items(): + outputs[name] = { + "path": str(path), + "bytes": path.stat().st_size, + "sha256": sha256_file(path), + } + manifest: dict[str, object] = { + "dataset": dataset_name or input_path.stem, + "source_url": source_url, + "source_path": str(input_path), + "source_bytes": input_path.stat().st_size, + "source_sha256": input_sha256, + "distance": distance, + "warpfusion_metric": "cosine" if metric_code == 3 else "l2", + **shape, + "outputs": outputs, + } + manifest_output.parent.mkdir(parents=True, exist_ok=True) + manifest_output.write_text( + json.dumps(manifest, ensure_ascii=False, indent=2) + "\n", encoding="utf-8" + ) + return manifest + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--input", type=Path, required=True) + parser.add_argument("--database-output", type=Path, required=True) + parser.add_argument("--query-output", type=Path, required=True) + parser.add_argument("--groundtruth-output", type=Path, required=True) + parser.add_argument("--manifest-output", type=Path, required=True) + parser.add_argument("--query-limit", type=int) + parser.add_argument("--expected-sha256") + parser.add_argument("--source-url") + parser.add_argument( + "--dataset-name", + help="Name recorded in the provenance manifest (defaults to the input stem)", + ) + parser.add_argument("--expected-dim", type=int) + args = parser.parse_args() + manifest = convert( + args.input, + args.database_output, + args.query_output, + args.groundtruth_output, + args.manifest_output, + args.query_limit, + args.expected_sha256, + args.source_url, + args.dataset_name, + args.expected_dim, + ) + print( + f"{manifest['dataset']}: {manifest['database_rows']} x {manifest['dim']}, " + f"queries={manifest['query_rows']}, groundtruth_k={manifest['groundtruth_k']}, " + f"distance={manifest['distance']}" + ) + print(f"manifest: {args.manifest_output}") + + +if __name__ == "__main__": + main() diff --git a/09_vector_retrieval/PurplePulse/scripts/import_mnist.py b/09_vector_retrieval/PurplePulse/scripts/import_mnist.py new file mode 100644 index 00000000..d798dd86 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/scripts/import_mnist.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python3 +"""Convert the public Keras MNIST archive to the project vector format.""" + +from __future__ import annotations + +import argparse +import struct +from pathlib import Path + +import numpy as np + + +def images_to_vectors(images: np.ndarray) -> np.ndarray: + if images.ndim < 2: + raise ValueError("MNIST image array must have at least two dimensions") + rows = images.shape[0] + return np.ascontiguousarray(images.reshape(rows, -1), dtype=" None: + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("wb") as destination: + destination.write(b"PPVEC001") + destination.write( + struct.pack(" None: + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("wb") as destination: + destination.write(b"PPQRY001") + destination.write(struct.pack(" tuple[tuple[int, int], tuple[int, int]]: + with np.load(input_path) as archive: + if "x_train" not in archive or "x_test" not in archive: + raise ValueError("MNIST archive must contain x_train and x_test") + database = images_to_vectors(archive["x_train"]) + queries = images_to_vectors(archive["x_test"]) + if database.shape[1] != queries.shape[1]: + raise ValueError("MNIST train and test dimensions differ") + if database_size is not None: + database = database[:database_size] + if query_count is not None: + queries = queries[:query_count] + if len(database) == 0 or len(queries) == 0: + raise ValueError("database and query counts must be positive") + write_database(database_path, database) + write_queries(query_path, queries) + return database.shape, queries.shape + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--input", type=Path, required=True) + parser.add_argument("--database", type=Path, required=True) + parser.add_argument("--queries", type=Path, required=True) + parser.add_argument("--database-size", type=int) + parser.add_argument("--query-count", type=int) + args = parser.parse_args() + database_shape, query_shape = convert( + args.input, + args.database, + args.queries, + args.database_size, + args.query_count, + ) + print(f"database: {database_shape[0]} x {database_shape[1]}") + print(f"queries: {query_shape[0]} x {query_shape[1]}") + print("dtype/metric: fp32/l2") + + +if __name__ == "__main__": + main() diff --git a/09_vector_retrieval/PurplePulse/scripts/import_sift1m.py b/09_vector_retrieval/PurplePulse/scripts/import_sift1m.py new file mode 100644 index 00000000..66359cf0 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/scripts/import_sift1m.py @@ -0,0 +1,231 @@ +#!/usr/bin/env python3 +"""Convert TexMex SIFT1M fvecs/ivecs files to WarpFusion input files.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import shutil +import struct +from dataclasses import asdict, dataclass +from pathlib import Path + +import numpy as np + + +OFFICIAL_SOURCE = "ftp://ftp.irisa.fr/local/texmex/corpus/sift.tar.gz" +DATABASE_MAGIC = b"PPVEC001" +QUERY_MAGIC = b"PPQRY001" +FLOAT32 = 1 +L2 = 1 + + +@dataclass(frozen=True) +class VecsInfo: + path: str + kind: str + rows: int + dim: int + bytes: int + sha256: str + + +def sha256_file(path: Path, chunk_bytes: int = 8 * 1024 * 1024) -> str: + digest = hashlib.sha256() + with path.open("rb") as source: + while chunk := source.read(chunk_bytes): + digest.update(chunk) + return digest.hexdigest() + + +def record_dtype(dim: int, kind: str) -> np.dtype: + value_dtype = " VecsInfo: + if kind not in {"fvecs", "ivecs"}: + raise ValueError("kind must be fvecs or ivecs") + if not path.is_file() or path.stat().st_size < 4: + raise ValueError(f"{path}: missing or too small") + dim = int(np.fromfile(path, dtype=" 1_000_000: + raise ValueError(f"{path}: invalid row dimension {dim}") + record_bytes = (dim + 1) * 4 + size = path.stat().st_size + if size % record_bytes != 0: + raise ValueError( + f"{path}: size {size} is not divisible by record size {record_bytes}" + ) + rows = size // record_bytes + if rows == 0: + raise ValueError(f"{path}: no rows") + records = np.memmap(path, dtype=record_dtype(dim, kind), mode="r", shape=(rows,)) + for start in range(0, rows, chunk_rows): + if not np.all(records["dim"][start : start + chunk_rows] == dim): + raise ValueError(f"{path}: inconsistent row dimensions") + del records + return VecsInfo( + path=str(path), + kind=kind, + rows=rows, + dim=dim, + bytes=size, + sha256=sha256_file(path), + ) + + +def write_warpfusion_vectors( + source: Path, + destination: Path, + info: VecsInfo, + rows: int, + database: bool, + chunk_rows: int = 16384, +) -> None: + if info.kind != "fvecs": + raise ValueError("WarpFusion vectors must be imported from fvecs") + if rows <= 0 or rows > info.rows: + raise ValueError(f"requested rows must be in [1, {info.rows}]") + records = np.memmap( + source, + dtype=record_dtype(info.dim, "fvecs"), + mode="r", + shape=(info.rows,), + ) + destination.parent.mkdir(parents=True, exist_ok=True) + with destination.open("wb") as output: + if database: + output.write(DATABASE_MAGIC) + output.write(struct.pack(" None: + if info.kind != "ivecs": + raise ValueError("ground truth must use ivecs") + if info.rows < query_rows: + raise ValueError( + f"ground truth has {info.rows} rows but {query_rows} queries were requested" + ) + records = np.memmap( + path, + dtype=record_dtype(info.dim, "ivecs"), + mode="r", + shape=(info.rows,), + ) + values = np.asarray(records["values"][:query_rows]) + if np.any(values < 0) or np.any(values >= database_rows): + raise ValueError("ground truth contains an out-of-range vector ID") + if np.any(np.diff(np.sort(values, axis=1), axis=1) == 0): + raise ValueError("ground truth contains duplicate IDs in one query") + del records + + +def convert( + base_path: Path, + query_path: Path, + groundtruth_path: Path, + database_output: Path, + query_output: Path, + manifest_output: Path, + query_limit: int | None = None, + source_url: str = OFFICIAL_SOURCE, + groundtruth_output: Path | None = None, +) -> dict[str, object]: + base = inspect_vecs(base_path, "fvecs") + queries = inspect_vecs(query_path, "fvecs") + groundtruth = inspect_vecs(groundtruth_path, "ivecs") + if base.dim != queries.dim: + raise ValueError(f"base dim {base.dim} differs from query dim {queries.dim}") + if query_limit is not None and query_limit <= 0: + raise ValueError("query_limit must be positive") + query_rows = queries.rows if query_limit is None else min(query_limit, queries.rows) + validate_groundtruth(groundtruth_path, groundtruth, query_rows, base.rows) + write_warpfusion_vectors(base_path, database_output, base, base.rows, True) + write_warpfusion_vectors(query_path, query_output, queries, query_rows, False) + if groundtruth_output is not None: + groundtruth_output.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(groundtruth_path, groundtruth_output) + + output_paths = { + "database": database_output, + "queries": query_output, + } + if groundtruth_output is not None: + output_paths["groundtruth"] = groundtruth_output + + manifest: dict[str, object] = { + "dataset": "SIFT1M", + "source_url": source_url, + "metric": "l2", + "database_rows": base.rows, + "query_rows": query_rows, + "dim": base.dim, + "groundtruth_k": groundtruth.dim, + "inputs": { + "base": asdict(base), + "queries": asdict(queries), + "groundtruth": asdict(groundtruth), + }, + "outputs": { + name: { + "path": str(path), + "bytes": path.stat().st_size, + "sha256": sha256_file(path), + } + for name, path in output_paths.items() + }, + } + manifest_output.parent.mkdir(parents=True, exist_ok=True) + manifest_output.write_text( + json.dumps(manifest, ensure_ascii=False, indent=2) + "\n", encoding="utf-8" + ) + return manifest + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--base", type=Path, required=True) + parser.add_argument("--queries", type=Path, required=True) + parser.add_argument("--groundtruth", type=Path, required=True) + parser.add_argument("--database-output", type=Path, required=True) + parser.add_argument("--query-output", type=Path, required=True) + parser.add_argument("--manifest-output", type=Path, required=True) + parser.add_argument("--groundtruth-output", type=Path) + parser.add_argument("--query-limit", type=int) + parser.add_argument("--source-url", default=OFFICIAL_SOURCE) + args = parser.parse_args() + manifest = convert( + args.base, + args.queries, + args.groundtruth, + args.database_output, + args.query_output, + args.manifest_output, + args.query_limit, + args.source_url, + args.groundtruth_output, + ) + print( + f"SIFT1M: {manifest['database_rows']} x {manifest['dim']}, " + f"queries={manifest['query_rows']}, groundtruth_k={manifest['groundtruth_k']}" + ) + print(f"manifest: {args.manifest_output}") + + +if __name__ == "__main__": + main() diff --git a/09_vector_retrieval/PurplePulse/scripts/plot_results.py b/09_vector_retrieval/PurplePulse/scripts/plot_results.py new file mode 100644 index 00000000..dd7513b3 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/scripts/plot_results.py @@ -0,0 +1,243 @@ +#!/usr/bin/env python3 +"""Generate dependency-free SVG figures from project benchmark CSV files.""" + +from __future__ import annotations + +import argparse +import csv +import html +from pathlib import Path + + +WIDTH = 1180 +HEIGHT = 430 +COLORS = {"WarpFusion": "#0072B2", "FAISS GPU": "#D55E00"} +K_COLORS = {10: "#0072B2", 50: "#009E73", 100: "#D55E00"} + + +def read_csv(path: Path) -> list[dict[str, str]]: + with path.open(newline="") as source: + rows = list(csv.DictReader(source)) + if not rows: + raise ValueError(f"empty CSV: {path}") + return rows + + +def esc(value: object) -> str: + return html.escape(str(value), quote=True) + + +def line(x1: float, y1: float, x2: float, y2: float, **attrs: object) -> str: + attributes = " ".join(f'{key.rstrip("_").replace("_", "-")}="{esc(value)}"' for key, value in attrs.items()) + return f'' + + +def text(x: float, y: float, value: object, **attrs: object) -> str: + attributes = " ".join(f'{key.rstrip("_").replace("_", "-")}="{esc(attr)}"' for key, attr in attrs.items()) + return f'{esc(value)}' + + +def padded_domain(values: list[float], fraction: float = 0.06) -> tuple[float, float]: + low, high = min(values), max(values) + span = high - low + padding = span * fraction if span else max(abs(high) * fraction, 1.0) + return low - padding, high + padding + + +def scale(value: float, domain: tuple[float, float], start: float, end: float) -> float: + low, high = domain + return start + (value - low) / (high - low) * (end - start) + + +def ticks(domain: tuple[float, float], count: int = 5) -> list[float]: + low, high = domain + return [low + index * (high - low) / (count - 1) for index in range(count)] + + +def base_svg(title: str, body: list[str], width: int = WIDTH, height: int = HEIGHT) -> str: + return "\n".join([ + f'', + f'{esc(title)}', + 'Benchmark curves generated from the checked-in raw CSV data.', + '', + '', + *body, + '', + '', + ]) + + +def load_quality_series(results_dir: Path) -> dict[int, dict[str, list[dict[str, float]]]]: + formal_files = {top_k: f"ivf_k{top_k}.csv" for top_k in (10, 50, 100)} + purple_files = ( + formal_files + if all((results_dir / filename).exists() for filename in formal_files.values()) + else {top_k: f"optimized_v1_k{top_k}.csv" for top_k in (10, 50, 100)} + ) + series: dict[int, dict[str, list[dict[str, float]]]] = {} + for top_k, filename in purple_files.items(): + rows = read_csv(results_dir / filename) + series[top_k] = { + "WarpFusion": [ + { + "recall": float(row["recall_at_k"]), + "latency": float(row["average_ms"]), + "qps": float(row["qps"]), + "nprobe": float(row["nprobe"]), + } + for row in rows + ] + } + faiss_filename = ( + "faiss_gpu.csv" + if (results_dir / "faiss_gpu.csv").exists() + else "faiss_gpu_formal_v3.csv" + ) + faiss_rows = read_csv(results_dir / faiss_filename) + for top_k in purple_files: + series[top_k]["FAISS GPU"] = [ + { + "recall": float(row["recall_at_k"]), + "latency": float(row["average_query_ms"]), + "qps": float(row["qps"]), + "nprobe": float(row["nprobe"]), + } + for row in faiss_rows + if row["backend"] == "gpu" + and row["index_type"] == "ivf_flat" + and int(row["top_k"]) == top_k + ] + if not series[top_k]["FAISS GPU"]: + raise ValueError(f"missing FAISS GPU IVF rows for K={top_k}") + return series + + +def quality_figure(series: dict[int, dict[str, list[dict[str, float]]]], metric: str, + title: str, y_label: str) -> str: + body = [text(WIDTH / 2, 27, title, text_anchor="middle", class_="figure-title")] + body.extend([ + line(430, 47, 455, 47, stroke=COLORS["WarpFusion"], stroke_width=3), + text(462, 51, "WarpFusion", class_="label"), + line(590, 47, 615, 47, stroke=COLORS["FAISS GPU"], stroke_width=3), + text(622, 51, "FAISS GPU", class_="label"), + ]) + panel_width = 350 + panel_gap = 30 + plot_top, plot_bottom = 86, 355 + for panel_index, top_k in enumerate((10, 50, 100)): + panel_left = 55 + panel_index * (panel_width + panel_gap) + plot_left, plot_right = panel_left + 48, panel_left + panel_width - 10 + all_points = [point for values in series[top_k].values() for point in values] + x_domain = padded_domain([point["recall"] for point in all_points], 0.04) + y_domain = padded_domain([point[metric] for point in all_points], 0.08) + body.append(text((plot_left + plot_right) / 2, 73, f"K = {top_k}", text_anchor="middle", class_="panel-title")) + for value in ticks(y_domain): + y = scale(value, y_domain, plot_bottom, plot_top) + body.append(line(plot_left, y, plot_right, y, class_="grid")) + body.append(text(plot_left - 7, y + 4, f"{value:.0f}", text_anchor="end", class_="tick")) + for value in ticks(x_domain): + x = scale(value, x_domain, plot_left, plot_right) + body.append(line(x, plot_top, x, plot_bottom, class_="grid")) + body.append(text(x, plot_bottom + 17, f"{value:.2f}", text_anchor="middle", class_="tick")) + body.extend([ + line(plot_left, plot_top, plot_left, plot_bottom, class_="axis"), + line(plot_left, plot_bottom, plot_right, plot_bottom, class_="axis"), + text((plot_left + plot_right) / 2, 397, "Recall@K", text_anchor="middle", class_="label"), + ]) + if panel_index == 0: + body.append(text(15, (plot_top + plot_bottom) / 2, y_label, text_anchor="middle", class_="label", transform=f"rotate(-90 15 {(plot_top + plot_bottom) / 2})")) + for name in ("WarpFusion", "FAISS GPU"): + points = sorted(series[top_k][name], key=lambda item: item["recall"]) + coordinates = [ + (scale(point["recall"], x_domain, plot_left, plot_right), + scale(point[metric], y_domain, plot_bottom, plot_top), point) + for point in points + ] + path = " ".join(("M" if index == 0 else "L") + f" {x:.2f} {y:.2f}" for index, (x, y, _) in enumerate(coordinates)) + body.append(f'') + for x, y, point in coordinates: + body.append(f'') + body.append(text(x + 5, y - 6, int(point["nprobe"]), class_="note")) + body.append(text(WIDTH - 10, HEIGHT - 8, "Matched batch=64 · point labels: nprobe", text_anchor="end", class_="note")) + return base_svg(title, body) + + +def batch_figure(results_dir: Path) -> str: + rows = read_csv(results_dir / "optimized_v1_batch_sweep.csv") + body = [text(WIDTH / 2, 28, "Batch size vs throughput", text_anchor="middle", class_="figure-title")] + plot_left, plot_right, plot_top, plot_bottom = 85, 1135, 78, 350 + x_values = sorted({int(row["batch_size"]) for row in rows}) + qps_values = [float(row["qps"]) for row in rows] + x_domain = padded_domain([float(value) for value in x_values], 0.03) + y_domain = padded_domain(qps_values, 0.08) + for value in ticks(y_domain): + y = scale(value, y_domain, plot_bottom, plot_top) + body.append(line(plot_left, y, plot_right, y, class_="grid")) + body.append(text(plot_left - 9, y + 4, f"{value:.0f}", text_anchor="end", class_="tick")) + for value in x_values: + x = scale(value, x_domain, plot_left, plot_right) + body.append(line(x, plot_top, x, plot_bottom, class_="grid")) + body.append(text(x, plot_bottom + 18, value, text_anchor="middle", class_="tick")) + body.extend([ + line(plot_left, plot_top, plot_left, plot_bottom, class_="axis"), + line(plot_left, plot_bottom, plot_right, plot_bottom, class_="axis"), + text((plot_left + plot_right) / 2, 397, "Batch size (queries)", text_anchor="middle", class_="label"), + text(20, (plot_top + plot_bottom) / 2, "Throughput (QPS)", text_anchor="middle", class_="label", transform=f"rotate(-90 20 {(plot_top + plot_bottom) / 2})"), + ]) + legend_x = 430 + for index, top_k in enumerate((10, 50, 100)): + color = K_COLORS[top_k] + body.append(line(legend_x + index * 135, 50, legend_x + 25 + index * 135, 50, stroke=color, stroke_width=3)) + body.append(text(legend_x + 32 + index * 135, 54, f"K={top_k}", class_="label")) + points = sorted( + ((int(row["batch_size"]), float(row["qps"])) for row in rows if int(row["top_k"]) == top_k), + key=lambda item: item[0], + ) + coordinates = [(scale(x, x_domain, plot_left, plot_right), scale(y, y_domain, plot_bottom, plot_top)) for x, y in points] + path = " ".join(("M" if item == 0 else "L") + f" {x:.2f} {y:.2f}" for item, (x, y) in enumerate(coordinates)) + body.append(f'') + for x, y in coordinates: + body.append(f'') + return base_svg("Batch size vs throughput", body) + + +def generate_figures( + results_dir: Path, + output_dir: Path, + batch_results_dir: Path | None = None, +) -> list[Path]: + output_dir.mkdir(parents=True, exist_ok=True) + quality = load_quality_series(results_dir) + figures = { + "recall-latency.svg": quality_figure( + quality, "latency", "Recall–latency frontier", "1000-query latency (ms)" + ), + "recall-qps.svg": quality_figure( + quality, "qps", "Recall–throughput frontier", "Throughput (QPS)" + ), + } + batch_root = batch_results_dir if batch_results_dir is not None else results_dir + if (batch_root / "optimized_v1_batch_sweep.csv").exists(): + figures["batch-qps.svg"] = batch_figure(batch_root) + paths = [] + for filename, contents in figures.items(): + path = output_dir / filename + path.write_text(contents) + paths.append(path) + return paths + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--results-dir", type=Path, default=Path("results/million")) + parser.add_argument("--output-dir", type=Path, default=Path("results/figures")) + parser.add_argument("--batch-results-dir", type=Path) + args = parser.parse_args() + for path in generate_figures( + args.results_dir, args.output_dir, args.batch_results_dir + ): + print(path) + + +if __name__ == "__main__": + main() diff --git a/09_vector_retrieval/PurplePulse/scripts/prepare_real_datasets.py b/09_vector_retrieval/PurplePulse/scripts/prepare_real_datasets.py new file mode 100644 index 00000000..b6aed852 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/scripts/prepare_real_datasets.py @@ -0,0 +1,196 @@ +#!/usr/bin/env python3 +"""Download, verify, and convert the two Stage-6 public datasets.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import sys +import tarfile +import urllib.request +from pathlib import Path + +from import_ann_benchmarks import convert as convert_hdf5 +from import_sift1m import convert as convert_sift + + +SIFT_URL = "ftp://ftp.irisa.fr/local/texmex/corpus/sift.tar.gz" +SIFT_MD5 = "b23d1b3b2ee8469d819b61ca900ef0ed" +VIBE_NAME = "yahoo-minilm-384-normalized" +VIBE_URL = ( + "https://huggingface.co/datasets/vector-index-bench/vibe/resolve/main/" + f"{VIBE_NAME}.hdf5" +) +VIBE_MIRROR_URL = ( + "https://hf-mirror.com/datasets/vector-index-bench/vibe/resolve/main/" + f"{VIBE_NAME}.hdf5" +) +VIBE_SHA256 = "8b0519850249d7ea4aea258eb43ae837dca1bb2f7a85ee0a5145afdb699c4136" + + +def file_hash(path: Path, algorithm: str) -> str: + digest = hashlib.new(algorithm) + with path.open("rb") as source: + while chunk := source.read(8 * 1024 * 1024): + digest.update(chunk) + return digest.hexdigest() + + +def verify(path: Path, algorithm: str, expected: str) -> None: + actual = file_hash(path, algorithm) + if actual.lower() != expected.lower(): + raise ValueError( + f"{path}: {algorithm} mismatch: expected {expected}, got {actual}" + ) + + +def download(url: str, destination: Path, algorithm: str, expected: str) -> Path: + destination.parent.mkdir(parents=True, exist_ok=True) + if destination.exists(): + verify(destination, algorithm, expected) + print(f"verified cached download: {destination}") + return destination + + partial = destination.with_suffix(destination.suffix + ".part") + request = urllib.request.Request(url, headers={"User-Agent": "WarpFusion/0.1"}) + downloaded = 0 + with urllib.request.urlopen(request, timeout=120) as response, partial.open("wb") as out: + total = int(response.headers.get("Content-Length", "0")) + while chunk := response.read(8 * 1024 * 1024): + out.write(chunk) + downloaded += len(chunk) + if total: + print( + f"\r{destination.name}: {downloaded / (1024**2):.1f}/" + f"{total / (1024**2):.1f} MiB", + end="", + flush=True, + ) + print() + verify(partial, algorithm, expected) + partial.replace(destination) + return destination + + +def safe_extract(archive: Path, destination: Path) -> None: + destination.mkdir(parents=True, exist_ok=True) + root = destination.resolve() + with tarfile.open(archive, "r:gz") as source: + for member in source.getmembers(): + target = (destination / member.name).resolve() + if root != target and root not in target.parents: + raise ValueError(f"unsafe archive member: {member.name}") + source.extractall(destination) + + +def prepare_sift(raw_dir: Path, output_dir: Path, skip_download: bool) -> dict: + archive = raw_dir / "sift1m" / "sift.tar.gz" + if not skip_download: + download(SIFT_URL, archive, "md5", SIFT_MD5) + elif not archive.is_file(): + raise FileNotFoundError(archive) + else: + verify(archive, "md5", SIFT_MD5) + + extracted = raw_dir / "sift1m" / "extracted" + required = { + "base": extracted / "sift" / "sift_base.fvecs", + "queries": extracted / "sift" / "sift_query.fvecs", + "groundtruth": extracted / "sift" / "sift_groundtruth.ivecs", + } + if not all(path.is_file() for path in required.values()): + safe_extract(archive, extracted) + + destination = output_dir / "sift1m" + manifest = convert_sift( + required["base"], + required["queries"], + required["groundtruth"], + destination / "database.bin", + destination / "queries.bin", + destination / "manifest.json", + source_url=SIFT_URL, + groundtruth_output=destination / "groundtruth.ivecs", + ) + manifest["archive"] = { + "path": str(archive), + "bytes": archive.stat().st_size, + "md5": file_hash(archive, "md5"), + "sha256": file_hash(archive, "sha256"), + } + (destination / "manifest.json").write_text( + json.dumps(manifest, ensure_ascii=False, indent=2) + "\n", + encoding="utf-8", + ) + return manifest + + +def prepare_vibe(raw_dir: Path, output_dir: Path, skip_download: bool) -> dict: + source = raw_dir / "vibe" / f"{VIBE_NAME}.hdf5" + if not skip_download: + try: + download(VIBE_URL, source, "sha256", VIBE_SHA256) + except Exception as primary_error: + print( + f"primary Hugging Face download failed ({primary_error}); " + "trying the checksum-equivalent mirror", + file=sys.stderr, + ) + download(VIBE_MIRROR_URL, source, "sha256", VIBE_SHA256) + elif not source.is_file(): + raise FileNotFoundError(source) + else: + verify(source, "sha256", VIBE_SHA256) + + destination = output_dir / VIBE_NAME + return convert_hdf5( + source, + destination / "database.bin", + destination / "queries.bin", + destination / "groundtruth.ivecs", + destination / "manifest.json", + expected_sha256=VIBE_SHA256, + source_url=VIBE_URL, + dataset_name=VIBE_NAME, + expected_dim=384, + ) + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument( + "--dataset", + choices=("all", "sift1m", VIBE_NAME), + default="all", + ) + parser.add_argument("--raw-dir", type=Path, default=Path("data/real/raw")) + parser.add_argument("--output-dir", type=Path, default=Path("data/real")) + parser.add_argument( + "--skip-download", + action="store_true", + help="Require already-downloaded archives, but still verify their checksums", + ) + args = parser.parse_args() + + selected = ( + ("sift1m", VIBE_NAME) if args.dataset == "all" else (args.dataset,) + ) + for name in selected: + try: + manifest = ( + prepare_sift(args.raw_dir, args.output_dir, args.skip_download) + if name == "sift1m" + else prepare_vibe(args.raw_dir, args.output_dir, args.skip_download) + ) + except Exception as error: + print(f"failed to prepare {name}: {error}", file=sys.stderr) + raise + print( + f"prepared {manifest['dataset']}: {manifest['database_rows']} x " + f"{manifest['dim']}, queries={manifest['query_rows']}" + ) + + +if __name__ == "__main__": + main() diff --git a/09_vector_retrieval/PurplePulse/scripts/reproduce.py b/09_vector_retrieval/PurplePulse/scripts/reproduce.py new file mode 100644 index 00000000..63c64bd8 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/scripts/reproduce.py @@ -0,0 +1,259 @@ +#!/usr/bin/env python3 +"""Run the complete WarpFusion build, benchmark, report, and validation flow.""" + +from __future__ import annotations + +import argparse +import json +import os +import subprocess +import sys +from datetime import datetime, timezone +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] +DATASETS = ("sift1m", "yahoo-minilm-384") +PREPARE_NAMES = { + "sift1m": "sift1m", + "yahoo-minilm-384": "yahoo-minilm-384-normalized", +} + + +def rooted(root: Path, value: Path) -> Path: + return value.resolve() if value.is_absolute() else (root / value).resolve() + + +def parse_args(argv: list[str] | None = None) -> argparse.Namespace: + parser = argparse.ArgumentParser( + description=( + "Build, test, prepare public data, run WarpFusion/FAISS, plot, " + "build the report, and validate release artifacts." + ) + ) + parser.add_argument("--project-root", type=Path, default=ROOT) + parser.add_argument("--dataset", action="append", choices=DATASETS) + parser.add_argument("--raw-dir", type=Path, default=Path("data/real/raw")) + parser.add_argument("--data-root", type=Path, default=Path("data/real")) + parser.add_argument("--results-root", type=Path, default=Path("results/real")) + parser.add_argument("--build-dir", type=Path, default=Path("build-reproduce")) + parser.add_argument( + "--report-output", + type=Path, + default=Path("output/pdf/WarpFusion_Technical_Report.pdf"), + ) + parser.add_argument("--cuda-architectures", default="native") + parser.add_argument("--nlist", type=int, default=1024) + parser.add_argument("--nprobes", default="32,64,128,160,192,224") + parser.add_argument("--training-samples", type=int, default=100000) + parser.add_argument("--iterations", type=int, default=15) + parser.add_argument("--threads", type=int, default=16) + parser.add_argument("--batch-size", type=int, default=64) + parser.add_argument("--warmup", type=int, default=1) + parser.add_argument("--repeat", type=int, default=5) + parser.add_argument("--skip-download", action="store_true") + parser.add_argument("--skip-index-build", action="store_true") + parser.add_argument("--skip-faiss", action="store_true") + parser.add_argument("--resume", action="store_true") + parser.add_argument("--dry-run", action="store_true") + return parser.parse_args(argv) + + +def make_step(name: str, command: list[str], log: Path) -> dict[str, object]: + return {"name": name, "command": command, "log": log} + + +def build_plan(args: argparse.Namespace) -> tuple[Path, list[dict[str, object]]]: + root = args.project_root.resolve() + build = rooted(root, args.build_dir) + raw = rooted(root, args.raw_dir) + data = rooted(root, args.data_root) + results = rooted(root, args.results_root) + report = rooted(root, args.report_output) + state_root = results / "reproducibility" + logs = state_root / "logs" + python = sys.executable + datasets = tuple(dict.fromkeys(args.dataset or DATASETS)) + + steps = [ + make_step( + "configure", + [ + "cmake", "-S", str(root), "-B", str(build), "-G", "Ninja", + "-DCMAKE_BUILD_TYPE=Release", + f"-DCMAKE_CUDA_ARCHITECTURES={args.cuda_architectures}", + ], + logs / "configure.log", + ), + make_step( + "build", ["cmake", "--build", str(build)], logs / "build.log" + ), + make_step( + "test", + ["ctest", "--test-dir", str(build), "--output-on-failure"], + logs / "ctest.log", + ), + ] + + prepare = [ + python, + str(root / "scripts" / "prepare_real_datasets.py"), + "--dataset", + "all" if set(datasets) == set(DATASETS) else PREPARE_NAMES[datasets[0]], + "--raw-dir", str(raw), + "--output-dir", str(data), + ] + if args.skip_download: + prepare.append("--skip-download") + if set(datasets) == set(DATASETS) or len(datasets) == 1: + steps.append(make_step("prepare", prepare, logs / "prepare.log")) + else: + raise ValueError("unsupported dataset selection") + + for dataset in datasets: + command = [ + python, + str(root / "scripts" / "run_real_benchmarks.py"), + "--dataset", dataset, + "--project-root", str(root), + "--data-root", str(data), + "--results-root", str(results), + "--build-dir", str(build), + "--nlist", str(args.nlist), + "--nprobes", args.nprobes, + "--training-samples", str(args.training_samples), + "--iterations", str(args.iterations), + "--threads", str(args.threads), + "--batch-size", str(args.batch_size), + "--warmup", str(args.warmup), + "--repeat", str(args.repeat), + ] + if args.skip_index_build: + command.append("--skip-index-build") + if args.skip_faiss: + command.append("--skip-faiss") + steps.append(make_step(f"benchmark:{dataset}", command, + logs / f"benchmark-{dataset}.log")) + steps.append( + make_step( + f"plot:{dataset}", + [ + python, + str(root / "scripts" / "plot_results.py"), + "--results-dir", str(results / dataset), + "--output-dir", str(results / dataset / "figures"), + ], + logs / f"plot-{dataset}.log", + ) + ) + + steps.extend( + [ + make_step( + "report", + [ + python, + str(root / "scripts" / "build_final_report_pdf.py"), + "--source", str(root / "docs" / "FINAL_REPORT.md"), + "--output", str(report), + ], + logs / "report.log", + ), + make_step( + "validate", + [ + python, + str(root / "scripts" / "validate_release.py"), + "--project-root", str(root), + "--report", str(report), + ], + logs / "validate.log", + ), + ] + ) + return state_root, steps + + +def load_checkpoint(path: Path) -> dict[str, object]: + if not path.is_file(): + return {"completed": [], "steps": {}} + value = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(value.get("completed"), list): + raise ValueError(f"invalid checkpoint: {path}") + return value + + +def write_checkpoint(path: Path, state: dict[str, object]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_suffix(".tmp") + temporary.write_text( + json.dumps(state, ensure_ascii=False, indent=2) + "\n", encoding="utf-8" + ) + temporary.replace(path) + + +def run_step(root: Path, step: dict[str, object]) -> int: + log = Path(step["log"]) + command = [str(item) for item in step["command"]] + log.parent.mkdir(parents=True, exist_ok=True) + print(f"[{step['name']}] {' '.join(command)}", flush=True) + with log.open("w", encoding="utf-8") as output: + completed = subprocess.run( + command, + cwd=root, + env=os.environ.copy(), + stdout=output, + stderr=subprocess.STDOUT, + text=True, + check=False, + ) + return completed.returncode + + +def main(argv: list[str] | None = None) -> None: + args = parse_args(argv) + state_root, steps = build_plan(args) + root = args.project_root.resolve() + checkpoint_path = state_root / "checkpoint.json" + if args.dry_run: + print(json.dumps({"project_root": str(root), "steps": steps}, + ensure_ascii=False, indent=2, default=str)) + return + + state = load_checkpoint(checkpoint_path) if args.resume else { + "completed": [], "steps": {} + } + completed_names = set(state["completed"]) + state["started_utc"] = state.get( + "started_utc", datetime.now(timezone.utc).isoformat() + ) + state["project_root"] = str(root) + for step in steps: + name = str(step["name"]) + if args.resume and name in completed_names: + print(f"[{name}] checkpoint complete; skipping", flush=True) + continue + returncode = run_step(root, step) + state.setdefault("steps", {})[name] = { + "returncode": returncode, + "log": str(step["log"]), + "finished_utc": datetime.now(timezone.utc).isoformat(), + } + if returncode != 0: + state["failed_step"] = name + write_checkpoint(checkpoint_path, state) + raise SystemExit(f"step failed: {name}; see {step['log']}") + completed_names.add(name) + state["completed"] = [ + str(item["name"]) for item in steps + if str(item["name"]) in completed_names + ] + state.pop("failed_step", None) + write_checkpoint(checkpoint_path, state) + state["finished_utc"] = datetime.now(timezone.utc).isoformat() + write_checkpoint(checkpoint_path, state) + print(f"reproduction complete: {checkpoint_path}") + + +if __name__ == "__main__": + main() diff --git a/09_vector_retrieval/PurplePulse/scripts/run_real_benchmarks.py b/09_vector_retrieval/PurplePulse/scripts/run_real_benchmarks.py new file mode 100644 index 00000000..227e1666 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/scripts/run_real_benchmarks.py @@ -0,0 +1,378 @@ +#!/usr/bin/env python3 +"""Run the Stage-6 benchmark matrix for one prepared public dataset.""" + +from __future__ import annotations + +import argparse +import csv +import hashlib +import json +import os +import re +import shutil +import subprocess +import sys +from datetime import datetime, timezone +from pathlib import Path + + +DATA_DIR_NAMES = { + "sift1m": "sift1m", + "yahoo-minilm-384": "yahoo-minilm-384-normalized", +} +TOP_KS = (10, 50, 100) +PROVENANCE_FILES = ( + "src/gpu_ivf_flat.cu", + "src/gpu_search.cu", + "src/ivf_flat.cpp", + "scripts/run_real_benchmarks.py", + "scripts/benchmark_exact.py", + "scripts/benchmark_ivf_sweep.py", + "scripts/benchmark_faiss.py", +) + + +def run(command: list[str], log_path: Path | None = None) -> str: + print("+", " ".join(command), flush=True) + completed = subprocess.run( + command, + check=True, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + ) + print(completed.stdout, end="") + if log_path is not None: + log_path.parent.mkdir(parents=True, exist_ok=True) + log_path.write_text(completed.stdout, encoding="utf-8") + return completed.stdout + + +def capture(command: list[str]) -> str: + try: + return subprocess.run( + command, + check=False, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + ).stdout.strip() + except OSError as error: + return f"unavailable: {error}" + + +def extract(text: str, pattern: str) -> float: + match = re.search(pattern, text) + if match is None: + raise ValueError(f"cannot parse command output: {pattern}") + return float(match.group(1)) + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as source: + while chunk := source.read(1024 * 1024): + digest.update(chunk) + return digest.hexdigest() + + +def parse_build_stats(text: str, dataset: str, nlist: int) -> dict[str, object]: + training = re.search(r"迭代/训练样本: (\d+)/(\d+)", text) + buckets = re.search( + r"桶大小 min/avg/max: ([0-9.eE+-]+)/([0-9.eE+-]+)/([0-9.eE+-]+)", + text, + ) + if training is None or buckets is None: + raise ValueError("cannot parse IVF build metadata") + return { + "dataset": dataset, + "nlist": nlist, + "training_iterations": int(training.group(1)), + "training_samples": int(training.group(2)), + "threads": int(extract(text, r"建库线程: (\d+)")), + "build_ms": extract(text, r"构建时间: ([0-9.eE+-]+) ms"), + "save_ms": extract(text, r"保存时间: ([0-9.eE+-]+) ms"), + "minimum_bucket": int(float(buckets.group(1))), + "average_bucket": float(buckets.group(2)), + "maximum_bucket": int(float(buckets.group(3))), + "empty_buckets": int(extract(text, r"空桶: (\d+)")), + "index_bytes": int(extract(text, r"索引大小: (\d+) bytes")), + } + + +def write_one_row_csv(path: Path, row: dict[str, object]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("w", newline="") as output: + writer = csv.DictWriter(output, fieldnames=list(row)) + writer.writeheader() + writer.writerow(row) + + +def read_csv_by_nprobe(path: Path) -> dict[int, dict[str, str]]: + with path.open(newline="") as source: + rows = list(csv.DictReader(source)) + return {int(row["nprobe"]): row for row in rows} + + +def choose_partitioned_result( + partitioned_csv: Path, + full_heap_csv: Path, + effective_csv: Path, + maximum_recall_loss: float, +) -> dict[str, object]: + partitioned = read_csv_by_nprobe(partitioned_csv) + full_heap = read_csv_by_nprobe(full_heap_csv) + if partitioned.keys() != full_heap.keys(): + raise ValueError("partitioned and full-heap scans use different nprobe values") + comparisons = [] + fallback = False + for nprobe in sorted(partitioned): + partitioned_recall = float(partitioned[nprobe]["recall_at_k"]) + full_heap_recall = float(full_heap[nprobe]["recall_at_k"]) + recall_loss = full_heap_recall - partitioned_recall + fallback = fallback or recall_loss > maximum_recall_loss + comparisons.append( + { + "nprobe": nprobe, + "partitioned_recall": partitioned_recall, + "full_heap_recall": full_heap_recall, + "partitioned_qps": float(partitioned[nprobe]["qps"]), + "full_heap_qps": float(full_heap[nprobe]["qps"]), + "recall_loss": recall_loss, + } + ) + selected = full_heap_csv if fallback else partitioned_csv + shutil.copyfile(selected, effective_csv) + return { + "selected_mode": "full_heap" if fallback else "partitioned", + "maximum_allowed_recall_loss": maximum_recall_loss, + "comparisons": comparisons, + } + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--dataset", choices=tuple(DATA_DIR_NAMES), required=True) + parser.add_argument("--project-root", type=Path, default=Path.cwd()) + parser.add_argument("--data-root", type=Path, default=Path("data/real")) + parser.add_argument("--results-root", type=Path, default=Path("results/real")) + parser.add_argument("--build-dir", type=Path, default=Path("build")) + parser.add_argument("--nlist", type=int, default=1024) + parser.add_argument("--nprobes", default="32,64,128,160,192,224") + parser.add_argument("--training-samples", type=int, default=100000) + parser.add_argument("--iterations", type=int, default=15) + parser.add_argument("--threads", type=int, default=16) + parser.add_argument("--batch-size", type=int, default=64) + parser.add_argument("--warmup", type=int, default=1) + parser.add_argument("--repeat", type=int, default=5) + parser.add_argument("--skip-index-build", action="store_true") + parser.add_argument("--skip-faiss", action="store_true") + parser.add_argument( + "--maximum-partition-recall-loss", + type=float, + default=1e-4, + help="Fall back to the complete heap if any nprobe exceeds this loss", + ) + return parser.parse_args() + + +def main() -> None: + args = parse_args() + if args.maximum_partition_recall_loss < 0: + raise ValueError("maximum-partition-recall-loss must be non-negative") + os.environ.setdefault("_FAISS_WHEEL_DISABLE_CUDA_PRELOAD", "1") + cuda_library = "/usr/local/cuda/lib64" + library_path = os.environ.get("LD_LIBRARY_PATH", "") + if Path(cuda_library).is_dir() and cuda_library not in library_path.split(":"): + os.environ["LD_LIBRARY_PATH"] = ":".join( + item for item in (cuda_library, library_path) if item + ) + root = args.project_root.resolve() + data = (root / args.data_root / DATA_DIR_NAMES[args.dataset]).resolve() + results = (root / args.results_root / args.dataset).resolve() + build = (root / args.build_dir).resolve() + scripts = root / "scripts" + configs = root / "configs" + results.mkdir(parents=True, exist_ok=True) + + for required in ( + data / "database.bin", + data / "queries.bin", + data / "groundtruth.ivecs", + build / "vector_search", + build / "ivf_build", + build / "ivf_search", + ): + if not required.is_file(): + raise FileNotFoundError(required) + + index = data / f"nlist{args.nlist}.ivf" + if not args.skip_index_build: + build_text = run( + [ + str(build / "ivf_build"), + "--database", str(data / "database.bin"), + "--output", str(index), + "--nlist", str(args.nlist), + "--iterations", str(args.iterations), + "--training-samples", str(args.training_samples), + "--threads", str(args.threads), + ], + results / "logs" / "ivf_build.log", + ) + write_one_row_csv( + results / "build.csv", + parse_build_stats(build_text, args.dataset, args.nlist), + ) + elif not index.is_file(): + raise FileNotFoundError(index) + + exact_configs = ",".join( + f"{top_k}:{configs / f'real_exact_k{top_k}.conf'}" + for top_k in TOP_KS + ) + run( + [ + sys.executable, + str(scripts / "benchmark_exact.py"), + "--search-binary", str(build / "vector_search"), + "--database", str(data / "database.bin"), + "--queries", str(data / "queries.bin"), + "--configs", exact_configs, + "--output-dir", str(results / "exact_outputs"), + "--csv", str(results / "exact_gpu.csv"), + "--warmup", str(args.warmup), + "--repeat", str(args.repeat), + ], + results / "logs" / "exact.log", + ) + + validations = [] + for top_k in TOP_KS: + output = run( + [ + sys.executable, + str(scripts / "evaluate_ivecs_groundtruth.py"), + "--groundtruth", str(data / "groundtruth.ivecs"), + "--results", str(results / "exact_outputs" / f"exact_k{top_k}.txt"), + "--database", str(data / "database.bin"), + "--queries", str(data / "queries.bin"), + "--json", + ] + ) + validations.append(json.loads(output)) + (results / "groundtruth_validation.json").write_text( + json.dumps(validations, ensure_ascii=False, indent=2) + "\n", + encoding="utf-8", + ) + + def run_ivf(top_k: int, config: Path, suffix: str) -> Path: + csv_path = results / f"ivf_k{top_k}{suffix}.csv" + run( + [ + sys.executable, + str(scripts / "benchmark_ivf_sweep.py"), + "--search-binary", str(build / "ivf_search"), + "--index", str(index), + "--queries", str(data / "queries.bin"), + "--params", str(config), + "--exact", str(results / "exact_outputs" / f"exact_k{top_k}.txt"), + "--output-dir", str(results / f"ivf_k{top_k}{suffix}_outputs"), + "--csv", str(csv_path), + "--nlist", str(args.nlist), + "--training-samples", str(args.training_samples), + "--training-iterations", str(args.iterations), + "--training-seed", "2026", + "--nprobes", args.nprobes, + "--warmup", str(args.warmup), + "--repeat", str(args.repeat), + ], + results / "logs" / f"ivf_k{top_k}{suffix}.log", + ) + return csv_path + + partition_validations = {} + for top_k in TOP_KS: + optimized_config = configs / f"real_ivf_nlist1024_k{top_k}.conf" + if top_k == 10: + run_ivf(top_k, optimized_config, "") + continue + partitioned_csv = run_ivf(top_k, optimized_config, "_partitioned") + full_heap_csv = run_ivf( + top_k, + configs / f"real_ivf_nlist1024_k{top_k}_safe.conf", + "_full_heap", + ) + partition_validations[str(top_k)] = choose_partitioned_result( + partitioned_csv, + full_heap_csv, + results / f"ivf_k{top_k}.csv", + args.maximum_partition_recall_loss, + ) + (results / "partitioned_topk_validation.json").write_text( + json.dumps(partition_validations, ensure_ascii=False, indent=2) + "\n", + encoding="utf-8", + ) + + if not args.skip_faiss: + run( + [ + sys.executable, + str(scripts / "benchmark_faiss.py"), + "--database", str(data / "database.bin"), + "--queries", str(data / "queries.bin"), + "--output-dir", str(results / "faiss_outputs"), + "--csv", str(results / "faiss_gpu.csv"), + "--index-dir", str(data / "faiss_indexes"), + "--reference-results", str(results / "exact_outputs" / "exact_k100.txt"), + "--backends", "gpu", + "--index-types", "flat,ivf_flat", + "--top-ks", "10,50,100", + "--nprobes", args.nprobes, + "--nlist", str(args.nlist), + "--batch-size", str(args.batch_size), + "--train-samples", str(args.training_samples), + "--iterations", str(args.iterations), + "--seed", "2026", + "--warmup", str(args.warmup), + "--repeat", str(args.repeat), + ], + results / "logs" / "faiss_gpu.log", + ) + run( + [ + sys.executable, + str(scripts / "plot_results.py"), + "--results-dir", str(results), + "--output-dir", str(results / "figures"), + ] + ) + + environment = { + "dataset": args.dataset, + "timestamp_utc": datetime.now(timezone.utc).isoformat(), + "git_commit": capture(["git", "-C", str(root), "rev-parse", "HEAD"]), + "source_sha256": { + name: sha256_file(root / name) for name in PROVENANCE_FILES + }, + "gpu": capture([ + "nvidia-smi", + "--query-gpu=name,driver_version,memory.total", + "--format=csv,noheader", + ]), + "nvcc": capture(["nvcc", "--version"]), + "python": sys.version, + "parameters": { + key: str(value) if isinstance(value, Path) else value + for key, value in vars(args).items() + }, + } + (results / "environment.json").write_text( + json.dumps(environment, ensure_ascii=False, indent=2) + "\n", + encoding="utf-8", + ) + print(f"Stage-6 results: {results}") + + +if __name__ == "__main__": + main() diff --git a/09_vector_retrieval/PurplePulse/scripts/slice_queries.py b/09_vector_retrieval/PurplePulse/scripts/slice_queries.py new file mode 100644 index 00000000..27373dd6 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/scripts/slice_queries.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python3 +"""从查询文件中复制前 N 条 query，用于较慢的 CPU 正确性验证。""" + +import argparse +import struct +from pathlib import Path + + +DTYPE_SIZES = {1: 4, 2: 2} + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--input", required=True) + parser.add_argument("--output", required=True) + parser.add_argument("--count", type=int, required=True) + args = parser.parse_args() + + if args.count <= 0: + parser.error("count 必须大于 0") + + with Path(args.input).open("rb") as source: + magic = source.read(8) + if magic != b"PPQRY001": + raise SystemExit("输入不是本项目的 query 文件") + num_queries, dim, dtype = struct.unpack(" num_queries: + raise SystemExit("count 不能大于原文件的 query 数量") + value_bytes = args.count * dim * DTYPE_SIZES[dtype] + values = source.read(value_bytes) + if len(values) != value_bytes: + raise SystemExit("输入文件数据不完整") + + output_path = Path(args.output) + output_path.parent.mkdir(parents=True, exist_ok=True) + with output_path.open("wb") as output: + output.write(b"PPQRY001") + output.write(struct.pack(" int: + if not path.is_file(): + raise FileNotFoundError(path) + with path.open(newline="", encoding="utf-8") as source: + reader = csv.DictReader(source) + fields = set(reader.fieldnames or ()) + if required_columns and not required_columns <= fields: + missing = sorted(required_columns - fields) + raise ValueError(f"{path}: missing columns {missing}") + rows = list(reader) + if not rows: + raise ValueError(f"{path}: no data rows") + return len(rows) + + +def validate_pdf(path: Path) -> int: + if not path.is_file() or path.stat().st_size < 1024: + raise ValueError(f"missing or undersized PDF: {path}") + with path.open("rb") as source: + if source.read(5) != b"%PDF-": + raise ValueError(f"invalid PDF header: {path}") + return path.stat().st_size + + +def validate(root: Path, report: Path) -> dict[str, object]: + required_docs = { + "README.md": "一条命令复现", + "docs/FINAL_REPORT.md": "7.4", + "docs/REPRODUCIBILITY.md": "第九阶段工程化入口", + "docs/STABILITY_MATRIX.md": "Exact 选择规则", + "docs/API.md": "C++ API", + } + for relative, marker in required_docs.items(): + path = root / relative + if not path.is_file() or marker not in path.read_text(encoding="utf-8"): + raise ValueError(f"{relative}: missing required marker {marker!r}") + + csv_rows: dict[str, int] = {} + for dataset in ("sift1m", "yahoo-minilm-384"): + base = root / "results" / "real" / dataset + for filename, columns in ( + ("exact_gpu.csv", {"top_k", "qps"}), + ("faiss_gpu.csv", {"top_k", "qps"}), + ("ivf_k10.csv", {"nprobe", "recall_at_k", "qps"}), + ("ivf_k50.csv", {"nprobe", "recall_at_k", "qps"}), + ("ivf_k100.csv", {"nprobe", "recall_at_k", "qps"}), + ): + path = base / filename + csv_rows[str(path.relative_to(root))] = validate_csv(path, columns) + + for filename, columns in ( + ("exact_gpu.csv", {"top_k", "qps", "run_samples"}), + ("faiss_cpu_flat.csv", {"top_k", "qps", "run_samples"}), + ): + path = root / "results" / "formal_100k15" / filename + csv_rows[str(path.relative_to(root))] = validate_csv(path, columns) + + nsys_summary = root / "evidence" / "2026-09-04" / "nsys_exact_k10.txt" + nsys_text = nsys_summary.read_text(encoding="utf-8") + for marker in ("99.9", "exactFusedWarpTopKKernel", "cuda_gpu_kern_sum"): + if marker not in nsys_text: + raise ValueError(f"{nsys_summary.relative_to(root)}: missing marker {marker!r}") + + coverage = root / "results" / "stage8_stability" / "coverage.csv" + csv_rows[str(coverage.relative_to(root))] = validate_csv( + coverage, {"suite", "cases", "status"} + ) + with coverage.open(newline="", encoding="utf-8") as source: + failed = [row["suite"] for row in csv.DictReader(source) + if row["status"] != "passed"] + if failed: + raise ValueError(f"stage8 coverage failures: {failed}") + + for path in ( + root / "results" / "figures" / "report-architecture.svg", + root / "results" / "figures" / "report-exact-fused.svg", + root / "results" / "figures" / "report-exact-breakdown.svg", + root / "results" / "figures" / "report-k50-scheduling.svg", + root / "results" / "figures" / "report-k100-partitioned.svg", + root / "results" / "figures" / "report-recall-qps.svg", + root / "results" / "figures" / "report-real-comparison.svg", + ): + if not path.is_file() or " None: + parser = argparse.ArgumentParser() + parser.add_argument("--project-root", type=Path, default=ROOT) + parser.add_argument( + "--report", type=Path, + default=Path("output/pdf/WarpFusion_Technical_Report.pdf"), + ) + parser.add_argument("--json-output", type=Path) + args = parser.parse_args() + root = args.project_root.resolve() + report = args.report.resolve() if args.report.is_absolute() else (root / args.report) + result = validate(root, report) + text = json.dumps(result, ensure_ascii=False, indent=2) + "\n" + if args.json_output: + output = (args.json_output if args.json_output.is_absolute() + else root / args.json_output) + output.parent.mkdir(parents=True, exist_ok=True) + output.write_text(text, encoding="utf-8") + print(text, end="") + + +if __name__ == "__main__": + main() diff --git a/09_vector_retrieval/PurplePulse/scripts/verify_results.py b/09_vector_retrieval/PurplePulse/scripts/verify_results.py new file mode 100644 index 00000000..9dc44349 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/scripts/verify_results.py @@ -0,0 +1,91 @@ +#!/usr/bin/env python3 +"""比较 CPU 与 GPU 的文本结果，检查 Top-K ID 和分数误差。""" + +import argparse +from collections import defaultdict +from pathlib import Path + + +def read_results(path): + rows = [] + for line_number, line in enumerate(Path(path).read_text().splitlines(), 1): + parts = line.split() + if len(parts) != 3: + raise ValueError(f"{path}:{line_number} 不是三列") + rows.append((int(parts[0]), int(parts[1]), float(parts[2]))) + return rows + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--expected", required=True) + parser.add_argument("--actual", required=True) + parser.add_argument( + "--tolerance", + type=float, + default=5e-5, + help="CPU 串行累加与 GPU warp 归约的允许绝对误差", + ) + args = parser.parse_args() + + expected = read_results(args.expected) + actual = read_results(args.actual) + if len(expected) != len(actual): + raise SystemExit( + f"失败：行数不同，expected={len(expected)}, actual={len(actual)}" + ) + + expected_by_query = defaultdict(list) + actual_by_query = defaultdict(list) + for row in expected: + expected_by_query[row[0]].append(row) + for row in actual: + actual_by_query[row[0]].append(row) + + if expected_by_query.keys() != actual_by_query.keys(): + raise SystemExit("失败：query ID 集合不同") + + max_error = 0.0 + error_sum = 0.0 + error_count = 0 + near_tie_reorders = 0 + for query_id in expected_by_query: + expected_rows = expected_by_query[query_id] + actual_rows = actual_by_query[query_id] + expected_scores = {row[1]: row[2] for row in expected_rows} + actual_scores = {row[1]: row[2] for row in actual_rows} + if expected_scores.keys() != actual_scores.keys(): + missing = expected_scores.keys() - actual_scores.keys() + unexpected = actual_scores.keys() - expected_scores.keys() + raise SystemExit( + f"失败：query {query_id} 的 Top-K 集合不同，" + f"missing={sorted(missing)}, unexpected={sorted(unexpected)}" + ) + for vector_id in expected_scores: + error = abs(expected_scores[vector_id] - actual_scores[vector_id]) + max_error = max(max_error, error) + error_sum += error + error_count += 1 + for rank, (left, right) in enumerate(zip(expected_rows, actual_rows)): + if left[1] != right[1]: + score_gap = abs(left[2] - right[2]) + if score_gap > args.tolerance: + raise SystemExit( + f"失败：query {query_id} rank {rank} 顺序不同且非近似并列，" + f"expected={left}, actual={right}" + ) + near_tie_reorders += 1 + + if max_error > args.tolerance: + raise SystemExit( + f"失败：最大分数误差 {max_error} 超过容差 {args.tolerance}" + ) + print( + "通过：Top-K 候选集合相同，" + f"平均绝对分数误差={error_sum / error_count if error_count else 0.0}，" + f"最大分数误差={max_error}，近似并列换位数={near_tie_reorders}" + ) + + +if __name__ == "__main__": + main() diff --git a/09_vector_retrieval/PurplePulse/src/build_ivf.cpp b/09_vector_retrieval/PurplePulse/src/build_ivf.cpp new file mode 100644 index 00000000..4e01abf5 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/src/build_ivf.cpp @@ -0,0 +1,137 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "file_io.h" +#include "ivf_flat.h" + +namespace { + +void printUsage(const char* program) { + std::cerr << "用法:\n " << program + << " --database <文件> --output <索引文件> --nlist <数量>" + " [--iterations 15] [--training-samples 100000]" + " [--threads N]\n"; +} + +std::unordered_map parseArguments(int argc, + char** argv) { + std::unordered_map arguments; + for (int index = 1; index < argc; index += 2) { + if (index + 1 >= argc || std::string(argv[index]).rfind("--", 0) != 0) { + throw std::runtime_error("命令行参数必须是 --名称 值 的形式"); + } + arguments[argv[index]] = argv[index + 1]; + } + for (const char* required : {"--database", "--output", "--nlist"}) { + if (arguments.count(required) == 0) { + throw std::runtime_error(std::string("缺少参数: ") + required); + } + } + return arguments; +} + +std::uint64_t readPositive( + const std::unordered_map& arguments, + const std::string& name, std::uint64_t default_value) { + const auto found = arguments.find(name); + if (found == arguments.end()) { + return default_value; + } + std::size_t parsed_chars = 0; + const unsigned long long parsed = std::stoull(found->second, &parsed_chars); + if (parsed_chars != found->second.size() || parsed == 0) { + throw std::runtime_error(name + " 必须是正整数"); + } + return parsed; +} + +} // namespace + +int main(int argc, char** argv) { + try { + if (argc == 2 && std::string(argv[1]) == "--help") { + printUsage(argv[0]); + return EXIT_SUCCESS; + } + if (argc == 1) { + printUsage(argv[0]); + return EXIT_FAILURE; + } + const auto arguments = parseArguments(argc, argv); + const std::uint64_t parsed_nlist = readPositive(arguments, "--nlist", 0); + const std::uint64_t parsed_iterations = + readPositive(arguments, "--iterations", 15); + if (parsed_nlist > UINT32_MAX || parsed_iterations > UINT32_MAX) { + throw std::runtime_error("nlist 或 iterations 超过 uint32 范围"); + } + const std::uint64_t training_samples = + readPositive(arguments, "--training-samples", 100000); + const std::uint32_t default_threads = std::max( + 1U, std::min(16U, std::thread::hardware_concurrency())); + const std::uint64_t parsed_threads = + readPositive(arguments, "--threads", default_threads); + if (parsed_threads > 1024) { + throw std::runtime_error("threads 必须位于 [1, 1024]"); + } + const VectorDatabase database = + readVectorDatabase(arguments.at("--database")); + + const auto build_start = std::chrono::steady_clock::now(); + const IvfFlatIndex index = buildIvfFlatIndex( + database, static_cast(parsed_nlist), + static_cast(parsed_iterations), training_samples, + static_cast(parsed_threads)); + const auto build_end = std::chrono::steady_clock::now(); + + const std::filesystem::path output_path(arguments.at("--output")); + if (output_path.has_parent_path()) { + std::filesystem::create_directories(output_path.parent_path()); + } + const auto save_start = std::chrono::steady_clock::now(); + writeIvfFlatIndex(output_path.string(), index); + const auto save_end = std::chrono::steady_clock::now(); + + std::uint64_t minimum_bucket = index.num_vectors; + std::uint64_t maximum_bucket = 0; + std::uint32_t empty_buckets = 0; + for (std::uint32_t center_id = 0; center_id < index.nlist; ++center_id) { + const std::uint64_t size = + index.offsets[center_id + 1] - index.offsets[center_id]; + minimum_bucket = std::min(minimum_bucket, size); + maximum_bucket = std::max(maximum_bucket, size); + empty_buckets += size == 0; + } + const double build_ms = + std::chrono::duration(build_end - build_start) + .count(); + const double save_ms = + std::chrono::duration(save_end - save_start) + .count(); + std::cout << "IVF-Flat 索引构建完成" << "\n向量库: " << index.num_vectors + << " x " << index.dim + << "\ndtype/metric: " << dataTypeName(index.dtype) << '/' + << metricName(index.metric) << "\nnlist: " << index.nlist + << "\n迭代/训练样本: " << parsed_iterations << '/' + << std::min(training_samples, index.num_vectors) + << "\n建库线程: " << parsed_threads + << "\n构建时间: " << build_ms << " ms" + << "\n保存时间: " << save_ms << " ms" + << "\n桶大小 min/avg/max: " << minimum_bucket << '/' + << static_cast(index.num_vectors) / index.nlist << '/' + << maximum_bucket << "\n空桶: " << empty_buckets + << "\n索引大小: " << std::filesystem::file_size(output_path) + << " bytes\n输出: " << output_path << '\n'; + return EXIT_SUCCESS; + } catch (const std::exception& error) { + std::cerr << "错误: " << error.what() << '\n'; + printUsage(argv[0]); + return EXIT_FAILURE; + } +} diff --git a/09_vector_retrieval/PurplePulse/src/cpu_search.cpp b/09_vector_retrieval/PurplePulse/src/cpu_search.cpp new file mode 100644 index 00000000..5670d17d --- /dev/null +++ b/09_vector_retrieval/PurplePulse/src/cpu_search.cpp @@ -0,0 +1,163 @@ +#include +#include +#include +#include +#include +#include +#include + +#include "file_io.h" +#include "search.h" + +namespace { + +float halfToFloat(std::uint16_t bits) { + const bool negative = (bits & 0x8000U) != 0; + const std::uint32_t exponent = (bits >> 10U) & 0x1fU; + const std::uint32_t mantissa = bits & 0x03ffU; + float value = 0.0F; + if (exponent == 0) { + value = std::ldexp(static_cast(mantissa), -24); + } else if (exponent == 31) { + value = mantissa == 0 ? std::numeric_limits::infinity() + : std::numeric_limits::quiet_NaN(); + } else { + value = std::ldexp(static_cast(1024U + mantissa), + static_cast(exponent) - 25); + } + return negative ? -value : value; +} + +float databaseValue(const VectorDatabase& database, std::uint64_t index) { + return database.dtype == DataType::kFloat32 + ? database.values[index] + : halfToFloat(database.half_values[index]); +} + +float queryValue(const QuerySet& queries, std::uint64_t index) { + return queries.dtype == DataType::kFloat32 + ? queries.values[index] + : halfToFloat(queries.half_values[index]); +} + +float computeScore(const VectorDatabase& database, const QuerySet& queries, + std::uint64_t query_id, std::uint64_t vector_id) { + float dot = 0.0F; + float query_norm = 0.0F; + float vector_norm = 0.0F; + float squared_l2 = 0.0F; + + for (std::uint32_t d = 0; d < database.dim; ++d) { + const float q = queryValue(queries, query_id * queries.dim + d); + const float x = databaseValue(database, vector_id * database.dim + d); + if (database.metric == Metric::kL2) { + const float difference = q - x; + squared_l2 = std::fma(difference, difference, squared_l2); + } else { + dot = std::fma(q, x, dot); + if (database.metric == Metric::kCosine) { + query_norm = std::fma(q, q, query_norm); + vector_norm = std::fma(x, x, vector_norm); + } + } + } + + if (database.metric == Metric::kL2) { + return squared_l2; + } + if (database.metric == Metric::kInnerProduct) { + return dot; + } + if (query_norm == 0.0F || vector_norm == 0.0F) { + return 0.0F; + } + return dot / std::sqrt(query_norm * vector_norm); +} + +bool isBetter(const Neighbor& left, const Neighbor& right, Metric metric) { + if (left.score == right.score) { + return left.id < right.id; + } + if (metric == Metric::kL2) { + return left.score < right.score; + } + return left.score > right.score; +} + +std::vector selectTopK(std::vector candidates, + std::uint32_t top_k, Metric metric) { + const auto comparator = [metric](const Neighbor& left, + const Neighbor& right) { + return isBetter(left, right, metric); + }; + + // nth_element 先把最好的 K 个移到前面，再只排序这 K 个。 + // 它比对全部候选排序更省时间，同时代码仍然很容易理解。 + if (top_k < candidates.size()) { + std::nth_element(candidates.begin(), candidates.begin() + top_k, + candidates.end(), comparator); + candidates.resize(top_k); + } + std::sort(candidates.begin(), candidates.end(), comparator); + return candidates; +} + +} // namespace + +SearchResults cpuExactSearch(const VectorDatabase& database, + const QuerySet& queries, + const SearchParams& params) { + validateInputs(database, queries, params); + SearchResults results(queries.num_queries); + + for (std::uint64_t query_id = 0; query_id < queries.num_queries; ++query_id) { + std::vector candidates(database.num_vectors); + + for (std::uint64_t vector_id = 0; vector_id < database.num_vectors; + ++vector_id) { + candidates[vector_id] = { + vector_id, + computeScore(database, queries, query_id, vector_id), + }; + } + results[query_id] = + selectTopK(std::move(candidates), params.top_k, database.metric); + } + return results; +} + +bool resultsMatch(const SearchResults& expected, const SearchResults& actual, + float absolute_tolerance, std::string* error_message) { + if (expected.size() != actual.size()) { + if (error_message != nullptr) { + *error_message = "query 数量不同"; + } + return false; + } + + for (std::size_t query_id = 0; query_id < expected.size(); ++query_id) { + if (expected[query_id].size() != actual[query_id].size()) { + if (error_message != nullptr) { + *error_message = "query " + std::to_string(query_id) + " 的 K 不同"; + } + return false; + } + for (std::size_t rank = 0; rank < expected[query_id].size(); ++rank) { + const Neighbor& left = expected[query_id][rank]; + const Neighbor& right = actual[query_id][rank]; + if (left.id != right.id || + std::fabs(left.score - right.score) > absolute_tolerance) { + if (error_message != nullptr) { + std::ostringstream message; + message << "query=" << query_id << ", rank=" << rank + << " 不一致，CPU(id=" << left.id << ", score=" << left.score + << ")，GPU(id=" << right.id << ", score=" << right.score + << ')'; + *error_message = message.str(); + } + return false; + } + } + } + return true; +} diff --git a/09_vector_retrieval/PurplePulse/src/file_io.cpp b/09_vector_retrieval/PurplePulse/src/file_io.cpp new file mode 100644 index 00000000..a99d7174 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/src/file_io.cpp @@ -0,0 +1,533 @@ +#include "file_io.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +constexpr std::array kDatabaseMagic = {'P', 'P', 'V', 'E', + 'C', '0', '0', '1'}; +constexpr std::array kQueryMagic = {'P', 'P', 'Q', 'R', + 'Y', '0', '0', '1'}; +constexpr std::array kMemoryMetadataMagic = {'P', 'P', 'M', 'E', + 'T', 'A', '0', '1'}; + +template +void writeValue(std::ofstream& output, const T& value) { + output.write(reinterpret_cast(&value), sizeof(T)); +} + +template +T readValue(std::ifstream& input, const char* field_name) { + T value{}; + input.read(reinterpret_cast(&value), sizeof(T)); + if (!input) { + throw std::runtime_error(std::string("读取字段失败: ") + field_name); + } + return value; +} + +std::string trim(std::string text) { + const auto not_space = [](unsigned char c) { return !std::isspace(c); }; + text.erase(text.begin(), std::find_if(text.begin(), text.end(), not_space)); + text.erase(std::find_if(text.rbegin(), text.rend(), not_space).base(), + text.end()); + if (text.size() >= 2 && text.front() == '"' && text.back() == '"') { + text = text.substr(1, text.size() - 2); + } + return text; +} + +std::uint32_t parsePositiveInt(const std::string& value, + const std::string& key) { + std::size_t parsed_chars = 0; + const unsigned long parsed = std::stoul(value, &parsed_chars); + if (parsed_chars != value.size() || parsed == 0 || parsed > UINT32_MAX) { + throw std::runtime_error("参数 " + key + " 必须是正整数"); + } + return static_cast(parsed); +} + +float parsePositiveFloat(const std::string& value, const std::string& key) { + std::size_t parsed_chars = 0; + const float parsed = std::stof(value, &parsed_chars); + if (parsed_chars != value.size() || !std::isfinite(parsed) || + parsed <= 0.0F) { + throw std::runtime_error("参数 " + key + " 必须是有限正数"); + } + return parsed; +} + +float parseNonnegativeFloat(const std::string& value, const std::string& key) { + std::size_t parsed_chars = 0; + const float parsed = std::stof(value, &parsed_chars); + if (parsed_chars != value.size() || !std::isfinite(parsed) || parsed < 0.0F) { + throw std::runtime_error("参数 " + key + " 必须是有限非负数"); + } + return parsed; +} + +std::uint64_t parseUint64(const std::string& value, const std::string& key) { + if (value.empty() || value.front() == '-') { + throw std::runtime_error("参数 " + key + " 必须是非负整数"); + } + std::size_t parsed_chars = 0; + const unsigned long long parsed = std::stoull(value, &parsed_chars); + if (parsed_chars != value.size()) { + throw std::runtime_error("参数 " + key + " 必须是非负整数"); + } + return static_cast(parsed); +} + +std::uint32_t parseOptionalUint32(const std::string& value, + const std::string& key) { + if (value == "any") { + return UINT32_MAX; + } + const std::uint64_t parsed = parseUint64(value, key); + if (parsed >= UINT32_MAX) { + throw std::runtime_error("参数 " + key + " 必须小于 UINT32_MAX"); + } + return static_cast(parsed); +} + +std::uint64_t elementSize(DataType dtype) { + if (dtype == DataType::kFloat32) { + return sizeof(float); + } + if (dtype == DataType::kFloat16) { + return sizeof(std::uint16_t); + } + throw std::runtime_error("不支持的 dtype 编号: " + + std::to_string(static_cast(dtype))); +} + +std::uint64_t checkedValueCount(std::uint64_t rows, std::uint32_t dim, + const std::string& path) { + if (dim != 0 && rows > UINT64_MAX / dim) { + throw std::runtime_error("向量数量与维度乘积溢出: " + path); + } + return rows * dim; +} + +void checkFileSize(const std::string& path, std::uint64_t header_bytes, + std::uint64_t value_count, DataType dtype) { + const std::uint64_t element_bytes = elementSize(dtype); + if (value_count > (UINT64_MAX - header_bytes) / element_bytes) { + throw std::runtime_error("文件大小计算溢出: " + path); + } + const std::uint64_t expected = header_bytes + value_count * element_bytes; + const std::uint64_t actual = std::filesystem::file_size(path); + if (actual != expected) { + throw std::runtime_error("文件大小不正确: " + path + "，期望 " + + std::to_string(expected) + " 字节，实际 " + + std::to_string(actual) + " 字节"); + } +} + +} // namespace + +const char* metricName(Metric metric) { + switch (metric) { + case Metric::kL2: + return "l2"; + case Metric::kInnerProduct: + return "inner_product"; + case Metric::kCosine: + return "cosine"; + } + throw std::runtime_error("未知的距离类型"); +} + +const char* dataTypeName(DataType dtype) { + switch (dtype) { + case DataType::kFloat32: + return "fp32"; + case DataType::kFloat16: + return "fp16"; + } + throw std::runtime_error("未知的数据类型"); +} + +Metric parseMetric(const std::string& text) { + if (text == "l2") { + return Metric::kL2; + } + if (text == "inner_product") { + return Metric::kInnerProduct; + } + if (text == "cosine") { + return Metric::kCosine; + } + throw std::runtime_error("不支持的 metric: " + text); +} + +VectorDatabase readVectorDatabase(const std::string& path) { + std::ifstream input(path, std::ios::binary); + if (!input) { + throw std::runtime_error("无法打开向量库: " + path); + } + + std::array magic{}; + input.read(magic.data(), magic.size()); + if (magic != kDatabaseMagic) { + throw std::runtime_error("向量库 magic/version 不正确: " + path); + } + + VectorDatabase database; + database.num_vectors = readValue(input, "num_vectors"); + database.dim = readValue(input, "dim"); + database.dtype = + static_cast(readValue(input, "dtype")); + database.metric = + static_cast(readValue(input, "metric")); + + (void)dataTypeName(database.dtype); + (void)metricName(database.metric); + if (database.num_vectors == 0 || database.dim == 0) { + throw std::runtime_error("向量库的数量和维度必须大于 0"); + } + + const std::uint64_t count = + checkedValueCount(database.num_vectors, database.dim, path); + checkFileSize(path, 8 + 8 + 4 + 4 + 4, count, database.dtype); + if (database.dtype == DataType::kFloat32) { + database.values.resize(count); + input.read(reinterpret_cast(database.values.data()), + static_cast(count * sizeof(float))); + } else { + database.half_values.resize(count); + input.read(reinterpret_cast(database.half_values.data()), + static_cast(count * sizeof(std::uint16_t))); + } + if (!input) { + throw std::runtime_error("读取向量库数据失败: " + path); + } + return database; +} + +QuerySet readQuerySet(const std::string& path) { + std::ifstream input(path, std::ios::binary); + if (!input) { + throw std::runtime_error("无法打开查询文件: " + path); + } + + std::array magic{}; + input.read(magic.data(), magic.size()); + if (magic != kQueryMagic) { + throw std::runtime_error("查询文件 magic/version 不正确: " + path); + } + + QuerySet queries; + queries.num_queries = readValue(input, "num_queries"); + queries.dim = readValue(input, "dim"); + queries.dtype = + static_cast(readValue(input, "dtype")); + (void)dataTypeName(queries.dtype); + if (queries.num_queries == 0 || queries.dim == 0) { + throw std::runtime_error("查询数量和维度必须大于 0"); + } + + const std::uint64_t count = + checkedValueCount(queries.num_queries, queries.dim, path); + checkFileSize(path, 8 + 8 + 4 + 4, count, queries.dtype); + if (queries.dtype == DataType::kFloat32) { + queries.values.resize(count); + input.read(reinterpret_cast(queries.values.data()), + static_cast(count * sizeof(float))); + } else { + queries.half_values.resize(count); + input.read(reinterpret_cast(queries.half_values.data()), + static_cast(count * sizeof(std::uint16_t))); + } + if (!input) { + throw std::runtime_error("读取查询数据失败: " + path); + } + return queries; +} + +void validateMemoryMetadata(const MemoryMetadata& metadata, + std::uint64_t expected_vectors) { + if (metadata.num_vectors == 0 || + (expected_vectors != 0 && metadata.num_vectors != expected_vectors) || + metadata.timestamps.size() != metadata.num_vectors || + metadata.importance.size() != metadata.num_vectors || + metadata.session_ids.size() != metadata.num_vectors || + metadata.source_types.size() != metadata.num_vectors) { + throw std::runtime_error("记忆元数据列长度或向量数量不一致"); + } + for (const float value : metadata.importance) { + if (!std::isfinite(value) || value < 0.0F || value > 1.0F) { + throw std::runtime_error("记忆 importance 必须位于 [0, 1]"); + } + } +} + +void writeMemoryMetadata(const std::string& path, + const MemoryMetadata& metadata) { + validateMemoryMetadata(metadata); + std::ofstream output(path, std::ios::binary); + if (!output) { + throw std::runtime_error("无法写入记忆元数据: " + path); + } + output.write(kMemoryMetadataMagic.data(), kMemoryMetadataMagic.size()); + writeValue(output, metadata.num_vectors); + output.write(reinterpret_cast(metadata.timestamps.data()), + static_cast(metadata.num_vectors * + sizeof(std::uint64_t))); + output.write( + reinterpret_cast(metadata.importance.data()), + static_cast(metadata.num_vectors * sizeof(float))); + output.write(reinterpret_cast(metadata.session_ids.data()), + static_cast(metadata.num_vectors * + sizeof(std::uint32_t))); + output.write(reinterpret_cast(metadata.source_types.data()), + static_cast(metadata.num_vectors * + sizeof(std::uint32_t))); + if (!output) { + throw std::runtime_error("写入记忆元数据失败: " + path); + } +} + +MemoryMetadata readMemoryMetadata(const std::string& path) { + std::ifstream input(path, std::ios::binary); + if (!input) { + throw std::runtime_error("无法打开记忆元数据: " + path); + } + std::array magic{}; + input.read(magic.data(), magic.size()); + if (magic != kMemoryMetadataMagic) { + throw std::runtime_error("记忆元数据 magic/version 不正确: " + path); + } + MemoryMetadata metadata; + metadata.num_vectors = readValue(input, "num_vectors"); + constexpr std::uint64_t kBytesPerVector = + sizeof(std::uint64_t) + sizeof(float) + 2 * sizeof(std::uint32_t); + if (metadata.num_vectors == 0 || + metadata.num_vectors > (UINT64_MAX - 16) / kBytesPerVector || + std::filesystem::file_size(path) != + 16 + metadata.num_vectors * kBytesPerVector) { + throw std::runtime_error("记忆元数据文件大小不正确: " + path); + } + metadata.timestamps.resize(metadata.num_vectors); + metadata.importance.resize(metadata.num_vectors); + metadata.session_ids.resize(metadata.num_vectors); + metadata.source_types.resize(metadata.num_vectors); + input.read(reinterpret_cast(metadata.timestamps.data()), + static_cast(metadata.num_vectors * + sizeof(std::uint64_t))); + input.read( + reinterpret_cast(metadata.importance.data()), + static_cast(metadata.num_vectors * sizeof(float))); + input.read(reinterpret_cast(metadata.session_ids.data()), + static_cast(metadata.num_vectors * + sizeof(std::uint32_t))); + input.read(reinterpret_cast(metadata.source_types.data()), + static_cast(metadata.num_vectors * + sizeof(std::uint32_t))); + if (!input) { + throw std::runtime_error("读取记忆元数据失败: " + path); + } + validateMemoryMetadata(metadata); + return metadata; +} + +SearchParams readSearchParams(const std::string& path) { + std::ifstream input(path); + if (!input) { + throw std::runtime_error("无法打开参数文件: " + path); + } + + SearchParams params; + std::string line; + while (std::getline(input, line)) { + const std::size_t comment = line.find('#'); + if (comment != std::string::npos) { + line.erase(comment); + } + line = trim(line); + if (line.empty()) { + continue; + } + + const std::size_t equal = line.find('='); + if (equal == std::string::npos) { + throw std::runtime_error("参数行缺少 '=': " + line); + } + const std::string key = trim(line.substr(0, equal)); + const std::string value = trim(line.substr(equal + 1)); + + if (key == "top_k") { + params.top_k = parsePositiveInt(value, key); + } else if (key == "search_mode") { + params.search_mode = value; + } else if (key == "batch_size") { + params.batch_size = parsePositiveInt(value, key); + } else if (key == "distance_mode") { + params.distance_mode = value; + } else if (key == "topk_mode") { + params.topk_mode = value; + } else if (key == "exact_query_tile") { + params.exact_query_tile = parsePositiveInt(value, key); + } else if (key == "exact_fused_layout") { + params.exact_fused_layout = value; + } else if (key == "exact_fused_blocks") { + params.exact_fused_blocks = parsePositiveInt(value, key); + } else if (key == "nlist") { + params.nlist = parsePositiveInt(value, key); + } else if (key == "nprobe") { + params.nprobe = parsePositiveInt(value, key); + } else if (key == "nprobe_policy") { + params.nprobe_policy = value; + } else if (key == "adaptive_execution") { + params.adaptive_execution = value; + } else if (key == "adaptive_nprobe_min") { + params.adaptive_nprobe_min = parsePositiveInt(value, key); + } else if (key == "adaptive_nprobe_step") { + params.adaptive_nprobe_step = parsePositiveInt(value, key); + } else if (key == "adaptive_target_mass") { + params.adaptive_target_mass = parsePositiveFloat(value, key); + } else if (key == "adaptive_temperature") { + params.adaptive_temperature = parsePositiveFloat(value, key); + } else if (key == "memory_mode") { + params.memory_mode = value; + } else if (key == "memory_semantic_weight") { + params.memory_semantic_weight = parsePositiveFloat(value, key); + } else if (key == "memory_importance_weight") { + params.memory_importance_weight = parseNonnegativeFloat(value, key); + } else if (key == "memory_recency_weight") { + params.memory_recency_weight = parseNonnegativeFloat(value, key); + } else if (key == "memory_time_scale") { + params.memory_time_scale = parsePositiveFloat(value, key); + } else if (key == "memory_now") { + params.memory_now = parseUint64(value, key); + } else if (key == "filter_min_timestamp") { + params.filter_min_timestamp = parseUint64(value, key); + } else if (key == "filter_session_id") { + params.filter_session_id = parseOptionalUint32(value, key); + } else if (key == "filter_source_type") { + params.filter_source_type = parseOptionalUint32(value, key); + } else if (key == "memory_rerank_factor") { + params.memory_rerank_factor = parsePositiveInt(value, key); + } else if (key == "pq_m") { + params.pq_m = parsePositiveInt(value, key); + } else { + throw std::runtime_error("未知参数: " + key); + } + } + return params; +} + +void writeSearchResults(const std::string& path, const SearchResults& results) { + std::ofstream output(path); + if (!output) { + throw std::runtime_error("无法写入结果文件: " + path); + } + output.precision(9); + for (std::size_t query_id = 0; query_id < results.size(); ++query_id) { + for (const Neighbor& neighbor : results[query_id]) { + output << query_id << ' ' << neighbor.id << ' ' << neighbor.score << '\n'; + } + } +} + +void validateInputs(const VectorDatabase& database, const QuerySet& queries, + const SearchParams& params) { + if (database.num_vectors == 0 || database.dim == 0) { + throw std::runtime_error("向量库的数量和维度必须大于 0"); + } + if (queries.num_queries == 0 || queries.dim == 0) { + throw std::runtime_error("查询数量和维度必须大于 0"); + } + if (database.dim != queries.dim) { + throw std::runtime_error("向量库维度与查询维度不一致"); + } + if (database.dtype != queries.dtype) { + throw std::runtime_error("向量库与查询的数据类型不一致"); + } + const std::uint64_t database_count = + checkedValueCount(database.num_vectors, database.dim, "向量库"); + const std::uint64_t query_count = + checkedValueCount(queries.num_queries, queries.dim, "查询集"); + if (database.dtype == DataType::kFloat32) { + if (database.values.size() != database_count || + queries.values.size() != query_count) { + throw std::runtime_error("FP32 数据长度与头部元数据不一致"); + } + } else if (database.dtype == DataType::kFloat16) { + if (database.half_values.size() != database_count || + queries.half_values.size() != query_count) { + throw std::runtime_error("FP16 数据长度与头部元数据不一致"); + } + } else { + (void)elementSize(database.dtype); + } + if (params.top_k == 0) { + throw std::runtime_error("top_k 必须大于 0"); + } + if (params.top_k > database.num_vectors) { + throw std::runtime_error("top_k 不能大于向量库大小"); + } + if (params.top_k > 100) { + throw std::runtime_error("当前按照项目要求支持 top_k <= 100"); + } + if (params.batch_size == 0) { + throw std::runtime_error("batch_size 必须大于 0"); + } + if (params.search_mode != "exact") { + throw std::runtime_error("当前只实现了 search_mode=exact，尚未实现: " + + params.search_mode); + } + if (params.topk_mode != "simple" && params.topk_mode != "block" && + params.topk_mode != "two_stage" && params.topk_mode != "fused") { + throw std::runtime_error( + "topk_mode 必须是 simple、block、two_stage 或 fused"); + } + if (params.topk_mode == "two_stage" && params.top_k > 10) { + throw std::runtime_error("当前 two_stage Top-K 支持 top_k <= 10"); + } + if (params.distance_mode != "simple" && params.distance_mode != "warp") { + throw std::runtime_error("distance_mode 必须是 simple 或 warp"); + } + if (params.topk_mode == "fused" && params.distance_mode != "warp") { + throw std::runtime_error("fused Top-K 要求 distance_mode=warp"); + } + if (params.exact_query_tile != 1 && params.exact_query_tile != 2 && + params.exact_query_tile != 4 && params.exact_query_tile != 8) { + throw std::runtime_error("exact_query_tile 必须是 1、2、4 或 8"); + } + if (params.exact_query_tile != 1 && params.topk_mode != "fused") { + throw std::runtime_error("exact_query_tile > 1 只支持 fused Top-K"); + } + if (params.exact_fused_blocks != 0 && params.topk_mode != "fused") { + throw std::runtime_error("exact_fused_blocks 只支持 fused Top-K"); + } + if (params.exact_query_tile == 8 && params.top_k > 10) { + throw std::runtime_error("exact_query_tile=8 当前只支持 top_k <= 10"); + } + if (params.exact_fused_layout != "warp_query" && + params.exact_fused_layout != "block_tile") { + throw std::runtime_error( + "exact_fused_layout 必须是 warp_query 或 block_tile"); + } + if (params.exact_fused_layout == "block_tile" && + (params.topk_mode != "fused" || params.exact_query_tile != 8 || + params.top_k > 10)) { + throw std::runtime_error( + "block_tile 当前要求 fused、exact_query_tile=8、top_k<=10"); + } + if (params.exact_fused_layout == "block_tile") { + const std::uint32_t maximum_dim = + database.dtype == DataType::kFloat32 ? 384 : 768; + if (database.dim > maximum_dim) { + throw std::runtime_error("block_tile 当前共享内存预算不支持该维度"); + } + } +} diff --git a/09_vector_retrieval/PurplePulse/src/gpu_ivf_flat.cu b/09_vector_retrieval/PurplePulse/src/gpu_ivf_flat.cu new file mode 100644 index 00000000..677b71cf --- /dev/null +++ b/09_vector_retrieval/PurplePulse/src/gpu_ivf_flat.cu @@ -0,0 +1,2098 @@ +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "file_io.h" +#include "ivf_flat.h" + +namespace { + +void checkCuda(cudaError_t status, const char* operation) { + if (status != cudaSuccess) { + throw std::runtime_error(std::string(operation) + ": " + + cudaGetErrorString(status)); + } +} + +class DeviceBuffer { + public: + explicit DeviceBuffer(std::size_t bytes) { + checkCuda(cudaMalloc(&data_, bytes), "cudaMalloc IVF"); + } + ~DeviceBuffer() { + if (data_ != nullptr) { + cudaFree(data_); + } + } + DeviceBuffer(const DeviceBuffer&) = delete; + DeviceBuffer& operator=(const DeviceBuffer&) = delete; + void* data() { return data_; } + + private: + void* data_ = nullptr; +}; + +class CudaEvent { + public: + CudaEvent() { checkCuda(cudaEventCreate(&event_), "cudaEventCreate IVF"); } + ~CudaEvent() { cudaEventDestroy(event_); } + CudaEvent(const CudaEvent&) = delete; + CudaEvent& operator=(const CudaEvent&) = delete; + void record() { checkCuda(cudaEventRecord(event_), "cudaEventRecord IVF"); } + void synchronize() { + checkCuda(cudaEventSynchronize(event_), "cudaEventSynchronize IVF"); + } + cudaEvent_t get() const { return event_; } + + private: + cudaEvent_t event_{}; +}; + +double elapsedMilliseconds(const CudaEvent& start, const CudaEvent& end) { + float milliseconds = 0.0F; + checkCuda(cudaEventElapsedTime(&milliseconds, start.get(), end.get()), + "cudaEventElapsedTime IVF"); + return milliseconds; +} + +__device__ __forceinline__ float loadInput(const float* values, + std::uint64_t index) { + return values[index]; +} + +__device__ __forceinline__ float loadInput(const __half* values, + std::uint64_t index) { + return __half2float(values[index]); +} + +__device__ __forceinline__ float warpReduceSum(float value) { + for (int offset = warpSize / 2; offset > 0; offset >>= 1) { + value += __shfl_down_sync(0xFFFFFFFFU, value, offset); + } + return value; +} + +__device__ bool isBetter(float candidate_score, std::uint64_t candidate_id, + float current_score, std::uint64_t current_id, + Metric metric) { + if (candidate_score == current_score) { + return candidate_id < current_id; + } + return metric == Metric::kL2 ? candidate_score < current_score + : candidate_score > current_score; +} + +struct DeviceMemoryView { + const std::uint64_t* timestamps = nullptr; + const float* importance = nullptr; + const std::uint32_t* session_ids = nullptr; + const std::uint32_t* source_types = nullptr; +}; + +struct DeviceMemoryOptions { + float semantic_weight = 1.0F; + float importance_weight = 0.0F; + float recency_weight = 0.0F; + float time_scale = 1.0F; + std::uint64_t now = 0; + std::uint64_t min_timestamp = 0; + std::uint32_t session_id = UINT32_MAX; + std::uint32_t source_type = UINT32_MAX; +}; + +__device__ __forceinline__ bool memoryCandidateMatches( + DeviceMemoryView metadata, std::uint64_t vector_id, + DeviceMemoryOptions options) { + return metadata.timestamps[vector_id] >= options.min_timestamp && + (options.session_id == UINT32_MAX || + metadata.session_ids[vector_id] == options.session_id) && + (options.source_type == UINT32_MAX || + metadata.source_types[vector_id] == options.source_type); +} + +template +__device__ __forceinline__ float applyMemoryScore(float semantic_score, + std::uint64_t vector_id, + DeviceMemoryView metadata, + DeviceMemoryOptions options) { + float recency = 0.0F; + if (options.recency_weight != 0.0F && options.now != 0) { + const std::uint64_t timestamp = metadata.timestamps[vector_id]; + const double age = timestamp >= options.now + ? 0.0 + : static_cast(options.now - timestamp); + recency = expf(-static_cast(age / options.time_scale)); + } + const float boost = + options.importance_weight * metadata.importance[vector_id] + + options.recency_weight * recency; + const float weighted_semantic = options.semantic_weight * semantic_score; + return MetricValue == Metric::kL2 ? weighted_semantic - boost + : weighted_semantic + boost; +} + +template +__global__ void ivfCenterScoresKernel(const float* centers, + const InputType* queries, + float* center_scores, std::uint32_t nlist, + std::uint32_t dim, + std::uint32_t batch_count, + Metric metric) { + const std::uint32_t center_id = blockIdx.x * blockDim.x + threadIdx.x; + const std::uint32_t query_id = blockIdx.y; + if (center_id >= nlist || query_id >= batch_count) { + return; + } + const std::uint64_t center_offset = + static_cast(center_id) * dim; + const std::uint64_t query_offset = static_cast(query_id) * dim; + float dot = 0.0F; + float center_norm = 0.0F; + float query_norm = 0.0F; + float squared_l2 = 0.0F; + for (std::uint32_t d = 0; d < dim; ++d) { + const float query = loadInput(queries, query_offset + d); + const float center = centers[center_offset + d]; + if (metric == Metric::kL2) { + const float difference = query - center; + squared_l2 = fmaf(difference, difference, squared_l2); + } else { + dot = fmaf(query, center, dot); + if (metric == Metric::kCosine) { + query_norm = fmaf(query, query, query_norm); + center_norm = fmaf(center, center, center_norm); + } + } + } + float score = dot; + if (metric == Metric::kL2) { + score = squared_l2; + } else if (metric == Metric::kCosine) { + score = query_norm == 0.0F || center_norm == 0.0F + ? 0.0F + : dot / sqrtf(query_norm * center_norm); + } + center_scores[static_cast(query_id) * nlist + center_id] = + score; +} + +// 并行选择 nprobe：每轮让 256 个线程共同从剩余中心中归约出最佳中心， +// 选中后把该分数改为哨兵，再进行下一轮。 +__global__ void ivfSelectProbesBlockKernel(float* center_scores, + std::uint32_t* selected_centers, + float* selected_scores, + std::uint32_t nlist, + std::uint32_t nprobe, + Metric metric) { + constexpr std::uint32_t kThreads = 256; + constexpr float kFloatMaximum = 3.402823466e+38F; + __shared__ float shared_scores[kThreads]; + __shared__ std::uint32_t shared_ids[kThreads]; + const std::uint32_t thread_id = threadIdx.x; + const std::uint32_t query_id = blockIdx.x; + const float worst_score = + metric == Metric::kL2 ? kFloatMaximum : -kFloatMaximum; + float* query_scores = + center_scores + static_cast(query_id) * nlist; + + for (std::uint32_t probe = 0; probe < nprobe; ++probe) { + float local_best_score = worst_score; + std::uint32_t local_best_id = UINT32_MAX; + for (std::uint32_t center_id = thread_id; center_id < nlist; + center_id += blockDim.x) { + const float score = query_scores[center_id]; + if (isBetter(score, center_id, local_best_score, local_best_id, metric)) { + local_best_score = score; + local_best_id = center_id; + } + } + shared_scores[thread_id] = local_best_score; + shared_ids[thread_id] = local_best_id; + __syncthreads(); + for (std::uint32_t stride = kThreads / 2; stride > 0; stride >>= 1) { + if (thread_id < stride && + isBetter(shared_scores[thread_id + stride], + shared_ids[thread_id + stride], shared_scores[thread_id], + shared_ids[thread_id], metric)) { + shared_scores[thread_id] = shared_scores[thread_id + stride]; + shared_ids[thread_id] = shared_ids[thread_id + stride]; + } + __syncthreads(); + } + if (thread_id == 0) { + const std::uint32_t selected_id = shared_ids[0]; + selected_centers[static_cast(query_id) * nprobe + probe] = + selected_id; + if (selected_scores != nullptr) { + selected_scores[static_cast(query_id) * nprobe + probe] = + shared_scores[0]; + } + query_scores[selected_id] = worst_score; + } + __syncthreads(); + } +} + +// 将已按优劣排序的中心分数转换为概率质量。分数差先除以当前 query 的 +// top-1 到第 max_nprobe 个中心的跨度，使 temperature 对查询模长和 metric +// 尺度不敏感。集中分布会较早达到 target_mass，使用较少 probe；平坦分布 +// 会自动接近 nprobe 上限。 +__global__ void chooseAdaptiveNprobeKernel( + const float* selected_scores, std::uint32_t* query_nprobes, + std::uint32_t batch_count, std::uint32_t max_nprobe, + std::uint32_t min_nprobe, std::uint32_t step, float target_mass, + float temperature, Metric metric) { + const std::uint32_t query_id = blockIdx.x * blockDim.x + threadIdx.x; + if (query_id >= batch_count) { + return; + } + const float* scores = + selected_scores + static_cast(query_id) * max_nprobe; + const float direction = metric == Metric::kL2 ? -1.0F : 1.0F; + const float best_utility = direction * scores[0]; + const float worst_utility = direction * scores[max_nprobe - 1]; + const float utility_span = fmaxf(best_utility - worst_utility, 1.0e-6F); + const float inverse_scale = 1.0F / (temperature * utility_span); + float total_mass = 0.0F; + for (std::uint32_t probe = 0; probe < max_nprobe; ++probe) { + total_mass += + expf((direction * scores[probe] - best_utility) * inverse_scale); + } + + std::uint32_t selected_nprobe = max_nprobe; + if (target_mass < 1.0F) { + float prefix_mass = 0.0F; + for (std::uint32_t probe = 0; probe < max_nprobe; ++probe) { + prefix_mass += + expf((direction * scores[probe] - best_utility) * inverse_scale); + const std::uint32_t count = probe + 1; + const bool selectable = + count >= min_nprobe && + (((count - min_nprobe) % step) == 0 || count == max_nprobe); + if (selectable && prefix_mass >= target_mass * total_mass) { + selected_nprobe = count; + break; + } + } + } + query_nprobes[query_id] = selected_nprobe; +} + +// 正确性 baseline：每个 query 使用一个 CUDA 线程，直接遍历选中桶的候选， +// 距离计算和有序 Top-K 在同一个 kernel 中完成。索引常驻显存，避免复制向量。 +// 下一版会让一个 block/多个 warp 共同扫描一个 query 的候选。 +template +__global__ void fusedIvfScanTopKKernel( + const InputType* vectors, const std::uint64_t* original_ids, + const InputType* queries, const std::uint64_t* candidate_positions, + const std::uint64_t* candidate_counts, float* top_scores, + std::uint64_t* top_ids, std::uint64_t candidate_stride, std::uint32_t dim, + std::uint32_t top_k, Metric metric) { + if (threadIdx.x != 0) { + return; + } + constexpr std::uint32_t kMaximumK = 100; + constexpr float kFloatMaximum = 3.402823466e+38F; + const float worst_score = + metric == Metric::kL2 ? kFloatMaximum : -kFloatMaximum; + float best_scores[kMaximumK]; + std::uint64_t best_ids[kMaximumK]; + for (std::uint32_t rank = 0; rank < top_k; ++rank) { + best_scores[rank] = worst_score; + best_ids[rank] = UINT64_MAX; + } + + const std::uint32_t query_id = blockIdx.x; + const std::uint64_t query_offset = static_cast(query_id) * dim; + float query_norm = 0.0F; + if (metric == Metric::kCosine) { + for (std::uint32_t d = 0; d < dim; ++d) { + const float query = loadInput(queries, query_offset + d); + query_norm = fmaf(query, query, query_norm); + } + } + + const std::uint64_t positions_offset = + static_cast(query_id) * candidate_stride; + for (std::uint64_t candidate = 0; candidate < candidate_counts[query_id]; + ++candidate) { + const std::uint64_t position = + candidate_positions[positions_offset + candidate]; + const std::uint64_t vector_offset = position * dim; + float dot = 0.0F; + float vector_norm = 0.0F; + float squared_l2 = 0.0F; + for (std::uint32_t d = 0; d < dim; ++d) { + const float query = loadInput(queries, query_offset + d); + const float vector = loadInput(vectors, vector_offset + d); + if (metric == Metric::kL2) { + const float difference = query - vector; + squared_l2 = fmaf(difference, difference, squared_l2); + } else { + dot = fmaf(query, vector, dot); + if (metric == Metric::kCosine) { + vector_norm = fmaf(vector, vector, vector_norm); + } + } + } + float score = dot; + if (metric == Metric::kL2) { + score = squared_l2; + } else if (metric == Metric::kCosine) { + score = query_norm == 0.0F || vector_norm == 0.0F + ? 0.0F + : dot / sqrtf(query_norm * vector_norm); + } + const std::uint64_t vector_id = original_ids[position]; + for (std::uint32_t rank = 0; rank < top_k; ++rank) { + if (isBetter(score, vector_id, best_scores[rank], best_ids[rank], + metric)) { + for (std::uint32_t shift = top_k - 1; shift > rank; --shift) { + best_scores[shift] = best_scores[shift - 1]; + best_ids[shift] = best_ids[shift - 1]; + } + best_scores[rank] = score; + best_ids[rank] = vector_id; + break; + } + } + } + + const std::uint64_t output_offset = + static_cast(query_id) * top_k; + for (std::uint32_t rank = 0; rank < top_k; ++rank) { + top_scores[output_offset + rank] = best_scores[rank]; + top_ids[output_offset + rank] = best_ids[rank]; + } +} + +// 并行版：一个 block 处理一个 query。每个线程扫描一部分候选并维护无序 +// 局部 Top-K，随后 block 用共享内存归约出最终 K 个结果。 +template +__global__ void fusedIvfBlockScanTopKKernel( + const InputType* vectors, const std::uint64_t* original_ids, + const InputType* queries, const std::uint64_t* candidate_positions, + const std::uint64_t* candidate_counts, float* top_scores, + std::uint64_t* top_ids, std::uint64_t candidate_stride, std::uint32_t dim, + std::uint32_t top_k, Metric metric) { + constexpr std::uint32_t kThreads = 256; + constexpr std::uint32_t kMaximumK = 10; + constexpr float kFloatMaximum = 3.402823466e+38F; + __shared__ float shared_scores[kThreads]; + __shared__ std::uint64_t shared_ids[kThreads]; + + const std::uint32_t thread_id = threadIdx.x; + const std::uint32_t query_id = blockIdx.x; + const float worst_score = + metric == Metric::kL2 ? kFloatMaximum : -kFloatMaximum; + float local_scores[kMaximumK]; + std::uint64_t local_ids[kMaximumK]; + for (std::uint32_t rank = 0; rank < top_k; ++rank) { + local_scores[rank] = worst_score; + local_ids[rank] = UINT64_MAX; + } + std::uint32_t local_worst_rank = 0; + + const std::uint64_t query_offset = static_cast(query_id) * dim; + float query_norm = 0.0F; + if (metric == Metric::kCosine) { + for (std::uint32_t d = 0; d < dim; ++d) { + const float query = loadInput(queries, query_offset + d); + query_norm = fmaf(query, query, query_norm); + } + } + const std::uint64_t positions_offset = + static_cast(query_id) * candidate_stride; + for (std::uint64_t candidate = thread_id; + candidate < candidate_counts[query_id]; candidate += blockDim.x) { + const std::uint64_t position = + candidate_positions[positions_offset + candidate]; + const std::uint64_t vector_offset = position * dim; + float dot = 0.0F; + float vector_norm = 0.0F; + float squared_l2 = 0.0F; + for (std::uint32_t d = 0; d < dim; ++d) { + const float query = loadInput(queries, query_offset + d); + const float vector = loadInput(vectors, vector_offset + d); + if (metric == Metric::kL2) { + const float difference = query - vector; + squared_l2 = fmaf(difference, difference, squared_l2); + } else { + dot = fmaf(query, vector, dot); + if (metric == Metric::kCosine) { + vector_norm = fmaf(vector, vector, vector_norm); + } + } + } + float score = dot; + if (metric == Metric::kL2) { + score = squared_l2; + } else if (metric == Metric::kCosine) { + score = query_norm == 0.0F || vector_norm == 0.0F + ? 0.0F + : dot / sqrtf(query_norm * vector_norm); + } + const std::uint64_t vector_id = original_ids[position]; + if (isBetter(score, vector_id, local_scores[local_worst_rank], + local_ids[local_worst_rank], metric)) { + local_scores[local_worst_rank] = score; + local_ids[local_worst_rank] = vector_id; + local_worst_rank = 0; + for (std::uint32_t rank = 1; rank < top_k; ++rank) { + if (isBetter(local_scores[local_worst_rank], + local_ids[local_worst_rank], local_scores[rank], + local_ids[rank], metric)) { + local_worst_rank = rank; + } + } + } + } + + for (std::uint32_t output_rank = 0; output_rank < top_k; ++output_rank) { + std::uint32_t local_best_rank = 0; + for (std::uint32_t rank = 1; rank < top_k; ++rank) { + if (isBetter(local_scores[rank], local_ids[rank], + local_scores[local_best_rank], local_ids[local_best_rank], + metric)) { + local_best_rank = rank; + } + } + shared_scores[thread_id] = local_scores[local_best_rank]; + shared_ids[thread_id] = local_ids[local_best_rank]; + __syncthreads(); + for (std::uint32_t stride = kThreads / 2; stride > 0; stride >>= 1) { + if (thread_id < stride && + isBetter(shared_scores[thread_id + stride], + shared_ids[thread_id + stride], shared_scores[thread_id], + shared_ids[thread_id], metric)) { + shared_scores[thread_id] = shared_scores[thread_id + stride]; + shared_ids[thread_id] = shared_ids[thread_id + stride]; + } + __syncthreads(); + } + if (thread_id == 0) { + const std::uint64_t output_offset = + static_cast(query_id) * top_k + output_rank; + top_scores[output_offset] = shared_scores[0]; + top_ids[output_offset] = shared_ids[0]; + } + const std::uint64_t selected_id = shared_ids[0]; + if (local_ids[local_best_rank] == selected_id) { + local_scores[local_best_rank] = worst_score; + local_ids[local_best_rank] = UINT64_MAX; + } + __syncthreads(); + } +} + +template +__global__ void ivfScanBucketsScalarKernel( + const InputType* vectors, const std::uint64_t* original_ids, + const std::uint64_t* bucket_offsets, const std::uint32_t* selected_centers, + const InputType* queries, float* probe_scores, std::uint64_t* probe_ids, + const std::uint32_t* active_query_ids, const std::uint32_t* query_nprobes, + std::uint32_t nprobe, std::uint32_t dim, std::uint32_t top_k, Metric metric, + bool apply_memory, DeviceMemoryView metadata, + DeviceMemoryOptions memory_options) { + constexpr std::uint32_t kThreads = 256; + constexpr float kFloatMaximum = 3.402823466e+38F; + __shared__ float shared_scores[kThreads]; + __shared__ std::uint64_t shared_ids[kThreads]; + + const std::uint32_t thread_id = threadIdx.x; + const std::uint32_t probe = blockIdx.x; + const std::uint32_t query_id = + active_query_ids == nullptr ? blockIdx.y : active_query_ids[blockIdx.y]; + if (query_nprobes != nullptr && probe >= query_nprobes[query_id]) { + return; + } + const std::uint32_t center_id = + selected_centers[static_cast(query_id) * nprobe + probe]; + const std::uint64_t bucket_start = bucket_offsets[center_id]; + const std::uint64_t bucket_end = bucket_offsets[center_id + 1]; + const float worst_score = + metric == Metric::kL2 ? kFloatMaximum : -kFloatMaximum; + float local_scores[MaximumK]; + std::uint64_t local_ids[MaximumK]; + for (std::uint32_t rank = 0; rank < top_k; ++rank) { + local_scores[rank] = worst_score; + local_ids[rank] = UINT64_MAX; + } + std::uint32_t local_worst_rank = 0; + + const std::uint64_t query_offset = static_cast(query_id) * dim; + float query_norm = 0.0F; + if (metric == Metric::kCosine) { + for (std::uint32_t d = 0; d < dim; ++d) { + const float query = loadInput(queries, query_offset + d); + query_norm = fmaf(query, query, query_norm); + } + } + for (std::uint64_t position = bucket_start + thread_id; position < bucket_end; + position += blockDim.x) { + const std::uint64_t vector_id = original_ids[position]; + if (apply_memory && + !memoryCandidateMatches(metadata, vector_id, memory_options)) { + continue; + } + const std::uint64_t vector_offset = position * dim; + float dot = 0.0F; + float vector_norm = 0.0F; + float squared_l2 = 0.0F; + for (std::uint32_t d = 0; d < dim; ++d) { + const float query = loadInput(queries, query_offset + d); + const float vector = loadInput(vectors, vector_offset + d); + if (metric == Metric::kL2) { + const float difference = query - vector; + squared_l2 = fmaf(difference, difference, squared_l2); + } else { + dot = fmaf(query, vector, dot); + if (metric == Metric::kCosine) { + vector_norm = fmaf(vector, vector, vector_norm); + } + } + } + float score = dot; + if (metric == Metric::kL2) { + score = squared_l2; + } else if (metric == Metric::kCosine) { + score = query_norm == 0.0F || vector_norm == 0.0F + ? 0.0F + : dot / sqrtf(query_norm * vector_norm); + } + if (apply_memory) { + const float boost = + memory_options.importance_weight * metadata.importance[vector_id] + + (memory_options.recency_weight == 0.0F || memory_options.now == 0 + ? 0.0F + : memory_options.recency_weight * + expf(-static_cast( + (metadata.timestamps[vector_id] >= memory_options.now + ? 0.0 + : static_cast( + memory_options.now - + metadata.timestamps[vector_id])) / + memory_options.time_scale))); + score = memory_options.semantic_weight * score + + (metric == Metric::kL2 ? -boost : boost); + } + if (isBetter(score, vector_id, local_scores[local_worst_rank], + local_ids[local_worst_rank], metric)) { + local_scores[local_worst_rank] = score; + local_ids[local_worst_rank] = vector_id; + local_worst_rank = 0; + for (std::uint32_t rank = 1; rank < top_k; ++rank) { + if (isBetter(local_scores[local_worst_rank], + local_ids[local_worst_rank], local_scores[rank], + local_ids[rank], metric)) { + local_worst_rank = rank; + } + } + } + } + + for (std::uint32_t output_rank = 0; output_rank < top_k; ++output_rank) { + std::uint32_t local_best_rank = 0; + for (std::uint32_t rank = 1; rank < top_k; ++rank) { + if (isBetter(local_scores[rank], local_ids[rank], + local_scores[local_best_rank], local_ids[local_best_rank], + metric)) { + local_best_rank = rank; + } + } + shared_scores[thread_id] = local_scores[local_best_rank]; + shared_ids[thread_id] = local_ids[local_best_rank]; + __syncthreads(); + for (std::uint32_t stride = kThreads / 2; stride > 0; stride >>= 1) { + if (thread_id < stride && + isBetter(shared_scores[thread_id + stride], + shared_ids[thread_id + stride], shared_scores[thread_id], + shared_ids[thread_id], metric)) { + shared_scores[thread_id] = shared_scores[thread_id + stride]; + shared_ids[thread_id] = shared_ids[thread_id + stride]; + } + __syncthreads(); + } + if (thread_id == 0) { + const std::uint64_t output_offset = + (static_cast(query_id) * nprobe + probe) * top_k + + output_rank; + probe_scores[output_offset] = shared_scores[0]; + probe_ids[output_offset] = shared_ids[0]; + } + const std::uint64_t selected_id = shared_ids[0]; + if (local_ids[local_best_rank] == selected_id) { + local_scores[local_best_rank] = worst_score; + local_ids[local_best_rank] = UINT64_MAX; + } + __syncthreads(); + } +} + +__global__ void countSelectedCenterTasksKernel( + const std::uint32_t* selected_centers, std::uint32_t* center_counts, + std::uint32_t task_count) { + const std::uint32_t task = blockIdx.x * blockDim.x + threadIdx.x; + if (task < task_count) { + atomicAdd(¢er_counts[selected_centers[task]], 1U); + } +} + +__global__ void prepareCenterTaskCursorsKernel( + const std::uint32_t* center_counts, std::uint32_t* center_cursors, + std::uint32_t nlist) { + if (blockIdx.x != 0 || threadIdx.x != 0) { + return; + } + std::uint32_t offset = 0; + for (std::uint32_t center = 0; center < nlist; ++center) { + center_cursors[center] = offset; + offset += center_counts[center]; + } +} + +__global__ void scatterSelectedCenterTasksKernel( + const std::uint32_t* selected_centers, std::uint32_t* center_cursors, + std::uint64_t* bucket_tasks, std::uint32_t nprobe, + std::uint32_t task_count) { + const std::uint32_t task = blockIdx.x * blockDim.x + threadIdx.x; + if (task >= task_count) { + return; + } + const std::uint32_t center = selected_centers[task]; + const std::uint32_t destination = atomicAdd(¢er_cursors[center], 1U); + const std::uint32_t query_id = task / nprobe; + const std::uint32_t probe = task - query_id * nprobe; + bucket_tasks[destination] = + (static_cast(query_id) << 32U) | probe; +} + +// 一个 block 包含若干 warp，每个 warp 协作计算一个候选向量。相邻 lane +// 读取相邻维度，适合行主序向量；lane 0 为该 warp 维护局部 Top-K，最后在 +// WarpsPerBlock 可取 1/2/4/8。大 K 使用更少的独立 heap，并让更多 probe block +// 并发以维持活跃 warp；旧 8-warp 路径和 scalar 路径都保留用于 A/B。 +template +__global__ void ivfScanBucketsWarpKernel( + const InputType* vectors, const std::uint64_t* original_ids, + const std::uint64_t* bucket_offsets, const std::uint32_t* selected_centers, + const std::uint64_t* bucket_tasks, const InputType* queries, + float* probe_scores, std::uint64_t* probe_ids, + const std::uint32_t* active_query_ids, const std::uint32_t* query_nprobes, + std::uint32_t nprobe, std::uint32_t dim, std::uint32_t top_k, + std::uint32_t heap_k, + bool cache_query, bool apply_memory, DeviceMemoryView metadata, + DeviceMemoryOptions memory_options) { + constexpr std::uint32_t kWarpSize = 32; + constexpr std::uint32_t kWarps = WarpsPerBlock; + static_assert(WarpsPerBlock == 1 || WarpsPerBlock == 2 || + WarpsPerBlock == 4 || WarpsPerBlock == 8, + "IVF warp block 只支持 1/2/4/8 个 warp"); + constexpr float kFloatMaximum = 3.402823466e+38F; + __shared__ float warp_scores[kWarps][MaximumK]; + __shared__ std::uint64_t warp_ids[kWarps][MaximumK]; + __shared__ float best_scores[kWarps]; + __shared__ std::uint64_t best_ids[kWarps]; + __shared__ std::uint32_t warp_ranks[kWarps]; + extern __shared__ __align__(16) unsigned char dynamic_shared[]; + auto* shared_query = reinterpret_cast(dynamic_shared); + + const std::uint32_t thread_id = threadIdx.x; + const std::uint32_t lane_id = thread_id % kWarpSize; + const std::uint32_t warp_id = thread_id / kWarpSize; + std::uint32_t query_slot = blockIdx.y; + std::uint32_t probe = blockIdx.x; + if constexpr (BucketMajor) { + const std::uint64_t task = bucket_tasks[blockIdx.x]; + query_slot = static_cast(task >> 32U); + probe = static_cast(task); + } + const std::uint32_t query_id = active_query_ids == nullptr + ? query_slot + : active_query_ids[query_slot]; + if (query_nprobes != nullptr && probe >= query_nprobes[query_id]) { + return; + } + const std::uint32_t center_id = + selected_centers[static_cast(query_id) * nprobe + probe]; + const std::uint64_t bucket_start = bucket_offsets[center_id]; + const std::uint64_t bucket_end = bucket_offsets[center_id + 1]; + const float worst_score = + MetricValue == Metric::kL2 ? kFloatMaximum : -kFloatMaximum; + float heap_root_score = worst_score; + std::uint64_t heap_root_id = UINT64_MAX; + + if (lane_id == 0) { + for (std::uint32_t rank = 0; rank < heap_k; ++rank) { + warp_scores[warp_id][rank] = worst_score; + warp_ids[warp_id][rank] = UINT64_MAX; + } + } + const std::uint64_t query_offset = static_cast(query_id) * dim; + const InputType* query_values = queries + query_offset; + if (cache_query) { + for (std::uint32_t d = thread_id; d < dim; d += blockDim.x) { + shared_query[d] = query_values[d]; + } + } + // 同时等待 heap 初始化和可选 query 缓存完成，不增加额外 block barrier。 + __syncthreads(); + if (cache_query) { + query_values = shared_query; + } + + float query_norm = 0.0F; + if constexpr (MetricValue == Metric::kCosine) { + for (std::uint32_t d = lane_id; d < dim; d += kWarpSize) { + const float query = loadInput(query_values, d); + query_norm = fmaf(query, query, query_norm); + } + query_norm = warpReduceSum(query_norm); + } + + for (std::uint64_t position = bucket_start + warp_id; position < bucket_end; + position += kWarps) { + const std::uint64_t vector_id = original_ids[position]; + const bool accepted = + !apply_memory || + (lane_id == 0 && + memoryCandidateMatches(metadata, vector_id, memory_options)); + if (!__shfl_sync(0xFFFFFFFFU, accepted, 0)) { + continue; + } + const std::uint64_t vector_offset = position * dim; + float dot = 0.0F; + float vector_norm = 0.0F; + float squared_l2 = 0.0F; + for (std::uint32_t d = lane_id; d < dim; d += kWarpSize) { + const float query = loadInput(query_values, d); + const float vector = loadInput(vectors, vector_offset + d); + if constexpr (MetricValue == Metric::kL2) { + const float difference = query - vector; + squared_l2 = fmaf(difference, difference, squared_l2); + } else { + dot = fmaf(query, vector, dot); + if constexpr (MetricValue == Metric::kCosine) { + vector_norm = fmaf(vector, vector, vector_norm); + } + } + } + // 每种 metric 只归约实际使用的累加器。旧代码在 inner-product 下还会 + // 对恒为 0 的 squared_l2 做一次完整 shuffle 归约，L2 下也会归约恒为 + // 0 的 dot；这在每个候选上浪费 5 次 warp shuffle。 + if constexpr (MetricValue == Metric::kL2) { + squared_l2 = warpReduceSum(squared_l2); + } else { + dot = warpReduceSum(dot); + if constexpr (MetricValue == Metric::kCosine) { + vector_norm = warpReduceSum(vector_norm); + } + } + if (lane_id == 0) { + float score = dot; + if constexpr (MetricValue == Metric::kL2) { + score = squared_l2; + } else if constexpr (MetricValue == Metric::kCosine) { + score = query_norm == 0.0F || vector_norm == 0.0F + ? 0.0F + : dot / sqrtf(query_norm * vector_norm); + } + if (apply_memory) { + score = applyMemoryScore(score, vector_id, metadata, + memory_options); + } + if (isBetter(score, vector_id, heap_root_score, heap_root_id, + MetricValue)) { + // 以空穴下沉代替逐层交换：候选保存在寄存器中，每层只把更差的 + // child 上移一次，最后写入候选。根节点也缓存在寄存器中，避免每个 + // 被扫描向量都读取 shared memory。 + std::uint32_t parent = 0; + while (true) { + const std::uint32_t left = parent * 2 + 1; + if (left >= heap_k) { + break; + } + const std::uint32_t right = left + 1; + std::uint32_t worse_child = left; + if (right < heap_k && + isBetter(warp_scores[warp_id][left], warp_ids[warp_id][left], + warp_scores[warp_id][right], warp_ids[warp_id][right], + MetricValue)) { + worse_child = right; + } + if (!isBetter(score, vector_id, warp_scores[warp_id][worse_child], + warp_ids[warp_id][worse_child], MetricValue)) { + break; + } + warp_scores[warp_id][parent] = warp_scores[warp_id][worse_child]; + warp_ids[warp_id][parent] = warp_ids[warp_id][worse_child]; + parent = worse_child; + } + warp_scores[warp_id][parent] = score; + warp_ids[warp_id][parent] = vector_id; + heap_root_score = warp_scores[warp_id][0]; + heap_root_id = warp_ids[warp_id][0]; + } + } + } + __syncthreads(); + + // 根节点始终是当前最差候选。原地 heap sort 后，每个 warp 的数组按 + // best -> worst 排列，随后只需合并少量 warp 有序列表。 + if (lane_id == 0) { + for (std::uint32_t heap_size = heap_k; heap_size > 1; --heap_size) { + const std::uint32_t last = heap_size - 1; + const float sorted_score = warp_scores[warp_id][0]; + const std::uint64_t sorted_id = warp_ids[warp_id][0]; + const float candidate_score = warp_scores[warp_id][last]; + const std::uint64_t candidate_id = warp_ids[warp_id][last]; + warp_scores[warp_id][last] = sorted_score; + warp_ids[warp_id][last] = sorted_id; + std::uint32_t parent = 0; + while (true) { + const std::uint32_t left = parent * 2 + 1; + if (left >= last) { + break; + } + const std::uint32_t right = left + 1; + std::uint32_t worse_child = left; + if (right < last && + isBetter(warp_scores[warp_id][left], warp_ids[warp_id][left], + warp_scores[warp_id][right], warp_ids[warp_id][right], + MetricValue)) { + worse_child = right; + } + if (!isBetter(candidate_score, candidate_id, + warp_scores[warp_id][worse_child], + warp_ids[warp_id][worse_child], MetricValue)) { + break; + } + warp_scores[warp_id][parent] = warp_scores[warp_id][worse_child]; + warp_ids[warp_id][parent] = warp_ids[warp_id][worse_child]; + parent = worse_child; + } + warp_scores[warp_id][parent] = candidate_score; + warp_ids[warp_id][parent] = candidate_id; + } + warp_ranks[warp_id] = 0; + } + __syncthreads(); + + if constexpr (kWarps == 1) { + if (lane_id == 0) { + const std::uint64_t output_offset = + (static_cast(query_id) * nprobe + probe) * top_k; + for (std::uint32_t rank = 0; rank < top_k; ++rank) { + probe_scores[output_offset + rank] = warp_scores[0][rank]; + probe_ids[output_offset + rank] = warp_ids[0][rank]; + } + } + } else { + for (std::uint32_t output_rank = 0; output_rank < top_k; ++output_rank) { + if (lane_id == 0) { + const std::uint32_t rank = warp_ranks[warp_id]; + best_scores[warp_id] = + rank < heap_k ? warp_scores[warp_id][rank] : worst_score; + best_ids[warp_id] = + rank < heap_k ? warp_ids[warp_id][rank] : UINT64_MAX; + } + __syncthreads(); + if (thread_id == 0) { + std::uint32_t best_warp = 0; + for (std::uint32_t candidate_warp = 1; candidate_warp < kWarps; + ++candidate_warp) { + if (isBetter(best_scores[candidate_warp], best_ids[candidate_warp], + best_scores[best_warp], best_ids[best_warp], + MetricValue)) { + best_warp = candidate_warp; + } + } + const std::uint64_t output_offset = + (static_cast(query_id) * nprobe + probe) * top_k + + output_rank; + probe_scores[output_offset] = best_scores[best_warp]; + probe_ids[output_offset] = best_ids[best_warp]; + ++warp_ranks[best_warp]; + } + __syncthreads(); + } + } +} + +template +void launchIvfBucketScanWarp( + dim3 grid, const InputType* vectors, const std::uint64_t* original_ids, + const std::uint64_t* bucket_offsets, const std::uint32_t* selected_centers, + const std::uint64_t* bucket_tasks, const InputType* queries, + float* probe_scores, std::uint64_t* probe_ids, + const std::uint32_t* active_query_ids, const std::uint32_t* query_nprobes, + std::uint32_t nprobe, std::uint32_t dim, std::uint32_t top_k, + std::uint32_t heap_k, Metric metric, bool apply_memory, + DeviceMemoryView metadata, + DeviceMemoryOptions memory_options) { + constexpr std::uint32_t kThreads = WarpsPerBlock * 32; + constexpr std::size_t kMaximumCachedQueryBytes = 8192; + const std::size_t query_bytes = + static_cast(dim) * sizeof(InputType); + const bool cache_query = query_bytes <= kMaximumCachedQueryBytes; + const std::size_t shared_bytes = cache_query ? query_bytes : 0; + if (metric == Metric::kL2) { + ivfScanBucketsWarpKernel + <<>>( + vectors, original_ids, bucket_offsets, selected_centers, + bucket_tasks, queries, probe_scores, probe_ids, active_query_ids, + query_nprobes, nprobe, dim, top_k, heap_k, cache_query, apply_memory, + metadata, memory_options); + } else if (metric == Metric::kInnerProduct) { + ivfScanBucketsWarpKernel + <<>>( + vectors, original_ids, bucket_offsets, selected_centers, + bucket_tasks, queries, probe_scores, probe_ids, active_query_ids, + query_nprobes, nprobe, dim, top_k, heap_k, cache_query, apply_memory, + metadata, memory_options); + } else { + ivfScanBucketsWarpKernel + <<>>( + vectors, original_ids, bucket_offsets, selected_centers, bucket_tasks, + queries, probe_scores, probe_ids, active_query_ids, query_nprobes, + nprobe, dim, top_k, heap_k, cache_query, apply_memory, metadata, + memory_options); + } +} + +template +void launchIvfBucketScan( + dim3 grid, const InputType* vectors, const std::uint64_t* original_ids, + const std::uint64_t* bucket_offsets, const std::uint32_t* selected_centers, + const std::uint64_t* bucket_tasks, const InputType* queries, + float* probe_scores, std::uint64_t* probe_ids, + const std::uint32_t* active_query_ids, const std::uint32_t* query_nprobes, + std::uint32_t nprobe, std::uint32_t dim, std::uint32_t top_k, Metric metric, + std::uint32_t warps_per_block, bool bucket_major, + std::uint32_t partitioned_topk_warps, + std::uint32_t partitioned_topk_heap_k, + bool apply_memory, DeviceMemoryView metadata, + DeviceMemoryOptions memory_options) { + constexpr std::uint32_t kThreads = 256; + if (partitioned_topk_warps == 2) { + constexpr std::uint32_t kPartitionHeapK = 32; + if (bucket_major) { + launchIvfBucketScanWarp( + grid, vectors, original_ids, bucket_offsets, selected_centers, + bucket_tasks, queries, probe_scores, probe_ids, active_query_ids, + query_nprobes, nprobe, dim, top_k, kPartitionHeapK, metric, + apply_memory, metadata, memory_options); + } else { + launchIvfBucketScanWarp( + grid, vectors, original_ids, bucket_offsets, selected_centers, + bucket_tasks, queries, probe_scores, probe_ids, active_query_ids, + query_nprobes, nprobe, dim, top_k, kPartitionHeapK, metric, + apply_memory, metadata, memory_options); + } + return; + } + if (partitioned_topk_warps == 4) { + if (partitioned_topk_heap_k == 16) { + constexpr std::uint32_t kPartitionHeapK = 16; + if (bucket_major) { + launchIvfBucketScanWarp( + grid, vectors, original_ids, bucket_offsets, selected_centers, + bucket_tasks, queries, probe_scores, probe_ids, active_query_ids, + query_nprobes, nprobe, dim, top_k, kPartitionHeapK, metric, + apply_memory, metadata, memory_options); + } else { + launchIvfBucketScanWarp( + grid, vectors, original_ids, bucket_offsets, selected_centers, + bucket_tasks, queries, probe_scores, probe_ids, active_query_ids, + query_nprobes, nprobe, dim, top_k, kPartitionHeapK, metric, + apply_memory, metadata, memory_options); + } + } else if (top_k <= 50) { + constexpr std::uint32_t kPartitionHeapK = 20; + if (bucket_major) { + launchIvfBucketScanWarp( + grid, vectors, original_ids, bucket_offsets, selected_centers, + bucket_tasks, queries, probe_scores, probe_ids, active_query_ids, + query_nprobes, nprobe, dim, top_k, kPartitionHeapK, metric, + apply_memory, metadata, memory_options); + } else { + launchIvfBucketScanWarp( + grid, vectors, original_ids, bucket_offsets, selected_centers, + bucket_tasks, queries, probe_scores, probe_ids, active_query_ids, + query_nprobes, nprobe, dim, top_k, kPartitionHeapK, metric, + apply_memory, metadata, memory_options); + } + } else { + constexpr std::uint32_t kPartitionHeapK = 40; + if (bucket_major) { + launchIvfBucketScanWarp( + grid, vectors, original_ids, bucket_offsets, selected_centers, + bucket_tasks, queries, probe_scores, probe_ids, active_query_ids, + query_nprobes, nprobe, dim, top_k, kPartitionHeapK, metric, + apply_memory, metadata, memory_options); + } else { + launchIvfBucketScanWarp( + grid, vectors, original_ids, bucket_offsets, selected_centers, + bucket_tasks, queries, probe_scores, probe_ids, active_query_ids, + query_nprobes, nprobe, dim, top_k, kPartitionHeapK, metric, + apply_memory, metadata, memory_options); + } + } + return; + } + if (partitioned_topk_warps == 8) { + constexpr std::uint32_t kPartitionHeapK = 8; + if (bucket_major) { + launchIvfBucketScanWarp( + grid, vectors, original_ids, bucket_offsets, selected_centers, + bucket_tasks, queries, probe_scores, probe_ids, active_query_ids, + query_nprobes, nprobe, dim, top_k, kPartitionHeapK, metric, + apply_memory, metadata, memory_options); + } else { + launchIvfBucketScanWarp( + grid, vectors, original_ids, bucket_offsets, selected_centers, + bucket_tasks, queries, probe_scores, probe_ids, active_query_ids, + query_nprobes, nprobe, dim, top_k, kPartitionHeapK, metric, + apply_memory, metadata, memory_options); + } + return; + } + if (warps_per_block == 1) { + if (bucket_major) { + launchIvfBucketScanWarp( + grid, vectors, original_ids, bucket_offsets, selected_centers, + bucket_tasks, queries, probe_scores, probe_ids, active_query_ids, + query_nprobes, nprobe, dim, top_k, top_k, metric, apply_memory, metadata, + memory_options); + } else { + launchIvfBucketScanWarp( + grid, vectors, original_ids, bucket_offsets, selected_centers, + bucket_tasks, queries, probe_scores, probe_ids, active_query_ids, + query_nprobes, nprobe, dim, top_k, top_k, metric, apply_memory, metadata, + memory_options); + } + } else if (warps_per_block == 2) { + if (bucket_major) { + launchIvfBucketScanWarp( + grid, vectors, original_ids, bucket_offsets, selected_centers, + bucket_tasks, queries, probe_scores, probe_ids, active_query_ids, + query_nprobes, nprobe, dim, top_k, top_k, metric, apply_memory, metadata, + memory_options); + } else { + launchIvfBucketScanWarp( + grid, vectors, original_ids, bucket_offsets, selected_centers, + bucket_tasks, queries, probe_scores, probe_ids, active_query_ids, + query_nprobes, nprobe, dim, top_k, top_k, metric, apply_memory, metadata, + memory_options); + } + } else if (warps_per_block == 4) { + if (bucket_major) { + launchIvfBucketScanWarp( + grid, vectors, original_ids, bucket_offsets, selected_centers, + bucket_tasks, queries, probe_scores, probe_ids, active_query_ids, + query_nprobes, nprobe, dim, top_k, top_k, metric, apply_memory, metadata, + memory_options); + } else { + launchIvfBucketScanWarp( + grid, vectors, original_ids, bucket_offsets, selected_centers, + bucket_tasks, queries, probe_scores, probe_ids, active_query_ids, + query_nprobes, nprobe, dim, top_k, top_k, metric, apply_memory, metadata, + memory_options); + } + } else if (warps_per_block == 8) { + if (bucket_major) { + launchIvfBucketScanWarp( + grid, vectors, original_ids, bucket_offsets, selected_centers, + bucket_tasks, queries, probe_scores, probe_ids, active_query_ids, + query_nprobes, nprobe, dim, top_k, top_k, metric, apply_memory, metadata, + memory_options); + } else { + launchIvfBucketScanWarp( + grid, vectors, original_ids, bucket_offsets, selected_centers, + bucket_tasks, queries, probe_scores, probe_ids, active_query_ids, + query_nprobes, nprobe, dim, top_k, top_k, metric, apply_memory, metadata, + memory_options); + } + } else { + ivfScanBucketsScalarKernel<<>>( + vectors, original_ids, bucket_offsets, selected_centers, queries, + probe_scores, probe_ids, active_query_ids, query_nprobes, nprobe, dim, + top_k, metric, apply_memory, metadata, memory_options); + } +} + +// 每个 probe 已按优劣输出有序 Top-K。最终阶段只需维护每个 probe 的当前 +// 游标，每轮在所有列表头之间归约出一个全局结果，避免为 K=50/100 给每个 +// CUDA 线程分配大型局部数组。 +__global__ void ivfMergeProbeTopKKernel(const float* probe_scores, + const std::uint64_t* probe_ids, + float* top_scores, + std::uint64_t* top_ids, + const std::uint32_t* query_nprobes, + std::uint32_t nprobe, + std::uint32_t top_k, Metric metric) { + constexpr std::uint32_t kThreads = 256; + constexpr float kFloatMaximum = 3.402823466e+38F; + __shared__ float shared_scores[kThreads]; + __shared__ std::uint64_t shared_ids[kThreads]; + __shared__ std::uint32_t shared_probes[kThreads]; + extern __shared__ std::uint32_t probe_ranks[]; + const std::uint32_t thread_id = threadIdx.x; + const std::uint32_t query_id = blockIdx.x; + const std::uint32_t active_nprobe = + query_nprobes == nullptr ? nprobe : query_nprobes[query_id]; + const float worst_score = + metric == Metric::kL2 ? kFloatMaximum : -kFloatMaximum; + for (std::uint32_t probe = thread_id; probe < active_nprobe; + probe += blockDim.x) { + probe_ranks[probe] = 0; + } + __syncthreads(); + + for (std::uint32_t output_rank = 0; output_rank < top_k; ++output_rank) { + float local_best_score = worst_score; + std::uint64_t local_best_id = UINT64_MAX; + std::uint32_t local_best_probe = UINT32_MAX; + for (std::uint32_t probe = thread_id; probe < active_nprobe; + probe += blockDim.x) { + const std::uint32_t rank = probe_ranks[probe]; + if (rank < top_k) { + const std::uint64_t input_offset = + (static_cast(query_id) * nprobe + probe) * top_k + + rank; + const float score = probe_scores[input_offset]; + const std::uint64_t id = probe_ids[input_offset]; + if (isBetter(score, id, local_best_score, local_best_id, metric)) { + local_best_score = score; + local_best_id = id; + local_best_probe = probe; + } + } + } + shared_scores[thread_id] = local_best_score; + shared_ids[thread_id] = local_best_id; + shared_probes[thread_id] = local_best_probe; + __syncthreads(); + for (std::uint32_t stride = kThreads / 2; stride > 0; stride >>= 1) { + if (thread_id < stride && + isBetter(shared_scores[thread_id + stride], + shared_ids[thread_id + stride], shared_scores[thread_id], + shared_ids[thread_id], metric)) { + shared_scores[thread_id] = shared_scores[thread_id + stride]; + shared_ids[thread_id] = shared_ids[thread_id + stride]; + shared_probes[thread_id] = shared_probes[thread_id + stride]; + } + __syncthreads(); + } + if (thread_id == 0) { + const std::uint64_t output_offset = + static_cast(query_id) * top_k + output_rank; + top_scores[output_offset] = shared_scores[0]; + top_ids[output_offset] = shared_ids[0]; + if (shared_probes[0] != UINT32_MAX && shared_ids[0] != UINT64_MAX) { + ++probe_ranks[shared_probes[0]]; + } + } + __syncthreads(); + } +} + +// Deliberately separate baseline: first retrieve semantic Top-(K * factor), +// then launch a second kernel to filter and rerank that bounded candidate set. +// This is kept beside the fused path so their latency and result quality can be +// compared without involving a CPU reranker. +__global__ void rerankMemoryTopKKernel( + const float* semantic_scores, const std::uint64_t* semantic_ids, + float* output_scores, std::uint64_t* output_ids, std::uint32_t semantic_k, + std::uint32_t top_k, Metric metric, DeviceMemoryView metadata, + DeviceMemoryOptions options) { + if (threadIdx.x != 0) { + return; + } + constexpr std::uint32_t kMaximumK = 100; + constexpr float kFloatMaximum = 3.402823466e+38F; + const std::uint32_t query_id = blockIdx.x; + const float worst_score = + metric == Metric::kL2 ? kFloatMaximum : -kFloatMaximum; + float best_scores[kMaximumK]; + std::uint64_t best_ids[kMaximumK]; + for (std::uint32_t rank = 0; rank < top_k; ++rank) { + best_scores[rank] = worst_score; + best_ids[rank] = UINT64_MAX; + } + std::uint32_t worst_rank = 0; + const std::uint64_t input_offset = + static_cast(query_id) * semantic_k; + for (std::uint32_t candidate = 0; candidate < semantic_k; ++candidate) { + const std::uint64_t vector_id = semantic_ids[input_offset + candidate]; + if (vector_id == UINT64_MAX || + !memoryCandidateMatches(metadata, vector_id, options)) { + continue; + } + const float semantic_score = semantic_scores[input_offset + candidate]; + float recency = 0.0F; + if (options.recency_weight != 0.0F && options.now != 0) { + const std::uint64_t timestamp = metadata.timestamps[vector_id]; + const double age = timestamp >= options.now + ? 0.0 + : static_cast(options.now - timestamp); + recency = expf(-static_cast(age / options.time_scale)); + } + const float boost = + options.importance_weight * metadata.importance[vector_id] + + options.recency_weight * recency; + const float score = options.semantic_weight * semantic_score + + (metric == Metric::kL2 ? -boost : boost); + if (isBetter(score, vector_id, best_scores[worst_rank], + best_ids[worst_rank], metric)) { + best_scores[worst_rank] = score; + best_ids[worst_rank] = vector_id; + worst_rank = 0; + for (std::uint32_t rank = 1; rank < top_k; ++rank) { + if (isBetter(best_scores[worst_rank], best_ids[worst_rank], + best_scores[rank], best_ids[rank], metric)) { + worst_rank = rank; + } + } + } + } + const std::uint64_t output_offset = + static_cast(query_id) * top_k; + for (std::uint32_t output_rank = 0; output_rank < top_k; ++output_rank) { + std::uint32_t best_rank = 0; + for (std::uint32_t rank = 1; rank < top_k; ++rank) { + if (isBetter(best_scores[rank], best_ids[rank], best_scores[best_rank], + best_ids[best_rank], metric)) { + best_rank = rank; + } + } + output_scores[output_offset + output_rank] = best_scores[best_rank]; + output_ids[output_offset + output_rank] = best_ids[best_rank]; + best_scores[best_rank] = worst_score; + best_ids[best_rank] = UINT64_MAX; + } +} + +void validateSearchConfiguration(const IvfFlatIndex& index, + const QuerySet& queries, + const SearchParams& params, + const MemoryMetadata* metadata) { + validateIvfFlatIndex(index); + if (queries.num_queries == 0 || queries.dim != index.dim || + queries.dtype != index.dtype) { + throw std::runtime_error("查询集与 GPU IVF 索引的维度或 dtype 不匹配"); + } + const std::uint64_t expected_values = queries.num_queries * queries.dim; + if ((index.dtype == DataType::kFloat32 && + queries.values.size() != expected_values) || + (index.dtype == DataType::kFloat16 && + queries.half_values.size() != expected_values)) { + throw std::runtime_error("GPU IVF 查询数据长度与元数据不一致"); + } + constexpr std::uint32_t kMaximumTopK = 100; + constexpr std::uint32_t kMaximumNprobe = 8192; + if (params.top_k == 0 || params.top_k > kMaximumTopK || + params.top_k > index.num_vectors || params.nprobe == 0 || + params.nprobe > index.nlist || params.nprobe > kMaximumNprobe || + params.batch_size == 0) { + throw std::runtime_error("GPU IVF 的 top_k、nprobe 或 batch_size 不合法"); + } + if (params.distance_mode != "simple" && params.distance_mode != "warp" && + params.distance_mode != "warp_compact" && + params.distance_mode != "bucket_major" && + params.distance_mode != "bucket_major_partitioned" && + params.distance_mode != "bucket_major_partitioned2" && + params.distance_mode != "bucket_major_partitioned4x16" && + params.distance_mode != "bucket_major_partitioned8x8" && + params.distance_mode != "warp1" && params.distance_mode != "warp2" && + params.distance_mode != "warp4" && params.distance_mode != "warp8") { + throw std::runtime_error( + "GPU IVF 的 distance_mode 必须是 simple、warp、warp_compact、" + "bucket_major、bucket_major_partitioned、" + "bucket_major_partitioned2、bucket_major_partitioned4x16、" + "bucket_major_partitioned8x8、warp1、warp2、warp4 或 warp8"); + } + const bool bucket_major = params.distance_mode == "bucket_major" || + params.distance_mode == + "bucket_major_partitioned" || + params.distance_mode == + "bucket_major_partitioned2" || + params.distance_mode == + "bucket_major_partitioned4x16" || + params.distance_mode == + "bucket_major_partitioned8x8"; + if (bucket_major && + (params.nprobe_policy != "fixed" || + static_cast(params.batch_size) * params.nprobe > + UINT32_MAX)) { + throw std::runtime_error( + "GPU IVF 的 bucket_major 当前只支持 fixed nprobe 和 32 位任务数"); + } + const bool partitioned = + params.distance_mode == "bucket_major_partitioned" || + params.distance_mode == "bucket_major_partitioned2" || + params.distance_mode == "bucket_major_partitioned4x16" || + params.distance_mode == "bucket_major_partitioned8x8"; + if (partitioned && params.top_k != 50 && params.top_k != 100) { + throw std::runtime_error( + "bucket_major_partitioned 仅支持 top_k=50 或 top_k=100"); + } + if (params.distance_mode == "bucket_major_partitioned2" && + params.top_k != 50) { + throw std::runtime_error( + "bucket_major_partitioned2 仅用于 top_k=50 的 2x32 实验"); + } + if ((params.distance_mode == "bucket_major_partitioned4x16" || + params.distance_mode == "bucket_major_partitioned8x8") && + params.top_k != 50) { + throw std::runtime_error( + "4x16 与 8x8 分区模式仅用于 top_k=50 实验"); + } + if (partitioned && params.memory_mode != "disabled") { + throw std::runtime_error( + "bucket_major_partitioned 要求 memory_mode=disabled"); + } + if (params.nprobe_policy != "fixed" && params.nprobe_policy != "score_mass") { + throw std::runtime_error( + "GPU IVF 的 nprobe_policy 必须是 fixed 或 score_mass"); + } + if (params.nprobe_policy == "score_mass" && + (params.adaptive_nprobe_min == 0 || + params.adaptive_nprobe_min > params.nprobe || + params.adaptive_nprobe_step == 0 || + !std::isfinite(params.adaptive_target_mass) || + params.adaptive_target_mass <= 0.0F || + params.adaptive_target_mass > 1.0F || + !std::isfinite(params.adaptive_temperature) || + params.adaptive_temperature <= 0.0F)) { + throw std::runtime_error("GPU IVF 的自适应 nprobe 参数不合法"); + } + if (params.adaptive_execution != "masked" && + params.adaptive_execution != "grouped") { + throw std::runtime_error( + "GPU IVF 的 adaptive_execution 必须是 masked 或 grouped"); + } + if (params.memory_mode != "disabled" && params.memory_mode != "fused" && + params.memory_mode != "rerank") { + throw std::runtime_error( + "GPU IVF 的 memory_mode 必须是 disabled、fused 或 rerank"); + } + if (params.memory_mode != "disabled") { + if (metadata == nullptr) { + throw std::runtime_error("GPU 记忆检索必须提供元数据"); + } + validateMemoryMetadata(*metadata, index.num_vectors); + if (!std::isfinite(params.memory_semantic_weight) || + params.memory_semantic_weight <= 0.0F || + !std::isfinite(params.memory_importance_weight) || + params.memory_importance_weight < 0.0F || + !std::isfinite(params.memory_recency_weight) || + params.memory_recency_weight < 0.0F || + !std::isfinite(params.memory_time_scale) || + params.memory_time_scale <= 0.0F || params.memory_rerank_factor == 0) { + throw std::runtime_error("GPU 记忆评分参数不合法"); + } + } +} + +} // namespace + +struct GpuIvfFlatSearchEngine::Impl { + Impl(const IvfFlatIndex& index, const QuerySet& initial_queries, + const SearchParams& search_params, const MemoryMetadata* metadata) + : num_vectors(index.num_vectors), + dim(index.dim), + nlist(index.nlist), + dtype(index.dtype), + metric(index.metric), + params(search_params), + adaptive_nprobe(search_params.nprobe_policy == "score_mass"), + group_adaptive_queries(search_params.nprobe_policy == "score_mass" && + search_params.adaptive_execution == "grouped"), + bucket_major(search_params.distance_mode == "bucket_major" || + search_params.distance_mode == + "bucket_major_partitioned" || + search_params.distance_mode == + "bucket_major_partitioned2" || + search_params.distance_mode == + "bucket_major_partitioned4x16" || + search_params.distance_mode == + "bucket_major_partitioned8x8"), + partitioned_topk_warps( + search_params.distance_mode == "bucket_major_partitioned2" + ? 2U + : search_params.distance_mode == + "bucket_major_partitioned8x8" + ? 8U + : search_params.distance_mode == "bucket_major_partitioned" + || search_params.distance_mode == + "bucket_major_partitioned4x16" + ? 4U + : 0U), + partitioned_topk_heap_k( + search_params.distance_mode == "bucket_major_partitioned2" + ? 32U + : search_params.distance_mode == + "bucket_major_partitioned4x16" + ? 16U + : search_params.distance_mode == + "bucket_major_partitioned8x8" + ? 8U + : search_params.top_k <= 50 ? 20U : 40U), + memory_enabled(search_params.memory_mode != "disabled"), + fused_memory(search_params.memory_mode == "fused"), + separate_rerank(search_params.memory_mode == "rerank"), + use_fp16(index.dtype == DataType::kFloat16), + input_element_bytes(use_fp16 ? sizeof(std::uint16_t) : sizeof(float)), + batch_capacity(search_params.batch_size), + host_index(&index), + host_metadata(metadata) { + validateSearchConfiguration(index, initial_queries, params, metadata); + scan_top_k = params.top_k; + if (separate_rerank) { + const std::uint64_t requested = static_cast(params.top_k) * + params.memory_rerank_factor; + scan_top_k = static_cast( + std::min({requested, 100, num_vectors})); + } + std::vector bucket_sizes(index.nlist); + for (std::uint32_t center_id = 0; center_id < index.nlist; ++center_id) { + bucket_sizes[center_id] = + index.offsets[center_id + 1] - index.offsets[center_id]; + } + std::sort(bucket_sizes.begin(), bucket_sizes.end()); + const std::uint32_t minimum_probe_count = + adaptive_nprobe ? params.adaptive_nprobe_min : params.nprobe; + const std::uint64_t minimum_candidate_count = std::accumulate( + bucket_sizes.begin(), bucket_sizes.begin() + minimum_probe_count, + std::uint64_t{0}); + if (minimum_candidate_count < scan_top_k) { + throw std::runtime_error( + "GPU IVF 的部分 nprobe 桶组合候选数可能少于 top_k"); + } + const std::size_t vector_bytes = + static_cast(num_vectors) * dim * input_element_bytes; + const std::size_t query_bytes = + static_cast(batch_capacity) * dim * input_element_bytes; + const std::size_t center_score_slots = + static_cast(batch_capacity) * nlist; + const std::size_t selected_center_slots = + static_cast(batch_capacity) * params.nprobe; + const std::size_t probe_result_slots = selected_center_slots * scan_top_k; + const std::size_t result_slots = + static_cast(batch_capacity) * scan_top_k; + device_bytes = + vector_bytes + + static_cast(num_vectors) * sizeof(std::uint64_t) + + (memory_enabled ? static_cast(num_vectors) * + (sizeof(std::uint64_t) + sizeof(float) + + 2 * sizeof(std::uint32_t)) + : 0) + + static_cast(nlist) * dim * sizeof(float) + + (static_cast(nlist) + 1) * sizeof(std::uint64_t) + + query_bytes + center_score_slots * sizeof(float) + + selected_center_slots * sizeof(std::uint32_t) + + (adaptive_nprobe ? selected_center_slots * sizeof(float) + + static_cast(batch_capacity) * + sizeof(std::uint32_t) + + (group_adaptive_queries + ? static_cast(batch_capacity) * + sizeof(std::uint32_t) + : 0) + : 0) + + (bucket_major + ? 2 * static_cast(nlist) * sizeof(std::uint32_t) + + selected_center_slots * sizeof(std::uint64_t) + : 0) + + probe_result_slots * (sizeof(float) + sizeof(std::uint64_t)) + + result_slots * (sizeof(float) + sizeof(std::uint64_t)); + device_vectors = std::make_unique(vector_bytes); + device_ids = std::make_unique( + static_cast(num_vectors) * sizeof(std::uint64_t)); + device_centers = std::make_unique( + static_cast(nlist) * dim * sizeof(float)); + device_offsets = std::make_unique( + (static_cast(nlist) + 1) * sizeof(std::uint64_t)); + if (memory_enabled) { + device_timestamps = std::make_unique( + static_cast(num_vectors) * sizeof(std::uint64_t)); + device_importance = std::make_unique( + static_cast(num_vectors) * sizeof(float)); + device_session_ids = std::make_unique( + static_cast(num_vectors) * sizeof(std::uint32_t)); + device_source_types = std::make_unique( + static_cast(num_vectors) * sizeof(std::uint32_t)); + } + device_queries = std::make_unique(query_bytes); + device_center_scores = + std::make_unique(center_score_slots * sizeof(float)); + device_selected_centers = std::make_unique( + selected_center_slots * sizeof(std::uint32_t)); + if (adaptive_nprobe) { + device_selected_scores = + std::make_unique(selected_center_slots * sizeof(float)); + device_query_nprobes = std::make_unique( + static_cast(batch_capacity) * sizeof(std::uint32_t)); + if (group_adaptive_queries) { + device_active_query_ids = std::make_unique( + static_cast(batch_capacity) * sizeof(std::uint32_t)); + } + } + if (bucket_major) { + device_bucket_task_counts = std::make_unique( + static_cast(nlist) * sizeof(std::uint32_t)); + device_bucket_task_cursors = std::make_unique( + static_cast(nlist) * sizeof(std::uint32_t)); + device_bucket_tasks = std::make_unique( + selected_center_slots * sizeof(std::uint64_t)); + } + device_probe_scores = + std::make_unique(probe_result_slots * sizeof(float)); + device_probe_ids = std::make_unique(probe_result_slots * + sizeof(std::uint64_t)); + device_top_scores = + std::make_unique(result_slots * sizeof(float)); + device_top_ids = + std::make_unique(result_slots * sizeof(std::uint64_t)); + + CudaEvent copy_start; + CudaEvent copy_end; + copy_start.record(); + const void* vectors = + use_fp16 ? static_cast(index.half_values.data()) + : static_cast(index.values.data()); + checkCuda(cudaMemcpy(device_vectors->data(), vectors, vector_bytes, + cudaMemcpyHostToDevice), + "复制 IVF 向量到 GPU"); + checkCuda(cudaMemcpy( + device_ids->data(), index.ids.data(), + static_cast(num_vectors) * sizeof(std::uint64_t), + cudaMemcpyHostToDevice), + "复制 IVF ID 到 GPU"); + checkCuda(cudaMemcpy(device_centers->data(), index.centers.data(), + static_cast(nlist) * dim * sizeof(float), + cudaMemcpyHostToDevice), + "复制 IVF centers 到 GPU"); + checkCuda(cudaMemcpy( + device_offsets->data(), index.offsets.data(), + (static_cast(nlist) + 1) * sizeof(std::uint64_t), + cudaMemcpyHostToDevice), + "复制 IVF offsets 到 GPU"); + if (memory_enabled) { + checkCuda( + cudaMemcpy( + device_timestamps->data(), metadata->timestamps.data(), + static_cast(num_vectors) * sizeof(std::uint64_t), + cudaMemcpyHostToDevice), + "复制记忆 timestamps 到 GPU"); + checkCuda( + cudaMemcpy(device_importance->data(), metadata->importance.data(), + static_cast(num_vectors) * sizeof(float), + cudaMemcpyHostToDevice), + "复制记忆 importance 到 GPU"); + checkCuda( + cudaMemcpy( + device_session_ids->data(), metadata->session_ids.data(), + static_cast(num_vectors) * sizeof(std::uint32_t), + cudaMemcpyHostToDevice), + "复制记忆 session IDs 到 GPU"); + checkCuda( + cudaMemcpy( + device_source_types->data(), metadata->source_types.data(), + static_cast(num_vectors) * sizeof(std::uint32_t), + cudaMemcpyHostToDevice), + "复制记忆 source types 到 GPU"); + } + copy_end.record(); + copy_end.synchronize(); + index_h2d_ms = elapsedMilliseconds(copy_start, copy_end); + } + + std::uint64_t num_vectors; + std::uint32_t dim; + std::uint32_t nlist; + DataType dtype; + Metric metric; + SearchParams params; + bool adaptive_nprobe; + bool group_adaptive_queries; + bool bucket_major; + std::uint32_t partitioned_topk_warps; + std::uint32_t partitioned_topk_heap_k; + bool memory_enabled; + bool fused_memory; + bool separate_rerank; + bool use_fp16; + std::size_t input_element_bytes; + std::uint32_t batch_capacity; + std::uint32_t scan_top_k = 0; + const IvfFlatIndex* host_index; + const MemoryMetadata* host_metadata; + double index_h2d_ms = 0.0; + std::size_t device_bytes = 0; + std::unique_ptr device_vectors; + std::unique_ptr device_ids; + std::unique_ptr device_centers; + std::unique_ptr device_offsets; + std::unique_ptr device_timestamps; + std::unique_ptr device_importance; + std::unique_ptr device_session_ids; + std::unique_ptr device_source_types; + std::unique_ptr device_queries; + std::unique_ptr device_center_scores; + std::unique_ptr device_selected_centers; + std::unique_ptr device_selected_scores; + std::unique_ptr device_query_nprobes; + std::unique_ptr device_active_query_ids; + std::unique_ptr device_bucket_task_counts; + std::unique_ptr device_bucket_task_cursors; + std::unique_ptr device_bucket_tasks; + std::unique_ptr device_probe_scores; + std::unique_ptr device_probe_ids; + std::unique_ptr device_top_scores; + std::unique_ptr device_top_ids; +}; + +GpuIvfFlatSearchEngine::GpuIvfFlatSearchEngine(const IvfFlatIndex& index, + const QuerySet& initial_queries, + const SearchParams& params, + const MemoryMetadata* metadata) + : impl_(std::make_unique(index, initial_queries, params, metadata)) {} + +GpuIvfFlatSearchEngine::~GpuIvfFlatSearchEngine() = default; +GpuIvfFlatSearchEngine::GpuIvfFlatSearchEngine( + GpuIvfFlatSearchEngine&&) noexcept = default; +GpuIvfFlatSearchEngine& GpuIvfFlatSearchEngine::operator=( + GpuIvfFlatSearchEngine&&) noexcept = default; + +double GpuIvfFlatSearchEngine::indexH2DMilliseconds() const { + return impl_->index_h2d_ms; +} + +std::size_t GpuIvfFlatSearchEngine::deviceBytes() const { + return impl_->device_bytes; +} + +SearchResults GpuIvfFlatSearchEngine::search(const QuerySet& queries, + SearchStats* stats) { + validateSearchConfiguration(*impl_->host_index, queries, impl_->params, + impl_->host_metadata); + SearchStats measured; + SearchResults results(queries.num_queries); + std::vector host_top_scores( + static_cast(impl_->batch_capacity) * impl_->params.top_k); + std::vector host_top_ids( + static_cast(impl_->batch_capacity) * impl_->params.top_k); + std::vector host_query_nprobes(impl_->batch_capacity, + impl_->params.nprobe); + + CudaEvent copy_start; + CudaEvent copy_end; + CudaEvent center_end; + CudaEvent scan_end; + CudaEvent merge_end; + CudaEvent rerank_end; + CudaEvent result_end; + for (std::uint64_t batch_start = 0; batch_start < queries.num_queries; + batch_start += impl_->batch_capacity) { + const std::uint32_t batch_count = + static_cast(std::min( + impl_->batch_capacity, queries.num_queries - batch_start)); + const std::size_t query_value_offset = + static_cast(batch_start) * impl_->dim; + const void* query_data = + impl_->use_fp16 ? static_cast(queries.half_values.data() + + query_value_offset) + : static_cast(queries.values.data() + + query_value_offset); + copy_start.record(); + checkCuda(cudaMemcpy(impl_->device_queries->data(), query_data, + static_cast(batch_count) * impl_->dim * + impl_->input_element_bytes, + cudaMemcpyHostToDevice), + "复制 IVF queries 到 GPU"); + copy_end.record(); + + constexpr std::uint32_t kThreads = 256; + const dim3 center_grid((impl_->nlist + kThreads - 1) / kThreads, + batch_count); + if (impl_->use_fp16) { + ivfCenterScoresKernel<<>>( + static_cast(impl_->device_centers->data()), + static_cast(impl_->device_queries->data()), + static_cast(impl_->device_center_scores->data()), + impl_->nlist, impl_->dim, batch_count, impl_->metric); + } else { + ivfCenterScoresKernel<<>>( + static_cast(impl_->device_centers->data()), + static_cast(impl_->device_queries->data()), + static_cast(impl_->device_center_scores->data()), + impl_->nlist, impl_->dim, batch_count, impl_->metric); + } + checkCuda(cudaGetLastError(), "启动 ivfCenterScoresKernel"); + ivfSelectProbesBlockKernel<<>>( + static_cast(impl_->device_center_scores->data()), + static_cast(impl_->device_selected_centers->data()), + impl_->adaptive_nprobe + ? static_cast(impl_->device_selected_scores->data()) + : nullptr, + impl_->nlist, impl_->params.nprobe, impl_->metric); + checkCuda(cudaGetLastError(), "启动 ivfSelectProbesBlockKernel"); + if (impl_->adaptive_nprobe) { + constexpr std::uint32_t kPolicyThreads = 128; + const std::uint32_t policy_blocks = + (batch_count + kPolicyThreads - 1) / kPolicyThreads; + chooseAdaptiveNprobeKernel<<>>( + static_cast(impl_->device_selected_scores->data()), + static_cast(impl_->device_query_nprobes->data()), + batch_count, impl_->params.nprobe, impl_->params.adaptive_nprobe_min, + impl_->params.adaptive_nprobe_step, + impl_->params.adaptive_target_mass, + impl_->params.adaptive_temperature, impl_->metric); + checkCuda(cudaGetLastError(), "启动 chooseAdaptiveNprobeKernel"); + } + center_end.record(); + + struct QueryTier { + std::uint32_t nprobe; + std::uint32_t offset; + std::uint32_t count; + }; + std::vector query_tiers; + std::vector active_query_ids; + if (impl_->group_adaptive_queries) { + // 自适应策略会产生多个 nprobe tier。先取回很小的计数数组，在 CPU + // 上稳定分组，再一次性上传 query ID；桶扫描因此不再启动空 block， + // 同一个 kernel launch 内也不会混入工作量悬殊的 query。 + center_end.synchronize(); + const auto policy_start = std::chrono::steady_clock::now(); + checkCuda( + cudaMemcpy( + host_query_nprobes.data(), impl_->device_query_nprobes->data(), + static_cast(batch_count) * sizeof(std::uint32_t), + cudaMemcpyDeviceToHost), + "复制自适应 nprobe 到 CPU"); + std::vector unique_nprobes( + host_query_nprobes.begin(), host_query_nprobes.begin() + batch_count); + std::sort(unique_nprobes.begin(), unique_nprobes.end()); + unique_nprobes.erase( + std::unique(unique_nprobes.begin(), unique_nprobes.end()), + unique_nprobes.end()); + active_query_ids.reserve(batch_count); + for (const std::uint32_t tier_nprobe : unique_nprobes) { + const std::uint32_t offset = + static_cast(active_query_ids.size()); + for (std::uint32_t query_id = 0; query_id < batch_count; ++query_id) { + if (host_query_nprobes[query_id] == tier_nprobe) { + active_query_ids.push_back(query_id); + } + } + query_tiers.push_back( + {tier_nprobe, offset, + static_cast(active_query_ids.size()) - offset}); + } + checkCuda( + cudaMemcpy( + impl_->device_active_query_ids->data(), active_query_ids.data(), + static_cast(batch_count) * sizeof(std::uint32_t), + cudaMemcpyHostToDevice), + "复制自适应 query 分层到 GPU"); + const auto policy_end = std::chrono::steady_clock::now(); + measured.adaptive_policy_ms += + std::chrono::duration(policy_end - policy_start) + .count(); + } + + const auto* device_ids = + static_cast(impl_->device_ids->data()); + const auto* device_offsets = + static_cast(impl_->device_offsets->data()); + const auto* selected_centers = static_cast( + impl_->device_selected_centers->data()); + auto* probe_scores = + static_cast(impl_->device_probe_scores->data()); + auto* probe_ids = + static_cast(impl_->device_probe_ids->data()); + DeviceMemoryView memory_view; + DeviceMemoryOptions memory_options; + if (impl_->memory_enabled) { + memory_view = { + static_cast(impl_->device_timestamps->data()), + static_cast(impl_->device_importance->data()), + static_cast(impl_->device_session_ids->data()), + static_cast( + impl_->device_source_types->data())}; + memory_options = {impl_->params.memory_semantic_weight, + impl_->params.memory_importance_weight, + impl_->params.memory_recency_weight, + impl_->params.memory_time_scale, + impl_->params.memory_now, + impl_->params.filter_min_timestamp, + impl_->params.filter_session_id, + impl_->params.filter_source_type}; + } + const auto* query_nprobes = impl_->adaptive_nprobe + ? static_cast( + impl_->device_query_nprobes->data()) + : nullptr; + std::uint32_t warps_per_block = 0; + const bool bucket_major = impl_->bucket_major; + const std::uint32_t partitioned_topk_warps = + impl_->partitioned_topk_warps; + const std::uint64_t* bucket_tasks = nullptr; + if (bucket_major) { + const std::uint32_t task_count = batch_count * impl_->params.nprobe; + auto* center_counts = static_cast( + impl_->device_bucket_task_counts->data()); + auto* center_cursors = static_cast( + impl_->device_bucket_task_cursors->data()); + auto* mutable_bucket_tasks = static_cast( + impl_->device_bucket_tasks->data()); + checkCuda(cudaMemsetAsync(center_counts, 0, + static_cast(impl_->nlist) * + sizeof(std::uint32_t)), + "清零 bucket-major 任务计数"); + const std::uint32_t task_blocks = + (task_count + kThreads - 1) / kThreads; + countSelectedCenterTasksKernel<<>>( + selected_centers, center_counts, task_count); + prepareCenterTaskCursorsKernel<<<1, 1>>>( + center_counts, center_cursors, impl_->nlist); + scatterSelectedCenterTasksKernel<<>>( + selected_centers, center_cursors, mutable_bucket_tasks, + impl_->params.nprobe, task_count); + bucket_tasks = mutable_bucket_tasks; + } + if (impl_->params.distance_mode == "warp" || + impl_->params.distance_mode == "warp8") { + warps_per_block = 8; + } else if (impl_->params.distance_mode == "warp1") { + warps_per_block = 1; + } else if (impl_->params.distance_mode == "warp4") { + warps_per_block = 4; + } else if (impl_->params.distance_mode == "warp2") { + warps_per_block = 2; + } else if (impl_->params.distance_mode == "warp_compact" || + bucket_major) { + // K10 保持高距离计算并行度；K50/K100 逐步减少每个 probe 重复维护的 + // heap 数。K100 的单 warp 版本在 Ada 上减少了候选插入与排序开销。 + warps_per_block = impl_->scan_top_k <= 10 ? 8 + : impl_->scan_top_k <= 50 ? 2 + : 1; + } + const auto launch_scan = [&](dim3 scan_grid, + const std::uint32_t* active_ids) { + if (impl_->use_fp16) { + const auto* vectors = + static_cast(impl_->device_vectors->data()); + const auto* device_queries = + static_cast(impl_->device_queries->data()); + if (impl_->scan_top_k <= 10) { + launchIvfBucketScan<__half, 10>( + scan_grid, vectors, device_ids, device_offsets, selected_centers, + bucket_tasks, device_queries, probe_scores, probe_ids, active_ids, + query_nprobes, impl_->params.nprobe, impl_->dim, + impl_->scan_top_k, impl_->metric, warps_per_block, + bucket_major, partitioned_topk_warps, + impl_->partitioned_topk_heap_k, impl_->fused_memory, memory_view, + memory_options); + } else if (impl_->scan_top_k <= 50) { + launchIvfBucketScan<__half, 50>( + scan_grid, vectors, device_ids, device_offsets, selected_centers, + bucket_tasks, device_queries, probe_scores, probe_ids, active_ids, + query_nprobes, impl_->params.nprobe, impl_->dim, + impl_->scan_top_k, impl_->metric, warps_per_block, + bucket_major, partitioned_topk_warps, + impl_->partitioned_topk_heap_k, impl_->fused_memory, memory_view, + memory_options); + } else { + launchIvfBucketScan<__half, 100>( + scan_grid, vectors, device_ids, device_offsets, selected_centers, + bucket_tasks, device_queries, probe_scores, probe_ids, active_ids, + query_nprobes, impl_->params.nprobe, impl_->dim, + impl_->scan_top_k, impl_->metric, warps_per_block, + bucket_major, partitioned_topk_warps, + impl_->partitioned_topk_heap_k, impl_->fused_memory, memory_view, + memory_options); + } + } else { + const auto* vectors = + static_cast(impl_->device_vectors->data()); + const auto* device_queries = + static_cast(impl_->device_queries->data()); + if (impl_->scan_top_k <= 10) { + launchIvfBucketScan( + scan_grid, vectors, device_ids, device_offsets, selected_centers, + bucket_tasks, device_queries, probe_scores, probe_ids, active_ids, + query_nprobes, impl_->params.nprobe, impl_->dim, + impl_->scan_top_k, impl_->metric, warps_per_block, + bucket_major, partitioned_topk_warps, + impl_->partitioned_topk_heap_k, impl_->fused_memory, memory_view, + memory_options); + } else if (impl_->scan_top_k <= 50) { + launchIvfBucketScan( + scan_grid, vectors, device_ids, device_offsets, selected_centers, + bucket_tasks, device_queries, probe_scores, probe_ids, active_ids, + query_nprobes, impl_->params.nprobe, impl_->dim, + impl_->scan_top_k, impl_->metric, warps_per_block, + bucket_major, partitioned_topk_warps, + impl_->partitioned_topk_heap_k, impl_->fused_memory, memory_view, + memory_options); + } else { + launchIvfBucketScan( + scan_grid, vectors, device_ids, device_offsets, selected_centers, + bucket_tasks, device_queries, probe_scores, probe_ids, active_ids, + query_nprobes, impl_->params.nprobe, impl_->dim, + impl_->scan_top_k, impl_->metric, warps_per_block, + bucket_major, partitioned_topk_warps, + impl_->partitioned_topk_heap_k, impl_->fused_memory, memory_view, + memory_options); + } + } + }; + if (impl_->group_adaptive_queries) { + const auto* grouped_query_ids = static_cast( + impl_->device_active_query_ids->data()); + for (const QueryTier& tier : query_tiers) { + launch_scan(bucket_major + ? dim3(tier.count, impl_->host_index->nlist) + : dim3(tier.nprobe, tier.count), + grouped_query_ids + tier.offset); + } + } else { + launch_scan(bucket_major + ? dim3(batch_count * impl_->params.nprobe) + : dim3(impl_->params.nprobe, batch_count), + nullptr); + } + checkCuda(cudaGetLastError(), "启动 IVF bucket scan kernel"); + scan_end.record(); + + const std::size_t merge_shared_bytes = + static_cast(impl_->params.nprobe) * sizeof(std::uint32_t); + ivfMergeProbeTopKKernel<<>>( + static_cast(impl_->device_probe_scores->data()), + static_cast(impl_->device_probe_ids->data()), + static_cast(impl_->device_top_scores->data()), + static_cast(impl_->device_top_ids->data()), + query_nprobes, impl_->params.nprobe, impl_->scan_top_k, impl_->metric); + checkCuda(cudaGetLastError(), "启动 ivfMergeProbeTopKKernel"); + merge_end.record(); + + const float* result_scores = + static_cast(impl_->device_top_scores->data()); + const std::uint64_t* result_ids = + static_cast(impl_->device_top_ids->data()); + if (impl_->separate_rerank) { + rerankMemoryTopKKernel<<>>( + result_scores, result_ids, + static_cast(impl_->device_probe_scores->data()), + static_cast(impl_->device_probe_ids->data()), + impl_->scan_top_k, impl_->params.top_k, impl_->metric, memory_view, + memory_options); + checkCuda(cudaGetLastError(), "启动 rerankMemoryTopKKernel"); + result_scores = + static_cast(impl_->device_probe_scores->data()); + result_ids = + static_cast(impl_->device_probe_ids->data()); + } + rerank_end.record(); + + const std::size_t result_count = + static_cast(batch_count) * impl_->params.top_k; + checkCuda(cudaMemcpy(host_top_scores.data(), result_scores, + result_count * sizeof(float), cudaMemcpyDeviceToHost), + "复制 IVF Top-K 分数到 CPU"); + checkCuda(cudaMemcpy(host_top_ids.data(), result_ids, + result_count * sizeof(std::uint64_t), + cudaMemcpyDeviceToHost), + "复制 IVF Top-K ID 到 CPU"); + if (stats != nullptr && impl_->adaptive_nprobe && + !impl_->group_adaptive_queries) { + checkCuda( + cudaMemcpy( + host_query_nprobes.data(), impl_->device_query_nprobes->data(), + static_cast(batch_count) * sizeof(std::uint32_t), + cudaMemcpyDeviceToHost), + "复制自适应 nprobe 到 CPU"); + } + result_end.record(); + result_end.synchronize(); + + measured.query_h2d_ms += elapsedMilliseconds(copy_start, copy_end); + measured.center_selection_ms += elapsedMilliseconds(copy_end, center_end); + measured.distance_kernel_ms += elapsedMilliseconds(center_end, scan_end); + measured.topk_kernel_ms += elapsedMilliseconds(scan_end, merge_end); + if (impl_->separate_rerank) { + measured.memory_rerank_ms += elapsedMilliseconds(merge_end, rerank_end); + } + measured.result_d2h_ms += elapsedMilliseconds(rerank_end, result_end); + measured.batch_latency_ms.push_back( + elapsedMilliseconds(copy_start, result_end)); + if (stats != nullptr) { + for (std::uint32_t query_in_batch = 0; query_in_batch < batch_count; + ++query_in_batch) { + const std::uint32_t selected = impl_->adaptive_nprobe + ? host_query_nprobes[query_in_batch] + : impl_->params.nprobe; + if (measured.selected_probe_queries == 0) { + measured.selected_probe_min = selected; + } else { + measured.selected_probe_min = + std::min(measured.selected_probe_min, selected); + } + measured.selected_probe_max = + std::max(measured.selected_probe_max, selected); + measured.selected_probe_sum += selected; + ++measured.selected_probe_queries; + measured.selected_probe_counts.push_back(selected); + } + } + for (std::uint32_t query_in_batch = 0; query_in_batch < batch_count; + ++query_in_batch) { + std::vector& query_results = + results[batch_start + query_in_batch]; + query_results.reserve(impl_->params.top_k); + const std::size_t offset = + static_cast(query_in_batch) * impl_->params.top_k; + for (std::uint32_t rank = 0; rank < impl_->params.top_k; ++rank) { + if (host_top_ids[offset + rank] == UINT64_MAX) { + break; + } + query_results.push_back( + {host_top_ids[offset + rank], host_top_scores[offset + rank]}); + } + } + } + if (stats != nullptr) { + *stats = measured; + } + return results; +} + +SearchResults gpuIvfFlatSearch(const IvfFlatIndex& index, + const QuerySet& queries, + const SearchParams& params, SearchStats* stats, + const MemoryMetadata* metadata) { + GpuIvfFlatSearchEngine engine(index, queries, params, metadata); + SearchResults results = engine.search(queries, stats); + if (stats != nullptr) { + stats->database_h2d_ms = engine.indexH2DMilliseconds(); + } + return results; +} diff --git a/09_vector_retrieval/PurplePulse/src/gpu_search.cu b/09_vector_retrieval/PurplePulse/src/gpu_search.cu new file mode 100644 index 00000000..8a1390e2 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/src/gpu_search.cu @@ -0,0 +1,1902 @@ +#include +#include + +#include +#include +#include +#include +#include +#include + +#include "file_io.h" +#include "search.h" + +// 这是第一版 baseline：一个 CUDA 线程计算一对 query/vector 的完整距离。 +// 优点是代码直观，缺点是暂时没有在线程间共同处理一个向量。 +__device__ __forceinline__ float loadInput(const float* values, + std::uint64_t index) { + return values[index]; +} + +__device__ __forceinline__ float loadInput(const __half* values, + std::uint64_t index) { + return __half2float(values[index]); +} + +template +__global__ void exactDistanceKernel(const InputType* database, + const InputType* queries, float* scores, + std::uint64_t num_vectors, + std::uint32_t dim, + std::uint32_t batch_count, Metric metric) { + const std::uint64_t vector_id = + static_cast(blockIdx.x) * blockDim.x + threadIdx.x; + const std::uint32_t query_in_batch = blockIdx.y; + if (vector_id >= num_vectors || query_in_batch >= batch_count) { + return; + } + + const std::uint64_t vector_offset = vector_id * dim; + const std::uint64_t query_offset = + static_cast(query_in_batch) * dim; + + float dot = 0.0F; + float query_norm = 0.0F; + float vector_norm = 0.0F; + float squared_l2 = 0.0F; + for (std::uint32_t d = 0; d < dim; ++d) { + const float q = loadInput(queries, query_offset + d); + const float x = loadInput(database, vector_offset + d); + if (metric == Metric::kL2) { + const float difference = q - x; + squared_l2 = fmaf(difference, difference, squared_l2); + } else { + dot = fmaf(q, x, dot); + if (metric == Metric::kCosine) { + query_norm = fmaf(q, q, query_norm); + vector_norm = fmaf(x, x, vector_norm); + } + } + } + + float score = dot; + if (metric == Metric::kL2) { + score = squared_l2; + } else if (metric == Metric::kCosine) { + score = (query_norm == 0.0F || vector_norm == 0.0F) + ? 0.0F + : dot * rsqrtf(query_norm * vector_norm); + } + scores[static_cast(query_in_batch) * num_vectors + vector_id] = + score; +} + +// 优化版距离计算：一个 warp（32 个线程）共同处理一个向量。 +// lane 0 计算维度 0、32、64...，lane 1 计算 1、33、65...，因此访问是连续的。 +template +__global__ void exactDistanceWarpKernel(const InputType* database, + const InputType* queries, float* scores, + std::uint64_t num_vectors, + std::uint32_t dim, + std::uint32_t batch_count, + Metric metric) { + constexpr std::uint32_t kWarpSize = 32; + const std::uint32_t lane = threadIdx.x % kWarpSize; + const std::uint32_t warp_in_block = threadIdx.x / kWarpSize; + const std::uint32_t warps_per_block = blockDim.x / kWarpSize; + const std::uint64_t first_vector_id = + static_cast(blockIdx.x) * warps_per_block + warp_in_block; + const std::uint32_t query_in_batch = blockIdx.y; + if (query_in_batch >= batch_count) { + return; + } + const std::uint64_t query_offset = + static_cast(query_in_batch) * dim; + + // grid-stride 循环让同一个 warp 连续处理多个向量，避免启动过多 block。 + const std::uint64_t vector_stride = + static_cast(gridDim.x) * warps_per_block; + for (std::uint64_t vector_id = first_vector_id; vector_id < num_vectors; + vector_id += vector_stride) { + const std::uint64_t vector_offset = vector_id * dim; + float dot = 0.0F; + float query_norm = 0.0F; + float vector_norm = 0.0F; + float squared_l2 = 0.0F; + + for (std::uint32_t d = lane; d < dim; d += kWarpSize) { + const float q = loadInput(queries, query_offset + d); + const float x = loadInput(database, vector_offset + d); + if (metric == Metric::kL2) { + const float difference = q - x; + squared_l2 = fmaf(difference, difference, squared_l2); + } else { + dot = fmaf(q, x, dot); + if (metric == Metric::kCosine) { + query_norm = fmaf(q, q, query_norm); + vector_norm = fmaf(x, x, vector_norm); + } + } + } + + // 每次让后半部分 lane 的结果加到前半部分，最终总和位于 lane 0。 + for (std::uint32_t offset = kWarpSize / 2; offset > 0; offset >>= 1) { + squared_l2 += __shfl_down_sync(0xffffffff, squared_l2, offset); + dot += __shfl_down_sync(0xffffffff, dot, offset); + query_norm += __shfl_down_sync(0xffffffff, query_norm, offset); + vector_norm += __shfl_down_sync(0xffffffff, vector_norm, offset); + } + + if (lane == 0) { + float score = dot; + if (metric == Metric::kL2) { + score = squared_l2; + } else if (metric == Metric::kCosine) { + score = (query_norm == 0.0F || vector_norm == 0.0F) + ? 0.0F + : dot * rsqrtf(query_norm * vector_norm); + } + scores[static_cast(query_in_batch) * num_vectors + + vector_id] = score; + } + } +} + +__device__ bool isBetter(float candidate_score, std::uint64_t candidate_id, + float current_score, std::uint64_t current_id, + Metric metric) { + if (candidate_score == current_score) { + return candidate_id < current_id; + } + if (metric == Metric::kL2) { + return candidate_score < current_score; + } + return candidate_score > current_score; +} + +// 教学版 GPU Top-K：每个 query 使用一个线程，按顺序维护有序的前 K 个结果。 +// 它的并行度不高，但逻辑清楚，并且避免把完整距离矩阵复制回 CPU。 +// 下一版会用一个 block 共同处理一个 query，作为明确的性能优化步骤。 +__global__ void simpleTopKKernel(const float* scores, float* top_scores, + std::uint64_t* top_ids, + std::uint64_t num_vectors, std::uint32_t top_k, + Metric metric) { + const std::uint32_t query_in_batch = blockIdx.x; + if (threadIdx.x != 0) { + return; + } + + constexpr std::uint32_t kMaximumK = 100; + float best_scores[kMaximumK]; + std::uint64_t best_ids[kMaximumK]; + // 使用 float 最大值作为哨兵，避免依赖额外的 CUDA 数学常量头文件。 + constexpr float kFloatMaximum = 3.402823466e+38F; + const float worst_score = + (metric == Metric::kL2) ? kFloatMaximum : -kFloatMaximum; + for (std::uint32_t rank = 0; rank < top_k; ++rank) { + best_scores[rank] = worst_score; + best_ids[rank] = UINT64_MAX; + } + + const float* query_scores = + scores + static_cast(query_in_batch) * num_vectors; + for (std::uint64_t vector_id = 0; vector_id < num_vectors; ++vector_id) { + const float score = query_scores[vector_id]; + for (std::uint32_t rank = 0; rank < top_k; ++rank) { + if (isBetter(score, vector_id, best_scores[rank], best_ids[rank], + metric)) { + for (std::uint32_t shift = top_k - 1; shift > rank; --shift) { + best_scores[shift] = best_scores[shift - 1]; + best_ids[shift] = best_ids[shift - 1]; + } + best_scores[rank] = score; + best_ids[rank] = vector_id; + break; + } + } + } + + const std::uint64_t output_offset = + static_cast(query_in_batch) * top_k; + for (std::uint32_t rank = 0; rank < top_k; ++rank) { + top_scores[output_offset + rank] = best_scores[rank]; + top_ids[output_offset + rank] = best_ids[rank]; + } +} + +// 并行版 GPU Top-K：一个 block 共同处理一个 query。 +// 每一轮中，各线程先在自己负责的候选中找最好结果，再用共享内存归约出全局最好结果。 +// 找到一个结果后将其标记为“最差”，重复 K 轮。这不是最终最优算法，但比单线程版本 +// 更容易理解，也能直接展示线程协作带来的收益。 +__global__ void blockTopKKernel(float* scores, + const std::uint64_t* candidate_ids, + float* top_scores, std::uint64_t* top_ids, + std::uint64_t num_vectors, std::uint32_t top_k, + Metric metric) { + constexpr std::uint32_t kThreads = 256; + __shared__ float shared_scores[kThreads]; + __shared__ std::uint64_t shared_ids[kThreads]; + __shared__ std::uint64_t shared_positions[kThreads]; + + const std::uint32_t query_in_batch = blockIdx.x; + const std::uint32_t thread_id = threadIdx.x; + float* query_scores = + scores + static_cast(query_in_batch) * num_vectors; + constexpr float kFloatMaximum = 3.402823466e+38F; + const float worst_score = + (metric == Metric::kL2) ? kFloatMaximum : -kFloatMaximum; + + for (std::uint32_t rank = 0; rank < top_k; ++rank) { + float local_best_score = worst_score; + std::uint64_t local_best_id = UINT64_MAX; + std::uint64_t local_best_position = UINT64_MAX; + + for (std::uint64_t position = thread_id; position < num_vectors; + position += blockDim.x) { + const float score = query_scores[position]; + const std::uint64_t vector_id = + candidate_ids == nullptr + ? position + : candidate_ids[static_cast(query_in_batch) * + num_vectors + + position]; + if (isBetter(score, vector_id, local_best_score, local_best_id, metric)) { + local_best_score = score; + local_best_id = vector_id; + local_best_position = position; + } + } + + shared_scores[thread_id] = local_best_score; + shared_ids[thread_id] = local_best_id; + shared_positions[thread_id] = local_best_position; + __syncthreads(); + + for (std::uint32_t stride = kThreads / 2; stride > 0; stride >>= 1) { + if (thread_id < stride && + isBetter(shared_scores[thread_id + stride], + shared_ids[thread_id + stride], shared_scores[thread_id], + shared_ids[thread_id], metric)) { + shared_scores[thread_id] = shared_scores[thread_id + stride]; + shared_ids[thread_id] = shared_ids[thread_id + stride]; + shared_positions[thread_id] = shared_positions[thread_id + stride]; + } + __syncthreads(); + } + + if (thread_id == 0) { + const std::uint64_t output_offset = + static_cast(query_in_batch) * top_k + rank; + top_scores[output_offset] = shared_scores[0]; + top_ids[output_offset] = shared_ids[0]; + query_scores[shared_positions[0]] = worst_score; + } + __syncthreads(); + } +} + +// 两阶段 Top-K 的第一阶段：每个线程只扫描自己负责的候选，并保留本线程的 Top-K。 +// 局部数组不需要有序：绝大多数候选只与当前最差项比较一次；只有候选真正 +// 进入 Top-K 时，才扫描 K 个槽位重新找最差项。256 个线程产生 256*K 个候选后， +// 再交给 blockTopKKernel 做最终有序归并。 +__global__ void localTopKCandidatesKernel(const float* scores, + float* local_scores, + std::uint64_t* local_ids, + std::uint64_t num_vectors, + std::uint32_t top_k, Metric metric) { + constexpr std::uint32_t kMaximumLocalK = 10; + const std::uint32_t query_in_batch = blockIdx.x; + const std::uint32_t thread_id = threadIdx.x; + constexpr float kFloatMaximum = 3.402823466e+38F; + const float worst_score = + (metric == Metric::kL2) ? kFloatMaximum : -kFloatMaximum; + + float best_scores[kMaximumLocalK]; + std::uint64_t best_ids[kMaximumLocalK]; + for (std::uint32_t rank = 0; rank < top_k; ++rank) { + best_scores[rank] = worst_score; + best_ids[rank] = UINT64_MAX; + } + std::uint32_t worst_rank = 0; + + const float* query_scores = + scores + static_cast(query_in_batch) * num_vectors; + for (std::uint64_t vector_id = thread_id; vector_id < num_vectors; + vector_id += blockDim.x) { + const float score = query_scores[vector_id]; + if (isBetter(score, vector_id, best_scores[worst_rank], + best_ids[worst_rank], metric)) { + best_scores[worst_rank] = score; + best_ids[worst_rank] = vector_id; + worst_rank = 0; + for (std::uint32_t rank = 1; rank < top_k; ++rank) { + if (isBetter(best_scores[worst_rank], best_ids[worst_rank], + best_scores[rank], best_ids[rank], metric)) { + worst_rank = rank; + } + } + } + } + + const std::uint64_t output_offset = + (static_cast(query_in_batch) * blockDim.x + thread_id) * + top_k; + for (std::uint32_t rank = 0; rank < top_k; ++rank) { + local_scores[output_offset + rank] = best_scores[rank]; + local_ids[output_offset + rank] = best_ids[rank]; + } +} + +// K=1/10 是项目的主要评测档。将 K 变成编译期常量后，编译器可以展开 +// 局部槽位扫描，并尽量避免通用 kernel 因动态数组索引产生的 thread stack。 +template +__global__ void localTopKCandidatesFixedKernel(const float* scores, + float* local_scores, + std::uint64_t* local_ids, + std::uint64_t num_vectors, + Metric metric) { + static_assert(TopK >= 1 && TopK <= 10, "fixed Top-K 必须位于 [1, 10]"); + const std::uint32_t query_in_batch = blockIdx.x; + const std::uint32_t thread_id = threadIdx.x; + constexpr float kFloatMaximum = 3.402823466e+38F; + const float worst_score = + (metric == Metric::kL2) ? kFloatMaximum : -kFloatMaximum; + + float best_scores[TopK]; + std::uint64_t best_ids[TopK]; +#pragma unroll + for (std::uint32_t rank = 0; rank < TopK; ++rank) { + best_scores[rank] = worst_score; + best_ids[rank] = UINT64_MAX; + } + std::uint32_t worst_rank = 0; + + const float* query_scores = + scores + static_cast(query_in_batch) * num_vectors; + for (std::uint64_t vector_id = thread_id; vector_id < num_vectors; + vector_id += blockDim.x) { + const float score = query_scores[vector_id]; + if (isBetter(score, vector_id, best_scores[worst_rank], + best_ids[worst_rank], metric)) { + best_scores[worst_rank] = score; + best_ids[worst_rank] = vector_id; + worst_rank = 0; +#pragma unroll + for (std::uint32_t rank = 1; rank < TopK; ++rank) { + if (isBetter(best_scores[worst_rank], best_ids[worst_rank], + best_scores[rank], best_ids[rank], metric)) { + worst_rank = rank; + } + } + } + } + + const std::uint64_t output_offset = + (static_cast(query_in_batch) * blockDim.x + thread_id) * + TopK; +#pragma unroll + for (std::uint32_t rank = 0; rank < TopK; ++rank) { + local_scores[output_offset + rank] = best_scores[rank]; + local_ids[output_offset + rank] = best_ids[rank]; + } +} + +// Fused exact scan: each block owns one query/chunk pair and contains eight +// cooperative warps. A warp computes one vector score and maintains a shared +// Top-K heap. The block emits only K candidates, so the full batch*N score +// matrix is never materialized. A small final merge combines all chunks. +// +// The distance-only kernel immediately below is a diagnostic differential +// probe. It intentionally mirrors the fused traversal and arithmetic but +// replaces heap maintenance with one checksum per warp. Comparing its event +// time with the full fused kernel estimates local-selection cost without +// changing the production search path or materializing the score matrix. +template +__global__ void exactFusedDistanceOnlyKernel( + const InputType* database, const InputType* queries, float* warp_checksums, + std::uint64_t num_vectors, std::uint32_t dim, bool cache_query) { + constexpr std::uint32_t kWarpSize = 32; + constexpr std::uint32_t kWarps = 8; + extern __shared__ __align__(16) unsigned char dynamic_shared[]; + auto* shared_query = reinterpret_cast(dynamic_shared); + + const std::uint32_t thread_id = threadIdx.x; + const std::uint32_t lane = thread_id % kWarpSize; + const std::uint32_t warp = thread_id / kWarpSize; + const std::uint32_t query_id = blockIdx.y; + const InputType* query_values = + queries + static_cast(query_id) * dim; + if (cache_query) { + for (std::uint32_t d = thread_id; d < dim; d += blockDim.x) { + shared_query[d] = query_values[d]; + } + } + __syncthreads(); + if (cache_query) { + query_values = shared_query; + } + + float query_norm = 0.0F; + if constexpr (MetricValue == Metric::kCosine) { + for (std::uint32_t d = lane; d < dim; d += kWarpSize) { + const float query = loadInput(query_values, d); + query_norm = fmaf(query, query, query_norm); + } + for (std::uint32_t offset = kWarpSize / 2; offset > 0; offset >>= 1) { + query_norm += __shfl_down_sync(0xffffffffU, query_norm, offset); + } + } + + float checksum = 0.0F; + const std::uint64_t first_vector = + static_cast(blockIdx.x) * kWarps + warp; + const std::uint64_t vector_stride = + static_cast(gridDim.x) * kWarps; + for (std::uint64_t vector_id = first_vector; vector_id < num_vectors; + vector_id += vector_stride) { + const std::uint64_t vector_offset = vector_id * dim; + float dot = 0.0F; + float vector_norm = 0.0F; + float squared_l2 = 0.0F; + for (std::uint32_t d = lane; d < dim; d += kWarpSize) { + const float query = loadInput(query_values, d); + const float vector = loadInput(database, vector_offset + d); + if constexpr (MetricValue == Metric::kL2) { + const float difference = query - vector; + squared_l2 = fmaf(difference, difference, squared_l2); + } else { + dot = fmaf(query, vector, dot); + if constexpr (MetricValue == Metric::kCosine) { + vector_norm = fmaf(vector, vector, vector_norm); + } + } + } + for (std::uint32_t offset = kWarpSize / 2; offset > 0; offset >>= 1) { + if constexpr (MetricValue == Metric::kL2) { + squared_l2 += __shfl_down_sync(0xffffffffU, squared_l2, offset); + } else { + dot += __shfl_down_sync(0xffffffffU, dot, offset); + if constexpr (MetricValue == Metric::kCosine) { + vector_norm += __shfl_down_sync(0xffffffffU, vector_norm, offset); + } + } + } + if (lane == 0) { + float score = dot; + if constexpr (MetricValue == Metric::kL2) { + score = squared_l2; + } else if constexpr (MetricValue == Metric::kCosine) { + score = query_norm == 0.0F || vector_norm == 0.0F + ? 0.0F + : dot * rsqrtf(query_norm * vector_norm); + } + checksum += score; + } + } + if (lane == 0) { + const std::uint64_t output = + (static_cast(query_id) * gridDim.x + blockIdx.x) * + kWarps + + warp; + warp_checksums[output] = checksum; + } +} + +template +__global__ void exactFusedWarpTopKKernel( + const InputType* database, const InputType* queries, float* local_scores, + std::uint64_t* local_ids, std::uint64_t num_vectors, std::uint32_t dim, + std::uint32_t top_k, bool cache_query) { + constexpr std::uint32_t kWarpSize = 32; + constexpr std::uint32_t kWarps = 8; + constexpr float kFloatMaximum = 3.402823466e+38F; + __shared__ float warp_scores[kWarps][MaximumK]; + __shared__ std::uint64_t warp_ids[kWarps][MaximumK]; + __shared__ float merge_scores[kWarps]; + __shared__ std::uint64_t merge_ids[kWarps]; + __shared__ std::uint32_t warp_ranks[kWarps]; + extern __shared__ __align__(16) unsigned char dynamic_shared[]; + auto* shared_query = reinterpret_cast(dynamic_shared); + + const std::uint32_t thread_id = threadIdx.x; + const std::uint32_t lane = thread_id % kWarpSize; + const std::uint32_t warp = thread_id / kWarpSize; + const std::uint32_t query_id = blockIdx.y; + const float worst_score = + MetricValue == Metric::kL2 ? kFloatMaximum : -kFloatMaximum; + if (lane == 0) { + for (std::uint32_t rank = 0; rank < top_k; ++rank) { + warp_scores[warp][rank] = worst_score; + warp_ids[warp][rank] = UINT64_MAX; + } + } + + const InputType* query_values = + queries + static_cast(query_id) * dim; + if (cache_query) { + for (std::uint32_t d = thread_id; d < dim; d += blockDim.x) { + shared_query[d] = query_values[d]; + } + } + __syncthreads(); + if (cache_query) { + query_values = shared_query; + } + + float query_norm = 0.0F; + if constexpr (MetricValue == Metric::kCosine) { + for (std::uint32_t d = lane; d < dim; d += kWarpSize) { + const float query = loadInput(query_values, d); + query_norm = fmaf(query, query, query_norm); + } + for (std::uint32_t offset = kWarpSize / 2; offset > 0; offset >>= 1) { + query_norm += __shfl_down_sync(0xffffffffU, query_norm, offset); + } + } + + const std::uint64_t first_vector = + static_cast(blockIdx.x) * kWarps + warp; + const std::uint64_t vector_stride = + static_cast(gridDim.x) * kWarps; + for (std::uint64_t vector_id = first_vector; vector_id < num_vectors; + vector_id += vector_stride) { + const std::uint64_t vector_offset = vector_id * dim; + float dot = 0.0F; + float vector_norm = 0.0F; + float squared_l2 = 0.0F; + for (std::uint32_t d = lane; d < dim; d += kWarpSize) { + const float query = loadInput(query_values, d); + const float vector = loadInput(database, vector_offset + d); + if constexpr (MetricValue == Metric::kL2) { + const float difference = query - vector; + squared_l2 = fmaf(difference, difference, squared_l2); + } else { + dot = fmaf(query, vector, dot); + if constexpr (MetricValue == Metric::kCosine) { + vector_norm = fmaf(vector, vector, vector_norm); + } + } + } + for (std::uint32_t offset = kWarpSize / 2; offset > 0; offset >>= 1) { + if constexpr (MetricValue == Metric::kL2) { + squared_l2 += __shfl_down_sync(0xffffffffU, squared_l2, offset); + } else { + dot += __shfl_down_sync(0xffffffffU, dot, offset); + if constexpr (MetricValue == Metric::kCosine) { + vector_norm += __shfl_down_sync(0xffffffffU, vector_norm, offset); + } + } + } + if (lane == 0) { + float score = dot; + if constexpr (MetricValue == Metric::kL2) { + score = squared_l2; + } else if constexpr (MetricValue == Metric::kCosine) { + score = query_norm == 0.0F || vector_norm == 0.0F + ? 0.0F + : dot * rsqrtf(query_norm * vector_norm); + } + if (isBetter(score, vector_id, warp_scores[warp][0], warp_ids[warp][0], + MetricValue)) { + warp_scores[warp][0] = score; + warp_ids[warp][0] = vector_id; + std::uint32_t parent = 0; + while (true) { + const std::uint32_t left = parent * 2 + 1; + if (left >= top_k) { + break; + } + const std::uint32_t right = left + 1; + std::uint32_t worse_child = left; + if (right < top_k && + isBetter(warp_scores[warp][left], warp_ids[warp][left], + warp_scores[warp][right], warp_ids[warp][right], + MetricValue)) { + worse_child = right; + } + if (!isBetter(warp_scores[warp][parent], warp_ids[warp][parent], + warp_scores[warp][worse_child], + warp_ids[warp][worse_child], MetricValue)) { + break; + } + const float swap_score = warp_scores[warp][parent]; + const std::uint64_t swap_id = warp_ids[warp][parent]; + warp_scores[warp][parent] = warp_scores[warp][worse_child]; + warp_ids[warp][parent] = warp_ids[warp][worse_child]; + warp_scores[warp][worse_child] = swap_score; + warp_ids[warp][worse_child] = swap_id; + parent = worse_child; + } + } + } + } + __syncthreads(); + + // Heap sort each warp into best-to-worst order. + if (lane == 0) { + for (std::uint32_t heap_size = top_k; heap_size > 1; --heap_size) { + const std::uint32_t last = heap_size - 1; + const float swap_score = warp_scores[warp][0]; + const std::uint64_t swap_id = warp_ids[warp][0]; + warp_scores[warp][0] = warp_scores[warp][last]; + warp_ids[warp][0] = warp_ids[warp][last]; + warp_scores[warp][last] = swap_score; + warp_ids[warp][last] = swap_id; + std::uint32_t parent = 0; + while (true) { + const std::uint32_t left = parent * 2 + 1; + if (left >= last) { + break; + } + const std::uint32_t right = left + 1; + std::uint32_t worse_child = left; + if (right < last && + isBetter(warp_scores[warp][left], warp_ids[warp][left], + warp_scores[warp][right], warp_ids[warp][right], + MetricValue)) { + worse_child = right; + } + if (!isBetter(warp_scores[warp][parent], warp_ids[warp][parent], + warp_scores[warp][worse_child], + warp_ids[warp][worse_child], MetricValue)) { + break; + } + const float parent_score = warp_scores[warp][parent]; + const std::uint64_t parent_id = warp_ids[warp][parent]; + warp_scores[warp][parent] = warp_scores[warp][worse_child]; + warp_ids[warp][parent] = warp_ids[warp][worse_child]; + warp_scores[warp][worse_child] = parent_score; + warp_ids[warp][worse_child] = parent_id; + parent = worse_child; + } + } + warp_ranks[warp] = 0; + } + __syncthreads(); + + for (std::uint32_t output_rank = 0; output_rank < top_k; ++output_rank) { + if (lane == 0) { + const std::uint32_t rank = warp_ranks[warp]; + merge_scores[warp] = warp_scores[warp][rank]; + merge_ids[warp] = warp_ids[warp][rank]; + } + __syncthreads(); + if (thread_id == 0) { + std::uint32_t best_warp = 0; + for (std::uint32_t candidate_warp = 1; candidate_warp < kWarps; + ++candidate_warp) { + if (isBetter(merge_scores[candidate_warp], merge_ids[candidate_warp], + merge_scores[best_warp], merge_ids[best_warp], + MetricValue)) { + best_warp = candidate_warp; + } + } + const std::uint64_t output_offset = + (static_cast(query_id) * gridDim.x + blockIdx.x) * + top_k + + output_rank; + local_scores[output_offset] = merge_scores[best_warp]; + local_ids[output_offset] = merge_ids[best_warp]; + ++warp_ranks[best_warp]; + } + __syncthreads(); + } +} + +// Experimental cross-query tile. A block owns QueryTile queries for the same +// vector chunk, so every database element loaded by a lane is reused for all +// queries in the tile. QueryTile=1 continues to use the established kernel +// above; keeping the paths separate makes the A/B reversible. +template +__global__ void exactFusedQueryTiledTopKKernel( + const InputType* database, const InputType* queries, float* local_scores, + std::uint64_t* local_ids, std::uint64_t num_vectors, std::uint32_t dim, + std::uint32_t batch_count, std::uint32_t top_k, bool cache_queries) { + static_assert(QueryTile == 2 || QueryTile == 4 || QueryTile == 8, + "query tile must be 2, 4, or 8"); + constexpr std::uint32_t kWarpSize = 32; + constexpr std::uint32_t kWarps = 8; + constexpr float kFloatMaximum = 3.402823466e+38F; + __shared__ float warp_scores[QueryTile][kWarps][MaximumK]; + __shared__ std::uint64_t warp_ids[QueryTile][kWarps][MaximumK]; + __shared__ float merge_scores[QueryTile][kWarps]; + __shared__ std::uint64_t merge_ids[QueryTile][kWarps]; + __shared__ std::uint32_t warp_ranks[QueryTile][kWarps]; + extern __shared__ __align__(16) unsigned char dynamic_shared[]; + auto* shared_queries = reinterpret_cast(dynamic_shared); + + const std::uint32_t thread_id = threadIdx.x; + const std::uint32_t lane = thread_id % kWarpSize; + const std::uint32_t warp = thread_id / kWarpSize; + const std::uint32_t query_base = blockIdx.y * QueryTile; + const std::uint32_t remaining_queries = + query_base < batch_count ? batch_count - query_base : 0; + const std::uint32_t query_count = + remaining_queries < QueryTile ? remaining_queries : QueryTile; + const float worst_score = + MetricValue == Metric::kL2 ? kFloatMaximum : -kFloatMaximum; + + if (lane == 0) { + for (std::uint32_t query = 0; query < query_count; ++query) { + for (std::uint32_t rank = 0; rank < top_k; ++rank) { + warp_scores[query][warp][rank] = worst_score; + warp_ids[query][warp][rank] = UINT64_MAX; + } + } + } + + const InputType* query_values[QueryTile]; +#pragma unroll + for (std::uint32_t query = 0; query < QueryTile; ++query) { + query_values[query] = + query < query_count + ? queries + static_cast(query_base + query) * dim + : queries; + } + if (cache_queries) { + const std::uint32_t tile_values = query_count * dim; + for (std::uint32_t index = thread_id; index < tile_values; + index += blockDim.x) { + const std::uint32_t query = index / dim; + const std::uint32_t d = index - query * dim; + shared_queries[index] = query_values[query][d]; + } + } + __syncthreads(); + if (cache_queries) { +#pragma unroll + for (std::uint32_t query = 0; query < QueryTile; ++query) { + query_values[query] = shared_queries + query * dim; + } + } + + float query_norm[QueryTile] = {}; + if constexpr (MetricValue == Metric::kCosine) { +#pragma unroll + for (std::uint32_t query = 0; query < QueryTile; ++query) { + if (query < query_count) { + for (std::uint32_t d = lane; d < dim; d += kWarpSize) { + const float value = loadInput(query_values[query], d); + query_norm[query] = fmaf(value, value, query_norm[query]); + } + for (std::uint32_t offset = kWarpSize / 2; offset > 0; offset >>= 1) { + query_norm[query] += + __shfl_down_sync(0xffffffffU, query_norm[query], offset); + } + } + } + } + + const std::uint64_t first_vector = + static_cast(blockIdx.x) * kWarps + warp; + const std::uint64_t vector_stride = + static_cast(gridDim.x) * kWarps; + for (std::uint64_t vector_id = first_vector; vector_id < num_vectors; + vector_id += vector_stride) { + const std::uint64_t vector_offset = vector_id * dim; + float dot[QueryTile] = {}; + float squared_l2[QueryTile] = {}; + float vector_norm = 0.0F; + for (std::uint32_t d = lane; d < dim; d += kWarpSize) { + const float vector = loadInput(database, vector_offset + d); + if constexpr (MetricValue == Metric::kCosine) { + vector_norm = fmaf(vector, vector, vector_norm); + } +#pragma unroll + for (std::uint32_t query = 0; query < QueryTile; ++query) { + if (query < query_count) { + const float query_value = loadInput(query_values[query], d); + if constexpr (MetricValue == Metric::kL2) { + const float difference = query_value - vector; + squared_l2[query] = + fmaf(difference, difference, squared_l2[query]); + } else { + dot[query] = fmaf(query_value, vector, dot[query]); + } + } + } + } + for (std::uint32_t offset = kWarpSize / 2; offset > 0; offset >>= 1) { + if constexpr (MetricValue == Metric::kCosine) { + vector_norm += __shfl_down_sync(0xffffffffU, vector_norm, offset); + } +#pragma unroll + for (std::uint32_t query = 0; query < QueryTile; ++query) { + if (query < query_count) { + if constexpr (MetricValue == Metric::kL2) { + squared_l2[query] += __shfl_down_sync( + 0xffffffffU, squared_l2[query], offset); + } else { + dot[query] += + __shfl_down_sync(0xffffffffU, dot[query], offset); + } + } + } + } + if (lane == 0) { +#pragma unroll + for (std::uint32_t query = 0; query < QueryTile; ++query) { + if (query >= query_count) { + continue; + } + float score = dot[query]; + if constexpr (MetricValue == Metric::kL2) { + score = squared_l2[query]; + } else if constexpr (MetricValue == Metric::kCosine) { + score = query_norm[query] == 0.0F || vector_norm == 0.0F + ? 0.0F + : dot[query] * + rsqrtf(query_norm[query] * vector_norm); + } + if (isBetter(score, vector_id, warp_scores[query][warp][0], + warp_ids[query][warp][0], MetricValue)) { + warp_scores[query][warp][0] = score; + warp_ids[query][warp][0] = vector_id; + std::uint32_t parent = 0; + while (true) { + const std::uint32_t left = parent * 2 + 1; + if (left >= top_k) { + break; + } + const std::uint32_t right = left + 1; + std::uint32_t worse_child = left; + if (right < top_k && + isBetter(warp_scores[query][warp][left], + warp_ids[query][warp][left], + warp_scores[query][warp][right], + warp_ids[query][warp][right], MetricValue)) { + worse_child = right; + } + if (!isBetter(warp_scores[query][warp][parent], + warp_ids[query][warp][parent], + warp_scores[query][warp][worse_child], + warp_ids[query][warp][worse_child], MetricValue)) { + break; + } + const float swap_score = warp_scores[query][warp][parent]; + const std::uint64_t swap_id = warp_ids[query][warp][parent]; + warp_scores[query][warp][parent] = + warp_scores[query][warp][worse_child]; + warp_ids[query][warp][parent] = + warp_ids[query][warp][worse_child]; + warp_scores[query][warp][worse_child] = swap_score; + warp_ids[query][warp][worse_child] = swap_id; + parent = worse_child; + } + } + } + } + } + __syncthreads(); + + if (lane == 0) { + for (std::uint32_t query = 0; query < query_count; ++query) { + for (std::uint32_t heap_size = top_k; heap_size > 1; --heap_size) { + const std::uint32_t last = heap_size - 1; + const float swap_score = warp_scores[query][warp][0]; + const std::uint64_t swap_id = warp_ids[query][warp][0]; + warp_scores[query][warp][0] = warp_scores[query][warp][last]; + warp_ids[query][warp][0] = warp_ids[query][warp][last]; + warp_scores[query][warp][last] = swap_score; + warp_ids[query][warp][last] = swap_id; + std::uint32_t parent = 0; + while (true) { + const std::uint32_t left = parent * 2 + 1; + if (left >= last) { + break; + } + const std::uint32_t right = left + 1; + std::uint32_t worse_child = left; + if (right < last && + isBetter(warp_scores[query][warp][left], + warp_ids[query][warp][left], + warp_scores[query][warp][right], + warp_ids[query][warp][right], MetricValue)) { + worse_child = right; + } + if (!isBetter(warp_scores[query][warp][parent], + warp_ids[query][warp][parent], + warp_scores[query][warp][worse_child], + warp_ids[query][warp][worse_child], MetricValue)) { + break; + } + const float parent_score = warp_scores[query][warp][parent]; + const std::uint64_t parent_id = warp_ids[query][warp][parent]; + warp_scores[query][warp][parent] = + warp_scores[query][warp][worse_child]; + warp_ids[query][warp][parent] = warp_ids[query][warp][worse_child]; + warp_scores[query][warp][worse_child] = parent_score; + warp_ids[query][warp][worse_child] = parent_id; + parent = worse_child; + } + } + warp_ranks[query][warp] = 0; + } + } + __syncthreads(); + + for (std::uint32_t query = 0; query < query_count; ++query) { + const std::uint32_t query_id = query_base + query; + for (std::uint32_t output_rank = 0; output_rank < top_k; ++output_rank) { + if (lane == 0) { + const std::uint32_t rank = warp_ranks[query][warp]; + merge_scores[query][warp] = warp_scores[query][warp][rank]; + merge_ids[query][warp] = warp_ids[query][warp][rank]; + } + __syncthreads(); + if (thread_id == 0) { + std::uint32_t best_warp = 0; + for (std::uint32_t candidate_warp = 1; candidate_warp < kWarps; + ++candidate_warp) { + if (isBetter(merge_scores[query][candidate_warp], + merge_ids[query][candidate_warp], + merge_scores[query][best_warp], + merge_ids[query][best_warp], MetricValue)) { + best_warp = candidate_warp; + } + } + const std::uint64_t output_offset = + (static_cast(query_id) * gridDim.x + blockIdx.x) * + top_k + + output_rank; + local_scores[output_offset] = merge_scores[query][best_warp]; + local_ids[output_offset] = merge_ids[query][best_warp]; + ++warp_ranks[query][best_warp]; + } + __syncthreads(); + } + } +} + +// Block-tiled variant: one warp owns one query while all eight query warps +// reuse a shared tile of eight database vectors. This preserves query-level +// parallelism, unlike the warp-query tile above where one warp serially +// accumulates several queries. +template +__global__ void exactFusedBlockTiledTopKKernel( + const InputType* database, const InputType* queries, float* local_scores, + std::uint64_t* local_ids, std::uint64_t num_vectors, std::uint32_t dim, + std::uint32_t batch_count, std::uint32_t top_k) { + constexpr std::uint32_t kWarpSize = 32; + constexpr std::uint32_t kWarps = 8; + constexpr std::uint32_t kQueryTile = 8; + constexpr std::uint32_t kVectorTile = 8; + constexpr float kFloatMaximum = 3.402823466e+38F; + __shared__ float warp_scores[kWarps][MaximumK]; + __shared__ std::uint64_t warp_ids[kWarps][MaximumK]; + __shared__ float shared_vector_norms[kVectorTile]; + extern __shared__ __align__(16) unsigned char dynamic_shared[]; + auto* shared_queries = reinterpret_cast(dynamic_shared); + InputType* shared_database = shared_queries + kQueryTile * dim; + + const std::uint32_t thread_id = threadIdx.x; + const std::uint32_t lane = thread_id % kWarpSize; + const std::uint32_t warp = thread_id / kWarpSize; + const std::uint32_t query_base = blockIdx.y * kQueryTile; + const std::uint32_t remaining_queries = + query_base < batch_count ? batch_count - query_base : 0; + const std::uint32_t query_count = + remaining_queries < kQueryTile ? remaining_queries : kQueryTile; + const bool valid_query = warp < query_count; + const float worst_score = + MetricValue == Metric::kL2 ? kFloatMaximum : -kFloatMaximum; + + if (valid_query && lane == 0) { + for (std::uint32_t rank = 0; rank < top_k; ++rank) { + warp_scores[warp][rank] = worst_score; + warp_ids[warp][rank] = UINT64_MAX; + } + } + + const std::uint32_t query_values = query_count * dim; + for (std::uint32_t index = thread_id; index < query_values; + index += blockDim.x) { + const std::uint32_t query = index / dim; + const std::uint32_t d = index - query * dim; + shared_queries[index] = + queries[static_cast(query_base + query) * dim + d]; + } + __syncthreads(); + + float query_norm = 0.0F; + if constexpr (MetricValue == Metric::kCosine) { + if (valid_query) { + for (std::uint32_t d = lane; d < dim; d += kWarpSize) { + const float query = + loadInput(shared_queries + static_cast(warp) * dim, + d); + query_norm = fmaf(query, query, query_norm); + } + for (std::uint32_t offset = kWarpSize / 2; offset > 0; offset >>= 1) { + query_norm += __shfl_down_sync(0xffffffffU, query_norm, offset); + } + } + } + + const std::uint64_t first_tile = + static_cast(blockIdx.x) * kVectorTile; + const std::uint64_t tile_stride = + static_cast(gridDim.x) * kVectorTile; + for (std::uint64_t tile_base = first_tile; tile_base < num_vectors; + tile_base += tile_stride) { + const std::uint32_t tile_count = static_cast( + num_vectors - tile_base < kVectorTile ? num_vectors - tile_base + : kVectorTile); + const std::uint32_t database_values = tile_count * dim; + for (std::uint32_t index = thread_id; index < database_values; + index += blockDim.x) { + const std::uint32_t vector = index / dim; + const std::uint32_t d = index - vector * dim; + shared_database[index] = + database[(tile_base + vector) * static_cast(dim) + d]; + } + __syncthreads(); + + if constexpr (MetricValue == Metric::kCosine) { + if (warp < tile_count) { + float vector_norm = 0.0F; + for (std::uint32_t d = lane; d < dim; d += kWarpSize) { + const float vector = loadInput( + shared_database + static_cast(warp) * dim, d); + vector_norm = fmaf(vector, vector, vector_norm); + } + for (std::uint32_t offset = kWarpSize / 2; offset > 0; offset >>= 1) { + vector_norm += + __shfl_down_sync(0xffffffffU, vector_norm, offset); + } + if (lane == 0) { + shared_vector_norms[warp] = vector_norm; + } + } + __syncthreads(); + } + + float dot[kVectorTile] = {}; + float squared_l2[kVectorTile] = {}; + if (valid_query) { + const InputType* query_values_ptr = + shared_queries + static_cast(warp) * dim; + for (std::uint32_t d = lane; d < dim; d += kWarpSize) { + const float query = loadInput(query_values_ptr, d); +#pragma unroll + for (std::uint32_t vector = 0; vector < kVectorTile; ++vector) { + if (vector < tile_count) { + const float value = loadInput( + shared_database + static_cast(vector) * dim, d); + if constexpr (MetricValue == Metric::kL2) { + const float difference = query - value; + squared_l2[vector] = + fmaf(difference, difference, squared_l2[vector]); + } else { + dot[vector] = fmaf(query, value, dot[vector]); + } + } + } + } + for (std::uint32_t offset = kWarpSize / 2; offset > 0; offset >>= 1) { +#pragma unroll + for (std::uint32_t vector = 0; vector < kVectorTile; ++vector) { + if (vector < tile_count) { + if constexpr (MetricValue == Metric::kL2) { + squared_l2[vector] += __shfl_down_sync( + 0xffffffffU, squared_l2[vector], offset); + } else { + dot[vector] += + __shfl_down_sync(0xffffffffU, dot[vector], offset); + } + } + } + } + if (lane == 0) { + for (std::uint32_t vector = 0; vector < tile_count; ++vector) { + const std::uint64_t vector_id = tile_base + vector; + float score = dot[vector]; + if constexpr (MetricValue == Metric::kL2) { + score = squared_l2[vector]; + } else if constexpr (MetricValue == Metric::kCosine) { + const float vector_norm = shared_vector_norms[vector]; + score = query_norm == 0.0F || vector_norm == 0.0F + ? 0.0F + : dot[vector] * rsqrtf(query_norm * vector_norm); + } + if (isBetter(score, vector_id, warp_scores[warp][0], + warp_ids[warp][0], MetricValue)) { + warp_scores[warp][0] = score; + warp_ids[warp][0] = vector_id; + std::uint32_t parent = 0; + while (true) { + const std::uint32_t left = parent * 2 + 1; + if (left >= top_k) { + break; + } + const std::uint32_t right = left + 1; + std::uint32_t worse_child = left; + if (right < top_k && + isBetter(warp_scores[warp][left], warp_ids[warp][left], + warp_scores[warp][right], warp_ids[warp][right], + MetricValue)) { + worse_child = right; + } + if (!isBetter(warp_scores[warp][parent], warp_ids[warp][parent], + warp_scores[warp][worse_child], + warp_ids[warp][worse_child], MetricValue)) { + break; + } + const float swap_score = warp_scores[warp][parent]; + const std::uint64_t swap_id = warp_ids[warp][parent]; + warp_scores[warp][parent] = warp_scores[warp][worse_child]; + warp_ids[warp][parent] = warp_ids[warp][worse_child]; + warp_scores[warp][worse_child] = swap_score; + warp_ids[warp][worse_child] = swap_id; + parent = worse_child; + } + } + } + } + } + __syncthreads(); + } + + if (valid_query && lane == 0) { + for (std::uint32_t heap_size = top_k; heap_size > 1; --heap_size) { + const std::uint32_t last = heap_size - 1; + const float swap_score = warp_scores[warp][0]; + const std::uint64_t swap_id = warp_ids[warp][0]; + warp_scores[warp][0] = warp_scores[warp][last]; + warp_ids[warp][0] = warp_ids[warp][last]; + warp_scores[warp][last] = swap_score; + warp_ids[warp][last] = swap_id; + std::uint32_t parent = 0; + while (true) { + const std::uint32_t left = parent * 2 + 1; + if (left >= last) { + break; + } + const std::uint32_t right = left + 1; + std::uint32_t worse_child = left; + if (right < last && + isBetter(warp_scores[warp][left], warp_ids[warp][left], + warp_scores[warp][right], warp_ids[warp][right], + MetricValue)) { + worse_child = right; + } + if (!isBetter(warp_scores[warp][parent], warp_ids[warp][parent], + warp_scores[warp][worse_child], + warp_ids[warp][worse_child], MetricValue)) { + break; + } + const float parent_score = warp_scores[warp][parent]; + const std::uint64_t parent_id = warp_ids[warp][parent]; + warp_scores[warp][parent] = warp_scores[warp][worse_child]; + warp_ids[warp][parent] = warp_ids[warp][worse_child]; + warp_scores[warp][worse_child] = parent_score; + warp_ids[warp][worse_child] = parent_id; + parent = worse_child; + } + } + const std::uint32_t query_id = query_base + warp; + const std::uint64_t output_offset = + (static_cast(query_id) * gridDim.x + blockIdx.x) * + top_k; + for (std::uint32_t rank = 0; rank < top_k; ++rank) { + local_scores[output_offset + rank] = warp_scores[warp][rank]; + local_ids[output_offset + rank] = warp_ids[warp][rank]; + } + } +} + +namespace { + +void checkCuda(cudaError_t status, const char* operation) { + if (status != cudaSuccess) { + throw std::runtime_error(std::string(operation) + ": " + + cudaGetErrorString(status)); + } +} + +class DeviceBuffer { + public: + explicit DeviceBuffer(std::size_t bytes) { + checkCuda(cudaMalloc(&data_, bytes), "cudaMalloc"); + } + + ~DeviceBuffer() { + if (data_ != nullptr) { + cudaFree(data_); + } + } + + DeviceBuffer(const DeviceBuffer&) = delete; + DeviceBuffer& operator=(const DeviceBuffer&) = delete; + + void* data() { return data_; } + + private: + void* data_ = nullptr; +}; + +class CudaEvent { + public: + CudaEvent() { checkCuda(cudaEventCreate(&event_), "cudaEventCreate"); } + ~CudaEvent() { cudaEventDestroy(event_); } + + CudaEvent(const CudaEvent&) = delete; + CudaEvent& operator=(const CudaEvent&) = delete; + + void record() { checkCuda(cudaEventRecord(event_), "cudaEventRecord"); } + void synchronize() { + checkCuda(cudaEventSynchronize(event_), "cudaEventSynchronize"); + } + cudaEvent_t get() const { return event_; } + + private: + cudaEvent_t event_{}; +}; + +double elapsedMilliseconds(const CudaEvent& start, const CudaEvent& end) { + float milliseconds = 0.0F; + checkCuda(cudaEventElapsedTime(&milliseconds, start.get(), end.get()), + "cudaEventElapsedTime"); + return milliseconds; +} + +template +void launchExactFusedWarpTopK(dim3 grid, const InputType* database, + const InputType* queries, float* local_scores, + std::uint64_t* local_ids, + std::uint64_t num_vectors, std::uint32_t dim, + std::uint32_t top_k, Metric metric) { + constexpr std::uint32_t kThreads = 256; + constexpr std::size_t kMaximumCachedQueryBytes = 8192; + const std::size_t query_bytes = + static_cast(dim) * sizeof(InputType); + const bool cache_query = query_bytes <= kMaximumCachedQueryBytes; + const std::size_t shared_bytes = cache_query ? query_bytes : 0; + if (metric == Metric::kL2) { + exactFusedWarpTopKKernel + <<>>(database, queries, local_scores, + local_ids, num_vectors, dim, top_k, + cache_query); + } else if (metric == Metric::kInnerProduct) { + exactFusedWarpTopKKernel + <<>>(database, queries, local_scores, + local_ids, num_vectors, dim, top_k, + cache_query); + } else { + exactFusedWarpTopKKernel + <<>>(database, queries, local_scores, + local_ids, num_vectors, dim, top_k, + cache_query); + } +} + +template +void launchExactFusedQueryTiledTopK( + dim3 grid, const InputType* database, const InputType* queries, + float* local_scores, std::uint64_t* local_ids, std::uint64_t num_vectors, + std::uint32_t dim, std::uint32_t batch_count, std::uint32_t top_k, + Metric metric) { + constexpr std::uint32_t kThreads = 256; + constexpr std::size_t kMaximumCachedTileBytes = 8192; + const std::size_t tile_bytes = static_cast(QueryTile) * dim * + sizeof(InputType); + const bool cache_queries = tile_bytes <= kMaximumCachedTileBytes; + const std::size_t shared_bytes = cache_queries ? tile_bytes : 0; + if (metric == Metric::kL2) { + exactFusedQueryTiledTopKKernel + <<>>( + database, queries, local_scores, local_ids, num_vectors, dim, + batch_count, top_k, cache_queries); + } else if (metric == Metric::kInnerProduct) { + exactFusedQueryTiledTopKKernel + <<>>( + database, queries, local_scores, local_ids, num_vectors, dim, + batch_count, top_k, cache_queries); + } else { + exactFusedQueryTiledTopKKernel + <<>>( + database, queries, local_scores, local_ids, num_vectors, dim, + batch_count, top_k, cache_queries); + } +} + +template +void launchExactFusedBlockTiledTopK( + dim3 grid, const InputType* database, const InputType* queries, + float* local_scores, std::uint64_t* local_ids, std::uint64_t num_vectors, + std::uint32_t dim, std::uint32_t batch_count, std::uint32_t top_k, + Metric metric) { + constexpr std::uint32_t kThreads = 256; + constexpr std::uint32_t kQueryTile = 8; + constexpr std::uint32_t kVectorTile = 8; + const std::size_t shared_bytes = + static_cast(kQueryTile + kVectorTile) * dim * + sizeof(InputType); + if (metric == Metric::kL2) { + exactFusedBlockTiledTopKKernel + <<>>( + database, queries, local_scores, local_ids, num_vectors, dim, + batch_count, top_k); + } else if (metric == Metric::kInnerProduct) { + exactFusedBlockTiledTopKKernel + <<>>( + database, queries, local_scores, local_ids, num_vectors, dim, + batch_count, top_k); + } else { + exactFusedBlockTiledTopKKernel + <<>>( + database, queries, local_scores, local_ids, num_vectors, dim, + batch_count, top_k); + } +} + +template +void launchExactFusedWarpTopKWithQueryTile( + dim3 grid, const InputType* database, const InputType* queries, + float* local_scores, std::uint64_t* local_ids, std::uint64_t num_vectors, + std::uint32_t dim, std::uint32_t batch_count, std::uint32_t top_k, + std::uint32_t query_tile, bool block_tiled, Metric metric) { + if (block_tiled) { + if constexpr (MaximumK <= 10) { + launchExactFusedBlockTiledTopK( + grid, database, queries, local_scores, local_ids, num_vectors, dim, + batch_count, top_k, metric); + } else { + throw std::runtime_error( + "block_tile cannot be instantiated for MaximumK > 10"); + } + return; + } + if (query_tile == 1) { + launchExactFusedWarpTopK( + grid, database, queries, local_scores, local_ids, num_vectors, dim, + top_k, metric); + } else if (query_tile == 2) { + launchExactFusedQueryTiledTopK( + grid, database, queries, local_scores, local_ids, num_vectors, dim, + batch_count, top_k, metric); + } else if (query_tile == 4) { + launchExactFusedQueryTiledTopK( + grid, database, queries, local_scores, local_ids, num_vectors, dim, + batch_count, top_k, metric); + } else { + if constexpr (MaximumK <= 10) { + launchExactFusedQueryTiledTopK( + grid, database, queries, local_scores, local_ids, num_vectors, dim, + batch_count, top_k, metric); + } else { + throw std::runtime_error( + "exact_query_tile=8 cannot be instantiated for MaximumK > 10"); + } + } +} + +template +void launchExactFusedDistanceOnly(dim3 grid, const InputType* database, + const InputType* queries, + float* warp_checksums, + std::uint64_t num_vectors, + std::uint32_t dim, Metric metric) { + constexpr std::uint32_t kThreads = 256; + constexpr std::size_t kMaximumCachedQueryBytes = 8192; + const std::size_t query_bytes = + static_cast(dim) * sizeof(InputType); + const bool cache_query = query_bytes <= kMaximumCachedQueryBytes; + const std::size_t shared_bytes = cache_query ? query_bytes : 0; + if (metric == Metric::kL2) { + exactFusedDistanceOnlyKernel + <<>>(database, queries, warp_checksums, + num_vectors, dim, cache_query); + } else if (metric == Metric::kInnerProduct) { + exactFusedDistanceOnlyKernel + <<>>(database, queries, warp_checksums, + num_vectors, dim, cache_query); + } else { + exactFusedDistanceOnlyKernel + <<>>(database, queries, warp_checksums, + num_vectors, dim, cache_query); + } +} + +} // namespace + +struct GpuExactSearchEngine::Impl { + explicit Impl(const VectorDatabase& database, const QuerySet& queries, + const SearchParams& search_params) + : num_vectors(database.num_vectors), + dim(database.dim), + dtype(database.dtype), + metric(database.metric), + params(search_params), + fused_topk(search_params.topk_mode == "fused"), + use_fp16(database.dtype == DataType::kFloat16), + input_element_bytes(use_fp16 ? sizeof(std::uint16_t) : sizeof(float)), + batch_capacity(search_params.batch_size) { + validateInputs(database, queries, params); + constexpr std::uint32_t kThreads = 256; + const std::size_t database_value_count = + static_cast(num_vectors) * dim; + const std::size_t database_bytes = + database_value_count * input_element_bytes; + const std::size_t query_bytes = + static_cast(batch_capacity) * dim * input_element_bytes; + const std::size_t score_bytes = + fused_topk ? 0 + : static_cast(batch_capacity) * num_vectors * + sizeof(float); + const std::size_t result_count = + static_cast(batch_capacity) * params.top_k; + const std::uint32_t required_fused_blocks = + static_cast((num_vectors + 8 - 1) / 8); + // Keep enough query/chunk blocks in flight to occupy the GPU, without + // producing 256*K candidates per query for large batches. + // Query tiling reduces grid.y. Scale grid.x by the same factor so the + // experiment keeps roughly the established 1024 resident/work blocks and + // does not confuse data reuse with a simple loss of parallelism. + const std::uint32_t target_blocks = 1024U * params.exact_query_tile; + const std::uint32_t occupancy_blocks_per_query = std::max( + 1, (target_blocks + batch_capacity - 1) / batch_capacity); + const std::uint32_t selected_fused_blocks = + params.exact_fused_blocks == 0 ? occupancy_blocks_per_query + : params.exact_fused_blocks; + fused_blocks_x = + std::min(required_fused_blocks, selected_fused_blocks); + const std::uint32_t local_blocks = fused_topk ? fused_blocks_x : kThreads; + const std::size_t local_candidate_count = + uses_local_candidates ? result_count * local_blocks : 0; + device_bytes = + database_bytes + query_bytes + score_bytes + + result_count * (sizeof(float) + sizeof(std::uint64_t)) + + local_candidate_count * (sizeof(float) + sizeof(std::uint64_t)); + + device_database = std::make_unique(database_bytes); + device_queries = std::make_unique(query_bytes); + if (!fused_topk) { + device_scores = std::make_unique(score_bytes); + } + device_top_scores = + std::make_unique(result_count * sizeof(float)); + device_top_ids = + std::make_unique(result_count * sizeof(std::uint64_t)); + if (uses_local_candidates) { + device_local_scores = + std::make_unique(local_candidate_count * sizeof(float)); + device_local_ids = std::make_unique(local_candidate_count * + sizeof(std::uint64_t)); + } + + CudaEvent database_copy_start; + CudaEvent database_copy_end; + database_copy_start.record(); + const void* database_data = + use_fp16 ? static_cast(database.half_values.data()) + : static_cast(database.values.data()); + checkCuda(cudaMemcpy(device_database->data(), database_data, database_bytes, + cudaMemcpyHostToDevice), + "复制向量库到 GPU"); + database_copy_end.record(); + database_copy_end.synchronize(); + database_h2d_ms = + elapsedMilliseconds(database_copy_start, database_copy_end); + } + + std::uint64_t num_vectors; + std::uint32_t dim; + DataType dtype; + Metric metric; + SearchParams params; + bool fused_topk; + bool uses_local_candidates = + params.topk_mode == "two_stage" || params.topk_mode == "fused"; + bool use_fp16; + std::size_t input_element_bytes; + std::uint32_t batch_capacity; + std::uint32_t fused_blocks_x = 0; + double database_h2d_ms = 0.0; + std::size_t device_bytes = 0; + std::unique_ptr device_database; + std::unique_ptr device_queries; + std::unique_ptr device_scores; + std::unique_ptr device_top_scores; + std::unique_ptr device_top_ids; + std::unique_ptr device_local_scores; + std::unique_ptr device_local_ids; +}; + +GpuExactSearchEngine::GpuExactSearchEngine(const VectorDatabase& database, + const QuerySet& initial_queries, + const SearchParams& params) + : impl_(std::make_unique(database, initial_queries, params)) {} + +GpuExactSearchEngine::~GpuExactSearchEngine() = default; +GpuExactSearchEngine::GpuExactSearchEngine(GpuExactSearchEngine&&) noexcept = + default; +GpuExactSearchEngine& GpuExactSearchEngine::operator=( + GpuExactSearchEngine&&) noexcept = default; + +double GpuExactSearchEngine::databaseH2DMilliseconds() const { + return impl_->database_h2d_ms; +} + +std::size_t GpuExactSearchEngine::deviceBytes() const { + return impl_->device_bytes; +} + +double GpuExactSearchEngine::profileDistanceOnlyMilliseconds( + const QuerySet& queries, std::uint32_t warmup, std::uint32_t repeat) { + if (!impl_->fused_topk) { + throw std::runtime_error("distance-only 差分剖析只支持 fused Exact 路径"); + } + if (repeat == 0) { + throw std::runtime_error("distance-only 剖析 repeat 必须大于 0"); + } + if (impl_->params.exact_query_tile != 1) { + throw std::runtime_error( + "distance-only 差分剖析当前只支持 exact_query_tile=1"); + } + if (impl_->params.exact_fused_layout != "warp_query") { + throw std::runtime_error( + "distance-only 差分剖析当前只支持 warp_query layout"); + } + if (queries.num_queries == 0 || queries.dim != impl_->dim || + queries.dtype != impl_->dtype) { + throw std::runtime_error("剖析查询集与 GPU 常驻引擎不匹配"); + } + const std::uint64_t expected_query_values = queries.num_queries * queries.dim; + if ((impl_->use_fp16 && + queries.half_values.size() != expected_query_values) || + (!impl_->use_fp16 && queries.values.size() != expected_query_values)) { + throw std::runtime_error("剖析查询数据长度与元数据不一致"); + } + + constexpr std::uint32_t kWarps = 8; + const std::size_t checksum_count = + static_cast(impl_->batch_capacity) * impl_->fused_blocks_x * + kWarps; + DeviceBuffer device_checksums(checksum_count * sizeof(float)); + CudaEvent kernel_start; + CudaEvent kernel_end; + + const auto run_once = [&](bool measure) { + double milliseconds = 0.0; + for (std::uint64_t batch_start = 0; batch_start < queries.num_queries; + batch_start += impl_->batch_capacity) { + const std::uint32_t batch_count = + static_cast(std::min( + impl_->batch_capacity, queries.num_queries - batch_start)); + const std::size_t query_value_offset = + static_cast(batch_start) * queries.dim; + const std::size_t current_query_bytes = + static_cast(batch_count) * queries.dim * + impl_->input_element_bytes; + const void* query_data = + impl_->use_fp16 + ? static_cast(queries.half_values.data() + + query_value_offset) + : static_cast(queries.values.data() + + query_value_offset); + checkCuda(cudaMemcpy(impl_->device_queries->data(), query_data, + current_query_bytes, cudaMemcpyHostToDevice), + "复制剖析查询到 GPU"); + + const dim3 grid(impl_->fused_blocks_x, batch_count); + kernel_start.record(); + if (impl_->use_fp16) { + launchExactFusedDistanceOnly( + grid, static_cast(impl_->device_database->data()), + static_cast(impl_->device_queries->data()), + static_cast(device_checksums.data()), impl_->num_vectors, + impl_->dim, impl_->metric); + } else { + launchExactFusedDistanceOnly( + grid, static_cast(impl_->device_database->data()), + static_cast(impl_->device_queries->data()), + static_cast(device_checksums.data()), impl_->num_vectors, + impl_->dim, impl_->metric); + } + checkCuda(cudaGetLastError(), "启动 exactFusedDistanceOnlyKernel"); + kernel_end.record(); + kernel_end.synchronize(); + if (measure) { + milliseconds += elapsedMilliseconds(kernel_start, kernel_end); + } + } + return milliseconds; + }; + + for (std::uint32_t run = 0; run < warmup; ++run) { + (void)run_once(false); + } + double total_ms = 0.0; + for (std::uint32_t run = 0; run < repeat; ++run) { + total_ms += run_once(true); + } + return total_ms / repeat; +} + +SearchResults GpuExactSearchEngine::search(const QuerySet& queries, + SearchStats* stats) { + if (queries.num_queries == 0 || queries.dim != impl_->dim || + queries.dtype != impl_->dtype) { + throw std::runtime_error("查询集与 GPU 常驻引擎的维度或 dtype 不匹配"); + } + const std::uint64_t expected_query_values = queries.num_queries * queries.dim; + if ((impl_->use_fp16 && + queries.half_values.size() != expected_query_values) || + (!impl_->use_fp16 && queries.values.size() != expected_query_values)) { + throw std::runtime_error("查询数据长度与元数据不一致"); + } + + SearchStats measured_stats; + const SearchParams& params = impl_->params; + const bool use_fp16 = impl_->use_fp16; + const std::size_t input_element_bytes = impl_->input_element_bytes; + const std::uint32_t batch_capacity = impl_->batch_capacity; + const std::uint64_t num_vectors = impl_->num_vectors; + const std::uint32_t dim = impl_->dim; + const Metric metric = impl_->metric; + DeviceBuffer& device_database = *impl_->device_database; + DeviceBuffer& device_queries = *impl_->device_queries; + DeviceBuffer* device_scores = impl_->device_scores.get(); + DeviceBuffer& device_top_scores = *impl_->device_top_scores; + DeviceBuffer& device_top_ids = *impl_->device_top_ids; + DeviceBuffer* device_local_scores = impl_->device_local_scores.get(); + DeviceBuffer* device_local_ids = impl_->device_local_ids.get(); + constexpr std::uint32_t kThreads = 256; + + SearchResults results(queries.num_queries); + std::vector host_top_scores(static_cast(batch_capacity) * + params.top_k); + std::vector host_top_ids( + static_cast(batch_capacity) * params.top_k); + const std::uint32_t simple_blocks_x = + static_cast((num_vectors + kThreads - 1) / kThreads); + constexpr std::uint32_t kWarpsPerBlock = kThreads / 32; + const std::uint32_t required_warp_blocks = static_cast( + (num_vectors + kWarpsPerBlock - 1) / kWarpsPerBlock); + const std::uint32_t warp_blocks_x = + std::min(required_warp_blocks, 4096); + const std::uint32_t fused_blocks_x = impl_->fused_blocks_x; + const std::uint32_t exact_query_tile = params.exact_query_tile; + const bool block_tiled = params.exact_fused_layout == "block_tile"; + + CudaEvent query_copy_start; + CudaEvent query_copy_end; + CudaEvent distance_end; + CudaEvent topk_end; + CudaEvent result_copy_end; + + for (std::uint64_t batch_start = 0; batch_start < queries.num_queries; + batch_start += batch_capacity) { + const std::uint32_t batch_count = + static_cast(std::min( + batch_capacity, queries.num_queries - batch_start)); + const std::size_t current_query_bytes = + static_cast(batch_count) * queries.dim * + input_element_bytes; + const std::size_t query_value_offset = + static_cast(batch_start) * queries.dim; + const void* query_data = + use_fp16 ? static_cast(queries.half_values.data() + + query_value_offset) + : static_cast(queries.values.data() + + query_value_offset); + query_copy_start.record(); + checkCuda(cudaMemcpy(device_queries.data(), query_data, current_query_bytes, + cudaMemcpyHostToDevice), + "复制查询到 GPU"); + query_copy_end.record(); + + if (params.topk_mode == "fused") { + const dim3 grid(fused_blocks_x, + (batch_count + exact_query_tile - 1) / + exact_query_tile); + if (use_fp16) { + const auto* database_values = + static_cast(device_database.data()); + const auto* query_values = + static_cast(device_queries.data()); + if (params.top_k <= 10) { + launchExactFusedWarpTopKWithQueryTile<__half, 10>( + grid, database_values, query_values, + static_cast(device_local_scores->data()), + static_cast(device_local_ids->data()), + num_vectors, dim, batch_count, params.top_k, exact_query_tile, + block_tiled, metric); + } else if (params.top_k <= 50) { + launchExactFusedWarpTopKWithQueryTile<__half, 50>( + grid, database_values, query_values, + static_cast(device_local_scores->data()), + static_cast(device_local_ids->data()), + num_vectors, dim, batch_count, params.top_k, exact_query_tile, + block_tiled, metric); + } else { + launchExactFusedWarpTopKWithQueryTile<__half, 100>( + grid, database_values, query_values, + static_cast(device_local_scores->data()), + static_cast(device_local_ids->data()), + num_vectors, dim, batch_count, params.top_k, exact_query_tile, + block_tiled, metric); + } + } else { + const auto* database_values = + static_cast(device_database.data()); + const auto* query_values = + static_cast(device_queries.data()); + if (params.top_k <= 10) { + launchExactFusedWarpTopKWithQueryTile( + grid, database_values, query_values, + static_cast(device_local_scores->data()), + static_cast(device_local_ids->data()), + num_vectors, dim, batch_count, params.top_k, exact_query_tile, + block_tiled, metric); + } else if (params.top_k <= 50) { + launchExactFusedWarpTopKWithQueryTile( + grid, database_values, query_values, + static_cast(device_local_scores->data()), + static_cast(device_local_ids->data()), + num_vectors, dim, batch_count, params.top_k, exact_query_tile, + block_tiled, metric); + } else { + launchExactFusedWarpTopKWithQueryTile( + grid, database_values, query_values, + static_cast(device_local_scores->data()), + static_cast(device_local_ids->data()), + num_vectors, dim, batch_count, params.top_k, exact_query_tile, + block_tiled, metric); + } + } + checkCuda(cudaGetLastError(), "启动 exactFusedWarpTopKKernel"); + distance_end.record(); + const std::uint64_t candidates_per_query = + static_cast(fused_blocks_x) * params.top_k; + blockTopKKernel<<>>( + static_cast(device_local_scores->data()), + static_cast(device_local_ids->data()), + static_cast(device_top_scores.data()), + static_cast(device_top_ids.data()), + candidates_per_query, params.top_k, metric); + checkCuda(cudaGetLastError(), "启动 fused exact 最终归并"); + } else { + if (params.distance_mode == "simple") { + const dim3 grid(simple_blocks_x, batch_count); + if (use_fp16) { + exactDistanceKernel<<>>( + static_cast(device_database.data()), + static_cast(device_queries.data()), + static_cast(device_scores->data()), num_vectors, dim, + batch_count, metric); + } else { + exactDistanceKernel<<>>( + static_cast(device_database.data()), + static_cast(device_queries.data()), + static_cast(device_scores->data()), num_vectors, dim, + batch_count, metric); + } + checkCuda(cudaGetLastError(), "启动 exactDistanceKernel"); + } else { + const dim3 grid(warp_blocks_x, batch_count); + if (use_fp16) { + exactDistanceWarpKernel<<>>( + static_cast(device_database.data()), + static_cast(device_queries.data()), + static_cast(device_scores->data()), num_vectors, dim, + batch_count, metric); + } else { + exactDistanceWarpKernel<<>>( + static_cast(device_database.data()), + static_cast(device_queries.data()), + static_cast(device_scores->data()), num_vectors, dim, + batch_count, metric); + } + checkCuda(cudaGetLastError(), "启动 exactDistanceWarpKernel"); + } + distance_end.record(); + + if (params.topk_mode == "simple") { + simpleTopKKernel<<>>( + static_cast(device_scores->data()), + static_cast(device_top_scores.data()), + static_cast(device_top_ids.data()), num_vectors, + params.top_k, metric); + checkCuda(cudaGetLastError(), "启动 simpleTopKKernel"); + } else if (params.topk_mode == "block") { + blockTopKKernel<<>>( + static_cast(device_scores->data()), nullptr, + static_cast(device_top_scores.data()), + static_cast(device_top_ids.data()), num_vectors, + params.top_k, metric); + checkCuda(cudaGetLastError(), "启动 blockTopKKernel"); + } else { + if (params.top_k == 1) { + localTopKCandidatesFixedKernel<1><<>>( + static_cast(device_scores->data()), + static_cast(device_local_scores->data()), + static_cast(device_local_ids->data()), + num_vectors, metric); + } else if (params.top_k == 10) { + localTopKCandidatesFixedKernel<10><<>>( + static_cast(device_scores->data()), + static_cast(device_local_scores->data()), + static_cast(device_local_ids->data()), + num_vectors, metric); + } else { + localTopKCandidatesKernel<<>>( + static_cast(device_scores->data()), + static_cast(device_local_scores->data()), + static_cast(device_local_ids->data()), + num_vectors, params.top_k, metric); + } + checkCuda(cudaGetLastError(), "启动 localTopKCandidatesKernel"); + + const std::uint64_t candidates_per_query = + static_cast(kThreads) * params.top_k; + blockTopKKernel<<>>( + static_cast(device_local_scores->data()), + static_cast(device_local_ids->data()), + static_cast(device_top_scores.data()), + static_cast(device_top_ids.data()), + candidates_per_query, params.top_k, metric); + checkCuda(cudaGetLastError(), "启动两阶段归并 blockTopKKernel"); + } + } + topk_end.record(); + + const std::size_t result_count = + static_cast(batch_count) * params.top_k; + checkCuda(cudaMemcpy(host_top_scores.data(), device_top_scores.data(), + result_count * sizeof(float), cudaMemcpyDeviceToHost), + "复制 Top-K 分数到 CPU"); + checkCuda(cudaMemcpy(host_top_ids.data(), device_top_ids.data(), + result_count * sizeof(std::uint64_t), + cudaMemcpyDeviceToHost), + "复制 Top-K ID 到 CPU"); + result_copy_end.record(); + result_copy_end.synchronize(); + + measured_stats.query_h2d_ms += + elapsedMilliseconds(query_copy_start, query_copy_end); + measured_stats.distance_kernel_ms += + elapsedMilliseconds(query_copy_end, distance_end); + measured_stats.topk_kernel_ms += + elapsedMilliseconds(distance_end, topk_end); + measured_stats.result_d2h_ms += + elapsedMilliseconds(topk_end, result_copy_end); + measured_stats.batch_latency_ms.push_back( + elapsedMilliseconds(query_copy_start, result_copy_end)); + + for (std::uint32_t query_in_batch = 0; query_in_batch < batch_count; + ++query_in_batch) { + std::vector& query_results = + results[batch_start + query_in_batch]; + query_results.resize(params.top_k); + const std::size_t offset = + static_cast(query_in_batch) * params.top_k; + for (std::uint32_t rank = 0; rank < params.top_k; ++rank) { + query_results[rank] = {host_top_ids[offset + rank], + host_top_scores[offset + rank]}; + } + } + } + if (stats != nullptr) { + *stats = measured_stats; + } + return results; +} + +SearchResults gpuExactSearch(const VectorDatabase& database, + const QuerySet& queries, + const SearchParams& params, SearchStats* stats) { + GpuExactSearchEngine engine(database, queries, params); + SearchResults results = engine.search(queries, stats); + if (stats != nullptr) { + stats->database_h2d_ms = engine.databaseH2DMilliseconds(); + } + return results; +} diff --git a/09_vector_retrieval/PurplePulse/src/ivf_flat.cpp b/09_vector_retrieval/PurplePulse/src/ivf_flat.cpp new file mode 100644 index 00000000..e1868607 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/src/ivf_flat.cpp @@ -0,0 +1,722 @@ +#include "ivf_flat.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "file_io.h" + +namespace { + +constexpr std::array kIvfMagic = {'P', 'P', 'I', 'V', + 'F', '0', '0', '1'}; +constexpr std::uint64_t kHeaderBytes = 8 + 8 + 4 + 4 + 4 + 4; + +template +void writeValue(std::ofstream& output, const T& value) { + output.write(reinterpret_cast(&value), sizeof(T)); +} + +template +T readValue(std::ifstream& input, const char* field_name) { + T value{}; + input.read(reinterpret_cast(&value), sizeof(T)); + if (!input) { + throw std::runtime_error(std::string("读取 IVF 字段失败: ") + field_name); + } + return value; +} + +float halfToFloat(std::uint16_t bits) { + const bool negative = (bits & 0x8000U) != 0; + const std::uint32_t exponent = (bits >> 10U) & 0x1fU; + const std::uint32_t mantissa = bits & 0x03ffU; + float value = 0.0F; + if (exponent == 0) { + value = std::ldexp(static_cast(mantissa), -24); + } else if (exponent == 31) { + value = mantissa == 0 ? std::numeric_limits::infinity() + : std::numeric_limits::quiet_NaN(); + } else { + value = std::ldexp(static_cast(1024U + mantissa), + static_cast(exponent) - 25); + } + return negative ? -value : value; +} + +float databaseValue(const VectorDatabase& database, std::uint64_t index) { + return database.dtype == DataType::kFloat32 + ? database.values[index] + : halfToFloat(database.half_values[index]); +} + +float indexValue(const IvfFlatIndex& index, std::uint64_t index_position) { + return index.dtype == DataType::kFloat32 + ? index.values[index_position] + : halfToFloat(index.half_values[index_position]); +} + +float queryValue(const QuerySet& queries, std::uint64_t index) { + return queries.dtype == DataType::kFloat32 + ? queries.values[index] + : halfToFloat(queries.half_values[index]); +} + +bool isBetter(float left_score, std::uint64_t left_id, float right_score, + std::uint64_t right_id, Metric metric) { + if (left_score == right_score) { + return left_id < right_id; + } + return metric == Metric::kL2 ? left_score < right_score + : left_score > right_score; +} + +float centerScore(const float* vector, const float* center, std::uint32_t dim, + Metric metric) { + float dot = 0.0F; + float vector_norm = 0.0F; + float center_norm = 0.0F; + float squared_l2 = 0.0F; + for (std::uint32_t d = 0; d < dim; ++d) { + if (metric == Metric::kL2) { + const float difference = vector[d] - center[d]; + squared_l2 = std::fma(difference, difference, squared_l2); + } else { + dot = std::fma(vector[d], center[d], dot); + if (metric == Metric::kCosine) { + vector_norm = std::fma(vector[d], vector[d], vector_norm); + center_norm = std::fma(center[d], center[d], center_norm); + } + } + } + if (metric == Metric::kL2) { + return squared_l2; + } + if (metric == Metric::kInnerProduct) { + return dot; + } + return vector_norm == 0.0F || center_norm == 0.0F + ? 0.0F + : dot / std::sqrt(vector_norm * center_norm); +} + +std::uint32_t nearestCenter(const float* vector, + const std::vector& centers, + std::uint32_t nlist, std::uint32_t dim, + Metric metric) { + std::uint32_t best = 0; + float best_score = centerScore(vector, centers.data(), dim, metric); + for (std::uint32_t center_id = 1; center_id < nlist; ++center_id) { + const float score = centerScore( + vector, centers.data() + static_cast(center_id) * dim, + dim, metric); + if (isBetter(score, center_id, best_score, best, metric)) { + best = center_id; + best_score = score; + } + } + return best; +} + +void normalize(float* vector, std::uint32_t dim) { + float squared_norm = 0.0F; + for (std::uint32_t d = 0; d < dim; ++d) { + squared_norm = std::fma(vector[d], vector[d], squared_norm); + } + if (squared_norm == 0.0F) { + return; + } + const float inverse_norm = 1.0F / std::sqrt(squared_norm); + for (std::uint32_t d = 0; d < dim; ++d) { + vector[d] *= inverse_norm; + } +} + +std::uint64_t checkedMultiply(std::uint64_t left, std::uint64_t right, + const char* description) { + if (right != 0 && left > UINT64_MAX / right) { + throw std::runtime_error(std::string(description) + " 大小溢出"); + } + return left * right; +} + +template +void parallelFor(std::uint64_t count, std::uint32_t requested_threads, + Function function) { + const std::uint32_t threads = static_cast( + std::min(count, requested_threads)); + if (threads <= 1) { + function(0, 0, count); + return; + } + std::vector workers; + workers.reserve(threads - 1); + for (std::uint32_t worker = 1; worker < threads; ++worker) { + const std::uint64_t begin = count * worker / threads; + const std::uint64_t end = count * (worker + 1) / threads; + workers.emplace_back(function, worker, begin, end); + } + function(0, 0, count / threads); + for (std::thread& worker : workers) { + worker.join(); + } +} + +float candidateScore(const IvfFlatIndex& index, const QuerySet& queries, + std::uint64_t query_id, std::uint64_t position) { + float dot = 0.0F; + float query_norm = 0.0F; + float vector_norm = 0.0F; + float squared_l2 = 0.0F; + for (std::uint32_t d = 0; d < index.dim; ++d) { + const float query = queryValue(queries, query_id * index.dim + d); + const float vector = indexValue(index, position * index.dim + d); + if (index.metric == Metric::kL2) { + const float difference = query - vector; + squared_l2 = std::fma(difference, difference, squared_l2); + } else { + dot = std::fma(query, vector, dot); + if (index.metric == Metric::kCosine) { + query_norm = std::fma(query, query, query_norm); + vector_norm = std::fma(vector, vector, vector_norm); + } + } + } + if (index.metric == Metric::kL2) { + return squared_l2; + } + if (index.metric == Metric::kInnerProduct) { + return dot; + } + return query_norm == 0.0F || vector_norm == 0.0F + ? 0.0F + : dot / std::sqrt(query_norm * vector_norm); +} + +bool memoryCandidateMatches(const MemoryMetadata& metadata, + std::uint64_t vector_id, + const SearchParams& params) { + return metadata.timestamps[vector_id] >= params.filter_min_timestamp && + (params.filter_session_id == UINT32_MAX || + metadata.session_ids[vector_id] == params.filter_session_id) && + (params.filter_source_type == UINT32_MAX || + metadata.source_types[vector_id] == params.filter_source_type); +} + +float applyMemoryScore(float semantic_score, std::uint64_t vector_id, + Metric metric, const MemoryMetadata& metadata, + const SearchParams& params) { + float recency = 0.0F; + if (params.memory_recency_weight != 0.0F && params.memory_now != 0) { + const std::uint64_t timestamp = metadata.timestamps[vector_id]; + const double age = timestamp >= params.memory_now + ? 0.0 + : static_cast(params.memory_now - timestamp); + recency = static_cast( + std::exp(-age / static_cast(params.memory_time_scale))); + } + const float boost = + params.memory_importance_weight * metadata.importance[vector_id] + + params.memory_recency_weight * recency; + const float weighted_semantic = + params.memory_semantic_weight * semantic_score; + return metric == Metric::kL2 ? weighted_semantic - boost + : weighted_semantic + boost; +} + +void validateMemorySearch(const IvfFlatIndex& index, const SearchParams& params, + const MemoryMetadata* metadata) { + if (params.memory_mode != "disabled" && params.memory_mode != "fused" && + params.memory_mode != "rerank") { + throw std::runtime_error("memory_mode 必须是 disabled、fused 或 rerank"); + } + if (params.memory_mode == "disabled") { + return; + } + if (metadata == nullptr) { + throw std::runtime_error("启用记忆检索时必须提供元数据"); + } + validateMemoryMetadata(*metadata, index.num_vectors); + if (!std::isfinite(params.memory_semantic_weight) || + params.memory_semantic_weight <= 0.0F || + !std::isfinite(params.memory_importance_weight) || + params.memory_importance_weight < 0.0F || + !std::isfinite(params.memory_recency_weight) || + params.memory_recency_weight < 0.0F || + !std::isfinite(params.memory_time_scale) || + params.memory_time_scale <= 0.0F || params.memory_rerank_factor == 0) { + throw std::runtime_error("记忆评分参数不合法"); + } +} + +} // namespace + +IvfFlatIndex buildIvfFlatIndex(const VectorDatabase& database, + std::uint32_t nlist, std::uint32_t iterations, + std::uint64_t max_training_vectors, + std::uint32_t num_threads) { + if (database.num_vectors == 0 || database.dim == 0) { + throw std::runtime_error("IVF 建库输入不能为空"); + } + if (nlist == 0 || nlist > database.num_vectors) { + throw std::runtime_error("nlist 必须位于 [1, num_vectors]"); + } + if (iterations == 0 || max_training_vectors == 0 || num_threads == 0) { + throw std::runtime_error("IVF 训练迭代数、采样数和线程数必须大于 0"); + } + const std::uint64_t value_count = + checkedMultiply(database.num_vectors, database.dim, "向量库"); + if ((database.dtype == DataType::kFloat32 && + database.values.size() != value_count) || + (database.dtype == DataType::kFloat16 && + database.half_values.size() != value_count)) { + throw std::runtime_error("IVF 建库数据长度与元数据不一致"); + } + + const std::uint64_t sample_count = std::min( + database.num_vectors, + std::max(nlist, max_training_vectors)); + std::vector samples(sample_count * database.dim); + for (std::uint64_t sample_id = 0; sample_id < sample_count; ++sample_id) { + const std::uint64_t vector_id = + sample_id * database.num_vectors / sample_count; + for (std::uint32_t d = 0; d < database.dim; ++d) { + samples[sample_id * database.dim + d] = + databaseValue(database, vector_id * database.dim + d); + } + } + + std::vector centers(static_cast(nlist) * database.dim); + for (std::uint32_t center_id = 0; center_id < nlist; ++center_id) { + const std::uint64_t sample_id = + static_cast(center_id) * sample_count / nlist; + std::copy_n( + samples.data() + sample_id * database.dim, database.dim, + centers.data() + static_cast(center_id) * database.dim); + if (database.metric != Metric::kL2) { + normalize( + centers.data() + static_cast(center_id) * database.dim, + database.dim); + } + } + + std::vector sums(centers.size()); + std::vector counts(nlist); + const std::uint32_t training_threads = static_cast( + std::min(num_threads, sample_count)); + std::vector partial_sums( + static_cast(training_threads) * centers.size()); + std::vector partial_counts( + static_cast(training_threads) * nlist); + for (std::uint32_t iteration = 0; iteration < iterations; ++iteration) { + std::fill(sums.begin(), sums.end(), 0.0F); + std::fill(counts.begin(), counts.end(), 0); + std::fill(partial_sums.begin(), partial_sums.end(), 0.0F); + std::fill(partial_counts.begin(), partial_counts.end(), 0); + parallelFor(sample_count, training_threads, + [&](std::uint32_t worker, std::uint64_t begin, + std::uint64_t end) { + float* worker_sums = + partial_sums.data() + + static_cast(worker) * centers.size(); + std::uint64_t* worker_counts = + partial_counts.data() + static_cast(worker) * nlist; + for (std::uint64_t sample_id = begin; sample_id < end; ++sample_id) { + const std::uint32_t center_id = + nearestCenter(samples.data() + sample_id * database.dim, centers, + nlist, database.dim, database.metric); + ++worker_counts[center_id]; + for (std::uint32_t d = 0; d < database.dim; ++d) { + worker_sums[static_cast(center_id) * database.dim + + d] += samples[sample_id * database.dim + d]; + } + } + }); + for (std::uint32_t worker = 0; worker < training_threads; ++worker) { + const float* worker_sums = + partial_sums.data() + + static_cast(worker) * centers.size(); + const std::uint64_t* worker_counts = + partial_counts.data() + static_cast(worker) * nlist; + for (std::uint32_t center_id = 0; center_id < nlist; ++center_id) { + counts[center_id] += worker_counts[center_id]; + } + for (std::size_t value = 0; value < centers.size(); ++value) { + sums[value] += worker_sums[value]; + } + } + for (std::uint32_t center_id = 0; center_id < nlist; ++center_id) { + float* center = + centers.data() + static_cast(center_id) * database.dim; + if (counts[center_id] == 0) { + const std::uint64_t replacement = + (static_cast(center_id) + iteration) % sample_count; + std::copy_n(samples.data() + replacement * database.dim, database.dim, + center); + } else { + const float inverse_count = + 1.0F / static_cast(counts[center_id]); + for (std::uint32_t d = 0; d < database.dim; ++d) { + center[d] = + sums[static_cast(center_id) * database.dim + d] * + inverse_count; + } + } + // 对 Cosine 和 Inner Product 使用 spherical k-means。若不归一化， + // 高范数中心会在点积比较中吸收过多向量，nlist 较大时桶会严重倾斜。 + if (database.metric != Metric::kL2) { + normalize(center, database.dim); + } + } + } + + std::vector database_assignments(database.num_vectors); + counts.assign(nlist, 0); + parallelFor(database.num_vectors, num_threads, + [&](std::uint32_t, std::uint64_t begin, std::uint64_t end) { + std::vector vector(database.dim); + for (std::uint64_t vector_id = begin; vector_id < end; ++vector_id) { + for (std::uint32_t d = 0; d < database.dim; ++d) { + vector[d] = databaseValue(database, vector_id * database.dim + d); + } + database_assignments[vector_id] = nearestCenter( + vector.data(), centers, nlist, database.dim, database.metric); + } + }); + for (const std::uint32_t center_id : database_assignments) { + ++counts[center_id]; + } + + IvfFlatIndex index; + index.num_vectors = database.num_vectors; + index.dim = database.dim; + index.nlist = nlist; + index.dtype = database.dtype; + index.metric = database.metric; + index.centers = std::move(centers); + index.offsets.resize(static_cast(nlist) + 1, 0); + for (std::uint32_t center_id = 0; center_id < nlist; ++center_id) { + index.offsets[center_id + 1] = index.offsets[center_id] + counts[center_id]; + } + std::vector positions = index.offsets; + index.ids.resize(database.num_vectors); + if (database.dtype == DataType::kFloat32) { + index.values.resize(value_count); + } else { + index.half_values.resize(value_count); + } + for (std::uint64_t vector_id = 0; vector_id < database.num_vectors; + ++vector_id) { + const std::uint64_t position = positions[database_assignments[vector_id]]++; + index.ids[position] = vector_id; + const std::uint64_t source = vector_id * database.dim; + const std::uint64_t destination = position * database.dim; + if (database.dtype == DataType::kFloat32) { + std::copy_n(database.values.data() + source, database.dim, + index.values.data() + destination); + } else { + std::copy_n(database.half_values.data() + source, database.dim, + index.half_values.data() + destination); + } + } + validateIvfFlatIndex(index); + return index; +} + +void validateIvfFlatIndex(const IvfFlatIndex& index) { + if (index.num_vectors == 0 || index.dim == 0 || index.nlist == 0 || + index.nlist > index.num_vectors) { + throw std::runtime_error("IVF 索引元数据不合法"); + } + const std::uint64_t center_count = + checkedMultiply(index.nlist, index.dim, "IVF centers"); + const std::uint64_t value_count = + checkedMultiply(index.num_vectors, index.dim, "IVF vectors"); + if (index.centers.size() != center_count || + index.offsets.size() != static_cast(index.nlist) + 1 || + index.ids.size() != index.num_vectors || index.offsets.front() != 0 || + index.offsets.back() != index.num_vectors) { + throw std::runtime_error("IVF 索引数组长度与元数据不一致"); + } + if ((index.dtype == DataType::kFloat32 && + (index.values.size() != value_count || !index.half_values.empty())) || + (index.dtype == DataType::kFloat16 && + (index.half_values.size() != value_count || !index.values.empty()))) { + throw std::runtime_error("IVF 索引向量 dtype 或长度不正确"); + } + if (index.dtype != DataType::kFloat32 && index.dtype != DataType::kFloat16) { + throw std::runtime_error("IVF 索引 dtype 不支持"); + } + if (index.metric != Metric::kL2 && index.metric != Metric::kInnerProduct && + index.metric != Metric::kCosine) { + throw std::runtime_error("IVF 索引 metric 不支持"); + } + for (std::uint32_t center_id = 0; center_id < index.nlist; ++center_id) { + if (index.offsets[center_id] > index.offsets[center_id + 1]) { + throw std::runtime_error("IVF 桶 offsets 必须单调递增"); + } + } + std::vector seen(index.num_vectors, false); + for (const std::uint64_t id : index.ids) { + if (id >= index.num_vectors || seen[id]) { + throw std::runtime_error("IVF 原始向量 ID 越界或重复"); + } + seen[id] = true; + } +} + +void writeIvfFlatIndex(const std::string& path, const IvfFlatIndex& index) { + validateIvfFlatIndex(index); + std::ofstream output(path, std::ios::binary); + if (!output) { + throw std::runtime_error("无法写入 IVF 索引: " + path); + } + output.write(kIvfMagic.data(), kIvfMagic.size()); + writeValue(output, index.num_vectors); + writeValue(output, index.dim); + writeValue(output, static_cast(index.dtype)); + writeValue(output, static_cast(index.metric)); + writeValue(output, index.nlist); + output.write( + reinterpret_cast(index.centers.data()), + static_cast(index.centers.size() * sizeof(float))); + output.write(reinterpret_cast(index.offsets.data()), + static_cast(index.offsets.size() * + sizeof(std::uint64_t))); + output.write( + reinterpret_cast(index.ids.data()), + static_cast(index.ids.size() * sizeof(std::uint64_t))); + const void* values = index.dtype == DataType::kFloat32 + ? static_cast(index.values.data()) + : static_cast(index.half_values.data()); + const std::uint64_t element_bytes = + index.dtype == DataType::kFloat32 ? sizeof(float) : sizeof(std::uint16_t); + output.write(reinterpret_cast(values), + static_cast(index.num_vectors * index.dim * + element_bytes)); + if (!output) { + throw std::runtime_error("写入 IVF 索引失败: " + path); + } +} + +IvfFlatIndex readIvfFlatIndex(const std::string& path) { + std::ifstream input(path, std::ios::binary); + if (!input) { + throw std::runtime_error("无法打开 IVF 索引: " + path); + } + std::array magic{}; + input.read(magic.data(), magic.size()); + if (magic != kIvfMagic) { + throw std::runtime_error("IVF 索引 magic/version 不正确: " + path); + } + IvfFlatIndex index; + index.num_vectors = readValue(input, "num_vectors"); + index.dim = readValue(input, "dim"); + index.dtype = static_cast(readValue(input, "dtype")); + index.metric = static_cast(readValue(input, "metric")); + index.nlist = readValue(input, "nlist"); + if (index.num_vectors == 0 || index.dim == 0 || index.nlist == 0 || + index.nlist > index.num_vectors) { + throw std::runtime_error("IVF 索引元数据不合法: " + path); + } + const std::uint64_t center_count = + checkedMultiply(index.nlist, index.dim, "IVF centers"); + const std::uint64_t value_count = + checkedMultiply(index.num_vectors, index.dim, "IVF vectors"); + const std::uint64_t element_bytes = + index.dtype == DataType::kFloat32 ? sizeof(float) + : index.dtype == DataType::kFloat16 ? sizeof(std::uint16_t) + : 0; + if (element_bytes == 0) { + throw std::runtime_error("IVF 索引 dtype 不支持: " + path); + } + const std::uint64_t expected_size = + kHeaderBytes + center_count * sizeof(float) + + (static_cast(index.nlist) + 1) * sizeof(std::uint64_t) + + index.num_vectors * sizeof(std::uint64_t) + value_count * element_bytes; + if (std::filesystem::file_size(path) != expected_size) { + throw std::runtime_error("IVF 索引文件大小不正确: " + path); + } + index.centers.resize(center_count); + index.offsets.resize(static_cast(index.nlist) + 1); + index.ids.resize(index.num_vectors); + input.read(reinterpret_cast(index.centers.data()), + static_cast(center_count * sizeof(float))); + input.read(reinterpret_cast(index.offsets.data()), + static_cast(index.offsets.size() * + sizeof(std::uint64_t))); + input.read( + reinterpret_cast(index.ids.data()), + static_cast(index.ids.size() * sizeof(std::uint64_t))); + if (index.dtype == DataType::kFloat32) { + index.values.resize(value_count); + input.read(reinterpret_cast(index.values.data()), + static_cast(value_count * sizeof(float))); + } else { + index.half_values.resize(value_count); + input.read( + reinterpret_cast(index.half_values.data()), + static_cast(value_count * sizeof(std::uint16_t))); + } + if (!input) { + throw std::runtime_error("读取 IVF 索引数据失败: " + path); + } + validateIvfFlatIndex(index); + return index; +} + +SearchResults cpuIvfFlatSearch(const IvfFlatIndex& index, + const QuerySet& queries, + const SearchParams& params, + const MemoryMetadata* metadata) { + if (params.nprobe_policy != "fixed") { + throw std::runtime_error("自适应 nprobe 当前只支持 GPU backend"); + } + const std::vector> candidate_positions = + selectIvfCandidatePositions(index, queries, params.nprobe); + if (params.top_k == 0 || params.top_k > index.num_vectors) { + throw std::runtime_error("IVF 的 top_k 不合法"); + } + validateMemorySearch(index, params, metadata); + + SearchResults results(queries.num_queries); + for (std::uint64_t query_id = 0; query_id < queries.num_queries; ++query_id) { + std::vector candidates; + candidates.reserve(candidate_positions[query_id].size()); + for (const std::uint64_t position : candidate_positions[query_id]) { + const std::uint64_t vector_id = index.ids[position]; + if (params.memory_mode == "fused" && + !memoryCandidateMatches(*metadata, vector_id, params)) { + continue; + } + float score = candidateScore(index, queries, query_id, position); + if (params.memory_mode == "fused") { + score = + applyMemoryScore(score, vector_id, index.metric, *metadata, params); + } + candidates.push_back({vector_id, score}); + } + if (params.memory_mode == "disabled" && candidates.size() < params.top_k) { + throw std::runtime_error("IVF 探测桶的候选数少于 top_k，请增大 nprobe"); + } + const auto better = [&](const Neighbor& left, const Neighbor& right) { + return isBetter(left.score, left.id, right.score, right.id, index.metric); + }; + std::size_t selected_k = + std::min(params.top_k, candidates.size()); + if (params.memory_mode == "rerank") { + const std::uint64_t requested = static_cast(params.top_k) * + params.memory_rerank_factor; + const std::size_t rerank_k = static_cast( + std::min({requested, 100, candidates.size()})); + if (rerank_k < candidates.size()) { + std::nth_element(candidates.begin(), candidates.begin() + rerank_k, + candidates.end(), better); + candidates.resize(rerank_k); + } + candidates.erase(std::remove_if(candidates.begin(), candidates.end(), + [&](const Neighbor& candidate) { + return !memoryCandidateMatches( + *metadata, candidate.id, params); + }), + candidates.end()); + for (Neighbor& candidate : candidates) { + candidate.score = applyMemoryScore(candidate.score, candidate.id, + index.metric, *metadata, params); + } + selected_k = std::min(params.top_k, candidates.size()); + } + if (selected_k < candidates.size()) { + std::nth_element(candidates.begin(), candidates.begin() + selected_k, + candidates.end(), better); + candidates.resize(selected_k); + } + std::sort(candidates.begin(), candidates.end(), better); + results[query_id] = std::move(candidates); + } + return results; +} + +std::vector> selectIvfCandidatePositions( + const IvfFlatIndex& index, const QuerySet& queries, std::uint32_t nprobe) { + validateIvfFlatIndex(index); + if (queries.num_queries == 0 || queries.dim != index.dim || + queries.dtype != index.dtype) { + throw std::runtime_error("查询集与 IVF 索引的维度或 dtype 不匹配"); + } + const std::uint64_t query_value_count = + checkedMultiply(queries.num_queries, queries.dim, "IVF queries"); + if ((queries.dtype == DataType::kFloat32 && + queries.values.size() != query_value_count) || + (queries.dtype == DataType::kFloat16 && + queries.half_values.size() != query_value_count)) { + throw std::runtime_error("IVF 查询数据长度与元数据不一致"); + } + if (nprobe == 0 || nprobe > index.nlist) { + throw std::runtime_error("IVF 的 nprobe 不合法"); + } + + std::vector> selected(queries.num_queries); + std::vector query(index.dim); + for (std::uint64_t query_id = 0; query_id < queries.num_queries; ++query_id) { + for (std::uint32_t d = 0; d < index.dim; ++d) { + query[d] = queryValue(queries, query_id * index.dim + d); + } + std::vector center_ranking(index.nlist); + for (std::uint32_t center_id = 0; center_id < index.nlist; ++center_id) { + center_ranking[center_id] = { + center_id, + centerScore(query.data(), + index.centers.data() + + static_cast(center_id) * index.dim, + index.dim, index.metric)}; + } + const auto better = [&](const Neighbor& left, const Neighbor& right) { + return isBetter(left.score, left.id, right.score, right.id, index.metric); + }; + std::partial_sort(center_ranking.begin(), center_ranking.begin() + nprobe, + center_ranking.end(), better); + + std::vector& positions = selected[query_id]; + for (std::uint32_t probe = 0; probe < nprobe; ++probe) { + const std::uint64_t center_id = center_ranking[probe].id; + for (std::uint64_t position = index.offsets[center_id]; + position < index.offsets[center_id + 1]; ++position) { + positions.push_back(position); + } + } + } + return selected; +} + +double recallAtK(const SearchResults& exact, const SearchResults& approximate) { + if (exact.size() != approximate.size()) { + throw std::runtime_error("recall@K 的 query 数量不同"); + } + std::uint64_t matches = 0; + std::uint64_t total = 0; + for (std::size_t query_id = 0; query_id < exact.size(); ++query_id) { + if (exact[query_id].size() != approximate[query_id].size()) { + throw std::runtime_error("recall@K 的 K 不同"); + } + std::unordered_set exact_ids; + for (const Neighbor& neighbor : exact[query_id]) { + exact_ids.insert(neighbor.id); + } + for (const Neighbor& neighbor : approximate[query_id]) { + matches += exact_ids.count(neighbor.id); + } + total += exact[query_id].size(); + } + return total == 0 ? 1.0 : static_cast(matches) / total; +} diff --git a/09_vector_retrieval/PurplePulse/src/main.cpp b/09_vector_retrieval/PurplePulse/src/main.cpp new file mode 100644 index 00000000..6d1ebe8c --- /dev/null +++ b/09_vector_retrieval/PurplePulse/src/main.cpp @@ -0,0 +1,234 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "file_io.h" +#include "search.h" + +namespace { + +void printUsage(const char* program) { + std::cerr << "用法:\n " << program + << " --database <文件> --queries <文件> --params <文件>" + " --backend --output <文件>" + " [--warmup 1] [--repeat 5]" + " [--profile-breakdown 0|1] [--profile-repeat 5]\n"; +} + +std::unordered_map parseArguments(int argc, + char** argv) { + std::unordered_map arguments; + for (int index = 1; index < argc; index += 2) { + if (index + 1 >= argc || std::string(argv[index]).rfind("--", 0) != 0) { + throw std::runtime_error("命令行参数必须是 --名称 值 的形式"); + } + arguments[argv[index]] = argv[index + 1]; + } + for (const char* required : + {"--database", "--queries", "--params", "--backend", "--output"}) { + if (arguments.count(required) == 0) { + throw std::runtime_error(std::string("缺少参数: ") + required); + } + } + return arguments; +} + +double percentile(std::vector values, double fraction) { + if (values.empty()) { + return 0.0; + } + std::sort(values.begin(), values.end()); + const std::size_t index = + static_cast( + std::ceil(fraction * static_cast(values.size()))) - + 1; + return values[index]; +} + +std::uint32_t readCount( + const std::unordered_map& arguments, + const std::string& name, std::uint32_t default_value, bool allow_zero) { + const auto found = arguments.find(name); + if (found == arguments.end()) { + return default_value; + } + std::size_t parsed_chars = 0; + const unsigned long parsed = std::stoul(found->second, &parsed_chars); + if (parsed_chars != found->second.size() || (!allow_zero && parsed == 0) || + parsed > UINT32_MAX) { + throw std::runtime_error(name + " 的值不合法"); + } + return static_cast(parsed); +} + +void addStats(SearchStats* total, const SearchStats& current) { + total->host_selection_ms += current.host_selection_ms; + total->center_selection_ms += current.center_selection_ms; + total->database_h2d_ms += current.database_h2d_ms; + total->query_h2d_ms += current.query_h2d_ms; + total->distance_kernel_ms += current.distance_kernel_ms; + total->topk_kernel_ms += current.topk_kernel_ms; + total->result_d2h_ms += current.result_d2h_ms; + total->batch_latency_ms.insert(total->batch_latency_ms.end(), + current.batch_latency_ms.begin(), + current.batch_latency_ms.end()); +} + +} // namespace + +int main(int argc, char** argv) { + try { + if (argc == 2 && std::string(argv[1]) == "--help") { + printUsage(argv[0]); + return EXIT_SUCCESS; + } + if (argc == 1) { + printUsage(argv[0]); + return 1; + } + const auto arguments = parseArguments(argc, argv); + const VectorDatabase database = + readVectorDatabase(arguments.at("--database")); + const QuerySet queries = readQuerySet(arguments.at("--queries")); + const SearchParams params = readSearchParams(arguments.at("--params")); + validateInputs(database, queries, params); + const std::uint32_t warmup = readCount(arguments, "--warmup", 0, true); + const std::uint32_t repeat = readCount(arguments, "--repeat", 1, false); + const bool profile_breakdown = + readCount(arguments, "--profile-breakdown", 0, true) != 0; + const std::uint32_t profile_repeat = + readCount(arguments, "--profile-repeat", repeat, false); + + std::cout << "向量库: " << database.num_vectors << " x " << database.dim + << "\n查询数: " << queries.num_queries + << "\ndtype: " << dataTypeName(database.dtype) + << "\nmetric: " << metricName(database.metric) + << "\ntop_k: " << params.top_k + << "\nbatch_size: " << params.batch_size + << "\ndistance_mode: " << params.distance_mode + << "\ntopk_mode: " << params.topk_mode + << "\nexact_query_tile: " << params.exact_query_tile + << "\nexact_fused_layout: " << params.exact_fused_layout + << "\nexact_fused_blocks: " << params.exact_fused_blocks + << "\nwarmup/repeat: " << warmup << '/' << repeat << '\n'; + + SearchResults results; + SearchStats stats; + const std::string backend = arguments.at("--backend"); + if (backend != "cpu" && backend != "gpu") { + throw std::runtime_error("backend 必须是 cpu 或 gpu"); + } + if (profile_breakdown && + (backend != "gpu" || params.topk_mode != "fused")) { + throw std::runtime_error( + "--profile-breakdown 只支持 backend=gpu、topk_mode=fused"); + } + + std::unique_ptr gpu_engine; + double gpu_initialization_ms = 0.0; + if (backend == "gpu") { + const auto initialization_start = std::chrono::steady_clock::now(); + gpu_engine = + std::make_unique(database, queries, params); + const auto initialization_end = std::chrono::steady_clock::now(); + gpu_initialization_ms = std::chrono::duration( + initialization_end - initialization_start) + .count(); + } + + const auto run_search = [&](SearchStats* run_stats) { + if (backend == "cpu") { + return cpuExactSearch(database, queries, params); + } + return gpu_engine->search(queries, run_stats); + }; + + for (std::uint32_t run = 0; run < warmup; ++run) { + SearchStats ignored_stats; + results = run_search(&ignored_stats); + } + + std::vector run_times_ms; + for (std::uint32_t run = 0; run < repeat; ++run) { + SearchStats run_stats; + const auto start = std::chrono::steady_clock::now(); + results = run_search(&run_stats); + const auto end = std::chrono::steady_clock::now(); + run_times_ms.push_back( + std::chrono::duration(end - start).count()); + addStats(&stats, run_stats); + } + + const std::filesystem::path output_path(arguments.at("--output")); + if (output_path.has_parent_path()) { + std::filesystem::create_directories(output_path.parent_path()); + } + writeSearchResults(output_path.string(), results); + + double total_ms = 0.0; + for (double time_ms : run_times_ms) { + total_ms += time_ms; + } + const double average_ms = total_ms / repeat; + const double qps = + static_cast(queries.num_queries) * repeat / (total_ms / 1000.0); + std::cout << "backend: " << backend; + if (backend == "gpu") { + std::cout << "\nGPU 一次性初始化总时间: " << gpu_initialization_ms + << " ms\n数据库一次性 H2D: " + << gpu_engine->databaseH2DMilliseconds() << " ms\nGPU 缓冲区: " + << gpu_engine->deviceBytes() / (1024.0 * 1024.0) << " MiB" + << "\n估算冷启动端到端: " << gpu_initialization_ms + average_ms + << " ms\n平均常驻查询时间: " << average_ms; + } else { + std::cout << "\n平均单次总时间: " << average_ms; + } + std::cout << " ms\nrun P50: " << percentile(run_times_ms, 0.50) + << " ms\nrun P99: " << percentile(run_times_ms, 0.99) << " ms\n" + << "run samples: " << run_times_ms.size() << '\n' + << (backend == "gpu" ? "常驻 QPS: " : "QPS: ") << qps + << "\n结果: " << output_path << '\n'; + if (backend == "gpu") { + double distance_only_ms = 0.0; + if (profile_breakdown) { + distance_only_ms = gpu_engine->profileDistanceOnlyMilliseconds( + queries, 1, profile_repeat); + } + std::cout + << "平均查询 H2D: " << stats.query_h2d_ms / repeat << " ms\n" + << (params.topk_mode == "fused" ? "平均融合距离+局部 Top-K kernel: " + : "平均距离 kernel: ") + << stats.distance_kernel_ms / repeat + << " ms\n平均最终 Top-K kernel: " << stats.topk_kernel_ms / repeat + << " ms\n平均结果 D2H: " << stats.result_d2h_ms / repeat + << " ms\nbatch P50: " << percentile(stats.batch_latency_ms, 0.50) + << " ms\nbatch P99: " << percentile(stats.batch_latency_ms, 0.99) + << " ms\nbatch samples: " << stats.batch_latency_ms.size() << '\n'; + if (profile_breakdown) { + const double fused_ms = stats.distance_kernel_ms / repeat; + const double final_merge_ms = stats.topk_kernel_ms / repeat; + const double local_topk_estimated_ms = + std::max(0.0, fused_ms - distance_only_ms); + std::cout + << "剖析方法: distance-only differential\n" + << "剖析重复次数: " << profile_repeat << '\n' + << "剖析距离-only kernel: " << distance_only_ms << " ms\n" + << "剖析局部 Top-K 估算: " << local_topk_estimated_ms << " ms\n" + << "剖析融合 kernel: " << fused_ms << " ms\n" + << "剖析最终归并 kernel: " << final_merge_ms << " ms\n"; + } + } + return 0; + } catch (const std::exception& error) { + std::cerr << "错误: " << error.what() << '\n'; + printUsage(argv[0]); + return 1; + } +} diff --git a/09_vector_retrieval/PurplePulse/src/search_ivf.cpp b/09_vector_retrieval/PurplePulse/src/search_ivf.cpp new file mode 100644 index 00000000..71404d2a --- /dev/null +++ b/09_vector_retrieval/PurplePulse/src/search_ivf.cpp @@ -0,0 +1,391 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "file_io.h" +#include "ivf_flat.h" + +namespace { + +void printUsage(const char* program) { + std::cerr << "用法:\n " << program + << " --index <文件> --queries <文件> --params <文件>" + " --output <文件> [--nlist N] [--nprobe N]" + " [--nprobe-policy fixed|score_mass]" + " [--adaptive-nprobe-min N] [--adaptive-nprobe-step N]" + " [--adaptive-target-mass F] [--adaptive-temperature F]" + " [--memory-metadata <文件>]" + " [--memory-mode disabled|fused|rerank]" + " [--filter-session-id N|any] [--filter-source-type N|any]" + " [--backend cpu|gpu] [--warmup 1] [--repeat 5]\n"; +} + +std::unordered_map parseArguments(int argc, + char** argv) { + std::unordered_map arguments; + for (int index = 1; index < argc; index += 2) { + if (index + 1 >= argc || std::string(argv[index]).rfind("--", 0) != 0) { + throw std::runtime_error("命令行参数必须是 --名称 值 的形式"); + } + arguments[argv[index]] = argv[index + 1]; + } + for (const char* required : + {"--index", "--queries", "--params", "--output"}) { + if (arguments.count(required) == 0) { + throw std::runtime_error(std::string("缺少参数: ") + required); + } + } + return arguments; +} + +std::uint32_t readCount( + const std::unordered_map& arguments, + const std::string& name, std::uint32_t default_value, bool allow_zero) { + const auto found = arguments.find(name); + if (found == arguments.end()) { + return default_value; + } + std::size_t parsed_chars = 0; + const unsigned long parsed = std::stoul(found->second, &parsed_chars); + if (parsed_chars != found->second.size() || (!allow_zero && parsed == 0) || + parsed > UINT32_MAX) { + throw std::runtime_error(name + " 的值不合法"); + } + return static_cast(parsed); +} + +float readPositiveFloat( + const std::unordered_map& arguments, + const std::string& name, float default_value) { + const auto found = arguments.find(name); + if (found == arguments.end()) { + return default_value; + } + std::size_t parsed_chars = 0; + const float parsed = std::stof(found->second, &parsed_chars); + if (parsed_chars != found->second.size() || !std::isfinite(parsed) || + parsed <= 0.0F) { + throw std::runtime_error(name + " 的值必须是有限正数"); + } + return parsed; +} + +float readNonnegativeFloat( + const std::unordered_map& arguments, + const std::string& name, float default_value) { + const auto found = arguments.find(name); + if (found == arguments.end()) { + return default_value; + } + std::size_t parsed_chars = 0; + const float parsed = std::stof(found->second, &parsed_chars); + if (parsed_chars != found->second.size() || !std::isfinite(parsed) || + parsed < 0.0F) { + throw std::runtime_error(name + " 的值必须是有限非负数"); + } + return parsed; +} + +std::uint64_t readUint64( + const std::unordered_map& arguments, + const std::string& name, std::uint64_t default_value) { + const auto found = arguments.find(name); + if (found == arguments.end()) { + return default_value; + } + if (found->second.empty() || found->second.front() == '-') { + throw std::runtime_error(name + " 的值必须是非负整数"); + } + std::size_t parsed_chars = 0; + const unsigned long long parsed = std::stoull(found->second, &parsed_chars); + if (parsed_chars != found->second.size()) { + throw std::runtime_error(name + " 的值必须是非负整数"); + } + return static_cast(parsed); +} + +std::uint32_t readOptionalId( + const std::unordered_map& arguments, + const std::string& name, std::uint32_t default_value) { + const auto found = arguments.find(name); + if (found == arguments.end()) { + return default_value; + } + if (found->second == "any") { + return UINT32_MAX; + } + const std::uint64_t parsed = readUint64(arguments, name, default_value); + if (parsed >= UINT32_MAX) { + throw std::runtime_error(name + " 的值必须小于 UINT32_MAX"); + } + return static_cast(parsed); +} + +double percentile(std::vector values, double fraction) { + if (values.empty()) { + return 0.0; + } + std::sort(values.begin(), values.end()); + const std::size_t index = + static_cast( + std::ceil(fraction * static_cast(values.size()))) - + 1; + return values[index]; +} + +void addStats(SearchStats* total, const SearchStats& current) { + total->host_selection_ms += current.host_selection_ms; + total->center_selection_ms += current.center_selection_ms; + total->adaptive_policy_ms += current.adaptive_policy_ms; + total->database_h2d_ms += current.database_h2d_ms; + total->query_h2d_ms += current.query_h2d_ms; + total->distance_kernel_ms += current.distance_kernel_ms; + total->topk_kernel_ms += current.topk_kernel_ms; + total->memory_rerank_ms += current.memory_rerank_ms; + total->result_d2h_ms += current.result_d2h_ms; + total->batch_latency_ms.insert(total->batch_latency_ms.end(), + current.batch_latency_ms.begin(), + current.batch_latency_ms.end()); + if (current.selected_probe_queries != 0) { + if (total->selected_probe_queries == 0) { + total->selected_probe_min = current.selected_probe_min; + } else { + total->selected_probe_min = + std::min(total->selected_probe_min, current.selected_probe_min); + } + total->selected_probe_max = + std::max(total->selected_probe_max, current.selected_probe_max); + total->selected_probe_sum += current.selected_probe_sum; + total->selected_probe_queries += current.selected_probe_queries; + if (total->selected_probe_counts.empty()) { + total->selected_probe_counts = current.selected_probe_counts; + } + } +} + +} // namespace + +int main(int argc, char** argv) { + try { + if (argc == 2 && std::string(argv[1]) == "--help") { + printUsage(argv[0]); + return EXIT_SUCCESS; + } + if (argc == 1) { + printUsage(argv[0]); + return EXIT_FAILURE; + } + const auto arguments = parseArguments(argc, argv); + const auto load_start = std::chrono::steady_clock::now(); + const IvfFlatIndex index = readIvfFlatIndex(arguments.at("--index")); + const auto load_end = std::chrono::steady_clock::now(); + const QuerySet queries = readQuerySet(arguments.at("--queries")); + SearchParams params = readSearchParams(arguments.at("--params")); + if (params.search_mode != "ivf_flat") { + throw std::runtime_error("IVF 查询要求 search_mode=ivf_flat"); + } + if (arguments.count("--nlist") != 0) { + params.nlist = readCount(arguments, "--nlist", params.nlist, false); + } + if (arguments.count("--nprobe") != 0) { + params.nprobe = readCount(arguments, "--nprobe", params.nprobe, false); + } + if (arguments.count("--nprobe-policy") != 0) { + params.nprobe_policy = arguments.at("--nprobe-policy"); + } + if (arguments.count("--adaptive-execution") != 0) { + params.adaptive_execution = arguments.at("--adaptive-execution"); + } + params.adaptive_nprobe_min = readCount(arguments, "--adaptive-nprobe-min", + params.adaptive_nprobe_min, false); + params.adaptive_nprobe_step = readCount(arguments, "--adaptive-nprobe-step", + params.adaptive_nprobe_step, false); + params.adaptive_target_mass = readPositiveFloat( + arguments, "--adaptive-target-mass", params.adaptive_target_mass); + params.adaptive_temperature = readPositiveFloat( + arguments, "--adaptive-temperature", params.adaptive_temperature); + if (arguments.count("--memory-mode") != 0) { + params.memory_mode = arguments.at("--memory-mode"); + } + params.memory_semantic_weight = readPositiveFloat( + arguments, "--memory-semantic-weight", params.memory_semantic_weight); + params.memory_importance_weight = + readNonnegativeFloat(arguments, "--memory-importance-weight", + params.memory_importance_weight); + params.memory_recency_weight = readNonnegativeFloat( + arguments, "--memory-recency-weight", params.memory_recency_weight); + params.memory_time_scale = readPositiveFloat( + arguments, "--memory-time-scale", params.memory_time_scale); + params.memory_now = + readUint64(arguments, "--memory-now", params.memory_now); + params.filter_min_timestamp = readUint64( + arguments, "--filter-min-timestamp", params.filter_min_timestamp); + params.filter_session_id = readOptionalId(arguments, "--filter-session-id", + params.filter_session_id); + params.filter_source_type = readOptionalId( + arguments, "--filter-source-type", params.filter_source_type); + params.memory_rerank_factor = readCount(arguments, "--memory-rerank-factor", + params.memory_rerank_factor, false); + if (params.nlist != index.nlist) { + throw std::runtime_error("参数 nlist 与索引 nlist 不一致"); + } + const std::uint32_t warmup = readCount(arguments, "--warmup", 0, true); + const std::uint32_t repeat = readCount(arguments, "--repeat", 1, false); + const std::string backend = + arguments.count("--backend") == 0 ? "cpu" : arguments.at("--backend"); + if (backend != "cpu" && backend != "gpu") { + throw std::runtime_error("backend 必须是 cpu 或 gpu"); + } + if (backend == "cpu" && params.nprobe_policy != "fixed") { + throw std::runtime_error("自适应 nprobe 当前只支持 GPU backend"); + } + MemoryMetadata memory_metadata; + const MemoryMetadata* metadata = nullptr; + const auto metadata_argument = arguments.find("--memory-metadata"); + if (metadata_argument != arguments.end()) { + memory_metadata = readMemoryMetadata(metadata_argument->second); + validateMemoryMetadata(memory_metadata, index.num_vectors); + metadata = &memory_metadata; + } + if (params.memory_mode != "disabled" && metadata == nullptr) { + throw std::runtime_error("memory_mode 启用时必须传入 --memory-metadata"); + } + + std::unique_ptr gpu_engine; + double gpu_initialization_ms = 0.0; + if (backend == "gpu") { + const auto initialization_start = std::chrono::steady_clock::now(); + gpu_engine = std::make_unique(index, queries, + params, metadata); + const auto initialization_end = std::chrono::steady_clock::now(); + gpu_initialization_ms = std::chrono::duration( + initialization_end - initialization_start) + .count(); + } + + SearchResults results; + const auto run_search = [&](SearchStats* stats) { + return backend == "gpu" + ? gpu_engine->search(queries, stats) + : cpuIvfFlatSearch(index, queries, params, metadata); + }; + for (std::uint32_t run = 0; run < warmup; ++run) { + SearchStats ignored; + results = run_search(&ignored); + } + std::vector run_times; + SearchStats stats; + for (std::uint32_t run = 0; run < repeat; ++run) { + SearchStats run_stats; + const auto start = std::chrono::steady_clock::now(); + results = run_search(&run_stats); + const auto end = std::chrono::steady_clock::now(); + run_times.push_back( + std::chrono::duration(end - start).count()); + addStats(&stats, run_stats); + } + const std::filesystem::path output_path(arguments.at("--output")); + if (output_path.has_parent_path()) { + std::filesystem::create_directories(output_path.parent_path()); + } + writeSearchResults(output_path.string(), results); + double total_ms = 0.0; + for (const double time : run_times) { + total_ms += time; + } + const double average_ms = total_ms / repeat; + const double load_ms = + std::chrono::duration(load_end - load_start) + .count(); + std::cout << "IVF-Flat " << backend << " 查询完成" + << "\n索引: " << index.num_vectors << " x " << index.dim + << "\ndtype/metric: " << dataTypeName(index.dtype) << '/' + << metricName(index.metric) + << "\nnlist/nprobe/top_k: " << index.nlist << '/' << params.nprobe + << '/' << params.top_k + << "\nnprobe policy: " << params.nprobe_policy + << "\nmemory mode: " << params.memory_mode + << "\n索引加载: " << load_ms << " ms" + << (backend == "gpu" ? "\nGPU 初始化: " : "") + << (backend == "gpu" ? std::to_string(gpu_initialization_ms) + : std::string()) + << (backend == "gpu" ? " ms\n索引 H2D: " : "") + << (backend == "gpu" + ? std::to_string(gpu_engine->indexH2DMilliseconds()) + : std::string()) + << (backend == "gpu" ? " ms" : "") + << (backend == "gpu" ? "\nGPU 缓冲区: " : "") + << (backend == "gpu" ? std::to_string(gpu_engine->deviceBytes() / + (1024.0 * 1024.0)) + : std::string()) + << (backend == "gpu" ? " MiB" : "") + << "\n平均查询: " << average_ms << " ms" + << "\nrun P50/P99: " << percentile(run_times, 0.50) << '/' + << percentile(run_times, 0.99) << " ms" + << "\nrun samples: " << run_times.size() + << "\nQPS: " << queries.num_queries * repeat / (total_ms / 1000.0) + << "\n输出: " << output_path << '\n'; + if (params.nprobe_policy == "score_mass") { + std::cout << "自适应 min/step/target/temperature: " + << params.adaptive_nprobe_min << '/' + << params.adaptive_nprobe_step << '/' + << params.adaptive_target_mass << '/' + << params.adaptive_temperature << '\n'; + std::cout << "自适应执行方式: " << params.adaptive_execution << '\n'; + } + if (backend == "gpu") { + std::cout + << "平均 CPU 中心/桶选择: " << stats.host_selection_ms / repeat + << " ms\n平均 query H2D: " << stats.query_h2d_ms / repeat + << " ms\n平均 GPU 中心选择: " << stats.center_selection_ms / repeat + << " ms\n平均自适应分层开销: " << stats.adaptive_policy_ms / repeat + << " ms\n平均 GPU 桶扫描+局部 Top-K: " + << stats.distance_kernel_ms / repeat + << " ms\n平均 GPU 最终 Top-K 归并: " << stats.topk_kernel_ms / repeat + << " ms\n平均记忆独立重排: " << stats.memory_rerank_ms / repeat + << " ms\n平均结果 D2H: " << stats.result_d2h_ms / repeat + << " ms\nbatch P50/P99: " << percentile(stats.batch_latency_ms, 0.50) + << '/' << percentile(stats.batch_latency_ms, 0.99) + << " ms\nbatch samples: " << stats.batch_latency_ms.size() << '\n'; + if (stats.selected_probe_queries != 0) { + std::cout << "实际 nprobe min/avg/max: " << stats.selected_probe_min + << '/' + << static_cast(stats.selected_probe_sum) / + stats.selected_probe_queries + << '/' << stats.selected_probe_max << '\n'; + } + const auto probe_counts_argument = + arguments.find("--probe-counts-output"); + if (probe_counts_argument != arguments.end()) { + const std::filesystem::path counts_path(probe_counts_argument->second); + if (counts_path.has_parent_path()) { + std::filesystem::create_directories(counts_path.parent_path()); + } + std::ofstream counts_output(counts_path); + if (!counts_output) { + throw std::runtime_error("无法写入实际 nprobe 文件: " + + counts_path.string()); + } + for (std::size_t query_id = 0; + query_id < stats.selected_probe_counts.size(); ++query_id) { + counts_output << query_id << ' ' + << stats.selected_probe_counts[query_id] << '\n'; + } + } + } + return EXIT_SUCCESS; + } catch (const std::exception& error) { + std::cerr << "错误: " << error.what() << '\n'; + printUsage(argv[0]); + return EXIT_FAILURE; + } +} diff --git a/09_vector_retrieval/PurplePulse/tests/file_io_tests.cpp b/09_vector_retrieval/PurplePulse/tests/file_io_tests.cpp new file mode 100644 index 00000000..b8ab98b9 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/tests/file_io_tests.cpp @@ -0,0 +1,195 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "file_io.h" + +namespace { + +template +void writeValue(std::ofstream& output, const T& value) { + output.write(reinterpret_cast(&value), sizeof(T)); +} + +void require(bool condition, const std::string& message) { + if (!condition) { + throw std::runtime_error(message); + } +} + +template +void requireThrows(Operation operation, const std::string& message) { + try { + operation(); + } catch (const std::runtime_error&) { + return; + } + throw std::runtime_error(message); +} + +void writeDatabase(const std::filesystem::path& path) { + std::ofstream output(path, std::ios::binary); + const std::array magic = {'P', 'P', 'V', 'E', 'C', '0', '0', '1'}; + const std::uint64_t rows = 2; + const std::uint32_t dim = 2; + const std::uint32_t dtype = static_cast(DataType::kFloat16); + const std::uint32_t metric = static_cast(Metric::kL2); + const std::array values = {0x3c00, 0x0000, 0x4000, 0xbc00}; + output.write(magic.data(), magic.size()); + writeValue(output, rows); + writeValue(output, dim); + writeValue(output, dtype); + writeValue(output, metric); + output.write(reinterpret_cast(values.data()), + values.size() * sizeof(values[0])); +} + +void writeQueries(const std::filesystem::path& path) { + std::ofstream output(path, std::ios::binary); + const std::array magic = {'P', 'P', 'Q', 'R', 'Y', '0', '0', '1'}; + const std::uint64_t rows = 1; + const std::uint32_t dim = 2; + const std::uint32_t dtype = static_cast(DataType::kFloat16); + const std::array values = {0x3c00, 0x0000}; + output.write(magic.data(), magic.size()); + writeValue(output, rows); + writeValue(output, dim); + writeValue(output, dtype); + output.write(reinterpret_cast(values.data()), + values.size() * sizeof(values[0])); +} + +} // namespace + +int main() { + const auto unique = + std::chrono::steady_clock::now().time_since_epoch().count(); + const std::filesystem::path directory = + std::filesystem::temp_directory_path() / + ("purplepulse-file-io-test-" + std::to_string(unique)); + try { + std::filesystem::create_directories(directory); + const std::filesystem::path database_path = directory / "database.bin"; + const std::filesystem::path query_path = directory / "queries.bin"; + writeDatabase(database_path); + writeQueries(query_path); + + const VectorDatabase database = readVectorDatabase(database_path.string()); + const QuerySet queries = readQuerySet(query_path.string()); + require(database.dtype == DataType::kFloat16, "database dtype 错误"); + require(queries.dtype == DataType::kFloat16, "query dtype 错误"); + require(database.values.empty() && queries.values.empty(), + "FP16 文件不应展开为 FP32 存储"); + require( + database.half_values.size() == 4 && database.half_values[2] == 0x4000, + "database half 位模式错误"); + require(queries.half_values.size() == 2 && queries.half_values[0] == 0x3c00, + "query half 位模式错误"); + SearchParams params; + params.top_k = 1; + validateInputs(database, queries, params); + params.search_mode = "ivf_flat"; + bool rejected_unimplemented_ivf = false; + try { + validateInputs(database, queries, params); + } catch (const std::runtime_error&) { + rejected_unimplemented_ivf = true; + } + require(rejected_unimplemented_ivf, + "未实现的 IVF-Flat 不应静默执行 exact search"); + + const std::filesystem::path metadata_path = directory / "memory.bin"; + MemoryMetadata written_metadata; + written_metadata.num_vectors = 3; + written_metadata.timestamps = {100, 200, 300}; + written_metadata.importance = {0.1F, 0.5F, 1.0F}; + written_metadata.session_ids = {1, 1, 2}; + written_metadata.source_types = {0, 2, 1}; + writeMemoryMetadata(metadata_path.string(), written_metadata); + const MemoryMetadata loaded_metadata = + readMemoryMetadata(metadata_path.string()); + require(loaded_metadata.timestamps == written_metadata.timestamps && + loaded_metadata.importance == written_metadata.importance && + loaded_metadata.session_ids == written_metadata.session_ids && + loaded_metadata.source_types == written_metadata.source_types, + "记忆元数据保存/加载后内容变化"); + + const std::filesystem::path params_path = directory / "adaptive.conf"; + { + std::ofstream output(params_path); + output << "top_k = 10\n" + "search_mode = ivf_flat\n" + "exact_query_tile = 2\n" + "exact_fused_layout = warp_query\n" + "exact_fused_blocks = 96\n" + "nprobe = 224\n" + "nprobe_policy = score_mass\n" + "adaptive_execution = grouped\n" + "adaptive_nprobe_min = 64\n" + "adaptive_nprobe_step = 16\n" + "adaptive_target_mass = 0.85\n" + "adaptive_temperature = 0.2\n" + "memory_mode = fused\n" + "memory_importance_weight = 0.15\n" + "memory_recency_weight = 0.1\n" + "memory_time_scale = 100\n" + "memory_now = 300\n" + "filter_session_id = any\n" + "filter_source_type = 2\n"; + } + const SearchParams adaptive = readSearchParams(params_path.string()); + require(adaptive.nprobe_policy == "score_mass" && + adaptive.exact_query_tile == 2 && + adaptive.exact_fused_layout == "warp_query" && + adaptive.exact_fused_blocks == 96 && + adaptive.adaptive_execution == "grouped" && + adaptive.adaptive_nprobe_min == 64 && + adaptive.adaptive_nprobe_step == 16 && + adaptive.adaptive_target_mass == 0.85F && + adaptive.adaptive_temperature == 0.2F && + adaptive.memory_mode == "fused" && + adaptive.memory_importance_weight == 0.15F && + adaptive.memory_now == 300 && + adaptive.filter_session_id == UINT32_MAX && + adaptive.filter_source_type == 2, + "自适应 nprobe 或记忆参数解析错误"); + + const std::filesystem::path bad_magic_path = directory / "bad-magic.bin"; + { + std::ofstream output(bad_magic_path, std::ios::binary); + output << "NOTAVECTORFILE"; + } + requireThrows([&] { (void)readVectorDatabase(bad_magic_path.string()); }, + "错误的向量库 magic/version 应被拒绝"); + + const std::filesystem::path truncated_path = directory / "truncated.bin"; + writeQueries(truncated_path); + std::filesystem::resize_file( + truncated_path, std::filesystem::file_size(truncated_path) - 1); + requireThrows([&] { (void)readQuerySet(truncated_path.string()); }, + "截断的查询文件应被拒绝"); + + const std::filesystem::path extra_bytes_path = directory / "extra.bin"; + writeDatabase(extra_bytes_path); + { + std::ofstream output(extra_bytes_path, std::ios::binary | std::ios::app); + output.put('\0'); + } + requireThrows([&] { (void)readVectorDatabase(extra_bytes_path.string()); }, + "包含额外字节的向量库文件应被拒绝"); + + std::filesystem::remove_all(directory); + std::cout << "FP16 文件读取与损坏文件测试通过\n"; + return EXIT_SUCCESS; + } catch (const std::exception& error) { + std::filesystem::remove_all(directory); + std::cerr << "测试失败: " << error.what() << '\n'; + return EXIT_FAILURE; + } +} diff --git a/09_vector_retrieval/PurplePulse/tests/ivf_flat_tests.cpp b/09_vector_retrieval/PurplePulse/tests/ivf_flat_tests.cpp new file mode 100644 index 00000000..cfecb7b9 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/tests/ivf_flat_tests.cpp @@ -0,0 +1,351 @@ +#include +#include +#include +#include +#include +#include +#include + +#include "ivf_flat.h" +#include "search.h" + +namespace { + +void require(bool condition, const std::string& message) { + if (!condition) { + throw std::runtime_error(message); + } +} + +VectorDatabase makeDatabase(Metric metric, DataType dtype) { + VectorDatabase database; + database.num_vectors = 12; + database.dim = 2; + database.metric = metric; + database.dtype = dtype; + const std::vector values = { + 1.0F, 0.0F, 1.0F, 0.25F, 0.75F, 0.0F, 0.0F, 1.0F, + 0.25F, 1.0F, 0.0F, 0.75F, -1.0F, 0.0F, -1.0F, -0.25F, + -0.75F, 0.0F, 0.0F, -1.0F, -0.25F, -1.0F, 0.0F, -0.75F, + }; + if (dtype == DataType::kFloat32) { + database.values = values; + } else { + // 测试数据只使用 0、±1、±0.75、±0.25，均可被 binary16 精确表示。 + const auto half = [](float value) -> std::uint16_t { + if (value == 1.0F) return 0x3c00; + if (value == -1.0F) return 0xbc00; + if (value == 0.75F) return 0x3a00; + if (value == -0.75F) return 0xba00; + if (value == 0.25F) return 0x3400; + if (value == -0.25F) return 0xb400; + return 0x0000; + }; + for (const float value : values) { + database.half_values.push_back(half(value)); + } + } + return database; +} + +QuerySet makeQueries(DataType dtype) { + QuerySet queries; + queries.num_queries = 4; + queries.dim = 2; + queries.dtype = dtype; + if (dtype == DataType::kFloat32) { + queries.values = {1.0F, 0.0F, 0.0F, 1.0F, -1.0F, 0.0F, 0.0F, -1.0F}; + } else { + queries.half_values = {0x3c00, 0x0000, 0x0000, 0x3c00, + 0xbc00, 0x0000, 0x0000, 0xbc00}; + } + return queries; +} + +MemoryMetadata makeMemoryMetadata(std::uint64_t num_vectors) { + MemoryMetadata metadata; + metadata.num_vectors = num_vectors; + for (std::uint64_t id = 0; id < num_vectors; ++id) { + metadata.timestamps.push_back(900 + id * 10); + metadata.importance.push_back(static_cast(id) / + static_cast(num_vectors)); + metadata.session_ids.push_back(static_cast(id % 2)); + metadata.source_types.push_back(static_cast(id % 3)); + } + return metadata; +} + +std::uint16_t exactHalf(float value) { + if (value == 1.0F) return 0x3c00; + if (value == -1.0F) return 0xbc00; + if (value == 0.75F) return 0x3a00; + if (value == -0.75F) return 0xba00; + if (value == 0.25F) return 0x3400; + if (value == -0.25F) return 0xb400; + return 0x0000; +} + +VectorDatabase makeLargeKDatabase(Metric metric, DataType dtype) { + constexpr float levels[] = {-1.0F, -0.75F, -0.25F, 0.0F, 0.25F, 0.75F, 1.0F}; + VectorDatabase database; + database.num_vectors = 128; + database.dim = 4; + database.metric = metric; + database.dtype = dtype; + std::vector values; + values.reserve(database.num_vectors * database.dim); + for (std::uint64_t vector_id = 0; vector_id < database.num_vectors; + ++vector_id) { + std::uint64_t code = vector_id; + for (std::uint32_t d = 0; d < database.dim; ++d) { + values.push_back(levels[code % 7]); + code /= 7; + } + } + if (dtype == DataType::kFloat32) { + database.values = std::move(values); + } else { + database.half_values.reserve(values.size()); + for (const float value : values) { + database.half_values.push_back(exactHalf(value)); + } + } + return database; +} + +QuerySet makeLargeKQueries(DataType dtype) { + const std::vector values = { + 1.0F, 0.25F, -0.75F, 0.0F, -0.25F, 1.0F, + 0.75F, -1.0F, 0.0F, -0.75F, 1.0F, 0.25F, + }; + QuerySet queries; + queries.num_queries = 3; + queries.dim = 4; + queries.dtype = dtype; + if (dtype == DataType::kFloat32) { + queries.values = values; + } else { + for (const float value : values) { + queries.half_values.push_back(exactHalf(value)); + } + } + return queries; +} + +void testLargeK(Metric metric, DataType dtype) { + const VectorDatabase database = makeLargeKDatabase(metric, dtype); + const QuerySet queries = makeLargeKQueries(dtype); + const IvfFlatIndex index = buildIvfFlatIndex(database, 8, 5, 128); + for (const std::uint32_t top_k : {50U, 100U}) { + SearchParams params; + params.top_k = top_k; + params.search_mode = "ivf_flat"; + params.nlist = 8; + params.nprobe = 8; + params.batch_size = 4; + params.distance_mode = "warp"; + SearchParams exact_params = params; + exact_params.search_mode = "exact"; + exact_params.topk_mode = "block"; + const SearchResults exact = cpuExactSearch(database, queries, exact_params); + const SearchResults cpu_ivf = cpuIvfFlatSearch(index, queries, params); + std::string error; + require(resultsMatch(exact, cpu_ivf, 0.0F, &error), + "大 K CPU IVF 与 exact 不一致: " + error); + SearchResults warp8_results; + for (const std::string& distance_mode : + {"warp8", "warp4", "warp2", "warp1", "warp_compact", + "bucket_major"}) { + params.distance_mode = distance_mode; + GpuIvfFlatSearchEngine engine(index, queries, params); + const SearchResults gpu_ivf = engine.search(queries); + require(resultsMatch(cpu_ivf, gpu_ivf, 1e-5F, &error), + "大 K " + distance_mode + " GPU IVF 与 CPU IVF 不一致: " + error); + if (distance_mode == "warp8") { + warp8_results = gpu_ivf; + } else { + require(resultsMatch(warp8_results, gpu_ivf, 0.0F, &error), + "大 K compact warp 与 warp8 结果不一致: " + error); + } + } + } +} + +void testRoundTrip(Metric metric, DataType dtype, + const std::filesystem::path& directory) { + const VectorDatabase database = makeDatabase(metric, dtype); + const QuerySet queries = makeQueries(dtype); + const IvfFlatIndex built = buildIvfFlatIndex(database, 4, 8, 12); + const IvfFlatIndex parallel_built = + buildIvfFlatIndex(database, 4, 8, 12, 4); + const IvfFlatIndex parallel_repeated = + buildIvfFlatIndex(database, 4, 8, 12, 4); + require(parallel_built.centers == parallel_repeated.centers && + parallel_built.offsets == parallel_repeated.offsets && + parallel_built.ids == parallel_repeated.ids && + parallel_built.values == parallel_repeated.values && + parallel_built.half_values == parallel_repeated.half_values, + "固定线程数的并行 IVF 建库结果应可重复"); + const std::filesystem::path path = + directory / + (std::string("index-") + std::to_string(static_cast(metric)) + "-" + + std::to_string(static_cast(dtype)) + ".bin"); + writeIvfFlatIndex(path.string(), built); + const IvfFlatIndex loaded = readIvfFlatIndex(path.string()); + require(loaded.centers == built.centers && loaded.offsets == built.offsets && + loaded.ids == built.ids && loaded.values == built.values && + loaded.half_values == built.half_values, + "IVF 索引保存/加载后内容变化"); + + SearchParams exact_params; + exact_params.top_k = 3; + const SearchResults exact = cpuExactSearch(database, queries, exact_params); + SearchParams ivf_params = exact_params; + ivf_params.search_mode = "ivf_flat"; + ivf_params.nlist = 4; + ivf_params.nprobe = 4; + const SearchResults exhaustive_ivf = + cpuIvfFlatSearch(loaded, queries, ivf_params); + std::string error; + require(resultsMatch(exact, exhaustive_ivf, 0.0F, &error), + "nprobe=nlist 应与 exact 完全一致: " + error); + require(recallAtK(exact, exhaustive_ivf) == 1.0, + "nprobe=nlist 的 recall@K 应为 1"); + + ivf_params.nprobe = 2; + const SearchResults approximate = + cpuIvfFlatSearch(loaded, queries, ivf_params); + const double recall = recallAtK(exact, approximate); + require(recall >= 0.0 && recall <= 1.0, "IVF recall@K 必须位于 [0, 1]"); + + ivf_params.distance_mode = "simple"; + GpuIvfFlatSearchEngine scalar_gpu_engine(loaded, queries, ivf_params); + const SearchResults scalar_gpu = scalar_gpu_engine.search(queries); + require(resultsMatch(approximate, scalar_gpu, 1e-5F, &error), + "scalar GPU IVF 与 CPU IVF 不一致: " + error); + + ivf_params.distance_mode = "warp"; + GpuIvfFlatSearchEngine gpu_engine(loaded, queries, ivf_params); + SearchStats first_stats; + const SearchResults gpu_first = gpu_engine.search(queries, &first_stats); + const SearchResults gpu_second = gpu_engine.search(queries); + require(resultsMatch(approximate, gpu_first, 1e-5F, &error), + "GPU IVF 与 CPU IVF 不一致: " + error); + require(resultsMatch(gpu_first, gpu_second, 0.0F, &error), + "GPU IVF 常驻引擎重复查询不一致: " + error); + require(first_stats.host_selection_ms >= 0.0 && + first_stats.distance_kernel_ms >= 0.0 && + gpu_engine.indexH2DMilliseconds() >= 0.0 && + gpu_engine.deviceBytes() > 0, + "GPU IVF 计时不应为负数且设备缓冲区应非空"); + + SearchParams adaptive_params = ivf_params; + adaptive_params.nprobe = 4; + adaptive_params.nprobe_policy = "score_mass"; + adaptive_params.adaptive_nprobe_min = 2; + adaptive_params.adaptive_nprobe_step = 1; + adaptive_params.adaptive_target_mass = 0.5F; + adaptive_params.adaptive_temperature = 0.2F; + GpuIvfFlatSearchEngine adaptive_engine(loaded, queries, adaptive_params); + SearchStats adaptive_stats; + const SearchResults adaptive_results = + adaptive_engine.search(queries, &adaptive_stats); + require(adaptive_results.size() == queries.num_queries && + adaptive_stats.selected_probe_queries == queries.num_queries && + adaptive_stats.selected_probe_min >= 2 && + adaptive_stats.selected_probe_max <= 4 && + adaptive_stats.selected_probe_sum >= 2 * queries.num_queries && + adaptive_stats.selected_probe_sum <= 4 * queries.num_queries, + "自适应 nprobe 统计或边界错误"); + + adaptive_params.adaptive_nprobe_min = 4; + adaptive_params.adaptive_target_mass = 0.1F; + adaptive_params.adaptive_execution = "grouped"; + GpuIvfFlatSearchEngine capped_adaptive_engine(loaded, queries, + adaptive_params); + SearchStats capped_stats; + const SearchResults capped_adaptive = + capped_adaptive_engine.search(queries, &capped_stats); + require(resultsMatch(exhaustive_ivf, capped_adaptive, 1e-5F, &error), + "min_nprobe=max_nprobe 时自适应结果应与固定全桶一致: " + error); + require(capped_stats.selected_probe_min == 4 && + capped_stats.selected_probe_max == 4, + "自适应 nprobe 上下限相同时统计错误"); + + const MemoryMetadata metadata = makeMemoryMetadata(database.num_vectors); + SearchParams memory_params = ivf_params; + memory_params.nprobe = 4; + memory_params.top_k = 3; + memory_params.memory_mode = "fused"; + memory_params.memory_importance_weight = 0.2F; + memory_params.memory_recency_weight = 0.1F; + memory_params.memory_time_scale = 50.0F; + memory_params.memory_now = 1000; + memory_params.filter_session_id = 0; + const SearchResults cpu_fused = + cpuIvfFlatSearch(loaded, queries, memory_params, &metadata); + GpuIvfFlatSearchEngine fused_engine(loaded, queries, memory_params, + &metadata); + const SearchResults gpu_fused = fused_engine.search(queries); + require(resultsMatch(cpu_fused, gpu_fused, 1e-5F, &error), + "融合记忆评分的 GPU IVF 与 CPU baseline 不一致: " + error); + for (const auto& query_results : gpu_fused) { + require(query_results.size() == memory_params.top_k, + "过滤后候选充足时应返回完整 Top-K"); + for (const Neighbor& neighbor : query_results) { + require(neighbor.id % 2 == 0, "GPU session filter 返回了不匹配候选"); + } + } + + memory_params.filter_source_type = 99; + const SearchResults cpu_empty = + cpuIvfFlatSearch(loaded, queries, memory_params, &metadata); + GpuIvfFlatSearchEngine empty_filter_engine(loaded, queries, memory_params, + &metadata); + const SearchResults gpu_empty = empty_filter_engine.search(queries); + require(resultsMatch(cpu_empty, gpu_empty, 0.0F, &error), + "过滤后不足 K 个候选时 CPU/GPU 结果不一致: " + error); + for (const auto& query_results : gpu_empty) { + require(query_results.empty(), "完全不匹配的过滤条件应返回空结果"); + } + + memory_params.memory_mode = "rerank"; + memory_params.filter_source_type = UINT32_MAX; + memory_params.memory_rerank_factor = 4; + const SearchResults cpu_rerank = + cpuIvfFlatSearch(loaded, queries, memory_params, &metadata); + GpuIvfFlatSearchEngine rerank_engine(loaded, queries, memory_params, + &metadata); + SearchStats rerank_stats; + const SearchResults gpu_rerank = rerank_engine.search(queries, &rerank_stats); + require(resultsMatch(cpu_rerank, gpu_rerank, 1e-5F, &error), + "独立记忆重排的 GPU IVF 与 CPU baseline 不一致: " + error); + require(rerank_stats.memory_rerank_ms >= 0.0, "独立记忆重排计时不应为负数"); +} + +} // namespace + +int main() { + const auto unique = + std::chrono::steady_clock::now().time_since_epoch().count(); + const std::filesystem::path directory = + std::filesystem::temp_directory_path() / + ("purplepulse-ivf-test-" + std::to_string(unique)); + try { + std::filesystem::create_directories(directory); + for (const DataType dtype : {DataType::kFloat32, DataType::kFloat16}) { + for (const Metric metric : + {Metric::kL2, Metric::kInnerProduct, Metric::kCosine}) { + testRoundTrip(metric, dtype, directory); + testLargeK(metric, dtype); + } + } + std::filesystem::remove_all(directory); + std::cout << "IVF-Flat 建库、持久化、查询与 recall 测试通过\n"; + return EXIT_SUCCESS; + } catch (const std::exception& error) { + std::filesystem::remove_all(directory); + std::cerr << "测试失败: " << error.what() << '\n'; + return EXIT_FAILURE; + } +} diff --git a/09_vector_retrieval/PurplePulse/tests/stability_tests.cpp b/09_vector_retrieval/PurplePulse/tests/stability_tests.cpp new file mode 100644 index 00000000..3bfad033 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/tests/stability_tests.cpp @@ -0,0 +1,317 @@ +#include +#include +#include +#include +#include +#include +#include +#include + +#include "ivf_flat.h" +#include "search.h" + +namespace { + +void require(bool condition, const std::string& message) { + if (!condition) { + throw std::runtime_error(message); + } +} + +template +void requireThrows(Operation operation, const std::string& message) { + try { + operation(); + } catch (const std::runtime_error&) { + return; + } + throw std::runtime_error(message); +} + +constexpr std::array kLevels = { + -1.0F, -0.75F, -0.5F, -0.25F, 0.0F, + 0.25F, 0.5F, 0.75F, 1.0F, +}; + +std::uint16_t exactHalf(float value) { + if (value == -1.0F) return 0xbc00; + if (value == -0.75F) return 0xba00; + if (value == -0.5F) return 0xb800; + if (value == -0.25F) return 0xb400; + if (value == 0.25F) return 0x3400; + if (value == 0.5F) return 0x3800; + if (value == 0.75F) return 0x3a00; + if (value == 1.0F) return 0x3c00; + return 0x0000; +} + +float deterministicValue(std::uint64_t row, std::uint32_t dimension) { + const std::uint64_t mixed = row * 37U + dimension * 17U + + (row / 3U) * 5U + dimension / 7U; + return kLevels[mixed % kLevels.size()]; +} + +void appendValue(std::vector* fp32, std::vector* fp16, + DataType dtype, float value) { + if (dtype == DataType::kFloat32) { + fp32->push_back(value); + } else { + fp16->push_back(exactHalf(value)); + } +} + +VectorDatabase makeCoverageDatabase(std::uint32_t dim, Metric metric, + DataType dtype, + std::uint64_t num_vectors = 107) { + VectorDatabase database; + database.num_vectors = num_vectors; + database.dim = dim; + database.metric = metric; + database.dtype = dtype; + database.values.reserve(dtype == DataType::kFloat32 ? num_vectors * dim : 0); + database.half_values.reserve(dtype == DataType::kFloat16 ? num_vectors * dim + : 0); + for (std::uint64_t row = 0; row < num_vectors; ++row) { + // Rows 0 and 1 are deliberately identical. This validates stable ID + // tie-breaking for every dimension, dtype, metric, batch, and K. + const std::uint64_t source_row = row == 1 ? 0 : row; + for (std::uint32_t d = 0; d < dim; ++d) { + appendValue(&database.values, &database.half_values, dtype, + deterministicValue(source_row, d)); + } + } + return database; +} + +QuerySet makeCoverageQueries(std::uint32_t dim, DataType dtype, + std::uint64_t num_queries = 9) { + QuerySet queries; + queries.num_queries = num_queries; + queries.dim = dim; + queries.dtype = dtype; + queries.values.reserve(dtype == DataType::kFloat32 ? num_queries * dim : 0); + queries.half_values.reserve(dtype == DataType::kFloat16 ? num_queries * dim + : 0); + for (std::uint64_t row = 0; row < num_queries; ++row) { + for (std::uint32_t d = 0; d < dim; ++d) { + // The first query is all zero, making every IP/cosine score tie. The + // second query exactly matches duplicate database rows 0 and 1. + const float value = row == 0 ? 0.0F : deterministicValue(row - 1, d); + appendValue(&queries.values, &queries.half_values, dtype, value); + } + } + return queries; +} + +void compareExact(const VectorDatabase& database, const QuerySet& queries, + const SearchParams& params, const std::string& label) { + const SearchResults cpu = cpuExactSearch(database, queries, params); + GpuExactSearchEngine engine(database, queries, params); + const SearchResults gpu = engine.search(queries); + std::string error; + require(resultsMatch(cpu, gpu, 5e-5F, &error), label + ": " + error); + if ((database.metric == Metric::kInnerProduct || + database.metric == Metric::kCosine) && + !gpu.empty()) { + const std::uint32_t expected = std::min( + params.top_k, static_cast(database.num_vectors)); + for (std::uint32_t rank = 0; rank < expected; ++rank) { + require(gpu[0][rank].id == rank, + label + ": equal-score candidates must use ascending IDs"); + } + } +} + +void testExactCoverageMatrix() { + constexpr std::array dimensions = {64, 128, 384, 768}; + constexpr std::array batches = {1, 8, 32, 40, 64, 128}; + constexpr std::array top_ks = {1, 10, 50, 100}; + std::uint32_t cases = 0; + for (const DataType dtype : {DataType::kFloat32, DataType::kFloat16}) { + for (const Metric metric : + {Metric::kL2, Metric::kInnerProduct, Metric::kCosine}) { + for (const std::uint32_t dim : dimensions) { + const VectorDatabase database = makeCoverageDatabase(dim, metric, dtype); + const QuerySet queries = makeCoverageQueries(dim, dtype); + for (const std::uint32_t top_k : top_ks) { + for (const std::uint32_t batch_size : batches) { + SearchParams params; + params.top_k = top_k; + params.batch_size = batch_size; + params.distance_mode = "warp"; + params.topk_mode = "fused"; + compareExact(database, queries, params, + "Exact coverage dim=" + std::to_string(dim) + + " batch=" + std::to_string(batch_size) + + " K=" + std::to_string(top_k)); + ++cases; + } + } + } + } + } + require(cases == 576, "Exact coverage matrix case count changed"); + std::cout << "Exact coverage matrix: " << cases << " cases\n"; +} + +void testBatchCapacityAndNonWarpDimension() { + constexpr std::array batches = {1, 8, 32, 40, 64, 128}; + for (const std::uint32_t batch_size : batches) { + const VectorDatabase database = + makeCoverageDatabase(65, Metric::kL2, DataType::kFloat32, 101); + const QuerySet queries = + makeCoverageQueries(65, DataType::kFloat32, batch_size + 1U); + SearchParams params; + params.top_k = 10; + params.batch_size = batch_size; + params.distance_mode = "warp"; + params.topk_mode = "fused"; + compareExact(database, queries, params, + "non-warp dim=65 full batch=" + std::to_string(batch_size)); + } + + for (const DataType dtype : {DataType::kFloat32, DataType::kFloat16}) { + for (const Metric metric : + {Metric::kL2, Metric::kInnerProduct, Metric::kCosine}) { + const VectorDatabase database = makeCoverageDatabase(65, metric, dtype); + const QuerySet queries = makeCoverageQueries(65, dtype); + SearchParams params; + params.top_k = 100; + params.batch_size = 8; + params.distance_mode = "warp"; + params.topk_mode = "fused"; + compareExact(database, queries, params, + "non-warp dim=65 dtype/metric coverage"); + } + } +} + +void testExactRecommendedAndFallbackPaths() { + struct RecommendedCase { + std::uint32_t dim; + std::uint32_t top_k; + std::uint32_t query_tile; + std::uint32_t blocks; + }; + constexpr std::array cases = { + RecommendedCase{128, 10, 4, 64}, + RecommendedCase{384, 10, 8, 160}, + RecommendedCase{384, 50, 4, 64}, + }; + for (const RecommendedCase& item : cases) { + const VectorDatabase database = makeCoverageDatabase( + item.dim, Metric::kCosine, DataType::kFloat32); + const QuerySet queries = makeCoverageQueries(item.dim, DataType::kFloat32); + SearchParams params; + params.top_k = item.top_k; + params.batch_size = 64; + params.distance_mode = "warp"; + params.topk_mode = "fused"; + params.exact_query_tile = item.query_tile; + params.exact_fused_blocks = item.blocks; + compareExact(database, queries, params, + "recommended query-tile dim=" + std::to_string(item.dim) + + " K=" + std::to_string(item.top_k)); + } + + VectorDatabase database = makeCoverageDatabase( + 768, Metric::kCosine, DataType::kFloat32); + QuerySet queries = makeCoverageQueries(768, DataType::kFloat32); + SearchParams params; + params.top_k = 10; + params.batch_size = 64; + params.distance_mode = "warp"; + params.topk_mode = "fused"; + params.exact_query_tile = 8; + params.exact_fused_layout = "block_tile"; + requireThrows([&] { (void)GpuExactSearchEngine(database, queries, params); }, + "FP32 dim=768 block-tile must fall back instead of launching"); + + params.exact_fused_layout = "warp_query"; + params.exact_query_tile = 1; + compareExact(database, queries, params, + "FP32 dim=768 safe warp-query fallback"); + + params.top_k = 50; + params.exact_query_tile = 8; + requireThrows([&] { (void)GpuExactSearchEngine(database, queries, params); }, + "K=50 query-tile=8 must be rejected"); + params.exact_query_tile = 1; + compareExact(database, queries, params, + "K=50 query-tile=1 safe fallback"); +} + +IvfFlatIndex makePathologicalIndex(Metric metric, DataType dtype) { + constexpr std::uint64_t num_vectors = 128; + constexpr std::uint32_t dim = 65; + constexpr std::uint32_t nlist = 8; + const VectorDatabase database = + makeCoverageDatabase(dim, metric, dtype, num_vectors); + IvfFlatIndex index; + index.num_vectors = num_vectors; + index.dim = dim; + index.nlist = nlist; + index.dtype = dtype; + index.metric = metric; + index.offsets = {0, 0, 1, 1, 3, 3, 3, 3, num_vectors}; + index.ids.resize(num_vectors); + for (std::uint64_t id = 0; id < num_vectors; ++id) { + index.ids[id] = id; + } + index.values = database.values; + index.half_values = database.half_values; + index.centers.resize(static_cast(nlist) * dim); + for (std::uint32_t center = 0; center < nlist; ++center) { + for (std::uint32_t d = 0; d < dim; ++d) { + index.centers[static_cast(center) * dim + d] = + deterministicValue(center + 11U, d); + } + } + validateIvfFlatIndex(index); + return index; +} + +void testEmptyAndSkewedIvfBuckets() { + for (const DataType dtype : {DataType::kFloat32, DataType::kFloat16}) { + for (const Metric metric : + {Metric::kL2, Metric::kInnerProduct, Metric::kCosine}) { + const IvfFlatIndex index = makePathologicalIndex(metric, dtype); + const QuerySet queries = makeCoverageQueries(65, dtype); + for (const std::uint32_t top_k : {1U, 10U, 50U, 100U}) { + SearchParams params; + params.search_mode = "ivf_flat"; + params.top_k = top_k; + params.batch_size = 8; + params.nlist = index.nlist; + params.nprobe = index.nlist; + params.distance_mode = "warp"; + const SearchResults cpu = cpuIvfFlatSearch(index, queries, params); + std::string error; + for (const std::string& mode : {"warp", "bucket_major"}) { + params.distance_mode = mode; + GpuIvfFlatSearchEngine engine(index, queries, params); + const SearchResults gpu = engine.search(queries); + require(resultsMatch(cpu, gpu, 5e-5F, &error), + "empty/skewed IVF " + mode + ": " + error); + } + } + } + } + std::cout << "IVF empty/skewed buckets: 48 cases\n"; +} + +} // namespace + +int main() { + try { + testExactCoverageMatrix(); + testBatchCapacityAndNonWarpDimension(); + testExactRecommendedAndFallbackPaths(); + testEmptyAndSkewedIvfBuckets(); + std::cout << "Stage 8 stability coverage passed\n"; + return EXIT_SUCCESS; + } catch (const std::exception& error) { + std::cerr << "Stage 8 stability failure: " << error.what() << '\n'; + return EXIT_FAILURE; + } +} diff --git a/09_vector_retrieval/PurplePulse/tests/test_benchmark_adaptive_nprobe.py b/09_vector_retrieval/PurplePulse/tests/test_benchmark_adaptive_nprobe.py new file mode 100644 index 00000000..436d1267 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/tests/test_benchmark_adaptive_nprobe.py @@ -0,0 +1,29 @@ +#!/usr/bin/env python3 + +import importlib.util +import unittest +from pathlib import Path + + +SCRIPT = Path(__file__).parents[1] / "scripts" / "benchmark_adaptive_nprobe.py" +SPEC = importlib.util.spec_from_file_location("benchmark_adaptive_nprobe", SCRIPT) +MODULE = importlib.util.module_from_spec(SPEC) +assert SPEC.loader is not None +SPEC.loader.exec_module(MODULE) + + +class BenchmarkAdaptiveNprobeTests(unittest.TestCase): + def test_extracts_actual_probe_triple(self): + values = MODULE.extract_triple( + "实际 nprobe min/avg/max: 64/137.25/224", + r"实际 nprobe min/avg/max: ([0-9.eE+-]+)/([0-9.eE+-]+)/([0-9.eE+-]+)", + ) + self.assertEqual(values, (64.0, 137.25, 224.0)) + + def test_recall(self): + mean, minimum = MODULE.recall([[1, 2], [3, 4]], [[1, 9], [4, 3]]) + self.assertEqual((mean, minimum), (0.75, 0.5)) + + +if __name__ == "__main__": + unittest.main() diff --git a/09_vector_retrieval/PurplePulse/tests/test_benchmark_exact_breakdown.py b/09_vector_retrieval/PurplePulse/tests/test_benchmark_exact_breakdown.py new file mode 100644 index 00000000..543a7d28 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/tests/test_benchmark_exact_breakdown.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python3 + +import importlib.util +import tempfile +import unittest +from pathlib import Path + + +ROOT = Path(__file__).parents[1] +SPEC = importlib.util.spec_from_file_location( + "benchmark_exact_breakdown", ROOT / "scripts" / "benchmark_exact_breakdown.py" +) +MODULE = importlib.util.module_from_spec(SPEC) +assert SPEC.loader is not None +SPEC.loader.exec_module(MODULE) + + +class BenchmarkExactBreakdownTests(unittest.TestCase): + def test_parses_breakdown_and_shares(self): + text = """\ +向量库: 1000000 x 384 +查询数: 1000 +top_k: 50 +batch_size: 64 +GPU 缓冲区: 1005.2 MiB +平均常驻查询时间: 300.0 ms +run P50: 299.0 ms +run P99: 305.0 ms +常驻 QPS: 3333.3 +剖析距离-only kernel: 100.0 ms +剖析局部 Top-K 估算: 150.0 ms +剖析融合 kernel: 250.0 ms +剖析最终归并 kernel: 10.0 ms +""" + row = MODULE.parse_profile_output(text) + self.assertEqual(row["top_k"], 50) + self.assertEqual(row["kernel_pipeline_ms"], 260.0) + self.assertAlmostEqual(row["distance_share"], 100.0 / 260.0) + self.assertAlmostEqual(row["local_topk_share"], 150.0 / 260.0) + + def test_compares_reference_results(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + first = root / "first.txt" + second = root / "second.txt" + first.write_text("0 7 1.25\n0 8 1.0\n") + second.write_text("0 7 1.25001\n0 8 1.0\n") + MODULE.compare_results(first, second, 2e-5) + + def test_rejects_candidate_mismatch(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + first = root / "first.txt" + second = root / "second.txt" + first.write_text("0 7 1.25\n") + second.write_text("0 8 1.25\n") + with self.assertRaises(ValueError): + MODULE.compare_results(first, second, 1e-5) + + def test_rejects_rank_order_mismatch(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + first = root / "first.txt" + second = root / "second.txt" + first.write_text("0 7 1.25\n0 8 1.0\n") + second.write_text("0 8 1.0\n0 7 1.25\n") + with self.assertRaises(ValueError): + MODULE.compare_results(first, second, 1e-5) + + +if __name__ == "__main__": + unittest.main() diff --git a/09_vector_retrieval/PurplePulse/tests/test_benchmark_exact_query_tile.py b/09_vector_retrieval/PurplePulse/tests/test_benchmark_exact_query_tile.py new file mode 100644 index 00000000..3db35588 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/tests/test_benchmark_exact_query_tile.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python3 + +import importlib.util +import sys +import unittest +from pathlib import Path + + +ROOT = Path(__file__).parents[1] +sys.path.insert(0, str(ROOT / "scripts")) +SPEC = importlib.util.spec_from_file_location( + "benchmark_exact_query_tile", ROOT / "scripts" / "benchmark_exact_query_tile.py" +) +MODULE = importlib.util.module_from_spec(SPEC) +assert SPEC.loader is not None +SPEC.loader.exec_module(MODULE) + + +class BenchmarkExactQueryTileTests(unittest.TestCase): + def test_parses_variant(self): + label, configs = MODULE.parse_variant( + f"tile2=10:{ROOT / 'README.md'},50:{ROOT / 'CMakeLists.txt'}" + ) + self.assertEqual(label, "tile2") + self.assertEqual([top_k for top_k, _ in configs], [10, 50]) + + def test_rejects_unsafe_label(self): + with self.assertRaises(ValueError): + MODULE.parse_variant(f"../tile=10:{ROOT / 'README.md'}") + + def test_parses_search_output(self): + text = """\ +向量库: 1000000 x 128 +查询数: 1000 +top_k: 10 +batch_size: 64 +exact_query_tile: 4 +exact_fused_layout: block_tile +exact_fused_blocks: 128 +GPU 缓冲区: 488.4 MiB +平均常驻查询时间: 50.0 ms +run P50: 49.0 ms +run P99: 52.0 ms +常驻 QPS: 20000.0 +平均融合距离+局部 Top-K kernel: 48.0 ms +平均最终 Top-K kernel: 0.2 ms +""" + row = MODULE.parse_search_output(text, "tile4") + self.assertEqual(row["exact_query_tile"], 4) + self.assertEqual(row["exact_fused_layout"], "block_tile") + self.assertEqual(row["exact_fused_blocks"], 128) + self.assertEqual(row["qps"], 20000.0) + self.assertEqual(row["fused_kernel_ms"], 48.0) + + +if __name__ == "__main__": + unittest.main() diff --git a/09_vector_retrieval/PurplePulse/tests/test_benchmark_faiss.py b/09_vector_retrieval/PurplePulse/tests/test_benchmark_faiss.py new file mode 100644 index 00000000..ef82d465 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/tests/test_benchmark_faiss.py @@ -0,0 +1,94 @@ +#!/usr/bin/env python3 + +import importlib.util +import struct +import sys +import tempfile +import unittest +from pathlib import Path + +import numpy as np + + +SCRIPT = Path(__file__).parents[1] / "scripts" / "benchmark_faiss.py" +SPEC = importlib.util.spec_from_file_location("benchmark_faiss", SCRIPT) +MODULE = importlib.util.module_from_spec(SPEC) +assert SPEC.loader is not None +sys.modules[SPEC.name] = MODULE +SPEC.loader.exec_module(MODULE) + + +class BenchmarkFaissTests(unittest.TestCase): + def test_reads_custom_vector_files_as_memmap(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + database = root / "database.bin" + queries = root / "queries.bin" + values = np.arange(12, dtype=" None: + value_dtype = " None: + with path.open("w", newline="") as destination: + writer = csv.DictWriter(destination, fieldnames=fieldnames) + writer.writeheader() + writer.writerows(rows) + + +class PlotResultsTests(unittest.TestCase): + def test_generates_all_figures_from_minimal_fixture(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + results = root / "results" + output = root / "figures" + results.mkdir() + + quality_fields = ["nprobe", "average_ms", "qps", "recall_at_k"] + for top_k in (10, 50, 100): + write_csv( + results / f"ivf_k{top_k}.csv", + quality_fields, + [ + { + "nprobe": 16, + "average_ms": 2.0 + top_k / 100.0, + "qps": 5000 - top_k, + "recall_at_k": 0.9, + }, + { + "nprobe": 32, + "average_ms": 3.0 + top_k / 100.0, + "qps": 4000 - top_k, + "recall_at_k": 0.97, + }, + ], + ) + + faiss_fields = [ + "backend", + "index_type", + "top_k", + "nprobe", + "average_query_ms", + "qps", + "recall_at_k", + ] + write_csv( + results / "faiss_gpu.csv", + faiss_fields, + [ + { + "backend": "gpu", + "index_type": "ivf_flat", + "top_k": top_k, + "nprobe": nprobe, + "average_query_ms": 2.5 if nprobe == 16 else 3.5, + "qps": 4500 if nprobe == 16 else 3500, + "recall_at_k": 0.89 if nprobe == 16 else 0.96, + } + for top_k in (10, 50, 100) + for nprobe in (16, 32) + ], + ) + + batch_fields = ["top_k", "batch_size", "qps"] + write_csv( + results / "optimized_v1_batch_sweep.csv", + batch_fields, + [ + {"top_k": top_k, "batch_size": batch, "qps": 1000 + batch} + for top_k in (10, 50, 100) + for batch in (8, 64) + ], + ) + + paths = MODULE.generate_figures(results, output) + + self.assertEqual( + {path.name for path in paths}, + {"recall-latency.svg", "recall-qps.svg", "batch-qps.svg"}, + ) + for path in paths: + contents = path.read_text() + self.assertIn("", contents) + + +if __name__ == "__main__": + unittest.main() diff --git a/09_vector_retrieval/PurplePulse/tests/test_stage6_tools.py b/09_vector_retrieval/PurplePulse/tests/test_stage6_tools.py new file mode 100644 index 00000000..afe2ad84 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/tests/test_stage6_tools.py @@ -0,0 +1,87 @@ +#!/usr/bin/env python3 + +import importlib.util +import csv +import sys +import tempfile +import unittest +from pathlib import Path + + +ROOT = Path(__file__).parents[1] +sys.path.insert(0, str(ROOT / "scripts")) + + +def load_module(name: str, filename: str): + spec = importlib.util.spec_from_file_location(name, ROOT / "scripts" / filename) + module = importlib.util.module_from_spec(spec) + assert spec.loader is not None + spec.loader.exec_module(module) + return module + + +EXACT = load_module("benchmark_exact_stage6", "benchmark_exact.py") +RUNNER = load_module("run_real_benchmarks_stage6", "run_real_benchmarks.py") +PREPARE = load_module("prepare_real_datasets_stage6", "prepare_real_datasets.py") + + +class Stage6ToolTests(unittest.TestCase): + def test_partitioned_result_falls_back_on_recall_loss(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + fields = ["nprobe", "recall_at_k", "qps"] + for name, recall, qps in ( + ("partitioned.csv", 0.89, 1200), + ("full.csv", 0.90, 1000), + ): + with (root / name).open("w", newline="") as output: + writer = csv.DictWriter(output, fieldnames=fields) + writer.writeheader() + writer.writerow({"nprobe": 64, "recall_at_k": recall, "qps": qps}) + decision = RUNNER.choose_partitioned_result( + root / "partitioned.csv", + root / "full.csv", + root / "effective.csv", + maximum_recall_loss=1e-4, + ) + self.assertEqual(decision["selected_mode"], "full_heap") + self.assertEqual( + (root / "effective.csv").read_text(), + (root / "full.csv").read_text(), + ) + + def test_exact_config_parser_and_output_parser(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + first = root / "k10.conf" + second = root / "k100.conf" + first.write_text("top_k = 10\n") + second.write_text("top_k = 100\n") + self.assertEqual( + EXACT.parse_configs(f"10:{first},100:{second}"), + [(10, first), (100, second)], + ) + self.assertEqual(EXACT.extract("常驻 QPS: 1234.5", r"常驻 QPS: ([0-9.]+)"), 1234.5) + + def test_build_metadata_parser(self): + output = """\ +迭代/训练样本: 15/100000 +建库线程: 16 +构建时间: 1234.5 ms +保存时间: 67.8 ms +桶大小 min/avg/max: 10/1000.25/2048 +空桶: 3 +索引大小: 987654 bytes +""" + row = RUNNER.parse_build_stats(output, "fixture", 1024) + self.assertEqual(row["training_iterations"], 15) + self.assertEqual(row["maximum_bucket"], 2048) + self.assertEqual(row["index_bytes"], 987654) + + def test_known_public_checksums_are_pinned(self): + self.assertEqual(len(PREPARE.SIFT_MD5), 32) + self.assertEqual(len(PREPARE.VIBE_SHA256), 64) + + +if __name__ == "__main__": + unittest.main() diff --git a/09_vector_retrieval/PurplePulse/tests/test_stage9_tools.py b/09_vector_retrieval/PurplePulse/tests/test_stage9_tools.py new file mode 100644 index 00000000..fe3812ce --- /dev/null +++ b/09_vector_retrieval/PurplePulse/tests/test_stage9_tools.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python3 + +import importlib.util +import sys +import tempfile +import unittest +from pathlib import Path + + +ROOT = Path(__file__).parents[1] +sys.path.insert(0, str(ROOT / "scripts")) + + +def load_module(name: str, filename: str): + spec = importlib.util.spec_from_file_location(name, ROOT / "scripts" / filename) + module = importlib.util.module_from_spec(spec) + assert spec.loader is not None + spec.loader.exec_module(module) + return module + + +REPRODUCE = load_module("stage9_reproduce", "reproduce.py") +VALIDATE = load_module("stage9_validate", "validate_release.py") + + +class Stage9ToolTests(unittest.TestCase): + def test_reproduction_plan_is_project_rooted_and_complete(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory).resolve() + args = REPRODUCE.parse_args([ + "--project-root", str(root), + "--dataset", "sift1m", + "--skip-download", + "--dry-run", + ]) + state_root, steps = REPRODUCE.build_plan(args) + names = [step["name"] for step in steps] + self.assertEqual( + names, + ["configure", "build", "test", "prepare", + "benchmark:sift1m", "plot:sift1m", "report", "validate"], + ) + self.assertTrue(str(state_root).startswith(str(root))) + for step in steps: + self.assertTrue(str(step["log"]).startswith(str(root))) + + def test_checkpoint_round_trip(self): + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "checkpoint.json" + state = {"completed": ["configure"], "steps": {}} + REPRODUCE.write_checkpoint(path, state) + self.assertEqual(REPRODUCE.load_checkpoint(path), state) + + def test_csv_and_pdf_validation_helpers(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + csv_path = root / "rows.csv" + csv_path.write_text("name,status\nfixture,passed\n", encoding="utf-8") + self.assertEqual( + VALIDATE.validate_csv(csv_path, {"name", "status"}), 1 + ) + pdf_path = root / "report.pdf" + pdf_path.write_bytes(b"%PDF-1.4\n" + b"0" * 2048) + self.assertGreater(VALIDATE.validate_pdf(pdf_path), 1024) + + +if __name__ == "__main__": + unittest.main() diff --git a/09_vector_retrieval/PurplePulse/tests/unit_tests.cpp b/09_vector_retrieval/PurplePulse/tests/unit_tests.cpp new file mode 100644 index 00000000..a3055391 --- /dev/null +++ b/09_vector_retrieval/PurplePulse/tests/unit_tests.cpp @@ -0,0 +1,365 @@ +#include +#include +#include +#include +#include + +#include "file_io.h" +#include "search.h" + +namespace { + +void require(bool condition, const std::string& message) { + if (!condition) { + throw std::runtime_error(message); + } +} + +template +void requireThrows(Operation operation, const std::string& message) { + try { + operation(); + } catch (const std::runtime_error&) { + return; + } + throw std::runtime_error(message); +} + +VectorDatabase makeDatabase(Metric metric, DataType dtype) { + VectorDatabase database; + database.num_vectors = 5; + database.dim = 2; + database.metric = metric; + database.dtype = dtype; + if (dtype == DataType::kFloat32) { + database.values = { + 1.0F, 0.0F, // id 0 + 0.0F, 1.0F, // id 1 + 1.0F, 1.0F, // id 2 + 2.0F, 0.0F, // id 3 + -1.0F, 0.0F // id 4 + }; + } else { + // IEEE 754 binary16: 0=0x0000, 1=0x3c00, 2=0x4000, -1=0xbc00。 + database.half_values = {0x3c00, 0x0000, 0x0000, 0x3c00, 0x3c00, + 0x3c00, 0x4000, 0x0000, 0xbc00, 0x0000}; + } + return database; +} + +QuerySet makeQueries(DataType dtype) { + QuerySet queries; + queries.num_queries = 2; + queries.dim = 2; + queries.dtype = dtype; + if (dtype == DataType::kFloat32) { + queries.values = {1.0F, 0.0F, 0.0F, 1.0F}; + } else { + queries.half_values = {0x3c00, 0x0000, 0x0000, 0x3c00}; + } + return queries; +} + +void testMetric(Metric metric, DataType dtype) { + const VectorDatabase database = makeDatabase(metric, dtype); + const QuerySet queries = makeQueries(dtype); + SearchParams params; + params.top_k = 3; + params.batch_size = 2; + + const SearchResults cpu = cpuExactSearch(database, queries, params); + params.topk_mode = "simple"; + const SearchResults gpu_simple = gpuExactSearch(database, queries, params); + params.topk_mode = "block"; + const SearchResults gpu_block = gpuExactSearch(database, queries, params); + params.topk_mode = "two_stage"; + const SearchResults gpu_two_stage = gpuExactSearch(database, queries, params); + params.topk_mode = "fused"; + const SearchResults gpu_fused = gpuExactSearch(database, queries, params); + params.exact_query_tile = 2; + const SearchResults gpu_fused_tile2 = + gpuExactSearch(database, queries, params); + params.exact_query_tile = 4; + const SearchResults gpu_fused_tile4 = + gpuExactSearch(database, queries, params); + params.exact_query_tile = 8; + const SearchResults gpu_fused_tile8 = + gpuExactSearch(database, queries, params); + params.exact_fused_layout = "block_tile"; + const SearchResults gpu_fused_block_tile = + gpuExactSearch(database, queries, params); + params.exact_fused_layout = "warp_query"; + params.exact_query_tile = 1; + std::string error; + require(resultsMatch(cpu, gpu_simple, 1e-5F, &error), + "simple Top-K: " + error); + require(resultsMatch(cpu, gpu_block, 1e-5F, &error), "block Top-K: " + error); + require(resultsMatch(cpu, gpu_two_stage, 1e-5F, &error), + "two-stage Top-K: " + error); + require(resultsMatch(cpu, gpu_fused, 1e-5F, &error), "fused Top-K: " + error); + require(resultsMatch(cpu, gpu_fused_tile2, 1e-5F, &error), + "fused query-tile=2 Top-K: " + error); + require(resultsMatch(cpu, gpu_fused_tile4, 1e-5F, &error), + "fused query-tile=4 Top-K: " + error); + require(resultsMatch(cpu, gpu_fused_tile8, 1e-5F, &error), + "fused query-tile=8 Top-K: " + error); + require(resultsMatch(cpu, gpu_fused_block_tile, 1e-5F, &error), + "fused block-tile Top-K: " + error); + + QuerySet initial_queries = queries; + initial_queries.num_queries = 1; + if (dtype == DataType::kFloat32) { + initial_queries.values.resize(initial_queries.dim); + } else { + initial_queries.half_values.resize(initial_queries.dim); + } + // 首次校验只传一个 query，引擎仍应按 params.batch_size 分配， + // 并在后续查询中处理更多 query。 + GpuExactSearchEngine resident_engine(database, initial_queries, params); + SearchStats first_stats; + SearchStats second_stats; + const SearchResults resident_first = + resident_engine.search(queries, &first_stats); + const SearchResults resident_second = + resident_engine.search(queries, &second_stats); + require(resultsMatch(cpu, resident_first, 1e-5F, &error), + "常驻引擎首次查询: " + error); + require(resultsMatch(resident_first, resident_second, 0.0F, &error), + "常驻引擎重复查询: " + error); + require(resident_engine.databaseH2DMilliseconds() >= 0.0 && + resident_engine.deviceBytes() > 0, + "数据库 H2D 计时不应为负数且设备缓冲区应非空"); + require( + first_stats.database_h2d_ms == 0.0 && second_stats.database_h2d_ms == 0.0, + "常驻查询不应重复计入数据库 H2D"); + const double distance_only_ms = + resident_engine.profileDistanceOnlyMilliseconds(queries, 0, 1); + require(distance_only_ms >= 0.0, + "fused distance-only 差分剖析时间不应为负数"); + + require(cpu.size() == 2 && cpu[0].size() == 3, "结果形状不正确"); + if (metric == Metric::kL2 || metric == Metric::kCosine) { + require(cpu[0][0].id == 0, "L2 第一名应当是 id 0"); + } else { + require(cpu[0][0].id == 3, "内积第一名应当是 id 3"); + } +} + +void testBoundaryCases() { + VectorDatabase database; + database.num_vectors = 5; + database.dim = 2; + database.metric = Metric::kCosine; + database.dtype = DataType::kFloat32; + database.values = { + 0.0F, 0.0F, // id 0: zero vector + 1.0F, 0.0F, // id 1 + 1.0F, 0.0F, // id 2: tie with id 1 + 0.0F, 1.0F, // id 3 + -1.0F, 0.0F // id 4 + }; + + QuerySet queries; + queries.num_queries = 5; // 故意不能被 batch_size=2 整除。 + queries.dim = 2; + queries.dtype = DataType::kFloat32; + queries.values = { + 0.0F, 0.0F, // zero query: every score is zero + 1.0F, 0.0F, 0.0F, 1.0F, -1.0F, 0.0F, 1.0F, 1.0F, + }; + + SearchParams params; + params.top_k = static_cast(database.num_vectors); + params.batch_size = 2; + const SearchResults cpu = cpuExactSearch(database, queries, params); + require(cpu.size() == queries.num_queries && + cpu.front().size() == database.num_vectors, + "top_k=num_vectors 的结果形状不正确"); + for (std::uint64_t id = 0; id < database.num_vectors; ++id) { + require(cpu[0][id].id == id && cpu[0][id].score == 0.0F, + "零向量 cosine 应按较小 ID 打破平局"); + } + require(cpu[1][0].id == 1 && cpu[1][1].id == 2, + "相同 cosine 分数应按较小 ID 打破平局"); + + std::string error; + for (const std::string& distance_mode : {"simple", "warp"}) { + params.distance_mode = distance_mode; + for (const std::string& topk_mode : + {"simple", "block", "two_stage", "fused"}) { + if (topk_mode == "fused" && distance_mode != "warp") { + continue; + } + params.topk_mode = topk_mode; + const SearchResults gpu = gpuExactSearch(database, queries, params); + require(resultsMatch(cpu, gpu, 1e-5F, &error), + distance_mode + "/" + topk_mode + " 边界测试: " + error); + } + } +} + +VectorDatabase makeLargeKDatabase(DataType dtype) { + VectorDatabase database; + database.num_vectors = 128; + database.dim = 4; + database.metric = Metric::kL2; + database.dtype = dtype; + const std::uint16_t positive_one = 0x3c00; + const std::uint16_t negative_one = 0xbc00; + for (std::uint64_t vector_id = 0; vector_id < database.num_vectors; + ++vector_id) { + for (std::uint32_t d = 0; d < database.dim; ++d) { + const bool positive = ((vector_id >> d) & 1U) != 0; + if (dtype == DataType::kFloat32) { + database.values.push_back(positive ? 1.0F : -1.0F); + } else { + database.half_values.push_back(positive ? positive_one : negative_one); + } + } + } + return database; +} + +QuerySet makeLargeKQueries(DataType dtype) { + QuerySet queries; + queries.num_queries = 3; + queries.dim = 4; + queries.dtype = dtype; + const std::vector values = {1.0F, 1.0F, 1.0F, 1.0F, -1.0F, 1.0F, + -1.0F, 1.0F, 1.0F, -1.0F, 1.0F, -1.0F}; + if (dtype == DataType::kFloat32) { + queries.values = values; + } else { + for (const float value : values) { + queries.half_values.push_back(value > 0.0F ? 0x3c00 : 0xbc00); + } + } + return queries; +} + +void testFusedLargeK() { + for (const DataType dtype : {DataType::kFloat32, DataType::kFloat16}) { + const VectorDatabase database = makeLargeKDatabase(dtype); + const QuerySet queries = makeLargeKQueries(dtype); + for (const std::uint32_t top_k : {50U, 100U}) { + SearchParams params; + params.top_k = top_k; + params.batch_size = 2; + params.distance_mode = "warp"; + params.topk_mode = "fused"; + const SearchResults cpu = cpuExactSearch(database, queries, params); + GpuExactSearchEngine fused_engine(database, queries, params); + const SearchResults fused = fused_engine.search(queries); + std::string error; + require(resultsMatch(cpu, fused, 1e-5F, &error), + "fused large-K: " + error); + } + } +} + +void testFusedDeviceMemory() { + VectorDatabase database; + database.num_vectors = 10000; + database.dim = 1; + database.metric = Metric::kL2; + database.dtype = DataType::kFloat32; + database.values.resize(database.num_vectors, 0.0F); + + QuerySet queries; + queries.num_queries = 1; + queries.dim = 1; + queries.dtype = DataType::kFloat32; + queries.values = {0.0F}; + + SearchParams params; + params.top_k = 100; + params.batch_size = 64; + params.distance_mode = "warp"; + params.topk_mode = "block"; + GpuExactSearchEngine matrix_engine(database, queries, params); + params.topk_mode = "fused"; + GpuExactSearchEngine fused_engine(database, queries, params); + require(fused_engine.deviceBytes() < matrix_engine.deviceBytes(), + "规模化 fused exact 应减少设备缓冲区占用"); +} + +void testInvalidInputs() { + VectorDatabase database = makeDatabase(Metric::kL2, DataType::kFloat32); + QuerySet queries = makeQueries(DataType::kFloat32); + SearchParams params; + + params.top_k = 0; + requireThrows([&] { validateInputs(database, queries, params); }, + "top_k=0 应被拒绝"); + params.top_k = 1; + params.batch_size = 0; + requireThrows([&] { validateInputs(database, queries, params); }, + "batch_size=0 应被拒绝"); + params.batch_size = 1; + + params.distance_mode = "simple"; + params.topk_mode = "fused"; + requireThrows([&] { validateInputs(database, queries, params); }, + "fused Top-K 与 simple distance 组合应被拒绝"); + params.distance_mode = "warp"; + params.topk_mode = "two_stage"; + + params.exact_query_tile = 3; + requireThrows([&] { validateInputs(database, queries, params); }, + "不支持的 exact_query_tile 应被拒绝"); + params.exact_query_tile = 2; + requireThrows([&] { validateInputs(database, queries, params); }, + "非 fused Top-K 不应接受 query tile"); + params.exact_query_tile = 1; + params.exact_fused_blocks = 32; + requireThrows([&] { validateInputs(database, queries, params); }, + "非 fused Top-K 不应接受 exact_fused_blocks"); + params.exact_fused_blocks = 0; + params.topk_mode = "fused"; + params.exact_fused_layout = "invalid"; + requireThrows([&] { validateInputs(database, queries, params); }, + "未知 exact_fused_layout 应被拒绝"); + params.exact_fused_layout = "block_tile"; + requireThrows([&] { validateInputs(database, queries, params); }, + "block_tile 不应接受 query tile 1"); + params.exact_fused_layout = "warp_query"; + params.topk_mode = "two_stage"; + + VectorDatabase empty_database = database; + empty_database.num_vectors = 0; + empty_database.values.clear(); + requireThrows([&] { validateInputs(empty_database, queries, params); }, + "空向量库应被拒绝"); + + QuerySet empty_queries = queries; + empty_queries.num_queries = 0; + empty_queries.values.clear(); + requireThrows([&] { validateInputs(database, empty_queries, params); }, + "空查询集应被拒绝"); + + QuerySet wrong_length = queries; + wrong_length.values.pop_back(); + requireThrows([&] { validateInputs(database, wrong_length, params); }, + "数据长度与元数据不一致时应被拒绝"); +} + +} // namespace + +int main() { + try { + for (const DataType dtype : {DataType::kFloat32, DataType::kFloat16}) { + testMetric(Metric::kL2, dtype); + testMetric(Metric::kInnerProduct, dtype); + testMetric(Metric::kCosine, dtype); + } + testBoundaryCases(); + testFusedLargeK(); + testFusedDeviceMemory(); + testInvalidInputs(); + std::cout << "FP32/FP16 的 CPU/GPU 正确性与边界测试通过\n"; + return EXIT_SUCCESS; + } catch (const std::exception& error) { + std::cerr << "测试失败: " << error.what() << '\n'; + return EXIT_FAILURE; + } +}

BPUWJ=_7&l?Sp}0|zm@P)ylVB>l#I(D z+gtzaOZr{LV9Cd_Hjm@Gj)u)seHa&S3bmkPW3xaqxtP#AZlA7;i_(ucDY0# zF)P%f+GQl-R)7w*3VoNWV8ctxGBc4>5-rz5h90f7|^GB?!He+0(9ys_o=bm675 zK)XF1up}Ge7_iV*l>~Z_4{EkF)q5RnH^>czbe)>Jv zq&gPI`hrq07Td>$R#`H-Sq8% zu2bV$Xi}zo;q{oW%eHCe?SQdH6L<}`UOBBon$h=CA*R;MCrdS z(FH|Iu%{2`TAal94`(!qy$k_HI16ej19c_Ds^D&eWIs&WA>&s~`P(4XDY$8nkYU8{}p;|s$HjyFU{`V4|CQSh37-1_-jZ_)qu z;~zg)SuB;8N;Ahl(dAOCBX7NzrYkcZ&DZLlt!#cO5OWLWK*6u_EPwtdFT61GH2OF5 z!V9YZvE#}s>>sjSRFi`SFXJH~K2v9&3*hkg@GOk_!9Gc8eVtZ_vcxay{ zoLAS7YL|eNT$VJXB$iUTRMn-LE)DC_9J<%jCj;H(@k z@?1&I*!ElFe!9<*79EmNb~?(6lvb)U^c$h2Om}uM+VW*<_IT*pr;#=kvCwvtvFDZ3 zZpDg)S+rPb zn>|)gnY$ex(ZWi(n9DDu3s!oHW%rv*{$`GTCAL2sD=g<)Q+5}}+0ZHyP#PZ5WytJ( zCcZOXiB3l8dr}3-XCC>;4#5HFUzVF)k$qX8Eh_Iq^VVTnwI=CryL^(}Xp?1tCOd4e zk>(?4|BL*53VbVxoydOcy7CnJdFCfb7w@1ZfK(?(zz;bpjQeGRKu0|lEh`Y`^8p7{ zo=Oltk}8ojrU+J{0SvbFmB_ZAku_rw$LE)2bL zTu3Sq_k9m234Jf07Rt)Ue|m3WZhR=xlNcNt&(Yt4#5wfuATp0*6Ny%Lu#m|pN+weX z(z6qnr#&6Q%uiI?=w76|o@Rbc+0nech!)U)fLS_A0Hhd}yg)#NvN|_lVhrp<0j+kK zTF~bQErW7F)GB{qCS;hM2Z^jX_}lsQ;EuatrUthR+`y-%?U3L$tui+e#%jq?l^jNVufBI)k(5lKk&*;{n`> zc{g`zVc}E`|9zHbXhV3Qi&K<*&1>BJtHVYkYEpVuZdzHnX?M>b_zu--M|@!U_z%}= zYkt7heeQGgkKy-PGw{6OE4D)SaCGz6xsu;*3N?K!ZFWh_+0D@ONB7a?&~R@F=#7z9 z#c$q-ltj;tLSadU`^ig%;^Noer_S%mE6=cRA>Am1ezX+ou;-~u^vO5XpB$^@=hXe0 znks}DtsqVYezFKzJ7eTE5~m39y9`Hiix4jezS0pV?qka_}f;0;=vSHpWi4{%C-lzzyVC_tR(Jp{9g9qAdugPkJX5-|Hi>bNqYmXvPxXm7njR_#Vx zujnVgKD22VR*)gT7S0F1%3ku=YvkkMS_?S!8{<>3thvtN9F53Zr6ya4$WyhZ{8z=u z)c;+vMhuwdd9=@9^FYzb3Kgiyz+xN1i01|(seM$7LqWLOqoVi!W0f;H6QV8bNlbX; zxK-&89JFBcIC%^I1G=mAPWv_wL_opi0ghv4FP6%e=4r+fjs(pn)?~IBjy~0BK6{ek z+2u0{bY8d_aO7ui*thTEOu<>`r0<7&oI{2&eQmVc>k13GYnzp$S(_VK1ST$#w|cCt z!~$)ad3eV`Qz#ZTBfDOy`q4DC1NGH#-Kj=yQ z>lktUfF4%t8}n_PL+B zOyOMcL)6P^Z_E=3qb(sFV7Gp3RhN$HXF-ZAFswn&u|_R0LHZFuP%o|n;z>LEexheQ zQo%<7%kka{CZ54}euhY@U#H!PLvq1W+Ze9|)KY03Rm)4-8tTpHbvRp9cB9ky^ z*?3ohFPM13V@O@q?bVKq{Y39I8IXMkRm)hKV*+W1lch&b`17fR%Nfmf=W^ZIsKna@ z8*BI0N=b{YI=WQs+c#8j^gO=9P#LXd-0nr5fVc)Ow@me@>KUvqdXf|a%+&Eq|#{(RVB4l`l^9VZpCQoUj7IE1 z_Znwpj_~p1|1Fg}I^52VdR6**W5sFUtmClG5~Kphm;=$My$)0fd*F{+h0WZ09rnAy z63Pt}LuOA?4&Aamb9<+f5SHGrA{>AniShS_s!^AreynM>Ftml|oiZ9{Xb{qg#w+Pw zj}o?}Z=GqL$k_$fAe*X<%`h5nU^LU&Rp_aB{nSRmEb15! zZfqf}0KTII%s;e&lRG~o;ymhEG8KME#MD;jPd? z(Ehm!mOn(QFlmlWQoW0Br&IzD&Q_IrZSTR=xS$9)FcV7JLXh6R4zR}hRf!2u+u-V4 zg#r!tpkFd^%*3&YvWs`xQZu^_(?={}X{RT>`5Rd=N=4CT=f(7BGqo!x(l5&$qQN7C z6Zt@G)@yZ2iF8)U)dF757HpL0BM+i~D+V4-)p9)6s|bq0(-1{gV$FggeynG%7BZWC zh2d(tGwCszJju>O4w z|5?7$oUWT&(imBRrpa;#QFaYb8DEFI`Z1C$o+9VK2hgNmT^i8uFeYNJ#<2R> zTI=4n$KIOli-k;U%`Q8C=7*gd)4OV=LBNIuidUta!hATD9DMIjdXPHZSK zJO0i)s&B0C%BR^+vpz~fbN4>#1obrak8e_UQg;!!Rgh@pCyBa?dG0#V>x;V29Tzvk zdN$0>&<1ScccJ}5X#Y?J%UCVI1+4)y0t^|?LN9`gA=w2EK3)Of9QfB2^453`kCV4H zqjsDY?Ld~qY0_^rX_`p*VNQ}+`gXDhKm0}Yp)gy`c7=e~*y?mGWP7Pju!1qM9jlqj zUE030&6BQ$F5AgoF}-USIfLFWH&gzMOU`-yJ?U^&F^8E`Vyy4z(%gZnLx@#J%jMx> z*v=q>i#9mogAK;v9B2SDH_QB!CE8Q%nasKjcDGyE>~>`GIjP~uLD5(`I_n9`q}wIji5Nm>2D_ zSlBx3h=v1}U-xD>*2eQ@YnA2q^V}))kLI}jJR7RY97p3r*k6AAm6zBXnSVzeE=pZb zFJj76?Cf44i+4!RHvtk>AYj#ERUm69Or=fp1{o6&zd?n_{0X6I4ImA&mJ$!STbTR^ zT5+BMmd7`6kQ-FzePoXfFevl2qEZ;+Gc}V7-dAN8g$Dw^0Xz(?)+nZk3-fy7{Ks}< zjug{?Su(r<4?~lZOkf>C+L&e~f=UHxw(vk%5nG8d@?cs83ovev>vuSZdb_ekR=+io zVXWu?EyW}~{G(zDwQfJu7J%2N=GsXrG!|ntUZAcE?)TqVhHMm}wDQQc*FLgR5;%a< zZoWp>M~)^EM@JIV!!@gwwgl^W#^tEiSSzF*PmZ3F>8^GjG=yVF0}c@;!?ga$vm}rp8g^Y<-vG*nPqwA zMak>58|0kJVw3wTQ0;n8yaJq`b1H}2GR;FNF?v$HhNHa^D|C9<5`=c1BJ<;)p!qR@ zexVXoLz>=$P{}AyigI2cg+~17bzc3lp{E%)a1Vu;0MM*)gqJp~mZH~Kr#W+lrgAP*dIju5c#bT!=&#nrA=QH}vUSZ~gR8-;38acSNH*njGiJ z`u!e*@oRLs^)BB2P3oIp?U1VTk4!e^dyDQvXY#u;J62HVRf=vaP^Hi%j;$80P`>ow zN`7CtakA>l)<;Xud}FLwFlBd))of3p`G1N9Hi|`#Q~&HigKP?Q{7}=yvLf#`zdZ8E zfr0wi*#8U-4)q5*BxUaq=p{Q$G@U`Z$C7~!WhKD3S&T-f=uT7xv&Cq3*~5%+2`MYB zAJRM!pQ6#}%f`IsOw(QO8H}qj(U;lJtF@!gP|wjXy-7VxB02YvX|YU7k@BR@C1ziF zgL+uRIfmd4(~$KbYXscK+UEQfaVyC+g6@y!^rqg$K;e9xRHf|HrC$9^AQ#d&c_BFv zoU~POi*X#*nqBLFY}Kw-;{QFE5S*wXHNr+ywpku;+Z;(=iF!IM7^D+SydxEc+F zZ$Xo}7AKp~U&H*Ff=5=^b zjzo=a6E!L)oVi9z^@h&Xr&l(oRZw4Z8>Apa*QKi(4TRzmdkpng`USF*Xm)m`w#Sz- zGwIc!R)T1ym(QJ}@4AI$oDHX75R7J(eVqR4=)M@oFus!AY%ucX^RvrAc$ss+FgVAu z7Bq~EcGm2=$TRM$9ksFsZi%KhU(FACLrYy<`$NiT&QV?nbqG9K8>Y`K@oXSq^tzH= zxe$G0>)+`@>)y|$3L?W;4XkWx{XedvECdCXbM$;SSTLc*hPLvo+wx=jZkBa)e;V32 zhBh0_fvu5xsIKeVfN9ccq zdXt@(QbVP`DMgf`l$hLgOwGESyz&bBDx0RP)HroB$;<>o(xggYH$nnHVQM??Bcx-v zDTCS>GepAS4(Kl8707?|qP)No|S z_a49e{khbc+aH}ozZ-Lhm`=rPaUq_J3!5y{dOF(~cUwXu0ShC9-L!f3J;x6B1QMpe z!lCQ>i^uyRE$9H`!5n~JAwCdQeYfE&PqOc@1vEVjP!rTm)IZ@e5#Tt7`k7imcCfNL z(Bg;fV<^3HU_&GAg_O9n)B*KNJ+4dV(b>4l?L{IH)ErdmTcGw3uhVhDmc#0jq)RDX z%IZ=Fk=~&8h#GkSWZh}U*L3NeE}hq39DedXY}gYKlYG}*{Wp^SOX=xQw-;Ei%V z;4K8QVV}+I=j&C8m3YOe!HByVdI0d6Dz6@U$iUN;{l(VfQq&VF%CTNF zGdawXJFL7lKcNKd@iQ|+`_b6E=gK$O)6732J#jC!7k)-NrSTGKtCU?|r6{~qsVVUl zI0)D@Z0>kTWD(ej@d7eSq-u?CuT#3k76>COCdJTL(-MZs?`zz`^{^Oo%d)4P%{_M= zIB>`APN(WPrk(ZW+tKS?b*E6v>QI5KI@<hI%50DmbeR(m#M&4LoNs6`#_gix36usetN%Rh|=sxs4_B z_X@!za)gRnMsq_&!r4ihlpzvUJv1|5KWRndJm!uWG0_G3zp1f+vW=}dx~WsP8*su$ zA|@M*F85Uo2nKbf1~d?BY`N%POio?Oum&`|Xj`@s8_+QYf-Oujc8v>FU6f5v?D)N4jQI<|V( zYB(2=4BqaLi8DAIfyKSMp~14oal(60KG%j_4?Tn`w6G1O@mkw*0d{W=5zR*0(-+j1ddiVLc- zgTGphMo31JErjz`1PWnP3J98Buk9Ujz=f%D)=O_v0c@kQNU z3v5Fv946V4_o==$Ost^QwPD*-Y)hNn1~N`$g|8z{k`MsSrU5Hzx+Vc1CQcF-=FofB zBrP{w`$A9Xceqc)NnWtW*-`M3@X@Q#5M8uvz>W#fC0&UvSPBM1sOPX@ddoXUAdd-m z-H1DO8^tQCX_>Be%a84&w1K>Jhk1py+HIVLb-7OpZmavR?gZchaX|NcwLO_}2 z_q_X7eMhu%_uf6Xlx! zw=kl{M21)7HmXd}b}Js#h0TTGn{7eUSQ z0)zl1AiY2?`sstR#{u1MGrpi83YShCD=A|hn zIC(ZN2ds_bJ<9On{<0w;Ek9F9u5`#HDK!;IEHq`0DNu5Xw7@5n59Jqfq5h=ZI8@K?#ypx}sRYcZ9{jQPWJ7v6*@{l~+@N-R3)Q*F6C z8dHDD2GO#!pZXL`idbVg&^8rA!Bi9JaNtMOcm%+8)~j;RQYytJZqW*i^b)NF+IFt% z)m$LoW|pW*qr?tT)JiU(duURUNVw1d_zZwF6yj@rs8odTVAOIv+x#Kokm-u zWhIb4C~Zps)WGG+=R6t36-iA;EgrYX`zv0FaoUQxH5NFqJ~4C5&s!{Vp)s0|jScm9 ztcu;lS*4zsujD!KK)wFJfray>`h7=^+*dDOSUmJdqxtY5`s3N5cp+E0enx3_Rz!=P zXYGP{YUkzwZzflZGf|t3QDPpNHJXjQ-Qh5Lva(=gd8FAGx!=$n3%OdY?;~LiVC8G5 zr_cp8g$ok;STr9#&c4a^qE+rX>IUjr3VT#|H3j-2Q9~($0p_U_`lMzj6JT-!W)PS& z@KKl_G-*(m#&l^?zYF}RK)txE7W>tX@7JZnM1uI1UKLs=CDDC3#&$P<2N3tNB;wUR z5u~>hW9n2xv{9`;gKWlM36;g0R;*EZ{OnRg$PG8k9=X^x)4?%Jq!ZcXR)fnlad@KU zwXsThuFK^0mIjfcW_&I-J=zqV)!vc##Aj|2Jj^{`o9#V!_FRu4ddIsvvLnGrZ&)aM zV}s4SJuu)E>{h?>WPCZ^`f+UZ<;KEbBj_&y1=>IXmf-{7YYYuXO(uhh?@U9}XDb6~ zCl@LQ4UUL^jY)&fn4B8z^TrbvxmR`%S7IHhfH|9U4CkXeO4Yf9)604$SB}&K563td z#uxjxZ$4cbS7QK^S6*O0!b<3u524>@ntGo8R|wIRh-2=Ax__K{PTdE{Pk$p)uV8OS z?T@QJ9t=c!14H(;tb(iULrf(P;qg*i1(4hZXo@A0n%+mGH>k(av6&6LpC`#45Ey!1 z#JHI?{hB+`Qw8*EzzYeKIU4Nb5pwP24V;>;Rssdqi`y!YSBwk3GbB-B39Vr`wvRvP z^VCz6R_9jxSjM)G<=XbKZd0ST_z|J@OoJnu_M28-!QDes;a+%A`$u19LJt0VFP zYqYx~ZaM4s#=U}x3HOc6luEOs{ZUx1jKS^>lOiSyis_ZHjNFLLl>!SL9Sec{^iR$Y zgSHv!N<_vy=v>wkipJ5JKJIXfcB7mX9CevqV3}29ud&{VMC+g9Jt1G(L4PT|7%eFA zNX&DG8i!!$Yg&Igablsjt2^m%@R53F@7(cGmEJ=IwTX$!aAEUDk%Y@?P^vzbw_BO# zu3wze8}^E%fM8D6~Ap z@U}pA<&~Qs-FKrdwR$Ho2!MKS- zS6*S?i~7A;t$%Shg(CvJY6X%t5>&&TUqmAV8%{A@O6yXGE)D3?s4mUw(xNWy)ujVO z!dct+$J|z2PR$D1qF!x(iBI#O?x6;N@Fi_lNSAJ8(BC%-rrKJjYp2jkxQaegFkV|3 zt5^e_UOVGZvY*^{XFfNzczi-Gh*sJvD=*ERNyJ9x_H|wR{kx5Bci{Ye4|PU%m#3C3 z$!Pi5!3AGvcF$_b==M##H0SHht8YbTF@&Xh+Z$9*_1Eq=-Xoj?TozcqC}? z#@*IpM5)~g15^GAHz?F!Q+EO|8>kvoPtz3jGvJrOo~9{eY)&EL zatbM#Q)mgELTcX>n&GC~+QAZ$F@}Z0>Dp7U zz8O4<&X=nL=%BDknb}4)scZO}F4-3OM#q81TERaxk<=;?I0K&)Lh<0-Gtf5~wJMd~ zU(%rLePwVRJ(s2($IQ$OF|&Qm7-P)L%*@P=nVFeljyYy#W@ct)+MD;A*_xRjQ?s?T z|Mphtwz_rTIY(+q)g|e8TA;CulfSm_g2pru3O{R= zlVuCJz2KI{J@^o&)ishb-_>g{I{*tAH7!Byd%j@3c#RXe zM@ec59$(Dr!KheNG?AeS&WF@5VT)^wiKB-K?R{Ok&8Pc-i&D8WN5vK7j9x}8~XflME&k5{7gvMw&X<2Ox zz-7PJ1{Y2grnx5Heju%KW9*!5$4PpKEhV|67TS@s_%480Wkk^0GG2X{gaQhx-MZy* zG|vyo%Fz-?+x&OcT@{&I91Gy<2^h591KvS2h4EGM+lG=k1>Uh361izOnACeMD;qia z{f-BsMY~|2!_QTSrE=LVX8zDiqd=FtK{k|uDWa!V+I8Id>CjUDil6SKW22HHlVXKC z7Vdh3_;aC-oHCp`H|d8levEiC0Y4UBC!}MU)astUXD%FqCT?#o&o-XXQ@APytIn}} zhjCeOAfZ1JGE$f@NvQ! z2wLfuC)1h6giQ;l5D9k{rW!1&qwD%-B0cjVu6I`2*!eaM{j=*LW5VfDGTJSExITsc zg(llZSL7J+;Uzsv;S^IW=jc+3+u?p2ZW0*qeCXsP9(PJXG+UNBZ#$0wpphEgy|S3# zD3UI3=jU_2Gwutj6=24f(Q>2npnQ-bo^vhVJLdypbKs=JqIV6_rlR?+4&isreYuMx zB&b5a_#9~wbB(?g?i;la@-7b7cLC6*8n=^%?u9z%*I!%_ye-#EnJ!07-4WVzZ3{UOJW>4agNj^qpRrz9 zHC1b(Z|4r}*Ez&Nbj?`uNWO}LkK`Ochoi{Gikch|hv&uM1`H#$mx-S^kms%(#D=fSlD&?I+VsxiE4?zu}3?z$0UQpe@0TXkdv#U^f+sA2FXwF#IR(T zNxO^Y7xpC+e;m;+xkTV#v(0^x~5;u*<6AD6#B;B7R6rA68j_( zg)rF#0ff#D9MmgY6!eccAJ^HX+Ta3)+8tKi&;8e-6gU25OLcR`->gs1Uus^q)w2a( zO{qDa5j5Y#VI>W@9$jcrhe-=if8iHUT;0vA?I`JKyM2KK8-^$UoT`G?pJ?=)s-}Ox zUKNr8KbuK$<>gOVsd2GS*U!uX@kAMzJkODp;cMq+Y_4zGLW56gNHXsnWA=oTe4d9r zsQ>2c_W4%QyRS}lZrT2Zc+zQ3MFK5B&XU4DDgzmFXBZbJ6@F9QYNJ-M6f|?zj9&ox zlECg+Yx}v>x1u*Y z(^!4|MzHDr%2kE=P75rm77Z*L7#Vyb?2(e5Gx#r%C(UjaaHs{*)v~&*(F+T?`iT@F zDT9|1g)v7Z3GpCJA76uqbKudQo-RAHO%)r9W@}9zCutwkvYRfeC1S5RohF;>Sy`@^ zhS@T%kdD)$MyQpzZxT?WS>GduI)4c;MSno75C#wQ?t!x!XfpclyFbV-f2n zvLj|F-GnKe0OXNp@){e_ZKuwcU_bz^x2*i(7rwHV9I;>Gj@tn8ptaG}{D zfPh6ot5`TH^_e*@S;7!zwzfpK&!@Mh%>f2CE2CAu`C+Vx*wImWi#Nulu#)kH zuTHpqCw=W)IEG!MnAn@0&114nxe>$Om6dCcFqhmGvDf+|BmRm@O_nWmz~89g z6WK0GJV}%HbQ|6*5nGy_CVf1lx^2WqsJ}-|c_-nEGmA6#J={GHFDU(z=X}mFb1k_3 z<_{^KbiW_DYoO8rFpu%6$hnGkg3@Ya7v#pP8wG@rgWrs1;os*vKW=;fE4#i-==6`uH4V##eGXmD1kk-WeR%UifLZ+NBE3+ zczcA>^!kp(`VF{kk_B@;IWuYodI{3CnM7q8CFR-BOEEa$y56JiFYya>lrqnUK!%*jxECW}_NgVfNn|%V!J*?>a3D%GS^1>s> zh?r?l2Hus1Wcodqg-w?|81QL&^a$)$F={2rDmg3id#URmz4Tj06W2skT`N-_1@5 zA>Q5j-}PNp{D!56+%yi`!F?MZ8y4jlnB=upJBECspAnM|mn6@NR7!j2>`z#|Y5XKq zD5JbU%lVehxn!Ue3XV7sa{Qy=(ZvQ<6ScR?Bs%R720RRK(eKzeFp+f2 zjXr)Z#+ri4uL>99uT_MEdHu|EOw0tc8jQtv&0N#1tcP+zQgRF!X&j|yhbRd&86M=2 z;h=;oJUG-pfePXjy;^IN6wiKDu9CH4^fQ4BwG)q>frD~`XTlyLo8hA!ij9RImcMtF z%7D}{NvM{eW`Sr~aT|g>Zj_0Jgk^F@73Bph&0_zO_|1sV z>=P6g+L(62k^I4XF zdF|ujsSj|$n)op8w5&$KoA&UC=kqX-k%tbSUMOqOICoNv z!j>V@+$pd^zMrQ^kA^I5XN9MqWkM0^EsF5^nQThafL;I|h>JvvMbQ?vl?&43t72Od z#^DA5U9F)G3;~bhw*uTFolCIbR>{sel)jrELECHh_>RLhAm6RBN$cJv7@*!rob`o) z)|;zEOY?O!U&VV*x&fw2I4h;Z$=qLD6Hd6QoU>5b350g{_(3_dO1NuWdP=j?aKef# zHC525WsI9s{$T~-7Q(iP3};-$ux<15R90;^;XT->l*X!~;Z>a4y`ucSKy`Z)W|A?J z8NkJS>5r2`Q~gm?QcyqwxjwHUQVa~07a>HDngsasQ+5W=5vkoGP1syy3vz_k+G**C z(0)xY`t;*s&vj!MJ_xJ8R}M5lZ8EkqN&K!z)&4?lGRbs#lEhO%$NKdhwM z*~9fbj&fov@-!+26)j-s2R3mdxZd$)772zN9dKpLXh6KcQmInE1MT`mPGYSO5=ty{ zf^yQ&K?r~nxxsx($Gi{t$D`-oRbU#21 z>A?PJm0!Y`UK}FByMc@f|!@3xQ08DCl2}oY5npKLdMygq!y{9pqAj_DPU5^49KPq*0^UgEOT5I$fAg{~^XZ zQs{8wvP18xUEq<}h?$U=Lp z7lSEdTtcs|S#SiFV!29uNM+a~w$h++O<*&V8mkkl+@tRg-UWRy$7QUQ@p$I)c2ej1 z{GmfKGlVVKKJ6x>dXtQnCLJCwpg*Y_Hpyv&%fy)Abd3r$bKc~@aCp3=P@(d~MTMJs z=9-d0bH8mh+itqsD+g@6Hgdg-JF?QV&09N%m*17c#u>-)(6+6^B3=!+jL>?!ih1x- ze;MDEFz5Qf18@G~$V+U{OG*;@#-Q?}OaAdS4_C`Q{hc~Pwf7Mq?+Xrk4_q6a_3dH@ zq)7P=js5Cqma<8p!5<3i>ngFYsRRDlDsN$Ye=U6_GIWl6xS5qHfDvqsZpv|iI@S3! zKHa-c$|@!IWg*tE1oTcI!l4xT=Z~@^P+PaKC0sj2A}PPy$!c3=S{ZZ0#KPq~k=UgM zYK@#eab>+)m~z7?n|2 zToun45izPOkV5be;g5ekZB6Jbj#ICJu<$(%nz3RBDCWyn^SZAt7bBtEwpP7%d72>H z6|Yq+@7HB`>!dZUVh0~`f=$*`x2q78ss>1khDV)DB2VXM79+ol;T1M^5~~a z=eN(zg$`Dps&dJzj>?Dg`Ho-g%B4!4No9r9Nh+2NbUzx6mTxGyldaWV3^b$Z23<1o zhYY?f6aA#6uqUz>^0NxogMxO#Xk?Nc+RK3OU5CrM`*{JuET#99+Q@p?=Bito7+ZZf zD85<-7}O z*6(@b6p-3lRljMAY`XoE{y08OHP|2S;eB&>vWfI}V*b|JP5s94Q1BBfuC9yNwkpya zk;nf`?NO@XLI~ZX#>4b!MJFevfI_=)h746v7xiU@T*97-%LP|4zHJqUOo!0TVPu8o zG*+{xZ4O$Xr){V?RjoBgzEFV0nPXFVijnJ=}s!;R3@Y)HxkblM6RyUus-!NVSNJ z+IwfI)V$%drA|V8%de%{zb}k8pDY^)A3ZYa;En#`(%9d_*6)u0c5ZdKF$_TU@?v`f@iSAgNXOOoWR0cAg7ewimwHfSS zJ7wsH74o;uf;s3{6>4+6V?Hv-jq(!71~PcqW4{N?LaLLYVG7+}a`IXh$tX5nTH(y< zmf-=)Ytwr?Qp^>pG7Ga=HgmOmX)A~YBg5AA{NJ$&FZ?YJQf__uBCN>R)vcotwwknh zv=CGE&M_R?EYyP2ODr86V99@C?^I0o-s=`E>VnMmDnz}(+I^f(jXKWjTxI+EEkSc; zULG)t&Vt{F3_GB)lnamr#FV4A7E zCf19{7&S-$PiovvWKW}>)3uJW@fW^53M!e?8!9ibdzXKC`lT-PD#r=C4}Rm;kW#IZ zcSuQgXhjY7BIm89fu;8|v>?kmte&<%F&xxIj>pY>#Y19Sqa%Vw;5asU>rR7HhsZh++#c zM%CUuR(59+I9$L&-e$!;x&%hynILF9mFU)PVxmWlk%5D?gdTCO@dWmb)tc&FD{Z?u zSx2S7`Bmzzrw2!GZ)0VOvFbmNkz9!tcJr z=9Aj0rIu#m*Ia>ucI>ZM!K(I`Rab{zpX<86LEz=jBwFgJyA%W#M9)$fUTBb$vW@R6 zRxlnwM@|ggO}?9A(jlxvO^LPhxF)IDHiR3DKK@4btJ=87RRCBH?<&Mub(2 zEE73It{7;kah%&e=J^yGUc)`2VaDfA`3BhMN@`0;V-)upm9tLE!;_DVhS4A;OP9*y z*Qrak)4smwyP@$kj*|}7r410TXw^w?dI7(2DgL>cIyMr zUMaL6=AFobPp&F)tDze7McM49Q0)|S0aD%=Qs9(T!A(E@t+<0SRGdaug&?sa200)Y zZZlFr1SfvuB0e6C68USH&14NuwJFDCK*O~7VY?O8pD`mPdtHmjIjm5*;PCV+P6NAW z@1h#Ei&=-v5iTBw-seVq0>fXwTa+}kp|Cjwx1{{Fd>uzeDmL}5M4A0DeXk=v6V7j6 zA#W?XlGytP@y?^F?$?WRxmR8u5QtBipLrlLd`2bT0(8{M~U}wtm z=)R_$PN2cs25IakOjm0vtJmvIY0__zG?P|M##A-e@k$`m4Kt16 zdHdZpasz2aubL|90hmKl_vwmgB(-#)2-%5B14BhoQ(^?bxm|&R9rxY`I;AkNJEfBh zt%{}SQEs5qhGC|b5&{R^4{Ux@dOG)8{q}fJZ1wR|GfiXiF}1zIUjX?h)LSe{F#`An z$oL}Vpl{(fR>(C7fww=he=AqmlrK~G-4#77Dkrh-!&v6y&pX76oPcTWUZ(V{|EziH zQDGa!>fPtC#5_fnLGKmaHFRWbpil#bgSPpzRJ-CDw(o8`VQdoh@}f{ISV2gr)eMMR z;~sKM;NkXR&EeIz|CPK*<1YSg%!9o@E zQOaz>zD0xt?L4@ZDy%_a z=~XS!nMwcy-x%*?2VyM3!Dp+3xXs;+aF39@t7D3s^sETD2xSAcWbyk?iX!DN8sK?R znGUQH#Ldp`Ix2<+XA|>xaDvTwv}s}td^Gc4kkdse#2da_r1wBe@J1R%Xb>i1Q4|%; ziW*;on-vw~Wq-L>6z_l-?$2c{C^{2H$dJUJVCD>JmoG#sv)Hj%Hl(Fke*fVT!N%kX zJZ5Z(1r{-jF4Ks>0@oE7uqWQ!+e&wHPOkru4$W=k2KYYk}l#6S}-1-o8lvmcwC3pRk;RPIK;;#?3J;LOh z9u7+BO-k#OTFV3!S2=6-kqG4<4W!mlC&)tf*rF}p<4OHH0uyauzJ{wToDPLbrzcqwB00C!y5QsrVZDMW!i zF;%5EP!-_CME>y7`NM)thydDE$v^xu!s6`Avah_@cl&6w?4?WZ-7D@Mv)Ne^$?2YH ztpiq~S}L34EC%L7b8G7?O=4qPT)fqjCobl259N5-LtG5+GlflbjbQ?^MDy$G6B5>e zplEEF>cttHrMpZy^4XA2r8du~cU(jq$FfXzLMg#qeqKVvnG?m212%z&p|4QkQ~PFT zH$qgV$SCuv62?w|Lx@tVT6v0S$6 z@}&n$-+%*x!0ZCy{OO5C{Mo?)XbK1&8-=2`)pjjuhD3PPPvncSlrr&aM*qVP?ycq@ zGb^i+>U;qT^u>tP-TT9>_frFOBckgFIN!Ec=nm~7Z1Wpyx=*Sjroj?5 z&U{bF)A><_1u94BD!EYMgI5uo9QJ*!>vGi}E(XRfwX^!H*amROA_jNzm1L3K&Va7*->_(6XHwe%{gV>_0>N zc7i3kRokwiQozDE{NU7gvKx7?EwW$`VcgEOko;$5770!*jsJC*gxR~;&8&NMDr==Z z{`px5^>*)@uid48=6r$d*xL0NvXj}0zj(bTrMZ|edela}4ev-S!)LXEZGkZUmI9Q(u2-Y^HPyu`-pW%dXp=HX%(@6h%9Iv2c0#o4)Y<%^q7Bq8cmerL12EQ%kklNny;8W3UEp73U-T?Gf{XYc=9D^ zPT~XAf09&b9cmfiUcIAkD&%&Eeq8BK{yX0wXU6a5#&6W8P4IxWzGvXy2 z!W~)BG!-fej?~h>Dc5`vBUN?iH=f&!=iN~VFeUeZX`=aKEMjTT?5J)F1?+usDxw#h z&m%9J^22x?F6IWZ0~S}2!aUB9T?_lMB{l1~Spve{kNhDQ)|^v#<<;VtDZHZI&*`#n z%)XwJuQeZB4A3p2dsnUtn+v=}=l9d6(yiLiV%V;+dk{s;QajqAL9D1k_d^B3f z!l%oXbuYp7GrB+JPs!4!^m*jV5CpTt2SO=VD=YXNtEFu)#Y~h0mLz+Xyg`RSp`_=y zOXP1$f?>c?IOvA|ZFy=!V2z!u2F?8>g6G2z6avV zNMy<0Xhuae#mF%xz`~xpFu?e2pT7{{ARK;fThbt^T)mxV%ogKI{o7u;rD&k^a4I~M zSv2BZ^o5yi4$k@gj|Yaf*bPkyYxPCrng>EJz00xn*4C`(>>8Tz3SrG0aSh1xX+K4~ zWG$SB8&OBTM)sv01A42sou_@wwDwRT1Jz{BhXIb|)JymD=;f~_`ej(w+3`E#F`Kh<3om5$FTa3 z7xx+YVDnrQGeXZCXXKLTN%XfGXoH|Fe^WNnF(VU2Xfs)=gd_svqC0_~R0+K+n}rHF z$PVT2$lLey#!Zp9MD`4c6s2)5+1sa*P0kHyw-{d~kCxw9v*?kW3T0_n zIVI0l6tzE>s@%|A@L-vkVfGIH`SK@S3X9DngPfs1-7W;hmS#QYOlZhuu=ONP1QJeE< zQX?><X#B!#W3n@D#dlSg0_pY62KjN^*{m1op! ziE|M=#6Yft(ITSTF1)-O>j8nU2J7+tg*frUyE3+|W~%xw)vEPF{Eu@w?#gJFLt;&} zJ~!FkP^wm`xrianIC4KQ#SkWxPX~ z0aTlIP(@R_gBGhqWX5yv)P@YU{Ebz;d5b(p|3%!pHmvqHm;*D{l?pCbzP>96?%81r z=c+;x%!>^cgWB6R8NI0EFk)g22d=k(++DRA2Rq0Th0?D~bm{5^3vo4e$phUmD{@hE z8-44DckS?QxlFZjR>YrRSt99_$8@KkA?6?@bZDGD9JFaDb7&1nW(*A_W4p& z!m7io1SZ>#mneEf2==a`dRsK{o9V2oA*bQ0$>=>A{%nUF;OuI37Hc$o6P2x7HJ`Q3 zxdp>GX!5an2OG25bStWfm*$wA*l5<{BdAedUksae+Q zTy{rK_|@m_VT29Cjg5Z!%k6V*3iV85_z3gkC-^m>ccb~q{DM+7*jh!FbPeUMBGM@B zK%tpZo?&hJL0i}`gBx(D8MFR=`{?WaZdje2?E&pF(%di0vRbwXbElyRvO6GR^yN5X zB*{?A{dgrLj0_c?fp4H4;tsgf5AhitXJ&X$7g5?Gz+{LwAB6|gIv5?hJt+jRv9;MxKmlVC*H80bc@AnQl2;{U&YB5b3LEP?ly0Ii{&m| zC79FlT+82esQJ8NgRBsYB)RsB9rba1VS+_+mB~C7-cR!*`zr%o9Cja3LZ~NJ3YAG13-ncI zlDrx-wu%k%8=OiVh7$jB{m8{M6;Fjd%!-p%5TiK6Ft{7$MlyBQg3H2=iL*iy8dY;S`H36L@8RN z*mHHd!P|OA*8OD|umo>(Vi=&gfU$0f=e`Y4S zgNFuQUv{f@4jW5^&LJiPS1JXp2t+1Q8jgZIc1y6c*GI0$ve-*#s4T34gIwD{Lqtq% zZ;#F{bko5oOqYusv0@?G9j;|AJm{h3q`=gvR$EwBU=^+;Gefw^U=_$%(9Yw=@Cytx zQSe422F$&N)X+y%^BmAm@?v?^W9_@c0(LBo%u5pS_}QskVhnt<%W43sXOqVI?N&-L zy{o^(k~)La!{_7_Q-2D?NfXm@ON`t&r)gI=@QDf2H4oxUZpIu8M9wL~e+w}$C(DTJ z%z6lr=j;=YI?fV+kHSt#y^48Jlv#iyD;e3;-j*_;-ACHMEp<9Oz2I`{9L_QOL9rSS ziVD*y}Qb?dd802Af)hvwZn-7HVnUcwGKIBu2rTn^AoVFBO1QC3Q1^_CO zTIFc>M(zsSIK!nt9aQpkF`iT_Z&A{%hMD3+kSadafvd&hzTt5V$*o%b`Zqhc$Fx%q z8;E)Z5Uxia%;51;#x&AzL%{Fq4m9pT?O@CEtA&II5a~vH_GLPkiE}5l@t#R!sI5^D z$9LV>*x|qf*K?=gSP5;->~?0_dnhIOxw12C55n<{BoMFa*v8zctZeI4-?7(sBde)k4e< z)@FH}d(rTFrdQ+@-8_D}aD^$EQxA!1vQ0t6^GhNrxp)k^DAV#~BBFbYnyfJ_FRzU*^R(`I)77 z>V5{z>}Ye-HWt3~T#W&W*8i|(4GY;+%vIZD1zL7rsnmEK*D^5;o&NX*%#$Jb3fUi)O zdL0CJVH>R6l|snr$EzntITLP9!l&ay<_HUR@|4tDq-4_We^F{?LUKIflx-I)a7=%s z^-{5d@&u1&xyd()J%r%+V4QliRLdT=5vKqcUBgb|y8tw#AU5KL5_>dwM=hMRvW`|8 z%G(e>Mp|8%j6J&@`voIZZDEfRnr!DaIV!_o^{PxCe$d)JhQms4AThF6iB^wTwL2RP z*yQM7;^Cc@*EO?_BkG87)S6uLT_8W?kS$B{KZjSGZ5@48A+a-T#7oT0C6LM$i}b}L zL&rG@FIWXHD`%vpWj+WDmjaI}Wt=V(FKQ4YQu z9)|X7yD6?(F}PsdcBz~@crG)wh%t)12ea~=Oc#IdxIn%K z+M(%g!lrm(>x+5e!QKn?UYU>t%<#e{N6;!mUd%j4vwE2bf421{8 zWVjCPqNF>h5w3~|V}7WhQkC(x54yXq3l++*&U7=?=qHIlPH(jNFZX1y4kI@)(1$%T zHZjefW__FIgjjf2x*uU1`*5!HWRb-ls=Fr_Cgp6l?98TY_wQK(rTZE@Wzp-ns*8)& z8-H5CvfwO zj1TxdW}jZdH{f5Ydw%4qu;MO#6mSo=bttit(Q9cbGa0AX7KdZW_R&FNwI7WSg5iiQ z=}8?#&`boru_j96bP4+xjUk?^# zTYTCCXH$^5em)MB%X#E#!54u^;wOGh6r_}=)XcLARtSy{iO)YYJ$jVaSo~gHEnx1) z{I%$-G04cM+=+RPh=wYWZ;+-nS%DO6&F@>IOzC)b8R@aJoFPj0sCs7A1Vy>@)arP~ zR;H)ngc7P>(*@hy@t`o&pR=!dm3yqrc)E;*9VVRu>{SPfi;Nf zutitjz26TQMh!wd?ky|Q%wdr671%>VIaCmW3miStQ9J2xnWw=o>|Lbg60i~=is~R< z*a=fv)xm)zq9;RSGY45sISqR|FTuaORv60P_;Kk`q zWh`DP&711MGG2T-gFEwBoFhG1;+w-{6rXm=>^lrmYg$)r{23pyFY4HIpfmi=EPGS* z_VRZ8X8DYmJ|o?lu4DG%-L0Uen@Yo@{6}dHZMVOnU%P)b;xK%ZW&)6K;CB1{wsM@I z{+9-OeWtm4^ZsLzS!tz{>+$#u;dmo)z&WGJ#A@uk@S68#DlZ4(D7i%^A1cY24n=)@N z8sk)>r8!!Ud-sqwZimHhn&--Rl;TWgHc_N&ze_42Hua+>MPyn0o2>WXdd;)J>AZS z!RpSNadcT=oXXqf)tpj=YzujRSs=ToELKoXq(H*uZr%qvU* zrffcOd*gIxmFvUr4e%@GQgW%_7Iu(9oTV5rss11l?6x5^#{3he| z;inlR;d(;?rxj8`Gcj`+VNaW-^bxK`V6fu0-P;IF5_@R3^2gB*!ot!L_0Z>VQ)$St z?gKCmaG)22RArf(fNWY?xs3Iuo|an;Xa^v=)} zLpmh*@WDtpFB=R}NC8nySgbfkhOpVhL)K;f?xXoV@46%InD;dXU#TN#ulhW%*nz#X z#Jpt+0SqIZ`B|Y@?oF;h2s9n7IAZ`H{%zou*@5>x8omVKT)m~Klyo)^7$wQG7*Phl z`NYi3E?IG!GniMg0i03yp!kdWm3o(gv*??bZ&^r zRe)q1{c9S4z$?P23buMY(_PqK0;)3(nHm1noS#z^Azs-i9FM4VNat5Z;3psR z4}^q-J*dnae|)3ppM_qg&>R?gB8Np;KY~Yo@BZ8h_nsOgHuI(vsGG8cqTGL%WBI`@ zARCfr3U|i@-+q>0fhdsX&q8coUokhz7+DXwh4l6)5CLYV)e=%L(`3#e(|e6`{849j zIXh*50295`bk>~AzkB17XKcEc|uSy*E4HWJg= z{$vOUIHFQ0As zFqUyTV1KdqN%iURas9UZR-J@{qES}vele!_Hktid6PiOW+NcL0y~J|}I-Qldh`IB}|2Ud}5m;Te3_;859odF475=YVQqewWM90oS1T1ACV*2~X$;e5h{jYLSb1P$JB4!q# zA_XECqJKS<(YH1xVi5f2N290&ROMu@C`>19Y-H|iO~*vUAmOBMWo{^FV`^m#WMq&w zcXTwjF;z7;axx=gVq^q9Qgkx5Rw3d55HTpZ+X03B#qt-XgSnlPtpf}n-~YCe?LXKk zr0;0_m-4?=E^6-J=p<~W@9$e4(k`5#yO2iQam{}|>7bR*-xtoz5r|G^(n^8bM9zmWXj zGcmI<5;1eK5wQTP8Nkd)#Ky)-1OTuR0oVXUOq?8ltC@wJ^Pj}Z%tpk{$WFuz01&aT z02>1E%E|)xhvhFDko526ze|5v{w5&lUrGO-Sb+dI*a1WUV3TA6uo7`Fu>kWxQ7r6# zNm)67#Qizy@rNEC7ywG_eDZ?EA0cf|iwM`-`Y9-93>^w7ZZ!pYbH z*cJXAW}J+_8~@X30tXW*V;fUo@6XB(_{;ilLsvk8h=$MiA(4%ak6<$z&_jOS*X<)%)qay(}?R2Zo8j0Rw+bx!{hHQ&DX41hg)_?+qR$AEtRgs(SPlLhIkwQa4_(ix!vJecj=ukn-THwE}vP@fqufUEru z4#78i*H@d1b!T(#rpDQR@DR$8e*bVDJH^jQ0Qx0W_2EUOHep_zufya_-L&}^3OgYT zJWmy6>#lH4>h6`gDpY88J>!_Iug#JUHYc1+&M*@^iw%2g0jC7bD$SSbesqh(xiaol6 z$%YCCaL(j!4W(|rPo{CU38qh`qPJXc&KP-!C?679H(YPCwR^nF{A@>v=kMS8rg{mz zDITinCPc1eYjM0uADUpCC2rOK(ARWMOrfs4({P}^!DH)~`j$^aQZGeO6T}a)>S@`5 z^9|IrR+i@|b$sLTOl{G3AXGI^!!+7{gdD@z2F^kL>m&v6f6zRb0Bpcc^WPc^6L6Gb z1hD_7_R`<(>ZP^lPULZV=Wsce)jaFjtM{S@(T)rPiWEhL1N!jgfg(O%Tv;;ND2%$X z#fZuw$#7X-h`dpXIBb-wF+o~5+>l%FtYUQF`k=k-v3By>M0(4+WDK10)c97+rxV=W zuETSp`7rzC-m&X9T(4T1CETP{PCC5`-_@q*gzyOUPq2_lJf2CWHH2XO)~)Be?a>sH z5oKFk9$$!TZ?Q>2ZpY(@tm^LDOQy)*c4v3<1(>!QCx%$sKlKqzqZ#md44#wk4&fXP zzK$%$2nT#Xd>R5HLr#Y6n$l~#UopVzh8)Uv-=3CC;`5nhy)45Ab8DVeAVUL>lC)88WyT4id4J=lQ~5%E0cP$y5PD}s*MSjx6!+e*`UtAz_(n{ z1B%|g0UPFl^VSWM-Gz_ubI_(8ADlpqLGm1fKA~%zco-9t2DK>DRaHW%Tyxrme-UqM#sz zUt$9HP=B&dVdZ)yG4Kg*PzhE{cNT1O?dVeK68ni-aUiYJ}z+_j^9JHg4utV4dH*{hOD`-#?pK~#I~{{HshvW_6S#f zK*Bd4ZnErc_%6o2ZukzpPni(n+;n{o5F^^)IQ+D5GeNK$hqL`6j_6j=0*6cnU@)8u zHb9WW-yEYA?%V!~!AKZBF89-xJ}FC{a|OmfYO&1mYy}6KectxzvvC)ZBW2*O@SI1OG6K~aVKESh>Swih7J`=4`!vK4zZSIXjNO_ zokpb)nrNv_m9&gh{cu%ztklBRSH^2qUfxw~U9qulv3n2B7v)oDE-uDcULB<75^*#> zx4WUWX6hyposDdCTLgftgXEffuFCMOe9BU(@o_OH#?a;Y%|JdtGU($Z2}NTdo+eO+N|6)aK9xGo(mQT~x$;5M)un!OBO8b(xO zN|g|GfqH~-Yl2@G7HDALcjZ?-F#hsv_LRL_7GAOO0hv5~?KFGtG*PZyYh=exl*BSo zk5_EWA7yxp6hVbbmUN{~q2=`p>mIm-M#q+NXgB&BJ6idUe0Uq{BlbM5iM#5J-o9il z)!I%7w3%nIBNLwdy8JhBRdKF_TOy8(dw|gZZBwK_G@Wcq*J~eZrd5zLjTeU8p6Ktp zvW7Wh#f(D| zx=K`8844k^d#qp44=hqK1TTlkALiL6`d zc(|{|Z9~m^Ln|tEjC*x_Cak(98(jX9K4^@%KOsEmyee@#<9@h;mJ*7Q$TfeIhX6y^D$L5uu$k782 zWxGe*Fy#!;_;1+-uLJx8+~)>htxXt?)Bga41pq`1K;7hR9s+|9SO_2vfg*jFW0ik`Vee@ZALQc37;F>6+z8?d)qd=lZHz#e=b0dM86 zOXJF9u5yH)BvZ(qxjJtM0Rwl{Hh$?Pfn|~96k;)%i}Aq#AO3k?km?I&YHG^_&DpxMm#HxYb8Uum#pa#Z0m}~ z@>(+nJL=tkvMEyc^Daz}WlA&WH)Cl}3q8bVlr%0@=2~v(-@6|lRktb+YyJyOECzi^R@ly2n;ueU_+C&g`~pnHlw~ob&Zx^Je$-=GVH_ zOPj7f==&TM$E~O{aOhrdL&O1pUnu0mth$pqsDk0(<*j_bw_`4H)wqTaJ^ z1BP>r?Cn#kr`97MO1D!p*U1ZhO2FX~WhT9NR<2_B$1dtib?qw#j4l;bU`+OcaF=e$ zqlAj)HKk0+>FEU7l}g}4Wd}-<$dzWh^)AyyQkup<1|XY54MiNp zI1%p7AD~~FJ5nBnB(BB&EK_-!pf#+~&HSmRgEs)DU|VeJSVT*~d)Gb!^UgPRMLz02 zGJjbVvbBVydw?rI4mf+PJ?Hp2&Y{#J$}J#!=)USCu}}hAeL@!(p)E*{O+Y+wcbt|Z z2u^+$}e|noPe*yIKPAx-cG2tys(2nnk&O4cRb>5lak!^utT^rwn$wJbOd8j zt_B#HC7Q*c${AZrPgmu|S3sVQ$c*rer4eI?#1U3=^lAf41v<%~Y%Dibc2^7ZDE9Q0 zN)dFj$;{Ra7%$qL=?RDQVeKldew1}t=5aokS9?}Ys()V~oUUmf-_clu;uNW6R6rX5 zb!o2U1E9SXJDI40zOYuXARhz0y`l2&E7Ty#J2f9c_>z&)Q?zUzv*?#4cTG98uF&Sy zQxU{6*i(!@`X85&Ya-qz;?v5NC!~5P!Ef`HlU29(CF3-o?L%cWcU^f;CaX2WpUQ*U z>-yT62EQE43&sp>E)Ddk0G^p^>zyN9U2^M36XF~@XLSz6+KJ8I*H=ONmQD+l;UcD# zwY@{Vz)X_lkJwY(S$;;acqT3TZG&K=J@QfOm@iuQhQ^!U~TFL!*1|km2}2Z;TpZ0?uip z_O!DC7or(ngKqUQ)UK#crZMrr8@x& z9Uz2n=h!89+2`TjI8=TrMLSp#Lnw(B{4n7+5xOD|jHc9hyi@l{@j?jzW=HKq2w|z1 z@Xcf}Dh9NLo&6=S2#-JW);}YQQb_$%0r3C?f_4fJ#!8YNz4h6ecPgBSb5g9)_^xZ*8o8tzM81ghs||jdK@(~5#?-JN zlA+lMYVtPnd~CaCNY$+vapO(RqhBrJL_e2AA!_g}No1Nvq|h~_K<$dORGV3gk-w*1 zRp~KB{8skIKB6$=naeFLsUGDtMmp2_O z+8`*7CvpNOEHTTqvO`|8 z*7(pPC{G1WSC=KoDfyj6g=lqoj(<^vK}F(lpO#VBIg4>9+OeIX49`G=r69vDW-$P= zm|tjjZ)Wk>EnrSs?dI;kK?Y$PLeqL8W! zp(gsS2GF(#1Y0!(-vz~mwxhv1&SQP1Whr`XVIG60L$-P`(T4XlQAH)_Nyv9> zBV+1mpajT?Ug?GP#z~G{P0G`0VxAXD&mSvlL1_Wwhs-DzXUQ{R`hz(Ch0416&QGKu$Q|j&s z=Rg3nT8u-JRRaiEvI3}oKOEJbY~FBqsge@JfDwrZ`Ys(Xooq87OAZGVYs)<#Vk?bXN!G6B)C zoBeQmXZIjql=0k~^F$D(T}F+lJMHfvYREq56J>?sjBDDAvw*}>p*#$zBhi9T4dl(# zeDRCjaWW)aY;u3tsYn0u{+!E1M<~6zhTL$k?Ekvkt1p?l?+>e#NaGC*)>Pynf};5> ztZAPK)fhY7-)#lO4GDK7X?ZvOOyv9nSL}0!YL28IoTN>FgtrB7;pE+2A$9n!2FpVG4 zIR!VqnHA0L$lJ50AWHucS)to>9%7tX(LySEj{tK`*vNfC`Lwa}GlEygwp-7;tvK#s zGjGkiS5Zn`RdlqUS zHW4xBm?Ke@9CP;%s4p6sy$zF7n+{*reu6UktBi5z?-`cT;1o2FlE&5NS)p6f!#13m z841&}+Ag|kMpe1)``1;(s6lc1A7B08_ZVf~=In^TwtJAduUcmQy&Fx_p8hX0s^lqi zDFzgHr4ha{Zw>7th<67BY;_1Dy8w-(xE^QBxOG3%KzG(A3kC8e#_)W)4A(@=;D_%! zJ-_(AK>&E-kgQ;pTm*_}hBt3S3w5mK>&3i7tl;PdqENoFVqQpEvRzN43r|?JU2GPU z^!Jt(xZozg;4;7jZa%<$Yz=0_!EC*Pg(u^CDK^j!Z=rmLEkY!aNGGI{__uRhCAcQfn`%Pf`Rj3GTVYVKNu@QFIbv&D~a15;?7vX&WGQAr$` zOY~)8Id*3zkUeV(j}M+NUX6{jGqe0rety)eq|)1*Zez5VFbUjED|YuA)sUQNQ693E zBZj8cT#duFJz9}hZlXbekS$HjJ1Q}w)ak(k^V3b(IRHD~qZe7fcVWyXJ2k8ve$_#P~RQdwvE#T|qO{{G0B?4k|iR zMYYB5Q8`8$mGXzeydG3`#h%1Ef~6fMVfig;cPjSy2UNtBlnVUfzR!N_1YHz*_5#;{ zo&b`p=LhQ7i{LY|&dcP4^n`t#o+#kdq$^)GO^=?r(;)F~dM(8{>zxwBQkrYVosdP) zJd6R#0B`^jLY1DUflfSg!#^VpSO&lks5~V+!y-LBuY7W<3JiB8FA>XGFbHohuhh2kevq1kEJi;R zj&-72im(r-?z4gwu|;nyB;2oAbqSHT($v7O zeod5?(4L;xwb7^iHr#i z@&v|G_ufOa)XUk;YRHs2bfJ2hvXvr5Z^P{MtudjIGqZ_P$CcTv(FEt>MUT`5{^-EB zaNVqRvuH|!y3{k1OIa>bF+KJwN+?nNV~-6UpANbCEmLp~;TNd(sC)KMJbewdVjyBuArT?N%IsY(w9)^VPSl*$bnyne$N&vhLin)#e z(E^Ik<*Y3x@9MJvY+5!(@andHWV$U!&{g*Q!2GhEXuC1g*5iv$APZH=4-%2Qq+Pmu z5EtMC9IGUT7a;Pgt0B5s{h@N-&(G4!2nxTn2zRSNI!Ba@_d-CVOL!s%&<}fyRV^rL z7x_1XDCTcps{A3A;>07hh*fY36V*G{0h6-d4YHT^%+XqY^q-^j=yLbq251S-{aUxMZk*L9-5^V(Gk36&4CN5jHa<7E~6Ss5V zQdIOB+Y7f5Uj+m?i|1(h1RMI@m$`r*D^aY1X6bIJ2AT0b%TQNTkv2(?E%PQ3bFPn6 zO4g=nYEnR^^^1_gD6HHJVcB_X!q~dEeAP(CETVV_bm6)xnn(JC4$MQ-qAKrjrby~N zszE0#=7K?R%Hayo6)GfFK~U>Va&y5pRn8U^B$}`EjJ{PPZ^dlsul31)xmn#gbi1s4 zf20V#9BEb(?39*edeAm~?G_TSczAtw6A_D%(1WdqfgHEboBdR}4QWi^L<`ldR8kD* z{m0Xr#?QB^#=IsIvYd@G6I}IY2uG{gBG|_W>D@qO^C=*RGDdIfyZZ12Mk2|#7(jl&1kA=o%CDI*JRf5u3;8GeCo0p7bn4!>xkk`2)Bf<@1-E-}b;K{3f9**lOS zSs7%hWOP-C*e*brZluTLU;_p?hYSI{4Uju9OmB+Wu0$i9U4?nN1B-g5In4mCbw*CM zpr?rBMAhz25luz4aGopzamOz%3@12pLQjxqN~fInpBZ6v@}!2PuvJBK<`(4swRz1u z&1E^M%!l*&)~ekA!g{6%nRnk~s})6-N++``HTpg@}#vYo1#-cBEP4~J|FwHFz zO*ZcYNI2Kkp2TT?w&cK*d-?#!6V8+Ht+&wQ@T_sUtBV-7eBG8?9A*ou$;^nm0FggPXtT~j4uKotl~?zu(Uv-nLSZH@8vDSYy}`x%^c7(( zg;%(xxv+H_Q>eIC?sNU=)QmT(h2+ZCQ&=PzPa__7p0Y^f=(pXo*F}I*(N?su2psPKbiA0L~~x z5Ju|?cbPT^jaBd_XOh%mC=gVIwOq~9gWV9y&zuTxGN2H)a)8_8EJ7^25o&&=fH~)~ zM9|BD7&gakcL??77F<>%YUnv@H`C~&CKpNb4uD7fk`r9HbnTk!tn{vPx-McYx@r;2 zatV^fw^Rl%+}@9%<^5@Ypi_i+A`=~Ediv9C(L`-lZwh6ieygZ&PteJ$MqiX&-Y#g% zvxV}sCfaX}WHK7_Vm0_pd`iV0q#IBok?D2~*-!8mngA@tsA0L8kzuFT*~^mtQ!3%7 zG(GvADmMsaaLVxq3DkGbkb9-OrN$4+y&vY!H$89|*kEGdAAM0GxyX8}m8;NJ4O3Lk ztfe#YXCcXq6DD{McH`}5=wuNc^%t9SHPm#X=da$D!rgCES10o zDAlldmc61qU|{N@sdgVwMd}3LHDGx(c?5qL>udHX zHRYl*E@j5v{k}(BZMz+6PXCXNnnx`)092+af1% zk&^INf`vL!%HHeYdQex?cym-J!~+$5!lBAc$0gXJz2rDBtZ9$KZp!O!JQ_Ol&$uvN zmrhG98xH%QOg%q_%OhSofTwJ3$SkbOGI>1whGL!RuvhtZ`0gdvr%ZJ?-o=%dDMCS* zlP)GZ)DR!}KeExNNX>{{ehyVaKY9_y*_Z1+UX_F zr*HK4f_r=33?;f#+f(ZC#H$23iGmXi?*?NX#jv@oAg_R1;>|r#)F8hFPle_+alAF~ zpT;NU*D)MK5;FLU0pghN6JYzzf8zUwSmDO0W01x+WS{aSY_=qHhjIzpPyc|?&u5vK za|TG6m<`%k4L(MK8&Z{&F{8KWCGr&E_YI}kNRZq*OdMBg_(_my%8zlS>vZY=t|K&D zJA1`k@nZwuH!ZFlMY&0x*h?H&Uu|p$dbMhztygH)4fFemFFxfIW)}=%U~BVmFEKSX zGn+?NeWT9eBY8A-dRdllu2kjxI!|unOLcs_Z1)1+HE)!!fkv4hx$d;q&L(>v1DD$@ zf_r5dY>)5a8R|>l#&~FrEXrRFguIl^n^54;hqFb+qr#}S*fo;0l>Uwz?}J1Mn=xjB zsIQMZazzQ6HE+P)d{qNaL*Gx$e7RZv!|chDia3Ao=+s8#iN$GIje>?J(x(i^BA$RZ zJpyblC`Dv)Y)S8*aDz)}s(YvW;{O7obwS`PJzl96?Ga8;zETq{_E8_O%SO907hc;C_G(9Z4w^qG_PjQbHZ{C1JhmUxo5ap3U%YuP4vbB zIeOKd2+uRmqF%04XH`vwvpBrZ510;R8Okr<4Qy8`a5Be%oyghhgT(y;Oy@S?o)t+|^xs#fC zK{Y8UiIzoGyjX-0U%*P&j&*~k3M$-F1jSp{H%N6ZGv=es0p+6JtBY?OuRfDmPQOtZ z?j40O+0X?oZ2eG8S#H@%fo8^Zcelflt+A7eIQI3%^Otz-9^;Yu{rU8L_v8m!A(MP= zA%Vpx(FHYP4v9~!43^&%;GDX$8w+2WR{ ziWBNK8m->AF}=zGbt)4fG~Ukr>@Jhn8S4rHsUiDH+&O=z5Lv08+uNCe@W^c>G23Uc z8~g@Ka{MoCs5F~hk498zNeOw7oG5-~dTwmAk7=o3@~dxGt4Spj-79J(@PtIw^ee&Z zQ8Rt_$5}Y7DgEyAS0}mwfSXa6VvhbN<1Bn`IxYm~K!vJJ#QE2FS1Vt6nMc?{%B8Oh zVb^85<@px?#4O*z<3jt7;18`W?4~%k|7MOUU0p4Kbe}~<8N}Q^$%8HQOB(Tu{Ly+$ z(kampt&;F}X!wdnN!30geUT@nnDCcWQ8*hPWk8llRy1I$C~FfDuT=dm{PhMba|brm zjp+qKi%?pSJ<5Oy#9twI#DP-6F5rFae?9M?q9@t_Ul>5G=#EPVWO~Ab$VEG?{XSEu_oEKBLAEc-t>5 zB`uRRk>I*UO?rAyz7UK)Ok^kWtF*ecoSsTq0qVsz7ZnBXYFj{E9^3P6Y;ynjn^nr_oRSC2$ zG-md7-(b+>dabt=*4}v>N({Z4CSWRE0ohH zluOeluTuOUtY$_rTm~^*_gK>dt65msvN(JQ*li)}HJC&3>KF_2vsDQ4M;=Xb@%m#N z$u*m0{NoHwLDpKK8P9*j4=oi2w^6`}CNhgT&y+zHSVAm)f-VfuVm1B-q!89e?=s*` zDsqllPT*Wy1KbZ7_O8`q{Jp&lRYkN;l?`0C7E0vxCfas(%q)9-eRiqXJn}9+P&50a z^-zEOBOT4_bjki21w=1#{4V~Nj+aT_0Ro8r#%NdU0zSa(Uzke}Ky=Fd5wxZ%GyM&U z%X*wrMN*Qjdb^Huiv004Vs|AML+gv^%~Cab3+zcNYWqLuPDq6s==1_==OgONxCZs! zIQ!D>(e7TNSUn)#%m4ioR`49e32l_S?Dd22XcW3WlEpux4Mm5Pyagf1rzt#KGKn+N zNnT=z#~sbiib-}`>jo|&4SBBflW+$fS#x*|0uvIuI8N-HPU}LD!dF3 zbq6RJqr$zN&ejkI-2zl%r>8+EniC!XT!aJT0M6FPKn@z)5+R);SWztl>hO&Lg3^E8 z&+;+Mx`~g48aUnWpf%Dx3ry;u688P}RLX1K?R=U3D&@MA&(or$7B9T0@ZKNAiL~Oz zdM2Jf2RS6L5iN@WR0xpybFi7;*h=4uAK{dwbb*@(35)ZVU1Uma$C&cH@OwYRiniIR zGvgDA@lT(Jf$2588__SXRn=T}c6X~*yPd1tlY13grK-)NAY|cDa&`i1t}0QB$jBI) zrmxs6miKbWt#Ro9YB^!U~U5d^O=qa8@Kne$g>UaVc?T z9ZVoLIU&}_`E)66oaomR6p`wfMD_?9`_bf%{{xoF9Dy^(AqT4q>w>Egw%DCiW@`A~ zGg~dM!ELO5w}Rv_zNjbuGJuISO%ZUef3XtOKZPs*$i*f}kdAR2|HuXfig%9tp$ECNb1Z89@M79>&I-WLjo<)Ti)Mp9&vm`!jFyNj ziobt@Z*g4ThiV?n%|}>H?A#nZx4&cC)^F5t1TG({yW=(dv{wfDfmp<$x~B`Kwt2i> zrVmF?XLIh*6MeI_?jF=~GdL8CLiD#HU7A|NIOx_YAnu$l?TEAwVw2)moeZnH87}Uc zC@yMhHhiG1^qa(KEp;#%y=2FWDCdW0gDa`TWA7b$9M39qUKpyB=$HeZ(6$Nud{f{L zo6QGxc2FF=g{W^1$cp-Fc5XZ(+qj*5OY&C*Jkpn8C6{0paYmKR<_X|KAW}$1wc|oM z;&*EVxtkJTitU-7uwlE>EbL9mBaixqC{$PnuSzxMWS+H%Tn;O@!~k_==$(Mn_2b!s zFDFWFM2f`JB|Ho3ub*T|q}cNuU^nsw#rG!>W7sPRRUoZsu{Mrq1J)azX(MCT>?5iD zSr*4!pce|=^NEb3{$`C*M$*<<56Yo;U*FN_t#7%lgs!_OzI|WWn_t%^5>L*TM=sxP zdUVHCjm95LxxOV69)u-fW%4&KA2ipqH;RXv@w3&gd8MGavZ>GJa&C}RNImuwhe3}k zz66M{;x?=cm0KECv$CPJc62wonI?A)_F+256MQbDxOJ@1b!Y{~Pb77&U*4_r9Fpdp z)dwt8;?S=xw09lMiU`#mX?pc(@u$IVxco#~I+qESX50m|L0zujB=Z1o0chnbOBhPR zf@kYFo!=mK%kIhGl!`IzOaa@qz;tQC)b%T#fVM<$8T0v*J41c$5U+&F+!Sb z*kH)z3K_$zogx350mSS!&rFZ25ClyRYRqHK0gboQw2)JP@>Znw0E_`D=)Yx^6uuJ@ zK@pLwOUl|6yh2_9NBMkLFl>mxssAl#&J&qys+VWQ^iRYiV-<#sV2dq6%;2q$#G0pY z`S&}yAk{5^?F8EM%wmIJ$;bH`VaHxZwA)MYImz_|dyDs!I~b0DpBMI9y^&8JsLZax zg-CA?E~#;)u`CrYg!3C5CMG(&nw`JXCnb)`_%D`;M0Dt3c;w9ib^L z<=NFMs^ksV*hf^$o;VAxj{w_M#(WN1N&R-*j6V%UH0n_!Q!LJvVGs{-P_53SO>_ow zu#0DLgc7RlkrlwLOS>I&XtQZ=-zC|Ed7(%n;+n7>av|*%mOxtpz#Om-?U3` z=3L_!rdP&YA)ex7Pc#{&`-RAu@}r_Q2UN=Q>ZQ^?dBOqQ0%?>vLGOc-hAEuzs=SV2J8gAV8twD`a!Njv`o#|>fki|8xjIV!~WLfQJHn*nTFKkx`9)Gt_nKcX>v?-gZ zYP4uJE|6UM@oMLvwH@i2CRJ564ekUqoCD0QSJt0N zV7T1a#&4!lvd4-8Or}k{mj|@qdU?#1T|&IXGG2y23I)M2J^Lx}x<_l(x|7W?^Bjhs z)bmGazfKq*Z9}|j31y#yE4xz+BPNDv^r&^ttj&W z)Z2yQTXU!pwizg1C1J6vAYi1&#Mu<4_AGLYDWrz)!zZsxJ%(etrMKg6NO1rreS#G6 zDT?rA$Ds8TH6t)4u4P(l$Ge+tD~kjh#75robwp&?e_^3sTB+zk9SpLL)#&7NWqYOB zya9Vs2;Owu&GQCk9bsqu%hMz}K#U9fK#NFoj$7jk?-1yG!Xl#^bL&yOHGu8N6MR(; zhv`SAE-Q@bN~I9-9%5gE*f!T*ku*TXJxGo)Gv8VsUp}zt`gZ@=uo1#y zK-aKa|Bxn!#`csxI9ZQ7SUcG1w)0=JQL?^9UYXai*Os3Aqfkxij92a?gDUudwaYyX z{BRU`mE8+X{jT@6?s>kjQm^b`XSJ*0d+U?@^*(NGMn|^)=rQl@#%#tQU2@jflHQ;? zPus(kpgWQA?dYISrsU=ua59# zD9w$GLyjKEf7d$NaqBu_1j#-LN_|sI>PFi3)VST5i#S9%ge^z{^%#z;-f*EmQsBS9 zNv9$aat*gU5V=z!Kc%L$9s4f{Nlq*<+j3&JVB(nyymsuoPrt)S&K-UGgst~S@dtA! z>^oTR&#sWr8(#%C(Sj~MvqR=;9DsRRntsG8!oc?@1|WU#>no;^9Cr+o#A}dB*yg#? zRbseBQDr#XZM>31gLP7FX#!5)MxzmkDV)4fHLSm<|Ego{u3C@*b5Tge7S^^rVf%kudb|5d} z`|A_mzsBo7t6e#7>f2DUZzs5(!Lon{y?I5#ha;)Sbf5%_6FmPd9 zu{l3M9LV7{O4QX#UG|Re9-)qmHT6NcqiW)2lut^=+?1M@h&Eyssn0)Hm7Fyu3JrCG zIEX>G&=XN}a|s^xeJlv4p$mkS7_FG~Erc z=O|cT4YS|o+|X6@xD#Q=C*~;SvU49PaCdI%`h#>foL&|zYjwHFen(f)Kkz!9&Fa1pIn*am0O~V^2GCuInY1! zPN&W>a-6`fgtPIsS5-@Qe2b7eNI6?5WC<0KEEE!5Dt=NS^#jXr(>bMrb;}T@?Uxg^ zkC~x21LYXv?Km$WrPt$`%I&9C`IDENjfSS_X0$`qv((6jyJV#Rdy}y0?5Cy&Xnj3lT=KFE(x-+W?YSFnVRy2rNIhZ%YOU?7|1vdX+}?odjB#^ zu&O6VQk3pf^D*2y*yD_%@C3$D&&57}z>X$(KZk9Yav--;X{1fy#Y90S)bFe;YxyMh zs!-~nv`9qh)H%lr?|6WttjI{LSh%907A57x8oBf}|F-x^^hUGVfMc{H5ZsMj?banS z#Xw<4QJ6+j6v%fEKAEo0i^}_wVb)wWwLHxka<%0?wj(c(!Rq$^8jNQgppCbZ_%Np{ zD2Fu<1Dm#*(}QB=xfLKImg9das*9vga$wh?!Va%+=!whuT8u>+&y>fq&RTjE4OQ+w zw?TXvmy%~)IMw&MdC&$9>UB*&TsvtHZaNSCw6opzeDsy))w_75kj$Nu79*Y4rQ2y1 zhQ$t*zd3t46W(^2FDJ=R9R#KqxB2%Ee-^*Z$d+pD=HhM%$fhIl%;`8GuHj7)U}Gy^ zK~^|VltFtc>P($$_2pR3wMbK`RygG}q99((g7uQJRUR?yL~QlB%@;O(MjhhMuwK2J z!F%|~+O9)uNDC*uC*@(+`W^2-G2#z}6PQ1ybm3+_}Rs z6rHg_(JCGEVr7>{kGt-SL?-)Qh$ajCJPE#vy~sr1SeFAGp0PQ^EY;6tGJOF<3fq1_4B#9+Gy< z*C)a~m;P*%E`0ou4}*={Z8n#vBAxqFJ-$fQ!8vY;t~^01w*`9ku;S8l4kBk0bu@Z_JqbU43Bp+X~@;F>iyA;gwD364WbNd?DC^ zXp92d9flMAwI1-lx+60R5w6Hh$~O`hYES)#7Uk)RyW9b&3OR~uF*-e2`Cvs(MGYJ6 zv~12{l6&KZOK}%owTQ=JJlz)?f#eG)Cz;#0<~gJIPUG;!V;gL1kInst__Z%FHiAt< zk#rFg87cW?`z2ZJuHAV{s4(B5Ru~19hZ~pqT&t`I5wcAe13b{cN)_qVnO<~~SmOD0 z^%mPvRS&*JyeIl+WaPRk)>fmanL-p`DFx=RhV&YOr5E|m39Wl8NK^=KM06o%uKG`n zg%wXihQ#*ZCo`=tb+6Tq^|8hwc8+8Hcup$xae@&gV*!xbaUT6j^Dm?o@{zaeHS)WS ztZZS(zl`-NUCD{=+8baYZdxd--C7&`E7>8h)@U^ANaN+}eHgT`WS=rO?r?atAamy0 z5LmTE9y@}spO$|u-}?J%=@^NIU}>!gzSE1qG(%caIx`W`G}TBoac!FfD|B)mqQQ3Y zCsgz{ZEMbw*mP_XjUjYM0yS*Hl}~C&UiA*SfJn?^r%F?3)5lc~W`dI?QtUd({dA9BOV=g?RV7Duno2?SM3*` zO8I~8|Gp9g<@AB4l$4&TqFK?qey4e%w5Q()rxKqJeKR1p;zgf)i^*N9$QFzj+JpCe zEZcEtU!cBwT9bv-SvQbmW)xl4+r%TX>?}X0zRf#fU&L4Y&nGBQPNzQzon zd-#4|{d8g4xxAm1zo%v%j*=vHQA=CCZln*I-ZSxLu0;EASW!1F*HLiOJ6u zHrKxpMcxTBsnC&@d-AxFW_1`$9@k@hI(mKyeqm;3yp+Btf<@wVM1K5JbZh5=Oac#~A?0%babTx}p{1G->Q4z{Zv$Oc6p4p6oK%(gW+07n)}fE5JrDrh>_SBHZGxm-gz zrMXf}KkbVV3KMaSN7vj_*AQ5XIL0M+Dw>{_z!iLyp0gu{2-~c62X;takg@6&wzgkb z-4m!+pJS_awjGEW79gk45EOJBVf7kJHhVpX>iF@QGmw+CI;Xc%2$HJZk7kMCDc1aX zjdj{ArJHv3h&hYfG4^xV!Qaw+&n~F0JG=pPS|=X`ip+EBKc%Ufo>T68EZFS7?r3S9 z^$Py{szEn3^Mc1W)eOi4%HmAKy|BKo`IeBSu-rP14f94*U$=x4?N!Oj$r>}gTF)hRCWQ#&bRNZ-6>X__(HMd^o-{oH(Qa7luORJhWOFqJ z$PUo-i}}Uv!Bhk1qH#liq#G@;0`mikvNsMwF8|D(vkByq*cowl*>%lM0S|=8D##AV*?gc8x*C$jLE@=7sfQr}+eY4Se zwJY3a1BZteO?dMlF(Oz+E~6Sby1Uacz{?waL3*wwHP%mH8!~ZrQ!uhOY&tAfb@T*~ z%B%L8y< zE94F8=DX;TG3_enX}uN zTI%GZ&u{_mOcoG@(e#)C995nDn)E(7UZG;SMJDvX)A zFzUOABykj^#FBDN{T3q7NJ4&)C5KM)8Qs-yECeI9szSlFl0WJ40*;HfUI4AWxYHaq zB4-RO-fH3nrP$er{dEje%bXEo6VXgB3sg8DeJ`Z%`x2{ivIG>z+-%$Y02Y$%e7e=3 znf-M3N)I2Rh*D%*HKfzZymHZaJepAk+zD(w#f5XusOj|l2}a`7|X zB+Di;>wVXJ9mfvr(WyTkAs_%NPzFqU+R(*73?a_%vRXG!!mUclVw7$cY_7!d*#q7h*N;e)_C`;$&NpbcCI8TRa(D_9SL+hSb} z;+mxwgb+R-c|a5EukcyPIVObY(`y1j&gkh%XqxF;cT72xiT5ZRuyf@MO(Xtc&hZbu zVyQMu7K`sjnG&H1r)?}Hs~EJe&>5AeQ(CNitdo@yTWPXE zKrVtnZzkFf_&X*IgP}ms2(!U4DUyk)+P>s>4 zu!?C$(bm)<9%6>`x_|4s6us82QBE)dwnYsB>g&B2s)C%My^sby{YhV#LSqz`YwQ0i zrJ0b_2pLe!+N^`fD`6bb*v1b4As_IG8t^_s-ta4~^+|jo!eq|3zDZ-W09YU3kJJV4 zmom~6zC6+^V-)*E4*(V3YH^=-Ug&0h|^XJ#~xxe)e)gd=0{))p;WBhQ2ZkNen zz=C)Iwm`DMfGc7W`u_pO6DAZkz!1(C`L_YYRX3m|)0s^PaQcTAa5!AO6-%7hD;FtI zB|qF%vZ7;h_}*RBcmB=oWM=X?xN1+YJWVBHmJa)y%j@>BRbLq&;1r@`{Iw7-M6%%b z;S92B&**4yey|WSDJ>2~EKN?A9WL`vw8rIhI87f>4(4$co2U+1h<{KjM&{F?Je7X- zF8;_7N`^uyr%5g~vmbd9$SGPhQ@Pt)C&sqnb~m7OJRj&Mmi{L;aJVo!-kkNe0>@0X z7elAY;qbY0G5MV4N2SW?bUZFZIGM4!Qx+WC@w)9#4c{Y=!s&G&e{kF1Or<-S5pO(R z`n|;W8b7>>kw4b{Y66=QokgVoN%_N{{pZGbhJkoW@Px%FoD<3-b#E8IDVB3q$AnJc zhw+4?D32QwF326jfU2%g8Y2^-43$9=WQPOJB`ajX4cHmZGRB`GQ0Hluky&BUql^=1 z4r$2|r!*(j9>$UA%?eZpy5R^1kOS!VBY(&uoPEO{k)Kwu3*_Ep>j$XG2U1EW(7%Vn zUc1-!_VD_csP))&fcasLQs8eaKody!Wl?u^pA+5z*NxM&pW-6R(?hCsUOneSkC7w7 zDkB|GG8f^a|4*fg9 z|3k<99~o@yOq~CF)YktwF7+P-+y5`l>p#Ow%nbkOt^Cj6B}Nw3|F^spvbt@1$c6yA z=fxjFuZ*B+e?Pj~hpA~C||9gLX|8Bp&ZB4%G z=jWrH%b$y{@7mh>naSS@ufJQf`^qo-^?v*_`91l5e*JXr``+a9_H%9jZOngb>)Xp$ zwYvCz?gzfpt53JPa{Bz$g750)_8Io<@OhvQpSlY-d--$HJM&|9m3o`%!vEk`YJ#JW zx3W69-=+tDTI%A)4deE?xpV7!`M$588#C`bvpRaHA9gvqX7Jg*Y1leUx*qPnb%2p2 zeAUXKX&F9<7RE+0ggCl6v}S*!!hA4v$V)K8ag!3;W}h>4xQKSD$N3x0ILJe+*Ru8? zBtQJdwXoP$a(zg9!_ng$?FQUiY3NJNzk0AYOKRkM+K!fUx!1Y|%lO4cmSYv28jkU! zVNP_qM20=L+774~7@U54Sl-Tcfku*XH1T#B(=#bEjMQz(2slHJoOK zgKBhLsD6bAK0LOMf$D^(9PJV#$!Kk)7JbQtHhdSIcmR1{h0)@m7_Aed%J{&*He41^ zPc637^@Zn4-y60&dAtAm;Pux1zf%nQ0{kKH^XDJK--CYufBWAm7JecAQ1}J&PvGzW zeGq^99~B#0{=1|9#@_87vA@asz7Qqu?yYYb$Szh+a3-5(@9?1R0{W>utv_?sY3Y%lC9u5wp)SA*%_{8V%o ziDQzX{46hn=1C<69RD{k7lB zkwJay5iIGWEZ9f%7e6CEimTuMM%X!ZSHeZxHb%v^ZQB*AV%xTD+Z9)A+upG&)=qX< zv7OxS>E3qRc{qP!uGz*Kv-glkBlZ*XdAghNiO-rr6075inKwh?lP`@8@gL6(HiI5? z39s&X0}PRF#RD!PIl#Rq3H>3RijC&FeLA8a)VZpS!`JC+df??2h(C5r6$J%c` zn85pt@$7Fy<6NLp+;O;w#p)~6cl!ys)AIONoS?!(MdJ2c*zx!a-j~Dm$YcK5b41e# zZoSL#y|`6SK}OJ(1;>4%_x1c{5WyX*)hfE1@G?o)^$fkAu%&wRgAh@j@gHkLcLOL<@y!as-o`;kod(ej*=yxQ+kAupA$ zt|;c(LJQzWZDGXwT=7XE;PxRY!EK4;_jlPXuiqv_(A+k~zd<)27Ogj;=Qz19G z_UT<;Gi1d6t_`}^tGVIX9oUkT?41bK=Xz-Yx!Ie2Ju0kdfA^dCSM1UMs}tDu>QxeW zPrBXocp~8e_4qCX{7|y`N_gBpL#gd6z`W-z6rr%I!g+)R+~64er)^`r&@u=VPJ2jt z8*e+mISC2qIe*0n@b(jb@x~bJfiMgY>`f%xYgoHfjBt|oza3;4KkNPSF6!>pcg7C< zhABF_!2BP3$A3V{{|A5l-=;q&2j_o?%m2$&<7Q`P<^BJ-YVrQC!P|ekuK1QJJ5o4P z|Kt34d2Y@TL(Y?cZ>jG9dH*sx_*p& zjuLrjChkQI9&t5C8sl5>d4n+#!-46c(5>7xwh@CR;F3_e{9T3rmuJ(&L#%HR*dL$KjO)TR&^%n&Lr423af ze25DxIQ0{>84V=8Uv<00Edm|E!bLElDjd#Kz&%SLE>=&eD|Mo2>FL@9QD z7(xWk5YXT)5XzZ+9qB1JZP3t}ofkUq$5Y5Bkkgr;7cvii4#FzTZLkX{=q%g`wgHd? znmWV(*HwgJh9vl5jz5YZd;x?~*maoIAj)71FbXI#lr-oD^tmwtjst0c z6+kN>^bPq7-wU)i{0I8Sk6uJW@Bx^H(6zyHAP=wu=mmTRq5zYCNKCUQ>=%v~ zgcpVvf){}ok{j+9SZ~-5j1LxX9G&HX!FM33bue^k5TYyukj6AfN>dbEIaS=1kuqv- zO>kzEH6gmmgbO((26rMRZ{#Q$tlc$z0k;W-eT2%Y<2u?n|f6;Vw-np}HCg zWg!j5KjDmmXoHLv$#7&V$xQQyFvUcIxeBdL!lvwpVYLM5*J5MO?awKUq?4w3l$ixp zQ0XC5-i(4uI?k~0`;d{V&RC?GU%k!8RquV^SJg4Xmzgs~7L_H>!puypD~T+I6sX@V zJ{>b9auaHCKlr@lrb_)4xO|g@WOX0Vb{)%rue2TimJnu#ojA-gBx_!^JTVtcFW`>E z)v4NoZ<{{j7=5GcrA&kF^Skow@)SzY^s!pM=4o&&jjA#`fLkC}bjKH-BpHsN>8|_f zQkAnXVjO4OXV)s84p-+lAh8}^Y0peafHVwc=LoODh9(kT=d8f*Y6v1>Dsd$o-Q@!B zJVe@oh*@Y$>fY6-u_F%M`VrW&Z*CYL&ZU8hSzFh{cbM#*pPj4Ek)hb0MVF)cbh2$z z&~cy7^81sK>0;=57`gocCc0esAa3U)=(GVs9LpRf1NF(jggN~JN(xjnuXzgav+)uv z_z#SbZ#)J_G>7rMQ$r*fweL%mA`;&5_mhHNL~YYZyEz<6-2A5~V{}iz>$v6}5PU$_ z<)SdLWtp_T-+zMN$3o61$D9`zg!PVL%MhG`;*e>LKp{^*t|;^{tP5vDtD*t)X`nVp z(T3@<_0Gh^htG?YGmkahZY^)EdjLZAIq&(3cW?!k_#dFO8(ogf9W@_TGg102Rr}|c z(mkfJxSNZoO89_^53K#Z(&l@8egqN4gNMg|MVQ@?m?FUKtdgg1#OX?@nsoMXxJVZl8UhM5U1>Z6Z~4p9)ZX_ay{n(}J?JR}qOO4$=lZ^J+?^?}AbuV`; zAZAgFUrlGgMW(Yi3oyOE{8n2egm&Uis}=NLl3nY~hSSV9tl;f=tL@@?RY1A3@4EG5 z)NiHvnXVv~|=ekNwjaD5a znI&)N8r!g6JpA3-kS*|9?>VWT2JrLE$|@P&&~FEps8N0_0vB11-omHYHZiZ!5g|3t z`&w^GV&rg`gn9KYi%OmHtQ>g&U8RmAunp`+a(xVWLotW*1%~@(I^-~3BMqH`l*`j^ zN?ncGmVBYE*BdN}4U)!k{|@@+-w{hloekozyWe7EeJKu=WK@d(Ja-@|vTl#V0&t(? zPf@;QzG+9^AsuIVU78`TUth+r-c-YGsaA$oUx( z&8jE>c8!F1+Z(c0LzL_h%EMVE1iDt3ayU(5bk6=8_9c_PRLA=lS`y$OCHc_xU9+VL>R@hadJv9v8eyLUmW!P@E)t~{?Np97uAksdsw^Les?Q(l zFOdl=phj9PE2?X?U(AH=I1K3;5p8o-4S%M->^w{BHt2k-C(+||cX2^F3B%fZ>YX~h z<$!Azww6)*u(}kucrV^M#O*O_PqpR<)i#~) z4@UC2LTWQir8xrl!+8ji9xtVyq7u={81k$yzTBQS@B|_?NkKArYU(6r5+GOwmQ$oL z`ynZ&L!1oU-AG!o4FpT1wldTM7>GuMGW6dSptbV=NxYl)>m+E94dh9%3+VE^iNh#G zEM3kFjk%HNY$H}InshX#mRS_oy48WTe-o9#RAt;x{RHj7f3{KUnbab1m%;9R9y$os z(z3dGft*mo!amc|A23JXk7c*<9y`u_Vr`dpKy;lA`fkgPo|aoKy` z6KrRG+hH)zsmwP#t&~~xcH|WvmdE!D^!86rbLQXmyPNUN2{P9!Rg2~GLY(n`3emH3 zwE$H#79teU^Vl0F%bv^ zb0LMcAOcw$uEfu0HZvKFW8CW6unFd>Ot_ax^y2YN!E?Ft3Q*RP95sc5HjU$7$WJA9zw{sJ>NYM>4Cf2bpx5wnq~3e8<$DZ`I4kU9&w?X~pIx z0#{quKV+cDM0VG8&oqca=QbHoYtEo{+vV2_t4eodd|TR)$n9FS&eNTzdz23OQB(WU zZpJ_fLiuK;-ZusegW?|a!qfy7MFd#oM;%TjOm15C=Pr#e!ngu*-@p&yK6P>KwOqbdC;2vl=V~M57t%6 z8{9N-R^Al-LIDZhcMWrEx09c|Jz?gnb9-axboWh20C@XFFVZ3DL4fFyHJ}BUJZ06y z^BA}&h1Q3!?J57H@yr~#92#Bq29dO@OK5f`!-`8_92ZXFHF;mHwI@KFch(3?#3mRH z(d`711)XQJ-t%PX(J6SE=y{&dNCtAX$y@nwuUN>hZsDPg9Qb|uGc!rS@`uJsA70zc z!;}WAsl!dWROFAG(Nil*YeX7P|K>JY_^V!F3K!$6iQd!yClKCsGZQuv_Ha!tH9jkIx(1Sp^=v7^gMmLoj6~e<{N*BLxc~n}AKk9EIyT zdM8zU%9nWSt>0=Om8X!ZuEclDOdY^oy0#yIaeLnQE#`A9<$U4#B}q-cVvSKcB2%VF zL?F*ktw6-{i25CzsOZhD5tmDA=|gC8YXxPQx%A%=jch5}1awrMh{zG0_sA-DY{$u` z)=nPI$cax-8^hv!^B1rJA9nx#b$0mtKZ=A4%NebWg_}_0Dn|Fsy5S=AGoqmIq<5Y` zzx=l~e)Zaa?|BN2ek;tbtZXuaI9)IKS#YWuHTy6tZ;`x7WJ{gbz1vOzA*{>iI>Un; z4F_kUU}W3$H{!UO>cX^a>)|`!w2VK>-+Aj^@gL#sHXScjai_y?#VpjD%T~WNxY5nA zR}YfN8BN6mrqF(YWrf+LJmh4?FFdmn$8oCKiM_F-69_OS*b_PUCPoq>$hLVDi*cD1 zogqS!7Tugaj}&B`lE;@hR`F6CJspA4N% znVbV$%p{1=JR5Nt=UVO)QojfeJ;_(6IzUd-1CD=Fa|?>(x-WC4gjX0RNKhwh@GsW` zv_7i;6Rkgbket=967F|;Gb{`1n%fMy4UZRBFUqs!>zh8$cOkNMrvi&HrzNFL6}spp zPKJeSDoiop%JbvrKYzWgT$-?T--#5&U|b}`R5j9fW2W9l78V%cQCQ2_f9yDDMYVVG zyys&I*)6e`*03K~?2WLMR?(kIxOk%ae@rkM>cywR&1Du#QniN5pH^a%lV>d5b=>Yp z1XJXJqY0VCtXRtr;Ro;uaxrS`ClSkeQ`|Cs{A$Z>;t7P@QdRTss-gdjKmW5=YuFgH zq9F6s#cx3iVukW62RgkyQUOs^2UBQZ!1j2CqRd}PofUxs?V%9?}QB{tqaQ6BTfmYGkF#+W~USc9mvpmYeKGHG$@IprwWZ0j~~xcY{^?54+q zEGDZ}Q}sH_gH0_I3pjdTb0T5HRQbx%UXT06c_N-D)-Ofm7v=d`xMRs|D(j>|nK6># z?wz9_Qv`yU5<)*E64xQJk3gDUqpTgHN241g#6}^Fk!ZzMQa3}Bq}vabTMnM-hG*)IzIQq(FdFSPfl7W*OQX_;kCmy^g|bWcWF@` zKFIo9z_X-Y_AOFv;O`4#xEaoMrvUQvm*-b%n}>JrpnRgJy)*lI+FpX`&AMFdjzOg= zqX|oR0991l=lNBVuIzzEf)n}qUUKXPZ_YJgLHSF`n^4(lBY)NxZAod;y$^`sm+If{ zNjw&v_0Yn&&mW_rXtiMXB(;Tb4Z=&7^h`${+95shflX8jkWWB>kz zM=y_IH3sc&Ny8^Fq+U65!lsQO!G@=2H`R4=@Y`qpxXtkg4uW6&3)|alnktWMH4qQG zIZ`aicoK^(Dhrd2pn#@3$dc5*H7V>&JnrILT&biaO0C88KhuLS*s#w{AUkvm-R;Fh zUloeIt7bxw1}T9!|6P;fLZ{KTKuo`D+BQs(Or4T9p~!YmjOEF5xiNEqw@E~?5}?r9 zS)@}TnnF&V8Rc>^J-1)-=D11rz{>8E_OdQ0EiQ5^RI%iOcKVsU zg7SS61<1YbgH1b^Hn_Z+L@J^_S;9#i3ySYU(hLO+c3Dck@2T@A4(4XkVvC^29yj{h zk}I?icV!UZT0U4G2dSer_OXn}2~b|n zI^-OT&a8s&f4rpgLATn^bRCJY+QF~KEKZ~}LSyKYx4I8^1C9qIVI}%Gt#^+%!;D`D z;bIQ|#b^+ALn<-=uCQhWyGgrRQvI5b-1L1&Y|qMF^Rma;1NgRDMu-)?=q(~4WWnXQ zqo@Q|&7!7Z10l^qt&Mi}_!7+v$_PEu?~YNZVliGrE&S_yyc$OK9dD9N{A(km!qWm# zo8fuI_0WLNsbhdzAKYtLYJ42~-w%@{2iHTo@FkKOtTZUaMih6H9RgJLaG>JDUUpUansngM2wv3FJg z_~4Z(5!+n?Gl6slQMChMS(?@!=5-M*#F`8Ku#OVJHGO3njerr==$qN?Ca%Huo z=<*d*lKzp%`H9$?k*~wnB*{+J`>1LFw^V!;P`ntWEyEcVQ09gCcv;0QmoP>2Nfx!L zx#Pm$9{4%&!*X0zqqQiK_eV^eioCVl&h=NAS;;oTN2r4y>> zD2?@8RlU3MPJTdB2mnXzOvIMB-iqFMO(w73Qe$^4rAQslDuJ8nTrTQCK!r6=U!2V) zb%>G-Zw^2RIXM)Zj`#BgAFi#BfZ{BkcuzyVLMF2GYIBPtm#wAztd*gJKWVqe1*6`F zf*@)QIpL^A2Zv5M!j)!7WhrQk!pCWSo}u(hzl&C~?`LfXraUfkLq5hj@lu@gT1O*z z5Vio^@Wh=owhIrsF0px1aF{knpV*zcoz!wIO8(!!kWWWj|HPyJe09_r(cJnm4&Rnu zLKQp{eO%s@YlnJz8n}er{I&+2NtpAV$72`gn)acSgmmO)8-scTxYzT31-uU;Dv-hk zcsnYF99ob6v7)^GCHUEnFY`5|TB2WK zq_L*d;1|^W4m73l8`_Rfe)#95=?iz25<4`_y@|{bAau}v#U|#=ml1_ZC?m4sJ)#ut zYQ~bQ#)@AUal#)-7dv9_QDA)$u|{&+4?)v|^%Cn2GFyKnIqENb2si7}vUe`qne8>t zMdSS!8=?|B&OQHMX}Oa@{@#82G@mqEM#8nHTFk#$k@qN#E@#YMWo?~vnl5sFg~;Y& z!9pX&R-A-Med&sIDEY$&-bpNNJ?p&Yo#*M(ofD67%b?zEt)N#?ZD(t($QY}x3(oL| z*NY&O{K5W)kDO@N$09;d*a#3hxHW=sxiQxNcZL)I;~j9~Za~VHQm&I&GCV#2l<&$T z&REtUw!(T3=ZhJ^N5~s06z4vN^k79|wzK^Ylp}t2Vp`CNo14kGXg)P>k<#(pY-bR^%{j_7|&nH#xu|-e)De|as!2?LmUO#rj6KE2i zb)iUQvr*R=1HoN88ka4(yE}<|k|(82uEpoLdX>;1Y@C604u#lnyS%nN5RL<-*RL#E zFV6mvzz>YM9Xu_@f_FH(FfAvN{ilAPpae8+P`Zx5LF!LXrkN%~aXksoD$0Nl+QE;@ z%e+Ba8ijfj>qEKbhlc!f=O}4mo7QX4M4kObK)}d0Qx7Dk(?~Z!3*%x2xK5w55tzF7 zi$2ivy^4nO%Syj4@E>BD=WjJrFqmG16X5&-RBWmhQ_X%(fd{1puf$euYoB>0UCIrr z{%?sHdgrwCjQ!k4hjpkl3i|D47nzxibyGYTVd5%;PXHz;{S_hRG7q?EaAur-;&gpl zektNoCnM7e*EKlN=HUV9b%6KP)f7`v(3+6E5V++vr=!v_?yrHPn6^^W7pA%4=TJZ8 z_Vp6kb`hTaa#&!k5X6`_R28T|P(a4gCn9@en(I<*#VyU6?rpP&O0zA@34jk;^KLV< zZQQBC@d<)MFZ|HgS(R-tH@D7JRA`S=WdfvzjUd*N!LL>e%Jbv1yz}=v^DO}`_Xy7r zu}v}V2M?^-FvSPrwt{gs9(8~}Zq6(^B~_Yv=cY~uz>9y4(OaoQiGtr~h|@i6-Vkfl zi|wyW39y>6_+m2oIF3b$J0}J1KnuJP`+LIn8owy#vn-4WQ!`J$HP*4pCiOw)2Y@Hy zvXvllg?DZNB>>IyHLR-f&m#@7^n(u^)kn{=PbS&#Eb)|h%o|qr$sYp_y*=#2YVKA- z%|Fl+NaBjcNs@wG{$vKQ{%raQ^rE?5M@2n9*?uFU4RoxG+(2r(xE^3rncL)@#<5JE z;|8Ku9$#R|aDgn%=>%4YBrNdWb>@(>j#Yti@Ckfr#gg>zGo2yx-nZa#q|TW91mirr z?T8q@%b%2IBR9lnZH$MhAkOm=gh>tL=%lBE@n2HS>L4!|vPDS)^Ik%}kbtegn`0sZ zXQ~6vislcy5qAJ!;MK!CFoQU!y(+h)zj%iM^Qziijgd|%(1Cu0W+~RxOL&*$&nczd zsL~WwY-NK1DUxuU5R^Ai;xu6#$F33G-YKJv;ppES?XerLASDHR{MGaM>1V(TQpNnq z;hpr3iy_kG?dyV}dms3VP$#w@5*)2ABo=&;YJ0tPnjo;dP zs=wC6Y?=QW%a+92Y#44ktRI>ImvN%ps$BZxD*38_YRhhhDw8Sk#9oTc{0F7l6k!jK z$1+is>#KktBXV`<%^6w8!-;92P4`E-dfzKCE-EBzJK6pCAVTuG`M!>pTO!5H! zVSl{!_@xh9%tn1!LQtH?yoRUmm=G)6GY1kHeWx9=keDt(qaLj96FiJbVI{Yn;ntxyfRvd_?QtzT3^nPZkwc{e#IU4JK9R zp2Z3Z)%6R73c&*P8Q3ksSJcfkuO2rO*f-9kPNAwwl%5kHXGj2&+tEA5ZGY%i_PHV@ z?2ju*h7%gC`aI&x_rVa28fwkD9Q-zS5DebmBs)<&L{djE)8kH)vho<-sdtr5y2qbH zl7!Z)d9wqS({?`oE`s3uReZaI(tS5(r=0hYpFh4#nLiz@1lROre!=`yiB{vzE3P?v zQ7>~pQ1&gI{jA*2h(0SQSS(4ecVEb*Ojj=6J)iS{S$pj5FFOSJh?VBQR$b|CB#Eh} zwmLT%)^#F$-p2U~!Vl=#pPw<;CrpRz>!Xd3J1*W(TeWqy8`D)I|8{EzJ`ZALHW2R9 zwUAt)9d}N?4C6J1_Qf>9iUk#VpMSlFH$WrKDRiE+|H=HB7CLakUP{7 zCCUi<4P@9u_FZ%xsRc;wv)zwCb*KYvLJDKY485*PY$hQDqdSM{kq1B?4GU~r>A`e# zV4d}$7L9SB@EOzO{%ae!QIF3zTO+JqLOwL_vK9jsQc>_(fti!$u5Zm9bua#B00_z5 z&~qREW;TdtF8%YH$SRZ+IS@solFW{?i3~mHoT(CP;fw!g{CsHcjOSm*TKj)Dlq1(X z3j!nyvrIz9?Og%;=Qk=B4)^$tP!u<@t{{1;i_7Oi(HCqeofkMyu(Q#mE9Y>W>SN`u zqv)rOr8J@Dg4(E`L}_O+wjQRvkwmn0B4*w<4@gH#Z_80qbNYLMH)e$|Nx8z0 zw|e*@Z4X??{QJxoCT@oAj3^_N7LpwDCRAx`g9caqU&3Bou928Jd)Tq99zPMo*u`Oi zxg0rcsUj~stg<}gSBT0{d!i1rf9$e3NPTaGTP#i67J5yqR{Y(F4XgTaYBGRY0G4j| z^p&4uoow9v`t)_lc~AMvi~ZaU_zh$18Mh)f%#x*#{E6pH=6w5v0wtS5yxY>X8^==~ zQDJSiR+Jfg3R8w98<7yHYmk*lT+3eAPO_>il0##P_Nz!cy&_BemZ~%uwYhcM)!tf< z-diQQZN%5I=eI)%M=36vkLz}~#!NO7{iPgeCp!6iabPBxan2MY%`C?<@#Sn*^(?yj zI|N7strDs~0eix@@S5nl+7_6W8caK@mM(B8wtLSi?@$~Jp;TQKg&Ddm?%CpB&VO6% z*1SP8+mR_HX6qpU6^A5Y{H~7c4*(Hs$0cY-lcZ;Xt3Or?ZS|JEayTK=PO7wW}}HQsQOE z`e)7nf4)<09W<^iNGGNJRj@$ci&lNU+gyN96xd1o++dm%TpA(bjiJC)M4l&9?1S5z ze6_)slMKl>FH6LyVJcqaEM(!mDZQNC9bBz?RIo~f<#j;Q1YxatJZw#NFYa1 z@+1wz?GbO}Km5aeUhp*X*~AM{V98@*YPC{Eqbgz;JsuM%>FVW3gyG?sZ`})uY)#x@ zdUxW#&fj!V{>(|{21%3H-jB%Os3=2^oZ^=|7{@u3<-m2Wo0k?!5GRG#CGR5QSYB{I zf8@6azJAfh4lz#_-X5O^u4)l0Rzz+IrdRkI(nA&<%6IM1HRRu~b*lQ7>Jnvu@|}a8 z>qUi57C}^nECm0J9_}oT;xNfVjxC}^jx&iuGjgX=$m#u9+fe$lLViLuXzUWDt9zqnc5R? z!OMWT()Rs@ik>C;fCX^b-ekua$@c2|KCn$oj}HZz-QV;bfB2j0tPZ%ubIc39pu8CF z@3yZ#GE1xpO!XQf86+=6ySylrSQb3AjT1BV<$0H7i&u!k{4z7@0JMoRnZAj)9c9z{ z(XR0qE&k{rXCAUxJg_%#smitd>pWM7b9u(BYdNLn5P6zttHAg#q*k&!p#S6wNUx#g zH^n58$_w~APMufZ9zGtqfh!rfVGcaD(bStlsM!CHNGRrSQ^PkNy+9b{Tx_#d>C%=O zO=6>zBB)u%EB!^KNmT~T-(b4V;GXDaoMNLZ^_3amvtn>kL z`mz!ndM);V6MRO(mT$#%M-QWm`Tfb^Dt^ZYM+Y-c!;O^E!^~V6a`cPI9NepDq1ST- zyg9CS>hv5vJwZ1wx88gGTkZ2kO)Z{dR3kLALna zA#)cl1l^H#44<;QM!)J6Xbv*4^k({5dQtwZU#{QNT2dsb(5a$`GC7e~;yu{f%07y> z(iGT44jD2U4pwQMpl;z?_)Q%Vx3f_8VtZ#FT(-4mXn0BCVO~O6=9+QWM%V?KWwHKb z%R-=oA_j01_v#(P>i|@0&DlcH*0c1l`{NkpBpgm9g$bmD|Ay(BKm@66M7P>@M^g<~ zf*%{Es~yjFp(%d;7NuvQpGMR$U0Y4N^qZ^KvN|#LHI=S&)wk%?Y7;NLmA?!dvi+&Y zmK}YCK>B;PU+tGMPc^TY!mn{Zr16~n2kpr?v4{>E;TQVZMed7+PKwQ|>nl@yr#|*e z1xT@?W2}M?thHJ6PliN1*EB2)z58W;(2-JKLyLjDisRyd4EXcflM=l z1jJ}OxjjX2B?6=uEf&=oSAHXLm@(eLN1;uE;N54e0d@l_r2>X-<%3uoP2tD`y&B-e>+ulZVo4bj`MqEPFq-b@)P_ACxq|-0Zg^ja zEWtgFu-sA%GqUf&Tc0@|iO&!xh0u;xC{u-$8kk{wVg@a;9KU&(AEDDscYo5be;39Y zw@jlRq)PFBplsl#oSzRa$QQB7Mtqmp;g;nDl8qu!&`Y>6>v9Fg;MZY?1>}z)zZr$f z93->z@in(~q|{~1{)>eRSO}Hyo;Ze3q+;AxN3qA3@k8?KQNm`OtD;G4m^Wx|+xFRB zxzBR=Q#Lqvo3gV!`S7>Cpvq;k-{L8)#W{G`+oge~@~*X%RF5!M*eO77>-FWYz2(^M ze!bBBnC&ZNOjP=aWs7+gnNEMqj@9wEbM^;k8uUcs5$zH}UCPpk7ZJ7l>_2TdJG0m3 z&8*1>x)O+4+U5Epb=T3Ik>!Lf>!NX?R5|3X^2=Hu%^>Er3I%x1?T|~iiR5&}Gjesb zmB>=M&d!&e{g0h%eR=@*KwM|>&;Uu`m)*JruQh)x!GW$E4 zi2O|^XNJHJhI>AI8@inJ#~45MkF)oxVxNWLrGVBp-W0=ad)blw)|8Go9Rby%R@0+i)etj<%SSX6N{%$yN@C^9KTz}%Eyd~qm7)#U*<)nN*F^HQ z&att|eS+7*Fu6zwR>5R)rkQe@?RN9#EDzfluZ{1{I3%Q zvBZ7HA=Ppb4Q}BNwbNpL>=&nkA&cEnj%0n?Xz>$k1HZV=C)qMf7N2BesjNqUVh85i z7ew@ZeAS})S3O~bdxNH8yFpNIJLBr{EU&pdF*hXE`2~oA@`avj&kDVb4i>kZmp|@} z9KeQyUu*zz_Q2%}nWN~q*#-AwJ+3l9N1j@qN%wCb)(C0fZ+%2A2&cmp#T~2q$_S09 zj&u9nIX6!K@RXI6-L8!OS_N^yhBAxmq@rYFsX^59(LJ}+m`tW46fkUem5iYL73B-8 zMNbgNWQ#$Q*hy-Mq7%!0c!L}vEbEs+W0m@@@8tG`PVUrYF0;Kq19xMS>y>265Yyw; zY`~EHR?f1t)ZC^veZ7F4{Y5;NLt{n|H?aE_Uk;#!EOU9gR+gI|*@f4a!ehfA&A+?*4Bp+#%=$_${yv^=CKa65~WGS<@ zhw0V{WS9*XEZ=AXwSSZl9JA{4BBR7T%ZS#z{nAgVv+r~EeR=R0ds%1eJIj9QPHZW# z{}5j9>n~~Vr@=fm)T?tmO*+i}7fV!@E#JO2b*@Bky3tpj#&G-)Uh5Fmft1IYm6UY- zuaMWxp{Vg6WgJPa>pNVk9{eBu9+Y-#ZSeiTGK1?dCevSJKTWk2QiegAf1Gr)synKQ zbjKErCW-4S=MG*fx=B92x^;9-sAct$hrd(Z zQ^x2*$w{U5N9?*I1C%IR_$@eM$^>pA)o6UiEW^&Lrr*nFBF<2c>UCZoHD}^kA_0=! z=2&I77Tu06UOx)~5l%rVkl>o^W1Uh` zJ~@W6??UM)`R?78S%ez9iR2wq438+*1VO$~n*byvTiXcb*5j+O3nh7^v!Z9NSR;#u zN;KPZ5g^2*Ihc6t5D2Fxx#TA9XY}@dF`!}r2c)aYm8p3?E%k8A-lx{2D{QB4< z=Pjfps(`)#BkgRpd>MJ?7fq%=F3Gl1UYT&x1=`B)1;OrdRcJ1I1}K%GzYyw(%)ghv ztI-i%10qJ@f3*=yf#d9o2YPb+(?cSwFzHr}3(3TIDK2BCuk&isdsA&-3`KIvU*B66 z(DnqR`>9E33iS(Kp5Cor$=6NE)f-xh*Uv?SW$4+s5>*^vyD}JwmQQ$sN{_S<$xag2%`g{@wUDKloENdf8|yO<(bNLGN$m` z!#a;6s~FyaYxv)A68>T+Yjx|mR@8`I+{YvZ>r4A&-ya_CKG=Gaz*35QZ*60|#PDQi zLjQOqPn=w~Aew_*l?z8wp7UF3Qu3*@CE7aJDbFPuQ z?m*q2y+EAL)#{`FN;yzG8I3)AL7VB1djFkD^v&v)ge11=g%sJA%VsR~?GbzO|8l|9A{#&G zz*`mFn-lZiW-c1Qx1F+ZD4|6CH|?31BY)pAU(B-*=dEPuUL447J#?S0aBn3~g$g zb|)+#*eH?J+BP+dD37=E=Alb(vK8=o4;&b?(k=_`0ycD-ON7<5S1@il3tq}KWjgyD z!fw5tR&AWx7t^P)WSEJQSx+X~BmHJQs;8$#!)-X=5!Cj_5;|@)Kq+&LOua=6t6I+Q zQ=l_tCL7M(eqt8_LK?GMeV$t0^lXQMKPl6>Q`2+qP>&L@aW^{76?Td}^TGTdP^l=c zkSo^0j(;XQX3~pc0V$m~ng;!)x#zLEcelfkI0nhYmF_Rprr$~S{YzQ4{ymI76UiQC7WdxPZ8BX3%n~l$KoAFwTuVb|Dv^%i^ zVT7YLSHH#n%MV}>7|*#*N&}!Aa zPTP?UvYG=ixxeMQx)h3C=x5_Gs-9_GqDLbKiu*MJy`Oxhb z13JBFQdM(#$Y%*QFici26FsJ$77N;Pu3W5h?x%+m%o=CwSv$iJ1PB=3lwyE2c{w8y zue0a5T%*V@{Hfm)&Huu-Vri_29qHIYm=>AG>H^sBegdcgam^;;Vp2pY1S+m*dO;jT#tw%P#ZiUfp zwVeH(ZYmPGZr*E+KTc0ek6D_E=g(w1P%y2_v^TZxyA8Tp^J2T9nm*Te3=M|)6kW6s zxj6Ech7H3|lG1A+pX_avTvI72KSL62*QRqK75-YL(aEh|v3SNrYJG&L4e@1{Tcj$t z;`o?rz;P9aTu$=Ry+6p9n!jY@{+j0QrDJ&!{+FA*2*9aZ`aVP7`R!?z9pp|>TN=3>11gG{%e)$R zzVfMpzZ$R0m|8VtMHcG^bohR?#-%{X4ym@mPQAou}M^{|*eqyU%;?HL`?785e;2>z5ngeBnoQquT z*4t5b3Kmhfv!V^RmgP9bKNCt$hCHqcW1d1~%|roL8|;c~Y_gWyXA0 z3u;9BG<~ak?z373kl@it!clF4dm8AclgbOSlWvF7T2AAq|{Qr{1=G(&Ouv6$)%K(ss+2z=&9m{ zJ0`q@yI}8<9rmCLxSzF8>B$L56a*aOU$bnkrLN*m)dOJin&c6|_DyfW_S-gY-X90v zP2RC7jPC%1T5{xZimp0w4>#6c9?4;(r$5uX*9lWq|K)zPJxX;?RG~y_fTDo$RoMCqs4l4Ky` zo!@(Ew?MWn4ukyPq_Mwvu9*OaVVlQSFZ){9)4hV3<~;WabI1;?>@k+A3bY9W|f7 zua;?d?S}J*7B*c0&%^@H;ra*GYC7CZNK_Ei=1##PM;9PouUA{6ywO}hstRusxLdg) z>8%SQ?%bq1P~gxTF*w69I?$LBN4JbsFiZ8B4+_Qgr1{94Y`f9S-jcgVVG#WtzUid1#n z9&Ng)NL_kBGh|rkC?BpArE_etEV9{rQHIQ3BIn5>-`1QWRJwpG$E8>fh?ib(;#{yo zlmLR{VH{B9Dx<95I8Vx(O=^xvBiVlo@H34@y@WEcVwH2Pjym=<+ac7iC@W;r+ zk#2>S6?F>bE6P3nk>&!uo+G^fi4SPg!AYiQ!1=MRHCr9Laokop>zCsYIcuXd+eKeq zs5>cI7vo2LRKq#ltQmoQ#YUj0laNTVdkSUCxnt{IwQ;J>ZIOhM`LlBIamgg%7uDt1 zufM0xHl((XbsMM0ADE{jn!_x){2Z{JdPPW|(ytb!k_E|0#ydF%(x<2pp8)DtWSn?m6E&R zmB%;d$A4~Uh7=p0w($=oM>xT&xWlV-5@o#o-2v60pY+}ODE-J~jlX}3e?Gu4RK_Eg ztp#X<$w8hE;Y!tUDkYhocPIEXz;b6#yw!pLV2_&vq8pxcjj)WH=(4}?!VB1+L*qds=Gt4Fr&v-vcxEb* zcJ%5B$=b2gxGXvNFxyyt>suHPE+6>;K;1KTlY-xldy%^r9f1}Gz z;xvke-+;l-TqeG9fXfh{Q*CO`z#zv!ri6r(iy2^Ya7sgZ~Nt&~P=^wtJBcJd3@9xc^#kD&l9YwZQ*wowc^O zE_?CGSFHz#iPrKGYtE)wzis(d%Q#vK+cHpIT5P;%Hxhj57@elc++!=!?no?uv~9D< zIup4vC0u+VdAWN?#@VgZP^XNED+fDyS8jcmZ`QAivdt^>e7dgMwu-d>cezZ75-@G*(?TU!*w7 z03^!E*W&kKe?hpCLsqm0TnRpAc~Z-fKKv;VA=%M-w3=xm+Onm8%bpo<#DkYwYlM%0 z8@ca8mUkci+{%kqmtq;5D#I1<^xzoFyVX(2M&zdPnQmv}oaiyfb8DxZI1KjZOXd@> zQXZs=KA)wLB0ejzd?8(=#Kz3XCI?5ybv0|kQf$Q2B?|g!FP!h{V$30pnzD~eqt9>b z!~POYcjQ7d^-8Wn3&>ZuL=!nV5Yi-Ga$>hPet7%+rw@q3C*1jY7%W#C|2nCokLHo0 zPC{>AM@7LIn%-RLruoJbBPXSV5!xAn(?S+67C4?@8;`AC5zo;a!6wY7!-m9oSP(V$ z*jOf2e&V8wtY^)X7FOV2!y#Z%G+BYL?F{S54Tqi{y7(vnGfF3#gAhc^N?S{4D~pTZ zKd6n?78Y?Br7Es3J?d<6#rWY5d?OROQLVu@J31m_0K-FnzpkF&r7I>05S&BG6_WNtm4t!ym_Y_5YnVzHo^Yr(@o=I{q; zw89R$L~|qYp-&}SeDp!o&AjTN+96$%SL1G@LsNSzCOW) z;ujXXN&Z1sBIp-zJDi^3IAOk{pAw$bM-g4@Zig$orN(&56^>=$vv|Xua!2ABI0GL^ zxgwF=bN`33_keG!zS_t2y?b9tS9@A}%hvXk<-J$z#94L{XAhD90YV@^5&~h50A+<8 zC?h~AtIVWyo=Q+bN#;?vzeeNrG8|6xcS*5riH?e)E2MkSn6RQO zNk^O8arKJ!KAJfg4Mf~HrB1dg^^?TRN$|r`Oiw9w`U*poyzkt=>zwvc(Fe_r3a5P@ z1D*msTl_Y!HwO7yM^OHSZLK3P`z2g&MB(>{bDqafINgEW7qI$#(HAA3@f!$#7dn%U zqa!OYO8jVKB}TuyVi_#BE$iAkUwGxIZbNqeWm~CgwC&Qa8%~$crf+5qd@oE$uTa*T z-KN?p>jw(6X7O!lib^$=vIb|_#5~6NUQ%|nC2fb|AgQZDbx?XHu6Jcn=~GaYnTwKW zC7cJJOF1@s z+6h9xf*~Lzo$kzQ#SZ|~Doge(rh!BHk7F|x)#8S(UV$;Px;Dt()Riuv)RhgL zIQ7niW^;|7p;A@hUrRUm%2=3q3xJ4Z7z?AhR+1RMX2EC_BM8GssWeJpTCqOY%vZTt zf#iif8$+!+mtjjs>t7cS)>-!My*sasaT5 zGgaZ1{zjHmBh3;f9BooBIZp?c?)?q-;%RGKZnZ}`j=AsT#X+vvu6wQk_VfQHzrWDTHuxziXUIsAeYx!66uIuX^HMauS=-g<96&}v z#xe?0h#2UsPBM0kYXizi0qQsGJw%?mS+`Zm7S6%pjVx)!$UmeCg!-?fXFOuy1jt5@ z9KXXT4m9<#J(pS*sC_~xc~F8h z9ax<2AxN>-q(RO7$3Z^FRyB_DnNR_|YFV~`-UgDJq9nscjvKU~w0z|@jKGq8ZS87= z+U?Ogi?yy%vL#_=O3I@!Ra9mv_=0)h>#QcRhGwNVWY#P2Im3~YIg z3F9;j5C7z~*8ulQyYf~1ym6gZZ6)YX71Xq9cUr<*wd1KI#-WBs-4gb9an2pxLpZy7 zsg4&6;M!IRW9<+a&S35AdO4U$+0HuaEZf;>a=Uq(UuLprQL!mhT%yvpZ+N59zBP$u` z`!~K9(<}t^HMJCtpu2C-vcPYnc3*KHCq6zlj2JEYsY2b`V~Z0qD_7Rluu3uGMHp2j zRSl|*sy$^aSIWcY0^0wjD?6L+&le0dmSxRLVg|~a&6PrrArFMPjBgghxTT*0eQlS) zOu_0D+`^$q{Z@WgrXgkX26Bthwefje+5OJEavL4KYRy zZyEi8*FQk{_{pDLO#IKPhK>Tsz!rUM0G+tp<^P-OqUWoniCF!QF=ve7Pc}(|%d?!T$OI^b1ZpnF`ll)uXFhd)p zOV1mF%aGi72Gat=W%slTmtBdIR)p$+9OsFBk*-2Dt(_V^*sg2&-T8G4$J4FF9C6{y zZulVdH|b2QRUI#L&t_F0^hMDOD4?&<>S_XI1p=e>EtMB&!s z&Yb?9iy)Y!NewS02?5%StRhJR(x9IHw7Fo@kvfY9BJMspo~CKkq-&-kUC|ncc3>cK zf5GDoR`CL*)*jB+hX^!t!C7@fw%w8|P4!bKjez9zb{|1S)MhPMwlx<+1dJ0C;KY%! zFwEj=@u^kgsNQOTmU^2K5v?vzA5!5e!_0d4VHlTk?Qzwlgndp>U8}lDLO$P>YtPnV zj?zu`&B^Z1;%u3GDl=m%(hx!39+Axjih7zQpi@M~LbgJ1Mp-kVuT;D$OI90|Kk(i% zMP%?WGDK1QrkV`Fy}YDBZ)JQb0M%$A(0YVE3*6;xpc|%i3t4Yk|3Ib%rMH_q2Co1@ zltM{+?Wn=lltuo?+7pYJd_=sv&Sx{Tkav@gX{Ttlc9$=Qk@~7!p4WzK zyY5D>+E{bE(=`~hczuyulEc2Z4a`tsY+qB{DFt@sNL@2U1xzmKHk#WUof@}BtFN{t znl3$oJwa$`##RI4sWFTr&PlZoNZI=*r>~B&4uRZ|-QA=f-+v--vEfHHpZ5MM6bJc- zsgV^%k3Z%nS)m~)3bC{uWpUCSO<*)a^)^CVi(F9{YXP61EWTPSN3KZJQu^T5KFI8(JyEo|_6Z`c zgK%UC7<1<^g5z3v81>hq?Q!O60I^XWz~8?5%`4*IMDbP$bgy9;skk|9HARc$Fjx$)e{OTj) zgW+%zA*J;GvVc9rn=M+7(wh3~94U^|n&_VX!^g`XUu~5?#8;huR#vj10R+iXHuVV_ zZ*34j10BPn5t7F=#ki4~>6PM!gyW=)=z!|NqhqR*_Ttw*rc2np z8$+qwt^-@P$au;j<)bganv%9`LDj3;TB;MsFc2@VPx}A6H^fkm{UBQ@p{1yvm*MoR zd>{B>EYF_O)z0#4lmpdGZ5aDXXSNAvf7nu=C%7MOZfcWIVx9|W5V++U`6Fvu+qKOPaW3=bW zliOA$qv6cTTgTZm?%8-QxlW0{XV0vIZ-hQouvk`}YeX|yueKaAA-otT^~N8j=b@h^ERMW&1bGQ}}g&~R1zBB#zmCabo; zQ!Th305~uOW>bFAF1SSx4(ps}7f6GnmKTRTU@Oq zBOMDaq7f(Ps2T@h=-)S%%fIQ7y+Bbm^*=Bd*4$_UyT8*Df(NJu+rNSR56voE}&# zAt)Rde|`PI2*o3md*fE`zygw!paK}HS>0d_(3%aaU9vB-&n$&sfxc8FROhNLQe9TA z&s0>hjw)%DlqItnPgz;ALYG-FZ|N5*QjTNP#pHuDPlv0bsWq$L?8->rl;r|J^*w%@ z*B|{cyp*AgA~Zsi+@WME&WZm5P%X+LB9bvcFxV$P!ubpka{6ib$p?^*6}956ew!~8 z0=`@PHhau>A2Pwxrd%+Asn3a@)0UzwF@Mj)xh{+r?`awP#M4PJMhL0bijROE@dG<2 zeU%u+10Fzy2b0H+oWb-{ZrRS}zxi0E;L7d6%ncm`MD787za!zwdof@%+ z?2U0Uf@~!H@PfD(lVENpp2Mk)$BpC|0yenn^8F~C%Eqd34yB&7=)H`U`OTD;m|82H zEk!?G%4Gwp_-y}2n6GIjL#w$8ucEA1!d3#BnHOP8K3*0t^BGIx`$c_t@OgAZ*cV^U zdPtr5%hmym*==6A`Ywl$hEV$Yx}D-%_zm}`hb6mnF748bEANb1eeuwbyiGW6!(ygG zKt(;(A0e@bfk%45EsNKaFM-iG#^LU^w&CR%DZX`Jqo0RNRmkejvzxfO&24)?+`{~J z_vt`&@;g)%Ph>F?5%&B0fmy@G#qaOp&q_zh21e)qXu(*l*bj_L_W2kLRQ=MGSgATw zwXYm4v9jqLyz>ZSnuPQb9!C_f&S$#b}Dn)37b2(q*RrMFQS-&Nq5* zvQbKjk33ORi!tH{K$~gCd2w$?$X;O8+TkY}+HwAyKl^PSZv>2pZwAa>UlcUG6g2t# zk#}ZPL5n94+OgXq;0#2k=|H{Fi{UN@IK|RJ5#aW4`eN_TKDh$ zAp*x>W`WCAqcgCDpuqA5v>~_3pM2xO3yb)qVP*_I0?nvMl}ixQH zTvAFq1G!55@cHWba`j1(LnxMF6u)28Gn9-LJa?MKEkIcKJ+#rS;kAAj!jS+P3Jx-< zOp;4W)+DwDtR8O+tjmZ;9`Rcx)D5j^=#aku5!a|&UTxH)PP*gbMS0U^`tBEr2lhZS zo~*$psDtU->j%tL8dqyLM8`WRjVDWNxK`O`(kgx?#kZa@f)anPrjfvyP0NzlGnFELJwgauG-RU3o7SsIcmJ{hwn97)pL(I2vYa3#MMiVB*y`r3(l>>1nFF zB`&>qW^Z8VM(p~AIBoEU{dUned@Xgsf=<>cU~qQL`C*IO7e9myQnMx?P ztCd9_pmLFiPpJYyMS)3(ZS$L4YJ@Y0=eLeYNeQV5IspN8H+4$x#}Tmm{NOI}%|#{# zQCIB?HL$M21X$?kA{R}W^cqI&Kiz=14R*m9Mw}i!eVh1sT~=+=k|g7{LQyS%G6(}f z$RcXLw#BSwI7s7>ZfT517PFiL{aHIeJj*80-d+KMrZx6@hYN6#?mY(iW zoS?(#$4)6&))1%Wy#zy64>R#P#@QM$1qAnP;=Zo+fjF6R+xd~SQ>#IA7O*E~^;)ZA zY*Ta9uer~|)@nPaud8Z0*uBV$65``}MrdE^?hR@+ev++n|9WK1gRwa6Ls?y5=iXr- z8=ZQ7)Gzxl!^{@=N$49XyBe16=UHV{&^ZQ0rd0c#$DcTwKH;kDS04kN2(yu7UPtTb zTtWHS9|pJZr*uy!71Ps3*S~aLl&D(ot;vx5zWN5eQH}6^$=5>wip{9#m==e-KMS-{ z^2&rJS<_rKqZ(frvbg=x4O@a%xz8|?ZN!+*m(>+8_UD(><#ATjNI$PNgXUXLWDb8W zS^4!Wp@|LVr9|wmx@L-AVX01gw0t@xzP+S_Hwi7lesD;y;WYX}@Qg$%I7)?YgEKbY`(bj zvDTt^Toqu(gW<9?J{SNshHeUD(Y#uGk%|fM!hQ6i1ODGA$oOlI;f0VC3*z8^C6njG zCv0xe**+vadf?KJJP{PejPJFdR5Jl4#22USH3WuIFGTW)#>FjeWLdpl$_@Y6+N{}p zaW@@92bM`Op=XpN*R&5WC&+s&JmS<-2hW~dD8Bcve@S?3$;{>OPod9BJ0?`6re)a$ zAFcPT`ET}1XgnI|(5?9{jGcZ&QF;Azt5PT22e0Kw%ckZ|oSjas2oDlOKwc_wTD=B^ zNaXp`z;R+<|0vE%HyAu5IX1C26GSOYBYrqAO0YkRH)0g57TAEaYi!f4 zwPFnH70)Q^5z89pA4^efy%Ntd#a&Kw;i&uukJ6Xdq)>w^nUf-2$%iS&N~tXvD^|hI zq0fq?vCwIo&H$<(rm<{jRaQZ9tg#{EG^syPJbXUD&%AC3BH69~ zvuCnq>=NRfW`8fI&vd(q<9VFG;<2-LDRrVt%zac?w|lw0on%Z}XwszC)Ng7gSjP6} zjv(zNHd9hEcWm3@p=&nfs4NlZnX-!41o$&iQsuLP50j5?i4ZAyZ%xGg~lAe zlguENCM4(b(_}S9V(bqk{OOw64ZjV&rqt1blC3YVWYK(U)j<)}$QMkuVOXZ2R^^OE zA*k=#NI0htf6;=4t~g=OgL6Oh^wG2hf-`#Q^sn4b_V*y5XGZmjK_9va3|s)Y0yM=P z_;^i6r^aQkx3XyJ99uraOWviorh(?oAhhchtHnHZGxSMn~I zLf$J*ki3y?W(fWntd=!>W*hi(!pNmJr-9JfretcafK@v1T2x=&DFT_6W}67^B>*)W zPhZ^}y;}T)-29Wmmm&FopbuHi(`KNSb7S$+UBKF^MG5|<6G9Lo?rQBJnBzlj ze9*eRry!q;Vd3IZ;zQNpix^)yDJ05;ps*cY@(W(nQa(uZ-s>OFox zFaeK`LooKHa}u|RZ{xsx*=~>#ALuwC*$#bWtMeGbo>Z;#ozgOJ@%NXmuU4am09ja& zy0V8`cjXO0D}J-La9=SGa!9ZB73f2iQ?kMq)s%8_i`JHz$8Ag{a|Q)~G=W*|@9*3@ z0Olu=@&cq|u!JB6?K{M*?}mF1`C3WRXiRkK{K10-{q#R>(Z?uD+omJ*n_841!m(7$ z=#BgOs@iC}3f#2xmyT*3f-!g0pCSH%0LaOGE>o3D!t!{aow5p4_OJQ+Za;L%ZKt$x zb=xnBw&ZHl(z#QgttJh10!iDdu3Zow(4|~sjniX0*`EYBwh;1wl?v$!rwG0HSWk#Y zj1GKR?Y0*6(j6=Bpd(naK#{0m-S^~mE#v!3`$gBnufPUXy{cK6VLyrvG*{ItRzqhg zD4?RHUNi`W>j9UcITY%5kF$>a_)i*J2=1K=TRUij_$Az`!(XF*@B{4N75Ed1Q-2)- z7ep+wAwTdEz;?B@8(l$!9D$5iY)%p!)4EY+;{0=Xl%)?YpMZ64>l2CH4?f^k$A54g z6~StDbxU8x&;EJ+BJhaFWIVFJ?^?!s%b{7?vXR)%3sn2hA4R+}(M|Y9md3 z&==}ps>)Od+)-F*aGH7+IMvBMO5Gf%c8i9@qF~;!blo$4_0Ve)8tHj&IEYicJ#c7X z!7_q!f&*+#B510wJKNu63>bSeGX8VUtb<>Jey6fa(Rz`xo8~C^RwO!>`x|)0nK%~H zXVzc)iWiB4WfWe*>N2T9>9hN1!z5(++S5Ukb_8u^#J@7tT0q19&MOt$7vl+%y)YR~ z5d4Ab(AQVnaRQ0oU!(l29o#U;X)##qUwrT!^4`m1CmAPvR$hy`Lv`2IqWv0*f$po$ z5L!(0K{VRIS8?ERaRJZijm(0nw|f&iB`e7_Q0$-5brkdWIJc(SgkpsZfFzjkzK>v8 zt^Fql&)bmYmOS=L@sLkr99fCfdkO|7s&VFS%r?8E6GbST_mgI}!skHmD)TdyO(MBR zRPFUS{4xyB?pr#)V&77k*ZW`YTRMtyI$O&1=;ij5@tAqC-cZ~f zEDu4$fnsm6c()Bk09}pUpetxlb`+{lnOwCR5)0m(QNVl;vYKUWiBuiNUIo~9{H+va zhLEiAHzvZ#OPLaX``h0ZJ}2cx5>$OOTOc^`vr|(I82N|3O9w#63$L0f^D5FfS&YP1<6j}aU&>mquxsqJMtgpQ4lMp2&NZ_T$8^h@z9PNUqp zZd*d*M8)Bn)5Z#mNILeFS+m&#Fn2MJrgN1F?~n)zo!w z`EZ7?g_n2~oHGu;4Rb1ss#m%LOXO|CLItg@+z3}LXm)m+-nZzd--3|E=LsH73YRmk z(v>Z@=pRfhVy@{P;XJjWFh#0ib+v{B?v&r*_e+;{EyLKphpeM%dt0_Zu@A4RYhtw1 z_b8l&Q{VmG_jb|_)5M}aV*iH2TebylI<@npI`GiYMnVL19YwPzbn~cMN{!B0xl~^9 z@pJ<0m7?v)wrn-VpzN&+h8D#T2Ip^U>7_{Vv-U2QkqhLkbc=Atw4x_Q@4f@8>q?{& z>ZS{0Domx7V(>H2j4GqrrrM=Cq`Fykcy0sGQ7q|-I*&Xmb#x~QotJsaiuSY93*W&P zDcfV9f}whh4E3lcZJ^Ygf{t99Z6sK6KuI4@n+%)0{^$v#0SR73*TG{nKnU`bS_y`I z)RU+s_`Ai)L?l6Q=MLu#dfHtL6Y>ZrMVttY;dDXHFvl9D zld~vUN6;B-EL~;k4*qNTh?+C&CiA`b+&a!2_`UR-&#Z0qqAaEP%hdW9!BAtZAA~#V zJUSm3sGok|`y8jYE$sb$Iu#G4q(qe#e-Uto)OLebgEN|CzaJglx>(tVL#UE7qwqs8 zrD|02xjxm9YD{&u>cBC$!j-$(E6F6DIf__`t(wkZxO5NX#bf~;p32NV0P~`Y@}y&V zsdV|qE34k$4|>T9+k0rM*>!ESk>nDS=X>i2isy4W*p>mlv}P}o)1k;7V*6fl#h7U- z7Rg+YKYs6{;Itp_{Sk7e*jiBXlu@Mm7N1$-KG zFt^_u!AaB;i#HM!Om7QUW3-e%5iUm{Y~VPU7alli95I>nt6IBH-dC^9tv~O7Kd6m3 z_lm#YvuDpr*BAC zTVn=!j)~qWFSUwZC=X4rqtzLlgSOW7crlXZGleda5#Q-sK+%_a)p(R#xuRw5At%?i zYmt=s0iWa#Cv7fFL$E#lwgKzHh2k9-bo5JSFN|Iw1@-1!#)h#ZyRJ4*(ZzYW@XTiT zN$3NGp*QdC&X2er1%WAPadBivvI{wZii5>}RxB6NHus{e_{4O_AjXM1T|+bpX>vKZ zuiXu{NH6PurO$!IIQ72^CZy!)!kT2nXSeYhhBXyAlfIc@_-^Q*Dw~wqud#IkZ7}0T=;?ntg@R9mQq9(zBoNv&QU%il zx2Ud}V3j(QeP&RZ^I9T#U)5awtirdtXV;V-wR!Wu7zr!yH%+0(*seAIR?Qv3PtX-C zV}x@i1Q}Ek`Ze7;&uU$ZK=^Z@DP>-&MJTxDNRxCsLg5i)8A}vn2t;^;K{tdWMo*$ebNyGZ;dW?hYk%fakl*ifW=Qd0rT919AX_$6L#u(lF|c1#)2a4ZO_tSq@#B{DA%K|Pt@Y<9T|-gxm!AX=DX4j1os08ywH6E zV=(8@ZR=?RpAL_B5gw(_scpp>=^E>1CgEpcNL8y^_+RZFlhfciW?%X8%8e{-xdnan zT>?+nWcUteKV{YM7b~FctZXyN{NuLz7K|1@0w8Tc;P*pjw>SDvac$=k>Vb|v#$gHv z6Ew-gjT#IFzuVJ_-Tkr@huBbL{o2ML%HZlp+XgY}-L65Lx#JXLP_UX;=G!ilP{6zC zEvqPi0c{OQk=e<1GfGGp@80E$VWCRDsiUFT!<2aqw?Lwl1J!@n*H{BK;<4f zQobY*KDQU_TlAG2JUg86XiqzL_2NO^Y{4Co07I+cf`B5>;^W1qpotrg&sITGJZ%!1 z^ZgDRjq0y{Hfd$>s8)+ka*W}v6KnEL!HkezaL1+&>U9vVp;)p?)7HZwGSB_o^#k&R zz{ZZe;lY$(TCnbvFYc~uuEuFxBNo2@3@M17JHQ3; z@oX}%X@Lq>HA?>YX(>J~QBA5=%d^+nz)>amV*$sE##0;=uizL%IZv6BDFaUFTV>!P zUmFR~dJM1}4S%;Ef}d904s=h0Ow)K9U>Dl@ zCEA8V^dXFzvY6;#;6jY5)&m>?#zXkc(CbT*&_&=6gG;6Kwrhx_raD6jRNhouh=*#_ zF1Mhe1RHp2bs;cNrLHz=)?}Vs$tSj+cinihq0Zz`tJ!qs{-wkLjyGHBzV#qJG@knY z*z~^^zPU|~{rZ_D@Pn`vFH~U(2QF2utvDs6_Jm?E^DAZ9oL}a+7=)wey^3uQf91O6 zRO#5rPysC`K45706zYoZJ~rS1wqy$xG-Kq;}z5- zWqMWS;}bNc-&D^v9yg>-XQNps*VEjk;3bRXQ+^Jk5_*==(s`8;h_PHNmB|qt#_Kn9b{beZ$SjCAr#01Sj6j9!tLzEA zog2>RHCk=T8Tr8HRc|=6@j8NCQ`@(0`tsLbhaQ!lNg~49fFq`}aJJhr%_Q+uh&{oj z$LJ0%z(Sb_LP?nDUjn#5s}ZyrNBHzWPW)6Y@E7dIp{m_Wqi+(v7d+KBIgH@ zZqjS$4esBq6D%XEL?*qH*|t?m-QCeh6-|;zBdNhq@YJ34EjTy*C#By1!n%-E-UXhl zZ6-MAaZ($sudUYXIUxSw^bz&I_s^zluw*;QV49VYNVYm)T-b7X^NG~K2aj)JIJ~8n zMQ9xR^VAYV>uS9|yjB!D2Wf_iqX42~Xh%QE7W-cH@{ABPt!hvd{SD=jk=fe!d}h$M zdKUUw!}cj<@yOL8@I_D<*L*d8mmKH7HA@O!c^EVLn)sKZhCmcvt!69$YUv_4@$^0Z zLCj@lJH+SnTh_PaH?exmY^Emq_9wJm=S)2*LE1Z{gEv~A3==e=-ZU^u(RYM8a1E+6 zo%xePD>fc1#x~c?O87;XRQXlos?$~H%yIUVSFArqOfS_hzk_@-yVo3=kFLa>EVDPkl(!h0_3vb;5-u;@=y)Ob(we z?1o~}1sls{11Lrk+;-_Qv)^m!R{EpsrGRnOQZLSt%q{tLg1z0#*NRtr?UA6(#PYP1 zGX-jmB-+%zQF>xqrwb)9s#kvE=TUc3<8kpCQvZ%2Sd|P9H@Ox2iOlSTuYx&MwPZoF z`_5AO#dAnxG&gGvsIkIknZ9Ed_KBu*zP67PA7&F z^+o~FV_-GMnn754_LpI!&ldrAHbBXtr8s#{I!{sveZ%3pu!~6q1Y`Q8Ga>|@&^V(X zcBVoogR#>ulZ5ovIu)DoN;dO6tWwpe_NewtKKmHk?EfE_I(Ib3T#9zzg(APJ2rCny zlwrj(j}FM*o;#B`mu*uMo+K8n(=IUK6dJy(LFt$bBa0=7?UR$%y?+FCp+=gvz;IqI zK9Ai1^|J(jdbS1QK3?LhC24sBy!K-%EI<%kKXH@yd3N~<1nd=`2g#4YL{=CUA9H!z z;uc1Y@B;9stRz(PaF$7hgR6{3DMP{f>2J z2mCDbzGOFfg(u3avN&hqr>TxVVSHq?8)_-6*dS1#)F+;Bblz)tOj zaz8u{J`8qJ78|;HZ>X7~!eeXCeAyAEG5Xg{6W#l*Bmu`ft3D_AzUcXJzV)(I|MR0y zr^p*Y18`AMEZIoWu%3D$P)E^-_V$fa4jwo{TQAvkgOx;UOz#~x@k=GQKY3;XJ|2c; z?xU;=HR6|~ z`Xi%;!KLVI|nj#8Jx!SBGsA*aL#~MLkkuN8O)s0 zYYYj6ZIP7++PEdpT|IpbO=~UvBjBlYR>m!cnYEJt|4f;i9hcDXwr?|oO2F@1(kcoy zXtu?DENPXwev+)MVk}P)&sMt?Lik6$yO=9-OB6SF!saIM+`(!b^_`TgZzTAe+uDt4 zBhF?3+{i-!^nL9vP_!O`>V>b=Au0YtpzzfgC9t;ml|9s<$*b0GaGR?*Oh4HGLbshn zzWz%o*d12G>c~I~jYGO0v=7kau%RheWoc{{e>G;WX3e^MUzg-Eel4x%hExcts}ZR}1x*6%;?>nOO$E05d9`szueO zm)0DXw zdPe-8ZQSc$g>e@B!o*=_J2iOuug~il!>Ip6JTZ=Odl|Ll06vz&#(C0bV;4sKn_zJF zhGqP!C+PZ-V8Kznaf^JG%`>auC!wz;oOSM;c89E3C%>bLWyNKUCdgB^Ie#9?b57PZ)jNbYL~f>WbDrO z7f8u_D>Xi4_Gvls-2se+bne{7?#&Y@341)DQRr(!O7HUZS64kvhYNmdwc5e@AFaz1 z($&27;(EqrX9e1O$vQiZhDnR-_(UT0I32?h)i{pot5A8%x|gI;)H%8mKCV6|-5)L} ze%Q@)kFeRj?a@NW?|`XpEOyQo)(PV+bmOJcIi{spb|s9I_}aX_>WbKVV&D@TxTOdp1PTS7QmAKT{}J)L@)x^JeqGI z)%PazD1&OJf8es!XmxaN!f4@i3;G~yZL9e2#~yqC*=Ij_;|+z^;G4N0ehzw3*&!%z z{;Qy-Rz8L+9zO@^dKTdP^fvk6tOnoj8AbW&cjePF2Hn*=L~!4{6|{MM(6;ea1V`7P z;&bC`P>xPu;+=~oX^yHv6@KL+=?r`&UH_bfgyiYl)X{numEpfa*`suU6)51*%WX2f z@0cD~R;ixP&YXW|A=?gQ8OLh3`Fjb<48eJBH{7U4;hnY0g@0=2&gXWEP}7O*zT^Uh zU$^b}dgP(Q)dWMw7RR#nB>%nI7NZ+u;z*+g0+6X`cwq#mN%oRdK^{uJR*LT$7n`_i z?J%lw7&ms-AM{yFGMVnA*4p-e|NGyX1LsliJV!^d+fAqt9x2cy|6rk$qWcZWrl6;> zPW;t)H{eXVynk510D+n9@H4Pl@*A}!mPXMr2&p;vIOiw64Gugm4BXf(E|4lz89=*`{VX=gMxT#C>hjtWJn=*ZoU~R%9o)E1i zNWysV)|y~k>-HD~t^0<1fBJf_dO5&Aro@Wxk!&k9vlMv@drx=_tJy{uMe z3~xWPcT-=#(IOyKY4MZ|5sORu1timebN79Pi1)0l(i2GJcR-}8(T%&@5Q*uA`}-D? z%mk* zJE0a$eM2zbX@YTZ*O6xFDop=TN|k)^hj%D9@O)$o!`RNq6>#n!pZ9ii3r<|#)X*=p zfwoM9F&ak9Te}yK#Qup-CWCF-NU-<7X#~1l(_CDmn{0gHr};*6c${<+ScBKK31*cg#~@PTWd2tD?8#CO$qmY z_St8WfA`KTmU0y-XHocC$EkLfU#qe@Ookhtc@vf8`URMq*e}!S@gn82Vm4D>!6>QF zajGoip>x3mO8z#~PI#Cb1_eZ&!FyO=pid)vuLa>4%;n9V>geOfj zA>|A#3Z{R4&N(Vvilg1|voNkArOZG^wJTL;$m&mXQ%`8B> zLFIsmu{=RyoL3tyl>&}k67{%CS%iGuv^(5p8s-AvWx8~Zw^mpr3@0GtA9ClVYV)!_SsDwrFLe{3INyi!y6aq zI31t8Dh7T(Jbs*9H}^+)H*f_FgoQex{)Bds9wo8p-Qc0q ze?;8Bu|ArU88yI^Y9VNkJzbc5qQ(wm?2&A~=KaL_sbn3_y?95aSqgN2hna*C5I0IV zD-d~TSxAE6(c{r2(lv%qI)XI~ja4BG&0i%QSmw?i^#<{=nqW-pck)`Apn<*CiQ~`Yo_#8wg&!T(VEU8LS9*6POe}S~TL-Y)Xwnk`X>EUJbn@{>{rX z7BfC_$GPHfFW0{ZyeXW4#ao&?X-awnH>c|{6471jwa0Z9CLdrpBd1^7G1XTmzWTuj z#d?c#W~-tTlvJJc|F$Pj1zV~JQG9YTYDhMhl(eK%-(1uJoL$}`!yR`VPRoKkW&NIF zuCI(jyiy38-L}jaGRZJzpWL)Pa4mVOwh<)(6mEf11)aN*#gOdXpl>K_bvh%=Bn(Zl z0f@ZV(?U@`B4^`WcHUsHt*vzm$R!xJlGjqMD%;6dVZRBlr9(La{D%soqZ1twl)<>e zjjaTGq?ukW{w5KO^Hm-#j|%2b4K)JX(*1o5AtDGMHtz~q5!#fvl8I*t2{279z3O1g zVtCVrus+i`xbllt?15E{35%a?^Bi~X685^+7Gv6~7R657GfSkLU!}~*B~^n;Z=syA zR~9zQ;9V9IDYp{JN(o2d!FdN6?7}?uU01R%Erv4E0Xa8hl8yw>HBR5$wwifjWfRR% zbzz&_US;D36b8TjJghw9vO*8eJ)!H>B6!^xyu3>XJ&Dtpd3_&y*?~PIA)SpbaYf6h z=K1H-q^>Vcq72S_xo%j)84T;%T7gM=f`pd{aeFEn;ax6`o@Q8;gwXvnr^0W-sPul@ zR9#AyW0h)?YKQ7t)$?=My>k4cJYikg<|==$vN9xJptAg_P*mJ12_c}xlt(EC;6(|L zY_8O8&G-Xl?=^cpbMu{6@u#3PZz2y07DZR&!+ea96X^b2-lk@7b3pvh0%w+{ba;|q z4|A>+rfL%7(HeG)4D)~ho(^stKD_>fG4bv3oX@0Vt1NJcIlE7~PkN;TnN8=9|=2Zk)ilGlP9VF!P}x z&uIi^LG$V;bLJ}io>$N6t^4yWw}>0wdJEbZOVx)b^VWqVB}Jn2$vB0x=b&eIF!6*r zq-DDj;=#IRnwl8Wv4U_v`FJZV*AxY1BA88 z{F($DYWu{Wy}IVZ+Wa41zt;!?uNejKN=+?JpxXYZRqVZGa2!pLC@5xT3rChMW@faQ znVFfH$zsW3X0n)>*yyZyCens-y8qG?N)MGE`It^b~2%j@Hda1HLqu+GpfEO~i) z`%W51;qWy#sFtC=npOYI4F-L`GO1Be4ve+KP;uy`{wh7jmMIEuW0UBnKGLcclh$-6 zKP$^>`zo4GP&!1m>~d!q{X`5{_i}v5)DM%@(}%KolKeya{>uM!Ur3>Z8j#P~*Wa64 zrPZJ)JdQLbu+S5pCqSen_G{1oB^B)(kvO>AP_2Bi#%@k}k`wapg=dD+mUbt<3bv`g zkz59zu$AYp$z@w}c^o=kEW9dTq%s1J*1-#Sypp>xNS&$n#cX@JKXs(< zDUaC~aFNnc)Cz+eI){qf`B&=Rg%17pYe#5|C^%Mmy}^OhX}YG-r8#ir}QQR4jS`Ml3pY^4UG#UW2By)oRa zgd0;X?O(t;d$|jgOo|n5{RoWt$(Yep(W@i%E#?d(`@E2^P=_A9hi7^ZrKWr%tColX zNVF~*cndRj{uuVBYi0||;;cowtN6CX>~VrjDioVP-x$*iZp~Zx@*N#^!0U3=5uFZCDwee3 zJNEP8mfem-<-2!#xyZc+jA&%?pOnFs*4dI>-7VP+m$JDUnmI9dRgYLds-()6exp=e zst}bKj3ygS*ogMCJf2YY-qKY+&tlINwjG&Dbxe2$#`d|;N#YmvhLr!X_G*i99Iz|t&QnXeoc1zhxzj<8mVVdG_SC;3lLXjvok`W}Q z8>yR4mQ@O>j&)5aB^P$N7=^VX}J#lkPYuST9rx*@Fl5#1-f?%4QOv zfz6OxmrVuc4VQ}zK30(X`+xS`8x+Qflm|L7O zIpcoHp>}GEi{jmlDmzMDnwMM-p?yuLX2UPH=5U58U2lW&lyOQe=2m{}lKP40-WHME z3J8Sql+v7LRmL*vO0eLA$6P90gN(o$74u}al1R~e<1wt>vS@L#v!8_CZ_0^~42R;U z%4awUjo&2$zvrN2G(#Yux));o;pj)aVaFc;=iOXbI&;HhymS+69-ty7&Fi;Sy>HdY z9<@RlCn?QgFyP&NG%pa7pIF`>n`^8^FV}0r@_v|Q6#jaLmn)M%JH}70$Crg%L8PcP zd80}|zkbt)F7E3bqm^iVEn^>pM|*w%9AD(?vmh*GyjWIkwsOUW{VhE>QnUF{n_FzO zMY}R3wv)j)nvmr&pu)udY_8TP5CSS3t5BnkEPD&g*8u2(dj(6 z9Ias2bt=6!N_Cq4y*O``OW=-YWm6PWHkgAR{X=Tw#--=U*^`qVp6;S?7Kzq-+pB(7 zt&~O8uickNv}IZ+7sn_&Mz zgN#Y7Y^@{?%>#^D3)-9{jXv(GlWZ4=pba5Jo}5#C-e-xaD$;s`SBko2yjIWMyoi(8 zE*O%ewLxQ*HAZ-TG8xU?B}eRKxu0`>wCRS|9aMVR$6td3p)wA6m2XD>DrB zODTDpabUSMN1W*RRLZDu_$^aM{|G-x&h3=w3-z~|wzN^;0L92REj7QpM1nn9v3%((rw7 zdwtuX9aLknOP2iL)iy%ywWS*9K~@}a4v!SS?CN8@ZocD|h{g|_0?5 zowaN!-Tb72SLa7eRQg$k5xcghtm!cnXUUXYUPp}U(>QuUV=HfYIo6|Y18#y8$vBOk zCWAm3gkpc+h@P}wd+?)_=5FbpJerIbr^2gIBry5RxIGkUKbB83GnM%lu3QgNwt>aw zjSJ*3e~-Yn+|3@+#&03gz318?p=Arh^1A9N0{qYi*`8*nto_QL_e51Yi(5m;`Ltpp zCU1eE%912r!SIEcMO&{>L<4H5WBl@6omF8wW-4c1zLYV}XBlg7mp``*BUkIN*tI7Qn>a7c}l+!$_8)1bzy97+z z@1v|`{LYMu;y?jTVcjhCeXK8YWw(lnCJM$h7jAa)7C+1lJ`0hIa%aKZNj|$JJ!2hJ z+*5W68PexQ*cfOMEU@SNwl6uM!C#a3o4el1hisNmcBK0tAPIYc&=d8t*kPvxKvUP( z<&o3d?h=%O(AVxz*lB7<-h?)YuU=EO)-tA)J&AP8^D?7)(+-{9tg$$rq+loZ29|#t zQ=yu+%wpK38VTRfna4qaH^$e<9Q(RY_xBj<)&o(q%5d^@jXW#6n{i1*dl7~kYt4tq zZC}u()T=L{_E7fF(?9Y)K1p|)GtSx$={GW(5qogU4^r$2v2z?b>9g}4+0k8Mj@DKf zbID1Oi!P|wh9j2p0`zGvMB??ajqBbW-Gx7v+6=`78YSqZhP-G3=%aT_>>lkf38LC3 zIje9VcrOa=FcDuL$|suUL}zkyIaYP8Jh9QC8%eY7fl#PB+xZc!~z#WFld zx-~9Ix$W0lrJ-Ur<_w`zctpmG5axsTbi)C1_PzM)y?v=DwYVSedZ)T~vAYZ7o3Dkj zR*|5OjMtkd@@nhdHkJ(E*=~&l(@`xt?Rwhr1o!cj&K)BAa|2_7bq_KFIJ6x_KwzUu zU`SEmH(Pj3EsO-=#ASBY5dP1F29oQYtg;P_mTNbMXAH8bHntdXJv?NqD}t$4kKX>B zvsV^V8*R?!m)UK{Io_9=36IMXZkqgm9=jntMWi$KHR(rXt4%F=$B(`!n-~~61g~E5 zm-@2%>x!NQ=osREqv>P`WIwdkrIS)G6KAMJ!daxo>KV3Dr@XwAtf-_=bSX<svRV=e3|3+ngOA-&XEF8RY~ z`C6_e7l_D5{F-#YRX@Db>#4P^)RIyvg&@_XN8S%PxeEm%AF7 zE$UA;Gy-+2k943|+xS@p#g$w#!%OQev$G(2OL1MoYB2UNbB{bsP1}!f{=gDWFh6r= zXQ3|XNPp~)bH~Kt(jnj8IOL|5+-O;5oFK98Uy1}e7D%oVRVv?WF6%h1MTP0Cqot29 zx8I~yw}%oV2^rdWX&b3kVUsnZxII0Y+oKx~lbh?}YTlArGmvO?=CDsx*RyS4?2W-oN z|4U?hc7h}nK7+jzk7kSF+qK_ccHek>GGQSF4GW#hBD{@{rrIJNlHcl;D!1ge=-)=y zx1mI~%-97oD(KKYt75bF_2R^@y}4;h+s%!%I7-<<85FY>Ih0nsG-DLX>3h6?rt#Hg z(Ly&t0S&Hmoe=b(7Ur5tEt-c^OSN!R?=H9Ed7%>tHB=-;WD)h{4~p|WqAEU3PI)T% zKGI*>$~q&zooGhktsMi{9EQ3CeenFkUcGK9Gfd*XWl%lLVE9|dwADYErE)iN&@^~0 zD~mPblA^y(Seq5Y2SgyEJN` zpKSUB|RLn*>2R79QuQD#y7N{z`|Gi9NqotB`E<|qN#XD0o_nj@q2@-iEi z{JW9bCh?PB2;Z;e6s~lBC@#^*{!p}MN->Vvj=1(KhA8%PJC;Y026NNkeO^s5>v{?U z6{k&=WLx}3leaU|2VrMMakRL1z-K7&t7PPz-te>M0#zM#zDZd525nyL-WQyGI0Ow1 z*0f&V<~$e(!AsIC6mZpOS=L!=@SZ)TxN7MQd{N&3O~lNRgPeLANB4gs&q0mCijynl z7K}E4-f9dPICzrE=ht!*f*&=hI|J|q)|D!_^=qS-k89IG0a%zN!qNsiOQnayucuMp zfLF3g%_h1}CYs0xoc>)Mof$iy?+!JDN4c`u5j@zZ9Kujku3wd4L{Q@GE`gorzUxZ1 zeGK$tmjFwYQ`epAujP~v?@gHci^26QJZK{mHK>4Q4$kb zBTXKsfz`-r5KCY_sZl%VR05+*3{j8pFHkFMzUtoUb@ASGgZ%pb#wDw?B2s8&CTn&`SKsXx%lpZSZ!= z;TUv>Equ_7WqoA@8<_jzPPSvAF0xvpqSbs0iMtW&DVD%}AX3&4+(o?iSZFedEq2QX zNJcn3Zp)o4H7sll9zmTCXi^j~)l$}^v!q+9)H!{V&bQ(|E05MdDB`;Wd$EXFtVyAe zP~C;xk?{TsUtXmx12=CyGyH<0l;*}o^)n*<4v$kM)xxmKi+Va;dDR{}T8ap_8?|GB$1biOIAqXq(<%zLVCc{@ z&xCf+rRE&nEB`j-Q#mSNE3YI2Bc@V5xa}asmxnz~OzPM6NAAW0I7`iSKtQ&kJ-UM^te6;_I?0ucDwYH9yxwrdqb?_c)MMYwpbPD#L z8{o+V^HY}}%Bv#{^G};!_x+$MK=KpB!25XnVy&W?Zebq-UXM~jS1)$wV1H3n;O5DE zVIB1!IMi`jr#yYV{Z3--nqFqq6qUX&g3~CtlLatg=Di!=hUh6*TR9%tSdFyG(8Feo(iE^_t2S_CpU50OC zD6EBCVid3CA!7qs zyu0jep&h{sleSm#R1J%%v9drVg*vac1=kpQwms)bVv zWl+Q#`Om6NatGmXy^f}bn0CtHj3(&xrXDEO^k?d1J3Z*kkPqli2IOX^BqJ)Lj%+@k!+6#1Z9x zP2^_}HYmI%ITf5?z!GeS7Gu9xKW)v5Nw?MVJ9A~))@9LK4!2L5tJk3#jbvH9 zm>^jbf5`5=qKZKT3hhA$(S!qK<5&E1=;$D_6{S^k)eP~7iN^~@noL%Z^&_l0cXdW0 zUJ+!C$5&vJZ}<>ffKp}f=k6HhrX14HL{YZCi#c8~tKz;wOeph@VilB^1Icw(S^nVl zRt}V&ic9Cd9U@!E5%kn)eSdjTFxVoCAve=-Vi#n^dyk2-+Y;o07T;dXiHydJghbecv)kAO3qqFA)! z!Srbx%f>Vf%$WUYq-0#1~lonHOrHE?I zEZTYWu*MeE1Y8OIkY_nZ(MtXjab3zx?M|p@vvMhi&hPtw@SNL=o1qx4&n_Qu5<7n5 z;+dLmIB+O}+?<5zaQpocm{5LPKl>C$nY-EVpa%VgL#AgklnRBBlbikh;nv-w!eTAN zkmD}SO5Vz*_@wO%?@S4DF`%fjAFHL%kY&O3`K1^O2Ek5k_T9*iZZYN_Sd zw-@Z_akI4lkbo)eX7s!0I$Wk z*wH|E`g}t|02Q?PU`}H0z3LzRZ(5RSVJuKA<>658(E;XPdJKj(GZX1& z#!heQlM8*(t!By}0>|ttx|jM2(nV?V-iVPByS8`M_eH_GaFy*+sp1JCU)6Q>^*^dI zWnE!{)Jy_(?wpNH>$WSU)Li<$RJwU(Z{sSgGIJ4G|M_uc?t~5fD*Lse8IvU~Mg&=P zXR7485cH%bVv=0O5>EP1{18u9NPtqwWvoMV=3pU_p0ccnP`|i)4T)#YHRYv)F>C&! zxYW2Ms8_8i5*JBGS(EnfnA?7m@Tu!Dnid`Q+M8H_3lL8;VJWyKM>FCb;$|zRo;da3 zyQmsVybat@+hr;r!em*<6#K&=n-%lWeV$(Fd9X}Lk(P}b2|3K_ecTYrOaV5 zZZ-XTEnm7V<~*I2AYDSK3G%{jt#-_J7%KX>t}lBqb~lC>4=ejT2KKG7<+mw# zcPDI4+`T@c$lEuYTd>~BP6>Dtnq|HZvx1owuom*EA-LGr^R)IRTxcr-g^ezu?~UE3bik zj>uz-i#@t}8MI7GTcnkmR+mSO_=S#ukn4<%v=)x5Y6;9L*CZt}Te=cn+4=!mzdtY$ zqP9sl-Z7isUX_Jw%%m)ZN6*OU;=|M&Xr}nHqF9P5czJ(IKVxbI>HgT39*NZ=m$X#Q zt~trDVsr`tlYElluHHmTT_-&h<0A@KGWgL$%Rkae_5EiPWNJ&Kp$ny20d_9r_XeQt zwV@^S_wbymz&Hjue$1!=x2LN=Lt%SJVh!H9%FJF{y- zF%%`f!b>7v2Nr`!3AX))i7fHU5vUrhH<{{V8qbL==FAO6ia8|Jw1-2Vmr})`kJFOZ zB7FJV`THcLvZf!;)OD-0pzhX$A?)Z~YZU8l zYVF^5z+V|6-6ra?bx&P!{0YZ-&n%cLC^C?*fw0#DZCn!OeS0Keo5-nw=lv)ijhla)#e>M-y87G(0@%>8YX|iD~Tu{gso745;NRE zzZQ})z7fxk2so@xfSa7@5APxdX8mkfCH{DpvYT|D;_LcG4LaRu6E{{oy$jx|Pyv#V7H+)2-$=xG~gI|($~F#9d5fxE;VhA(uC zutiDK>94f*(gURn%D#v0A^vqRQ}zCwIfGb1#O?94_Ox{7o$ZU6n{kiecIZ_xoF>^B zk?tsuP=A*Jlg;JpLcln8hL&UuH+DDkA@RWen|jXcY`im7*;W3SSOTuU@*kd+F)(Rn zU~s6iYV8@?N88HkR(jR(LV(pv0>1l($N7-?3_b=+*9$1NmMoZsvk6I=^Q2bNY9=t~ z9BN3bH%{C=z5b;0{lsoXJky(!JaJC>MXxGonCf~zWNT8(HB|FDiueL^06!p5OvKc5 zB~K40-3|xKtx%Ei5Qukmlac&A)J8KK<(no|ElJ<5u&RaYU`A%0E9D_WofJOg>^7Mv zFnh$L>XG1J^~yAk_%_}8tGAmkKl8C%G_BKL3CRW`Ig2QhVRF-9eCP7-Y;^9_zmch7 zNm8qIR#Kd$kpCvdKvSb_(D}(jOP+q?kl8CGR{e>$816@ijFIt6EVA z@3?6dRzQHeR(63oWYNtRxip22d>N4&n3(EKycA1lL=#=JH*1kQLOcm(^;PB+{>4c{ zUSCCOile<%-NNZb(_2=)=b(vsHWH=ZVF`GRL8M^*W1PvDcs8*zsp$14d2Ow~+Hz&# zCZ8Wax`d8SVR2qQ0Ud4!@v21S9KcWO;>|*I!X}}W{Ll2Q(P1V66 zz%8bQz>$bIPwvh_8Q|p_t2FFmE66T>MC?cPoN?YOi>A+^(1~x}lNRb?VY$KL_Db`1 z-{e|ln6=kUapRvifxc&wTmdx@Y(X+n0Uql7rne+Ep2h8dVGTy*J|XUovVet1?dL#8O=)NP9%-rVpex+^sK_4o+iHr`aw#Lzf;9bg zAZTu?8sGLx0uOPFvi-Z3tfDmanPq!Nanv4gQEnJ_pNI}xOkK%}WqCsDJcxG~u&fkK zaSR`Y?j}LIF z8=Q|v)d^g+#ZMqzJgrg7@Ou3$(GMX@P$ZB`lrkB_7_pEW_2vU9yDFdAWCyB4) zs!+9eE)9^B-vE3+oM1INW+bYA7}G$!{R;Z!j7W1UigerPePEjA|CbRrfjo(qF##*+ z`hp>qO;GJb;oS-C_pjFeT-Bb#8e^zZdMk&@*+L7~{DPK6+c9&0b91hZiuqbDzG*^( zfSV)_c5KM(wM1jb_F#9amxazYfDSd6i@(+K=1>-lyq)DqQF{5`68gwh1&wjEkf5rL zt@3fDwtBAOUM==R4ZMON6)XI1g>r8oKYRF<(d{+3J!|ToP@}EqFDDu0Gc~~$5W{^&@hxXO~3*b^h&dUJq5h(`R|IiFq)gpp2uOW zDCNxOpOASnWeqQ))Ue4LA35QAt3z6u2#P!3EnX~Yib%**gyPLVaLeN@Lgb2g66g~38 zvBM>!Ns+daR!Wp&3)Tt(1)t>MB;nf7zM^%aJqT++_koe}=%%>Hl0)%?fLyqiT-xaV zlme-?1jl-HCD^=>;DgK!SD8$(U!(`4eK4=Vap%BvSp2~$&|!J?QE!g|F(h8M1f_bz zB=Ft{>qIO93_^aP_Ce&+#=w%Q2muG)MCY1nqkn3G?MtAzK}F=DPW!}Gi$mSPctrQT z8xDqup_2{XDxl8>H-&#`oX85;?}3aVBV779gBJ^j2Rki}&iC^xxC`vWBhH?%I2yz| zX@Mw7I(l6di@*5vS5B{yiP96ga01GB@TUn;L9Scz)@c}$Ngo(3gSBlmR4FoY2ZoKp zTK+gPVuiSa{^N%{?r)HA;qaaWih;|9#Mnmt3&3X=v=73ZV|22y{n@VXFJ zQJDlLk|Rz4Nn~Cg6ckDb%Fc%HI3|d+0}&H91#*cQ*|?##=q?ul?J|NiFT~YMlnU8UeapH} z+MY@>|W6NU z7VU{@{l;G`%{XsMCQ@yP*J0K}X~`?-&!{@ayNCS7Ktb=WuOld9h7M3fu!Pg*g{!)& zVGY|ZV%aU+{wHrMO$$vU_vY>kU0d&!BbQ$uvVfgO*qz8*-o6OiPTNA;*4@%v@mb(c z-qQlR{sNB=ormD`K&u&C9}4|H+t>xd+QPXw3nB5qi2il5iYOb=X|c&!%({q`u%S|^ z{^Dxp{PWcti|2Rj_(*f=2AUSNFXaz|p-*K`9gA@bN#r&W?g^1a6wks{ko~5^WBZ_5ksp{arAyY`yQpGx9++l z9if+{UP01UDLOiTwO%$&2~r>F{Vj*qk{8Cz)mvlZwmYnQ zX6;V|_VJ7pL<9Dg-?*pZWi zKWP|#W#m$F|J&S8-i-c06X`cieAwD;nmJAc%V_!-V>o1|xmJjLLC%9r05<@#Z$LP|%Rwyz#|JyZCa3 znToKzzsAt)aI$2Qh^ruY<5Qr;eN7scJn$2h$5$uC9}2V7iGkxJ-A&TL)kz&T)CJg;y9l=^>CnF21!L9TY zJi;~Jr54A_l^i+B2He4*{6L_A;g?SHRsdQg41$QdDPq{640N_`nR4LN4}<1EE(u-s zMmslT54MDj^0wy$c!XE$ob99hZJ#@j#c%&8Y%iE;1P&fNJspJjUbf!0Pu9+WdvC8b z5x}-Y-^s?v$k~*yuPif=jmCH9k4ALYEsZY3z;wX0$gT2l+}+2?AT`Oz$=C%^c4|po z=HQ*?65=(g%ge~f6^}_q+nkeV|C&RwXJ=yLZM8I+4?5_JAxnh3RrZ46kc+u{jQNyN zv&qWF+@vX$Jpg;*e0)aE5o<5pjWUwXh!M1J0v9VxMT<0#;efIbCIb-p;}$O4uQ(Iy z@|BFy6ysF5nUvs{P}&^y$Lp>ql`B#|8ZVhH5B$giZ#Wy7@*l9l3!pl(^Ji8N(-=eR zG(%YZMuOb`4%n~{{BYV@9IEhJT;*=KwSTgsYkczj&PKCnI7Qcg+^@uQ~j~eM?JSiKfVyDM4~8lZ0lJUk<8(xKRzP$f-YztS??7Y&D~K7_Bl0duA8* z0o&VmTKl~rocO2xfKQzaDxMCX^8T^sA5)zy9bD|45cv83Pd{1yo1eml&ZhrR{y*s# zvvhKH5wS3I`bYc!EE@1{;FJIiGKT+ygqiI#O#k7P7%R;`wz(R){A-koldI{!=KbTl zioL3xrLnz3FY*6(CT2E905cO8fR&LQ zz|PJAU}fb5a5A$1IGDKrOdK5loNs0>=6`{end4uQf5=&wSpH>W`#g)R?4RtP@Jafg z=>OgEAJ%{1lk^{D{|PMYpBEg{XPbiyz|O=8U}s?iaBwgI*x5M$A!TD{|F_BUd1PT@ z1+cJu+Q-TSVB=s1aI$iIj(xT{8UKrv@lygjfc4XS_D`uCOrPz~VNOm?0LQ1j>}>yG z0NbYvpGy95;XjXTpKJ5~kDIX}7CK7j?m!v1;duz%Y8iL-sW#`?+jkK6wQi|tbyGxI0> zBjr=-Ke7BL0-wYG81i42!zb~7HU8gX|C@>NKce=(1M&ZBAawuxG|l?ors>bT;bQ9a zIT`+wZ(K}&nEpF$ekK(e)6Zn_nF83D82{n@9~sRfD>QY|j_7mw^`lp0N0yMm@6jU$ z12}^Q$=;}RsKU;Gue#n)gX;ZFoxt_dX?0-;6DG1=v50bdKVIHSv+W1@*V6IZ_0xyc zv+@1+=Zm}N=NlJ3fA42MU*23l9)Hj74NiYAFFzmekC+dV9{$d)4Pw;okEaJW(97w~ z^Wo9{$oz$Wtbm_)2Z-~%@YMYE6tw)BI-c5cjQaRKa6NL>BcLNtrbb)%sV-|yL)l_^OJ*?2hMs(x^v`PKF&^ioc!Ofz4$%-+?%6EQcG`J zmcvGF?{{wweQq5!FXc#%J3F}ab3hLqB#C^t>&GJ>{5sptsr$#5;hX5Gn(9NK@IqIqwxL@k4tauG4U98ePYw2G><=T&H$r9k$Cel?~Qz87KC|4UGF&^f1S_ z=WzGK1L3KFs$VQaEbNPph-{fp*-vMCc#jJk!F$PUmzs4?QJLlYgSn+-a+U74k-P5M z+ey#pL^j`IX+LXU(FamjP8?%QFZ79{;A~Med*yGF0~oi8FFvLeyI_#XH^GL z0!MkC%b{&SIl4sM&U-q$+dn(&st>S+hjLS^sD}lG0t~ zPso?J$`^1@w{;h|-#p#U2WuG>sCB0f?4v!B#brLG&FvO%e^0(|)L7`v(sqt6vZqoJ z2_2-QD&XXIOg$CuKU~etq|lHU3`%6dnB8u;V%m@S+N?1{r_htI-Gqw{a-9W2fciZ>bx?CKcmCIiRDLPlL9ajw zXVjCkqn5YOhgdGv=0hmqoj#_`UQGvCP`zGkgqXK4 zb@U1F?%$Eb@e_bFOI@}yF4(v>NX8K2x_UU5*X{6`m%TsY`3c_r_ij;owpB{k-zm;I z-Cz6*{BZr(*!k9jtKuDbdhXur-W%dYKJ>Zkx_Zu6t$Z_AUMSoXPUjh-+cL%PyQ2Al0mGJRUzZ%MiKYTQ~@cNww{eVKEL2l759N}5nF?VF_513}+e z>|UI^#db7HGlFk3S84+N318XHK?Cm&SZ8z#bs+gG{0#y3OGrr2rZpWTsFTI+IQ6v` zucVZfFl&3OF{N#Ve=Jw|&C}~*_5hULCLljmQz#&RmaD2IdudPOYq|EEeRB}vpZDAd z8t8J5+-BGSH)}{EoX>qdZhW*43j^7^=ZwDF*5hJ+%&^2Uggi2U`_i0#0k6bOE zT^IK+V~`NP9C&nReZ1v)`FbuYzdXX)%ir7oU!P#u{ufdP8#@ap^M57~CIBZBGb88! zltO%TU;_vin|$rPNbc#Yyi%Kg+g|xDcWia*xf49v*%%a0>|^Z{rkLH_QrHt3g1#}&igUPd#dXZIRFd{MF-AdhnSG6 zMysQw1@_{Zdrhr!tKP{QanlJ4j1nv$eaV*J+jduLqpywq|l&t>(NP*nkRN*^ZEnx?b;e<+c(~ z1JIYkrx*MS=%rQ~SFH5{Wo;83KcG2k8zqfGpJe z9O3TK*#(tKQ1JdIvHtKMg9r9G7Dl*?*)R0HsT1cA%X`wjssUgl{_o0n>vBf&htt(t zfPiOja0NW|mH3a~{A|Z9Aiac0PGAvx!SJ9E`_S-U$PBQe105N`$@{)CLgNH~H=xph zNeRX__^tu1R>-6UxfJATz_$X%9wckPvI5Z@WNW~)0`3;X(nnuQa1L%B*tKKhfWnD; z4tX4S+XwxJ{UJE27ulL*a|8Mk<}tv#Z@JfZ#}5crUkBSKvSSE@^T6VT=>{i; zl@FlkQ{J%zB0Z5m;c$ZIKvVQN@2u~50wJDIJkUF#x)JqYY68^z9(%2KT!Anin4Qqw zsD5C6a9c0}5CSj)-~!NB0l9&>!TJIELGFDuy)}LIz1};%Ky4ro&>IK~1b-rX!hAw| zLw#a-LVTimB6xy(Bl1B4f%AiZK;;H~Kz~5wl3KrTJVD4nLQmv9kIYDb`CvY%n?SRI zQB{c(04?~A4gItA!Y{J;!!KQ9R^;w7{LGj@yzDVYmYDe2A{IZ01$)3xx?KLKz&3C- zi~@|Q?&)DpRk(}z!-n@zjHhG>kWFU*LiXgTlM1YdbYX4L4F0*3&@)sBS3gP%PZSDY zCM@zSa9Ih0G44ErNu{ntyl=v+XHRdN|+NE_$?!2v->li)>*DW3GEX{sw?glA`VB0vYO99?KQS~Kz4r-=C7m(pLL3* z7+{bMFB%jZZv@Ji-w8GbrRc$}XS;Uw_anA@eL>9b@-JhPQ@T0ZLO0ZHgBbx%4dq~s zr-c^5yafH2_+B}_y#vwJt&3<#F4?Gl8ZP;ug{)XMbgf4$4P=Hz>b&#XIeoyDL|&${g|`U_wY zd8tP;I%CGe?U%CWJ#T<^)^E$0R924i!y*MMS$ce9Hau~exf14O#NFmCT%A5nbnKoV zl_84vWh~0e=v$%z{YhExkBWG!GYt~CA<5 zowP6SrHc`A7{EB!9-hj!=b^Poy;Vw0+K6~Dce8f)W<9e=6G0$ylS5JsqrDs+Hhc-y_ zm`z22!HaVILn!GiEjVLd^X^SH7|Ts6Bb6Wnqtybv z^TFanT5EcOm|+^t$xvMFjn1SXjqDH+k5MISXcqyYF`VXt!h6pv_?PtwN~lXkao z3}ZS88qu!NWSoO1kUjZ&k+W$o&)ddX-A8$^GJAW+P4+iG6awL1%%=u>Vpu5uwkl@f z2~+kE@7BCD<`QG1?Y!Lw{~_cI7yVStL_PWuFY>=XI;%Et5J$DO1 zM|F4g1eFB&`&|z`I&8`qt^VOJ8cZ{N<{30=L(S=2BgIUHU%bQ>21T1AC+cpQ7R){z zZ)s7=EetL@NTz?aHx~w&;@&RnQx2cTbj5OJIk_LByWlzSg!BIfls4x_7x?bha-18K zs&9O=X=kER;~BGYh)Vm0KSw*TrgUO*swJlY!kI%>w>m9>>u1G>W68WWuk|O@)(=>j z#2=KU7w?1=Tm&?DEpl5Xu7>B7fkE7tB4O$^y20Nj2N6YPh32inWlAY>DIit;b#$SE zFq%pm$Hk`=8Z3dCL*tzaNu5`zCC7$md`WD1*zJ@I47fU=Fnl`>&7R};@ z3tU(mK$!$g5t4<&Oykob*9l^>%^6$+joyY1rMF?`5`XL=QXf%Eg4qNrUGUh;)cqK3;KuRBn?VhZR`$oR{P`A^)GKZy3VB?~y;{j6s>*BJ zMD#A|4xZ7Fu|;*t*Q1E@9>$-=X~DzO%8{&|aKiCFsn3;;e~HDDninmM{oSsDp#oTi~TNi zZ$vaT6S2KKuX*mILlSeAw*mTyc!q^!>{mdJ=ay{o7N*FbI1+YO*n}=qXiS_?#4L6g){1}@_JmKlY`*u>(Y!(^HTfr50Uq@vE^od}78tR#CgP7N4o63#QEswwe$Z#O6+|LQ7%S%VL)*Puq82%#yu>HBay4E(HDGw6 z%W{)F4CzJ@HI~ZYP@h8ik-wrWKeQD_Pa|x9SPfpHe%o!wG5O)Y{cDOTV;PmLaenS@ z^BI@?`jF7NL!-@t!d*u^B5ArLO7xt#yk>2gB2w>M*SDT+cFkBf;;6?y%P_fTvE@3; z$@3U(JWm%GXhAJho1~~&>qha|(E9Dcg3z#4E8HkqQ!B42Ttt1MvM;wEtH`7Mmnc?t zLjTRJrjr!Bx~1)?f5WqWIBgIw(dnPBb9`>u4eDqzjq441{>1#vOMeG@#=diGd!aZU zBM!?-HuUg1po>+b)3N5WMvI}x$@;7;$VJI#8zl8yG&(eXe1EFZK^~vIK^(j4P*PHr zATt47KNLT1c2r5Ra6=-)#U&iedrH#TkEsAq|B{&fIEx#%+Tdy}3Y zzq?52{{S*T&A+b|3|!l8($B8feYp`Lkmt>W`pws6| zz`RADecb={0;^fDt!-RfkW=yUMKNnl70|3T^Qp=~K0@B* zK77KQv3a=8no}Qlw{-^xJtTgy? zdScE(@N5b9Ol5iERgB-oy8^14P`EP|Z4(=BY6^(xh^(?LfM{kg(Pq^O11x8XjHPO5 z@w;P-Fl0lyZnWd!A%0{8XR{yt~!6YV-z z#j5+UsFPvDuSLc_pkE&v(1aSfspdMt88bM)vTmbEWA;xuFvXRsC=a#PzkyJ`qpBrL z3AC`Ov7Hg$+hY#W0wuIoWI6sCzpg&IZkW*d4I4XJ;s#P^EWg#r2`F?t$*Jz&HT~YI zN5izhwe09y=zsABX8+yI1RNU#bCs0MVH#{3P}T z=m!(v0JsXxj!J0?ZlL0O6GS3G9W8Yv^Q0FIC3-D{UMV~IlGml+Zm_raJx52>oDkSzm`jHsNr~m^iSPm*46rOrkyp0Rw*BD z2)Zb`UY>7@|!SQYWc3!qHCi{p12C`Ijo6|_0BW=G>=xWRfLlGnll9=hRzgu9- zggV{BT*69l#6KU#?C0zZQW}bHU0PYqiWL9pC#^M_GOtd@XTJwUtmHfc|ci%@L z#FNRgJY`j`->?A_z0&BoC>h)<&xbn}S54oAanqk7+z8K{hrf-u{6a7aCc!Fj5Lu?r z=)zP(M95KO?<#Va=s7hPSBkK5N_g>k=E6)N`4B>Hq|r7Pr}Qp7uZq<%RrOpb=v7rg z9%N3Xw>Wj}mK`CZ#?pBahL{${Yc8GJ3F}EH_0?DY>Hd1X&6Lq05f$8N zOQtpcNVL{d%W*mw>-DBWrnseZVI~;sHtQUlw`J}f6wkQdBI~7kZ*wLa+<>UTqSi3Z zvBvJ^4o=(c&5Ic zhB-D$reZ1#<#W(0W-iB{$36pL(1B)Q&TF*@7^)D_Ou6s^|C`O^LT$iI&*lzhIZl2a zum$(GwDB5OzSZAh_trxZWAk}(FYX~we z7q)s8r7{HZ)&s3U^!MUczw&v@MJ;|sDfni1<}Sqheh+vMReclA4@qPZ+^|z9dES(R z{rUfkfoz6jCV#-O7&x=H7&cL6+KF{Zx_Jd2oA#e=b zspgrBI6kYWbETk*{+IGeI2xXySKWgIHlCejT}7#W>I3y{b19ZnqIpTJ6+zCdhAzC| z&1wRrkQq#3A-tf80uVbp$*{P^WvXi>yG4tgjZGf$m|A$fiR3&6;-Ns?rLhv$+tXR6Nh0h9%~Jzzf%L;s zc$djWvaOtc^Ucl_C*xMlzy(}mW1JBQ4`aEtU28Bzv}_x0@~wwGg3Mwb1EEj$DGweR zjx$<lt-YipL&tvb9_bM>2I9IvQAf0Tvg zsebrvOLe@gTnv{}*2+qcF5awF3?n`})36lbaP7=E{ve7uDKyV@$p5EQE%9aIUd*NV z=2+TXh{=Pp&sYZ$KZm6$Ay3cbBH-1-7q27MYDMc2e<#OTFrrCw9PiMR_%R{jz#06H z$BzB^`1E&M7P#CZ8#6=W^)6ADc9}d8T$gZapTXEgL!%K|U=4R8(B&!ZU0z30>lRyX zU9uF{dkpK^yDrZNT7!4E??7wyg1`LbFYsPxZELw9J!vUduIXQ~B%=3w^-F5XL=7~! z;YwuZjWdJz>o^4(KsQo18^CUG9XNu<;Iw>dku;$_$mj~B$uyrb;U2YI7HTX=^vs}N>fO>|TFGDId_R4+Mw=@V27H~)w6E)F(|;%IoWuTY~BFEiLxxjI_=2^)^+ zhFIR54w^cwbV@MWWa~B2nrn~HKi^%gQ~tdDLT>dEViDZ5*X!0`m~5S1HCFHQbA(j= z%P}pcEl2T*^!q4F(5m15&AwaYmGi98fhB28-;)QS@1qCa>gBp#yZTx#Q%udJabm3C+E4qiuRYe2ie6WfK>i%kQPmNA9i3iYv~l`)utgN%S- z<%9U-NST*Z{@m2+@w1rM(mm5_Aw|*$Z-6)33?x_2)ITv^%SxP$Te7WvpoS6o?3HJ@ z%vBD8Ya=atAH8)0KMc!ur8$|zBLnefTBODI|FF9vrt`XGy%>WOblw&js|)IySNN1) zY;IjL z2p^oP3I&;#D>^Uz&bB)Ue~{q$qc4mOw%cTjXw5oVhRm$%4BP(ojN*&hys2GkUD7XYdUyJMzfMDN1B|iz+07Y_p`F^=dlPiW)zZX) zN=xG?$k%!8GmG(;aS~L545&fK*agllGM*CPQ^n#)DcxQo-!VAvK2Xdgs+)cm24N}t z?laE%Y(t^kQVjE+QuHaF+C<_P(&F^Pd_s@W3zu>GuWk|@85%yYTMuEKh{564kJ9g7^+Cky_D5k4KEX0N z7^-hZXS)7uh(m^D9R`E0tM8e~MrIe(#8^S<{ML~%Pnc@K#$t)nwr|ZWXaB{ zDfqD%tSJ-piQE^z;VOiVl`ZGj+QO_uSLJNow_&eG9J-4SlZliS<0(1Nxq#-bh;r97p5ZUgF%UWdo&b>$5t$4ko7UPrkU zkb02dcnP8MgHX*t%H63LDKI+abr@?{>#>9mvD^b%1fxu zC6&srkNS)p>RX0crT$9xJ}W6Q;b$@kRX+g0D(vBLv4Qqfwqhz?Ck44pI3f4+^|BH^v0bF#-AG+ zjS}qiPj?B=MK~_!@xyUMx+;@2Pn%Ug3L)%&9%m4%>H*uqImP@-DaMydF9x;0W;U5Q zS3NAXQ+&C@$LB#@KyB2S3na5d{1h5a7Q#wCFIMcLAZPP0t)t;emX#*z8d>>!#(sgP zf;D(cjpH#5Cvpj`*;p1c2f}ez{felXEj|#@EMj>}9g7`pVNnQkrCz2pEr0Io(RIN z@a3uuLsOc|+Iu-(c|V>gkGgbPPI5waDJf)T8^bV{-(OvCg znN4%fHKx?+`RicotzN6b_ZSI8t8&r;jzX(cV z`S?+2B{1hx_1R3RXrL`sBtcK<⋘Gd|}E*@udAcf6QFwjlo^1eNU)2jB4vJ%O>w& z*M4F{edi#vvMb?xDrR!~qHp_itlyTs%@<~ITE1bwogr~Cie$~YTOjnlE8DQ_2fm^35HSrEQ-b!UeNJzZ^ER**=j9p z=0jB7*B39|Ff_ENWooOxR##yU)wVSV3tH1b(eJBi;=)uo!|;^1bNXG(>Snr+?7@z# z8P$riZe2&yIa*GxtV@%uL=H8Mrc#s!lBVR8=|PgkG^sMFx9UV-|r9OyKV4kz;Qv()qm)Xo+A#w#(B@`+Zo%O}=RZM8ovw__>JB2ZrVd6{F1zwjmDOx$c@<*S8Um+g7FF7u zhDyhK%6lQZ&!2z|GCt1HdM&JN?xrQ>yc;1vv7dRl8Q8#DW9ECYNM%{F&pG`AbX0A% z3zsOVyl;%*pP*#6OwVTauH19ftf?>Li$tU=UqhjlB7*t(p1y2d-9q)hr#Q&ViaknQOQ#DGOM43w|4(HsJUXn?Txru}$V!x;$d2 z&PD8Ae{xK;=@#;WEwVRNPfH(8ts|BTf}<*)rul<#DlxTC+}+tHImz4=s~t8Sr#pN} zciW&|5?MW@WY?xu`+CSxLC`NcBg@2`m)!2v$(v zPWd|~P=I7K>zlh+;e*bu7$Hz{LvEf@SPOu_D>^|Qh<@&oFg1~q> z`#YQMGOx?}>LJy#HbjaPcURrnO{`34`T+!~na%hUxCCOV%@%qgOGa0y^waw|>_UJA`f z$Km-|Q(woLbkQ;$Sv&7OYjapQE(8%}F{ z9*u#wV!KzihjB?!Xpwclf1!L5GMe1>WmWY-gXQdFy`az!GKIf^Yms8!0JZ?2FERLh zqIZ7jKYbv_-l=<6y>PyuG18`09cYQRmY7vLTrl ziDn%Q)6nK?;aLYpRO@$~L=;?B{^yKW>NEr??i{x9c$0)fa_WcN@$pL{b`;=#v$0VN zSpsfUehm*T?qENGzD7=<`1sf&k>qGvxl>I8d9CpATCBHHEkT&KxSbUmuarC7y?_;e zS}t%=YT4y$XFyvG#yj+BcgysTjtRHEgks-egVkmhTWc4!)F^*{@kLO`Gi=0P#NGoO zFrj!BpRH`>Wz4(;Fj*SMP`1Fw3*B{xhDX9iQ3uI>EJI6Q{7kK_B(Mj&re}s_*$}w# zS@HWYbNHb^kPYl!fxYMM9n9E_TRN3b7>-X;V{fKe0ViudJWqTXns@!E!c*lw``Pt{ zKHSUkpI~2rAX1xcNS*b8k<;=Dc@{V?tw0oCo7ESmHz~l9PD5L$luc!B(s3eTE4Qx+ zw2P9t;(|NUww&NHU)2gBR6h1l0%5U*mcYn6;TDm#*fLGDH2qY}>JElC;V|+m^|teq zRu@Nk;f3(4c9{`cHKC`*bG(2N3-7>3sw+8>a-Zxn>FW)I*h;$3h1C@)PNFr_HO;*Y zr?w;9SJ$NcIXQ1{HzIV0=vP5`B9h$kaXTw zgN5RlTJ*^`vdpq=H7}`J@jQtFPa}D!irP~f4?=V7dY%f~b-|td`C~mS(++~BF$-wB^F;t2r9>w!VZ5Rd0k7s5#&3`94#HV zxW0oD68V>36mjg7a4Lk^vWa*hpF{bBlrrArKw*|@d40lv|@L}AeIUV^q<3&UN~GeYyq z-@=iQ&hHj=ylA?MmD5c?~zgEHU)Aut7&fmL8F z*r@iym}Zle=2H z5NKo=D~@ON%7YhQ{MOZ1!y{K+r5wNq_R`AxF;gHEfjJBt=Q#rg?dc}OuXi)YJw2Rc z8fok##V=kCIQ@?F58yiGrc*b}VsRQO9KlE^6OU&pmXw`ZOGM)l%lD|aX&qdfEl&)w zK}6tW6E2W~sjiJ-hoTjfh!oP{{&AY+!UU}DdwlTm1?vY3`K|JqmG}d=fN;1ik0-^< zu0ya#v@gvELsLCt6DfMMrMCHGa^0@?`&=`2W$O?jzYdDj=0WS*k_3C7cxd_ zpAfQ!)5yzdm~(1fOr2G-IJ@+`WLb`!Yg~qVppIA}a^|txK1%xGc{MGJ=8Lz(Horgd z*qOdEbay$qMBuehr~LKQ?Qfz{u&v!X#H>5B4T9bK{8~vcM>cg}t1kY#(+D9l( z8L01JMXrj3kMR}Mj;a8~vhtoYZ$9_tOSkbFZF99xY+c?*23smqE>GP^BtM_JnWypR zvEL%zQLaY%|A?=%UBmNwFMSaAXyLnMZwGsM(Y>$Qxh_m^9kx>zLFZ4bECh>6ow72NNN*+>%H(?pJhLo7g0nMnx^&IAf2;WfIIGF#k_h97TQ@PhjgT#d zaJ!5hI&;wBrj<_?EEO*3o@6*nP+8X&MbSGsiLcg*j>_^ZEna(^de1SU<>@f|Y{6he z(zx27_weODhGRqU94r&9qd8iB@}ujjyk1R1RBulF0k&T@!5xQzEXm^Rm;&G!`rWUG za%oy%G_TdQAh&)tqFpXTc)kH{x7S$ugH0VjYpF4iV%=hOJYq8=__Npya3DoG0ag|m zP|3MY;XkS}&xep8l8=;Owxb@eYJ#ak*(c9_O2bm{Sc*2Lg7~bW|4v7TbZ7gBV6j$t z0s<@J^*U9F|KYqWV~=&DA$c&{NQs{b{_;kbWYh3Qo#*Nu@&gZ07uy>&3|(GB{(fli zKEC@}uU#ZfY53r#eg@K?S0d)@URx&-0xc?+tQjRtF4NYodQ4Z*+|G%ZOeGI>wxGZ{ zb?&0(%*w4Jap%gP->aPH7nbVMdMf2{IyiT9aGSbXFE{>6w|UVK;me z@W^6-KAlZf6&KCmjd&)VBr?TaXtZANn9Uw_W6Q!uMlx6plE)B8nnRU#c%<-(N3pVY zyOq&sS%Q&ey*(YURfe5JA#Ph{M)4E44n-3kXhgHIWj52V+Eyu3GQU&V=P9CKF}4?U zP=Ti)LoO|3I6X}WOXn(}W|5|0$>wH{ha=>;vZiCu9cFa(@U~qcr`p?fu(OxfD3{f8 zPV0#GwaP%dC#8kkg~E}u*6_4jgI5Pmv7W)3li zGlsr>Ee7jf>Eg5|xe?ddDjE{Jxv$o0&$U$ehMOKiUOO@+GG2;e4U2*u1C<H zp}*j3jBMn(%m^ATj*We9y&gFXKbyH}eb8({|i#|3(@7*y1HOPxc#%}BNMK{lF-5}X>_`exSSw_plLf-1x^d%!5-J0Rfs zfBIbKHzE}VD1ZW6DN&!7d7jyE&J}wl)t#o&8a3=CWuq=&#aVptQgXS$p!-XsN(7HS z6K&uq7nw0B8{rYOJvFhyZ{~PHOvjY-Qq|rN>wBxA*=y91l+~uZfOYs}vvo{**sd34 z&%*A$=3MTH1tT6>61YV@qcnTCJk7DVcG$BJn%rXTi^@CMiIBBit8ZRXo>&z3>B`n+ zsfv!uM6kLoaLF=LP~SFGSa;Zk^zG;BNWlNE6>F}|Wv=JYRLEMWNjutDT3Dnei}G2! zQkkQ8R``?Os&1M5426Bs@Zl`jgdn;nnm-TXDINA0H^aN;~GjSMfPxgEV@47WjD zhO(<`vaMF&FmlORNr`kh_TF(YSO150d3m{;Z{v)PTgN)IcRqv;-@wJm0{_{8{O?x8 z0o=e3A}HSWf`Pd_|NIrQNGBJ(Co?Cx4D^?ZKJ(l0^+jQZ?Rd)F&v8zDlaWeS;!V}s z_l*axqc1v(fUY0?=tsPA!4>5lH0{IitnQvV4TUrK-|o5R9_;aQ8`=(Rp4CU)3p=FAQ1YO@-d8vY9FT zRqS10L2N~}sW2MthB*fJCGlQbt~9%B=`>+lNLcj1#laDp_B6G$@|rK+LOL^$yatEQ zDA&QQbK4ruWxf0f`sH=M*ms@IB#C{OV_(Zu6D(;_sKG%|;$jqhx25L9jvZHS-+tAO z9T#udu$ti*?TQ+C`##<>5iASv+La6Pwvw7zfqxfq(4t4Es7b2^qS`+-FRffCPpSI6 z2p4m!xU~6*gXK{ZO+T_5SX2oJB%gITkp_f2FB5;i`_Ey!*Pnnw%rGfwH9`n3s%c?m z#jgz+VYRb^Ve#%Cvl?&n& zNm7JuplbogyBnRM46UJLWmUc2sOMRXqC9?4ph?=GhndUoI*{kxj+t}t*RbD%X3!26 zfV1)$iA>g?|AlJb0{vgQRU4qExB~6;E>{|D7gur4CjVR)XFF^Es`56>C2_ zbUu_jqKsK!h1(yf>tJ=tuh#o(8AgW@)mr64A!vaN-o5NZ)QSe~`>UX7`nIdwF`RDR z7o410^xc<;4ho8^Cmz`|Sgr9pYRhOkyFb*Lk&&B?C$a+X!p@Q~t+vTo!!a@rwa0pg zIN?wvN=X!d!0$@x?S>kcLDU*3CbP7+wK{o-Pg1Wx_Z)ntH|I_ShWe%l5H)tl<`@q# zObS;?Qhl6mZIWO$Lf2eY;3z`@bFQ1&f*-`*2QJ_ReTYDMO3OO(ykpi<3bvk6FQ8Cb zz*|G{#iQ&*fnVH%-4*!-|k2I zMGHoqlc}M^-x1Q!EM;=6Jy!h|d+w@4mBnghmY)aTAIrouofJb@6}g#xi0`mJQ11w-Cy+ z6A6+h9czh6meFn`mPryay9QE(0~eKdA`2(H{mKcwWb0X$8S!hhagO(Uu7v?D%|DYW%&9Mg1Xw6HSHv#m0)ip@ON%^>^EwA+1B^;&%Vrx+C|b=$!*BOi zc%2$Z#{LOC4vo_;~I3s$pE{UxP+^myQ6*`^E z8E=f1_X-jg3G$04Xdbg(oRQbwVXA0!C+zlB{mRv`1j7;hWgbVl#xK`cCCOk&xm&g; ztmTbj+i!POP#olj&eH0b3VQ9Xwk!IT&tH435HCD41Nh5`|Cjo8T9B8F=NZU6fc^hd zZ2v8eT*0FAsf3Dw;)BSqVAWv zShdC9P;9iAFJFqiNWQ`X;JkV|rSAfkYL@W&s@yBYVv*I~6X_9H^Kug;f-P8I+HwMK z&=F5Ttd?nOcaVi=v=34eR<~xv(Vh#CJT3RggY9ZQ~EM zER=msrijSvaBaju;jnLEwk(QcL5u5X#2RpA;1hMKX}n2d*5q(qBQMz}`d57mwipRk zchd@?p}tHK<&(r6DVF1@xW9K)Iu38&-ADfm8lyC;zcwln0wXGyt{OGU8vUk@2JEOc z*6a(Ldd%-|F|un2F%RnC+MzYoyZ{Gd-|ZBcD0~0M4=nE$Ns$wjm)EmbmkS)7HZ6eC zP!o51DF=HF{xlk|ra8PVMF^i~wmO|Ju+EjI@%(C=QqsEUS-DvN!%VBIyTu^-c}S~C z%GVePTHAXVS-HOE8@B~LMv9Rh-DEZD>h>I4L!5tqm!y0iL5wvH-8}KjsLdNlWcvDT z4htc6V&n}=vYZZUSl&Ct2s=iHCRpL}^Xw97_Hz3UubJ4isn4Q~A2@dmQP8gQ(VzE& z-5IY_*U$$@5Skgrzm0u{*!gl~3%7uu7CSFatK`kqG|Z~m=jPb@sduODJ-sbFHQTcR zON1SkmRe?f^OB2m;rH}5!5lf)#WBCrBY(EJ!npaA_PBf#b`F1%lNRH6xn^iZ6rtmj zTPxF){5_%qu@M4`Ma>?6q`AT3@khf+i`O5uwS{bMd(}kF;_*iP_%O?ux;(WMWnrS5 zhK7ceCohSJ6w8JTLW{}ci^kFcr*KB5R}u|2Ig_ZO*kfh>?d3Hjx80qrq{O{9VO3E( zi3x#^*SgxSp^wDMG+LY4DKwzSOnKZ39YH%fO(2w|tQ^;0<*lSRjhyvf^y*glzXIoX zRZ3RDx;0hH&}-|PSayZU=<)RML24}*q_%QlYAF{amI&y+vV!CYp(&9h7>u9HWR@)1 zy>zMa+FqZ7=OVJ%{3||09cl8@FF`!bb2!yqmTscS&gQyiia(OAr`dO>ZqXR^f<2wu z*ubn`yJ!o(vS#95Gf4)WuX7=?Dnm=OV16bUCm34z_U3eqF2wArnU(kx*ffgS0Tf$$ z!FsT{bR~hpWp)0;T3l*&A#W+LrLOzWT7-u%1b2WUXhkVp~%#mnf$QoYr5XtHbmbKKG-ipGFw-JSc~i z`FyJR)O@OT-s9&>nbguYmp}bu@qt=%^UZ9+U&Q`VRD%UIH4Jv<)y0>H&^#VgisO2O z-i1*`ex8>Lb4^!ys6(KTq17#y@sXtx|NNymvo1(eq5LH=;&6YYix;gJk<%;dgs28W z9R5aG2X9i=omdOk;aLkoVrM?1=V{Au^SH5@^USC)5e zqt@keMwz%$K$wiKh6!Uz66&>Z0Ux7!7c#===(geEpXw%8<#Z@2NX?0sINRRReI4WyR3yg9 zl*PLAhJCr-4TYHCn>iz&LuJt@gSoC$dj7JGA_e&I9=F~`3ZA#tY~=geiG`TGVVIGz znOMEaU)EXm?bn|CVpJIY;Q6N!ml&Dgq`WUR&+NqC!agYW5@eA++?w~LxocF-t(YR` z+6-uQ8BKm6S#cc=BDQ*Rw#{YlyG%rK)!;JPoq@bx}wHz;pun< z_5oGaX1nju8VrtJt$bM36fn{Z&ibC)?$L)W9z!67rGkb#mA_nTunU~G0`9v1Zsn~i zd$^(sjwTxDvLLto`m>fDeROHSM{ymmchn~(m)VanFJ&9knXwPb;}&a(6_=%&8G$C8 z64X}LUFfZjhjo^oim>w5;sZPRO20!GYxL?xXpY0B;Z(g`$(tv_O7_s3Jcj%X6Cxcf zgUkGy1B>f*oXlFCq*WU(muy`;)u5|n@^u9N%trh>IElv838J74>FwEyDi)e;OrM*; z;&s4~jzS|j|LQv_Pa~Wiz&TL@pQ~+lcv#NlcmWGFx3Ti{=)_u~rw)e=``so%#L663 zyxE3={PcLh6$m6?U)?~>Ldj^R+IyAjT3Y;G2I37ksVk3~eNuI~=W7cV^5VPCzca#* zJ@?kDsm!(aE_b$ZT1W3~HPb&E9A^bKMxFoTm&#ckZI$o14sPf=ySrX%K5}*ekGCKl zd? zkQhhyVe5S@6y>JLM&0yLY{T?XxMcGFoQ7n@YtsAfC2w0-Wm!1PpH)4|YVGW?Kr0*8 zr%uExDpOBU;Y6Fq#R!pYi!W+4T0MI%?y^aeNiuHT20vYvEKlTU4%bZoep!9J;FCtN z?<>D)G>LqG>#5kDlSbFo^sZ`T_dZ&N6sAp%RWlN1XFT%&#q0M=4#>ho}5}`b*cRQ^`w_5a&Ir*lLGlYQBGD z_F_-zQ%1CKhv>I@v5J&>juaYGvc!YF)M6DSv5dlaDnHN?wBp{SZ>XYfJS#sJdYSSYTo+IL>MXL=mZZ(~Kl~{4Z z`f{z^#cO%vx*Fr|3C(#sjHa;(>6~_hw!Kp9gJ)GmxHbPcv%0P$*}bxldk@l8j3CB) z8YP^k1ZDTig@n~;UfofbU$;9tb3OhL_Gf^e=}T8u$zO;j+XM^O`or>x5T7pZ)=1;| zO4mG@y--m)^a_@0=|jX|T29;RZGr}xt*qUXDj95J71_-G`+BQl;O-A8I` zGhuB9!nR;21}~~^W3|eMzH6NIyl8M{nkaGl&43f3)Y+j7QXYZ$kmcDUinXoR>t!+8 z>r$@JQ!*0_X5wm1v&Wqk1lJ=qeEB1=0Ua5oI zSFZN>WYRx!)xz^NXiCjn_pkd~*FrPF=zqLp&C55_yMNc+zar!4!#<9Tcb3b>%d4lq zw*`jxRg(g#RZiSAutwl&_}v>;tn+C?2L~7a;2l5BvYbN^(%9Q)<9&hy$P=PiFJmbyzyNw4CO+D3dDa#Xr6Nyvs%zUy!#X>vr3UT)JgDYVR%d!n zleAx2Nm9(c=u0dcZTK9?AXZZAq)H}e~j$kNjJ_jNRg zUE7R3l3=lzVHQzo>F48BC=z~iMXrIC-q_R7&dN$o4C)A?;fQ)OEKf;;RXu|ocSC&} z%irGG#R?~T7jk^WtkWAebvMD=Z@A&%+i(BY$&*;0CDULRO^vmMT$FKU6hDbGAO%)} z!*liFxe+|QSYB|EbT;EFUGf7zr7$#ajUiQPp${UIFE~%Sc*)V?Fo)&}t+N@6Q@XCy zWgrF`);|)q`2z75i6Ms7yn!ubIr(snnyUH)VpUq~4Tx13F~X5_y_;{rf>_90Cm0Pn zxG>j7ib|}f8m|)wl7M|z&{q!O@ay;I8YofuleSY2VYrs+%*aoK>^^Jtt7KK%Z(?Sj zdJb=n(*}~YjEJq~<5Xp+%7-zu*b}Mslp*2G_H++1oK0gFt#)tHD%nJfE24bbT8-Y@ zQ;6riIW; z=p_L|=p|qXkPsk13ZwvmByUIvq(BI1FB*L_vnnguyvcw4x~|d6tCg%hbIw!m`+hKH zYwOcXM-iQqD_-VU(?8Da*kI^>xuqLt?@amBdg>_-=;H`W zsCr%=Ic9ze7Uc-Cb6R<2tLBL2GR@aCni81N`CJGEn~@0&%nO?;)qE=rmrk~ZhB<;r z&S#ntgi2Myqdk?1i^t(br@MTAy9Rq^)I|`X^J5X^jy1uQYb!T@QJbIi1}iS?y=es%lpwwkn!(IHfnZtXCeaf`FFmr8BaLdD1Iv)XTbQJwvOWKDqBqM!V-%K!%3~sF^X(n1qBrzIg zt_-&UgMo2b1WwF2OP7Q*3F#l7JkvJn!&y4qulpqJs`vRF?S;Qvk}ko+rW4*Te(2*F zy$Nk@yDE?iq%s(TakqP%ah-*IXZ8L~YHm{_=Zg=?Ie%H{a*gKe70kf}US-}hh2RXP zM9wM8IPL|0rF#CqRlUC%^@cBVU$YGKFH<@VnIQfjo)V55Ah?EKb9SIe(s~Ha3m1P~ z%+iCy7T(0O(ygNw8gl{h03FrA5Ylqa5-uXhz6CV@d&z+VuU#x{+aHnLQ~WZ2k)Y+9 zQqZ0(tWVbBw2_Rg;4uiVPa%WdRy}o!7N?24j)f@%qhl_kImcuBV zE7&Xv!NXbaXCfU`W<^u9)7vVrT$|w`PAllev^_Bz$oan}XCYc`wJS6ftbX&Vt6shK z+G~OLY1v05<=;v2E8VIRr|!EeDYv<)|3iGfw$A(e?xSPN3JMa~O@&Px&1w zy^UMmCU9EDFw%Q=YeD+id+)W3*$r54Pq{WjPMrcjF6RVxO+jJzAWE>OE!!pxpQZBz zrfrKH4yyd0X&nd!@OOl=#@F0&Dt6J6BVd*2B(#76&Mb$~+nc4YcdxYiDSx9jcls*o z$_=v1C3%zpe?I4{dv>q$YHRNTi~kr3hcH&OPrh;*3mn$#{@mbR9=8SU+k4g=zP5wC z42%F)@kBhqHeDwBz^bXe@K2z3H91X5BcEoVvJt&XF-Bh@MLSdbsVsYY)JPW2h5~aD zuhFDx9~czmo%_o>{@_v43Yk|q`@5Ea9zz;I3c@Y`}?oe5|CXwcVN+g-epX92{aSfk&4@F)z-#P z3yM3S(=z!%lp<}$4WV>3P7(%US6i;lBLr7=9_nU>FD>MRs_^2~cRV)CMo0GyU;S*b zIv=ay9lc$X?KtH&M|pFOf(dtqp%c|;i#ZMAbQCExq6EULb}u`%6n;dG9~9LAB|_UZ zr&u?=YA4XFlWkEDUZ8|M4czr9S}aOrAV*p!K1VZv3q@UW7S}wiPqkZzWTV9oxXe3r zG*K@YS-mOvKy%A2`>>H?wVgWYhN#6CNPs;gvWk@L9`tHsE5T1`F63-l+>w?Bzv^qk zFeeOWv{zAKBSgSIy$aO6_`M)O$Tsk|km>USmtB$~c+B8E8cyxfUkYz-2n2E6#+`#3 zLY(J{zS{sV`~J;a{V;=au@+zNFhzgImY{jHdQMcg#28#iBpX+e){hNlZE zs5NC&hEk)1W|%e8JTO&Pwj8;sr8lFh<)IYYW<=Od1Yz0uw51Wl3`C<2#=L$w9_Q~O z0u~tcW%)(9I<>*P)W!6&HWzVWZ3`txf47BRyz8#JB;$T-CrNu;%kza1En&BVRD=$@ zmL8CuJirlP+~Nt6f^Os375guR%uX26H$8h&r{L++6KHs=+b`-JI!D3Wzz|mOW;l+J zxK4U07t69CGsB=J|G>j6!&{Tv02j?rC!Z@P8AE-3oFRI(ZM`JjklVj%)tReSf!Bi7 zC~0sjT|{G%25BM$yFvtxX*JV23+v&BVN$bNvtI4T?2(_Ga|hEgv8f`0p1p&f`7!nJ zYFwUqUsObQz~ITD)7Knn9c*CYJOUBRZzhk9$pNzV*AcVQ(1gs~T?vEy)r%q{eH^xV z<=FR7$~DLbQ|YH_o8%z-R$lvju1N6G_gNcQ)!HGag}mDWQm%G_67V{$bUzrGy>_2# zb+VW?wM57A2F@_lef9VCIdIu1=KzG_sCm>@C?^jGd&f za2ZG~M3sQKrZX6Yaf|V~#KCKxx(D|75S%UkcV%finP;aG1SO|Xc8{8nC-aK z9jGck7U-r~6ND7mUzI*RygKC6YH8LwvF!D~M!WXvf0`k#CL_D(JTGl>#XYSN-ODtM5PiH4 zLr8=?L?w}=MB25YPAA91UkT@;@e)R2rqv!tit}^Dw;q4I>`%N?WR+B_A2fC6UM-xj5-RIb^0qk5nf@x;90fdGPF;rpU%4R zkx`XSxfz*Y82!#{5#!!|$UBa5;Xs0OOILcG@;A?xZr>dyXxM3sFL(B^20c;C#;=MA zP4bTzYxcebej3LJ)a6U81h;KFJu2G8;*A@m&j=8$ep$BMk=XC8)C!t0>h)i3XvH|` zQ|{IJ1`8#ST(0AY#jZ`+4L@$S;UpRco1i`_jxBBSA~eB{Hg@CGj&vs?$3yl4xsr*a ztws|{6xsAQR_5h9re(%@>&fAC6-7fxV9hl;ZM6j^8&K=D(&LUqD3CKPUi8(Hu>BS` zfsVhNLcuMJf95)SW-r~;+o>Z#SRrK$VLz>Kjhg2bN^;Kd)85UJpBah9=oWZj> zCRCWwfQG)k_FF(#z_btQhC+4rc)zb2W4{uzc%q#`p(p5%Sp1<#o1R`qGCKc@)k?X0 zMIosqmu-jbwInIWXL;J?j-+vZ*G>4p)|znw_b-BnC{o{6xa1PBu_|owCA`3q%;U`e zMxKr+BQ+?esa@WV@mg%6+CYogt7bV>ug;KA_w!Y($&A0ab8|nMs>LWeSZHL52R6(1 zuzkV>F(`RbLmSTArxi?M%AVL4$z?(*425uRrPEoha|oWNsv8LE5tq2wt;d)a;cub5 z2gV?bmHx$oZyl5NQ8Botm7q6e8VLd=I`)O#oXKw#MBcMB>A;9QXDrv-YNpn~_d);A zXl3ivt73gPU{F&eNM7uP`cdC>1}lLP_j zw`wIB1jC&)<%HmRL3*UDDYbU;F$0_zVR)Q@-oixI?fuc;x2zrY^DP%V_vx-{r4~Al zeyU)5>nH{@NT@R9!pm<^Sl%I>^jLE}#8?ezdlzU6XJZ(n?p0xGE&MY%cCAyN$>1xJsDqIp!WC9LNG%?M?s97!KCLy?1;W#}$20_c7>CD&+W&qWI z59=EoaxCn-5!=;nCrI3HH9<`|#U=rLb3e*T-xn|Uc2Km%4zW&yPR9w5lg*ux=8jeo2mzlA1ovO4)jxGr!X{3-Owh@mB@<=fs=y=sc4$-G0VrZcAXg4G*h% z@ci){hB?r4K=lnO@b1F%!gBB&0PU^q6#s!GcSoq3AYBm5CFR3ZtAi-CCuxzc+`ONh z*j2n)eeJiybwtn*a&BznIno>e?!~+r1Bkizri+<<=Qo+8!AA9U-^(`P%*_u&(kCS= z$>P3mfzLOsb6NBpTiOEMK`Jvl7`kPskjDtz@JDATKnoTYTp(RyGP_rlUhGI0nid~! z(Y9T)SB}7{Y6u#0m%{$4wBM6syf`J=ZCC9$X;E0L8eVze$#bYU-qcRA@4JPi1}#RH z-05I8Smo!0(J{@m-ChU(Q1-i|Mkk+!<+Er9rLbz6K}pXj^311M=jL@IE2%iMDiQPg z-%4v5T2bdo!9>e1>Uc-|lJ(TZ;GG9s+DU%$`Gm71s-QS5+u?!qL+{m3hxuRSgo*09*9@9zf4QeiQ%hVuzF7$Uz zSQFP&X|l2f)aXut`mKbB&(a2D=cVw>7!W}igpN-e`KpE zR*uL)_Ja>V>O<+x4?ckOA424zy6Tw1I=&`VPY9SOT{6*FVtwqgT9z|e$u$ST(xuhi z|N7Uzq~8EYl9Wq{n=Nw8^oATaCp2s4Vx{By8^>{|lw1z{SFH`ajP{-3mw?k=3fqIh z#2>xB5!TK7L5D&ymjbOlaGe=NR&FQH8bfkbU_IZW10dB6wKVIWRHepVL&9Spt7Z!* zl~W;I>||)>hPq~qg_`LE*1Ix|vxK3uf1D)ut(5+>Y&C4On@?$PJi02+tlR1733N%% z|11-Ui#9oy#2E2iIfE#+>Nd%iKhjDu5b7zQD8;mN^y930AH7qj!^dD~8rL?i?rhKT z`?WH*C?osJk+*-g1~iQwonP)YOy6m=I-m2QD&g*3>7}SuUL5G6)&WA04jy%PGK5}E zDYzHuWY@b-fruLev`PB@XzHm4(I>C-7i##9jEi`lN(%^#j6M0%OF;9i^v@UJm%vNU z!q8RnvJ6iU)Ye3jz;XT!pFPQ1xZ?iycns%Ri;3uO%rsfGiMB+3J+U}kcanpmjkJ<1@c`XQeM>lA8s`JRMnCXBkUaLGQPnhlxdxKsc z=x2PPXqwTG2vzEn&$-AG?W2Wyb`@EzBGZ-)>1Nbz6p+-cmOjuFa)B^LTELai`fI2a z!)AD3^T~45WOQ;rhMucw#<_E#T0H`PBs;SEA{L)N28i^5fm9yj?$y?DD4LAH^+|Sh z(C)M6PTQ|&cg0_elD(Y6pXg23%bWGal|Cm6*?I{MH`VQC>N~y9cU2o{v)$l?A+0AC z@n!?k1;73I2;0B=9+17I(Ov8ySrYQ)zt-AMkkU_{)Y^`d>{-Q>|%~ckX}ebS@OY06wuAS2XCRHNB=CnzFKkm#9bg4rCL< z*k407%Afbvcj5eJ7k2m4>=!R9-%Y*=46nU*BOW%byJ|i5-risCJD9XNSWSD_TnUo|i=o=`}Nw{(ukOY+;XWelTGD(up458#Y;Xjzp1+SN#@zdDm+jKLeK z6bz1G>HvHU`nRe~sx(YzT>I=knkQaA5Bh*jXU1iHvS}6p1T8sp#ODS>g%Zxac4lo8 z&i)x-&5A3rmO=303sngGO_xV4j-@LoP~5+WH=*snJU?%Nap9ID(6xKcYd~7NDf-Fw zK9YX5ZRou@(BuWSzji3j1DR%RBaJ>=!3~AOQUsW*`cOmO2 zLjM`OiY9f^3vu@_$BSVe>@Btv!sNeeif+&r>mmp{gtUs%eN0jdfmN5#?`>6kGYP0y zx)0jgHi$BrFhn~{Samj)1hmth%gcARHJpnmGbu;pFFXi^3(_|_mQ|B9f;8&7MhSAD zh-I2E644zFTa$uUY>n_*qb}o!Jup7R-Tt8b8>G#zzpn1_w7kb3m*b7JW=!o9Yk)fS zMU4^`Fw=@)MzCdiWX!ZR(Jin!o)anjs-@xSbAVPbR-e8r*i6z!{g$|aGyuvO=)S5! zc!0m+P2}p!>C2*4Pc(c71{MiMe4OhL5HwN)4aO+xbtNuIf(~?CWH~@JcF4KlPnoEB z)zTj3vRxo{=@mvR%eCGFZ6Fdkf*|;P4vw!IOtjf`tQVz2*-MD?&Z*O7FW8O)Is!w) z(14F%XrpBQQ`{SuBPxz(dBZQ|aAmmDhjS!3&^?6ngCtJl#wDc`yYEh-YP@Ll@P;kR z)tXsRw&};@eVA5x;j=0#m6|{Q7hxJb;ZMx$&xye?r-65LAWb|9w;f?FYg;25+d%vP zvP@)6JvEIO`y^O;&IJKI9k>;)F=8;XN+;OCUNPpc_teLvzpfrdv|ins(f|O@Tt&V8 zcRGr$93J#y6eT>=-iI@dnRT@t{tbJi2Um~6dWZ3(wnp&9sunj+QKFiI)=sU4zY7~w zZ}pYAeBeK50#uBSz-9%oRgk4&9%a6)WeXK%9)0u84lMr$SXn*KEVC25^>NhV55@pM zCOAR=R-MAgdF0MqNq+bX2&(`-8;ATY%$3gH3R?4nhiI zX;HzCvIN!IHG)%1n|ldzsB;*n;zcA;l_Xh+r&p(RJ|AbRHPr-k;y07NNX()a8JfF? zCt0E%VH^FZK$01ZQR~}7vYnGc1cw-#IvHx&`8^#buO-oxm2Gw1)OPqguwI>fI2+CL z3r9H2t-3fpB2acW)0EO_NrKsR1-hDcVJy0s&=Y}&(2X=@{ce3TE=n&0w%Crd(kuSE zs`MPexw4C&YOq~G-uy6n^#SyoL7P7i2Tc&YW*EQgnUuUaxc0$Cy-t`R4YzmplgzW- zBP1z3rKP<0b$8fxLdrmUJz&WX-|(^Z&pjo-!)G;Wy>$cpGW0ju_jJh_rdnkV%uDU( z&Pa9cgh8S>FWM=Oo+!dTgTg0K*CZz(|@ zEYxEps+V5r8zZQ%r{f8}&MMMI*F`_RO!<($eVy{5ywA+k4)_%qQJFQ0LeuuS8GZ|> zR}1A6W(=fKXofj3O%JGG2h7X`0O-t-F;z~AEO#rQ(|%mvf^(BkN0ebQ-`@~cC>m!V z%LU4KVWAP{{vpAA{WceeaLy2b&NA^t*kokgX{a{hyiNLfTZh9z6SfHSuJop{+d$#I zbj?6r8|xG{_JK|DBC+~FfH&zuvU8ZE=_LA^e1Ffb$=NZE6z^#7C7HK%`G`HrYpXKh zlcejK`xlRKZMMY9b_Xdyo>s8yfFawO7D~COL>Ev4!*-A@`!>sG2iNcGHTE}ib$imb z)S@!~b$DtN{t5Jj?B^Th{9(D~EX}!^^G~4ZykKEZC|xo)EXq`#V4`wDXVw;tCu4pvNU1i z4Z2|b2g6oYG}aqD2~g!Qp8Hv;$DyZjLk&0r4B-Zf=EN2j#uLrZ;uIsj&BW!*1Z!)>^QtQ3%(?SYx*K#i}eplC*r}Jqf|#F>oxW9lKp%jE+V9Z)Wp14h?M|9WB=; zI^8$ zJhzW{;&)U?IyYd;=uK=xzsPABUDImtKub+7l2QZ&3)rG%y-|!LwRfhgN#V#RpGcp5 z`Y9xa`&RfQZ+ad3%X0E~jWZ_hQ!Uh|Vyqq(=ws?xSIAX-c~G&&nia{ugF-QE-mIJHpE95kG~;;V**@bKw;1dG=UD1bi#10 zR{9kaQy|Y^lk_X#mVR|U_+w2A!B0M+Bwpaf4NIuQzq+FeXV}3V0!+&3?HS8g!aA2_ zTVDoB#cK%~Ge~E+?P9zOX`}ceAz2DN0Q?lbwV<(@SqkI<@ zG#1$tD09UyWExs*(Bs}Y4b$Xo!%Ibk9Vd+jX2$@lz`t^pR8dBgnx)Ch@f&`d9%ghX z*74dcpvjKl%-t9K;vWSi9{N_u;SZeu`;bGK``1~gghr>tc+bu);LhlBa?gI7{O4<> z*RJ&9EaFPmUkW{P_9xBSO@Kcyf{_e>_FolXp=xs5P4CJ#SD8}zw0tSNeG?)g+s5^+ z?QD8?W3m(o>+}dc?e4cb^)Oe04UW7c!RYGq zcgufv0_%|;8v9BV4EcUt1N})u zYeMo49ankQWlrq0wpRr!HdtceEp<(+^Hvd0LMencNoqt zG;1O9G$`5ZrKI1Lm8)&h$?HY9S&+l~W%S)I1lvi{3dyGLRVJcE5TJea)mNcE*=n2P z9O+pU9v~Pw)7%#F$AuaL4-tY0T}Z{Tx+iFhmOmm->s9JB3~{qKeUU;4p;S#g`0!4(UK zje}{loNsX{ookM%U+#JNrP^e zyj|{*R$T5%pmhD_3+dK9f);a|^ifsXufg6)vQ}`65qF$t;Y>p98JnelqQG7MVaVw> z76EFZhba!jDASeo?Fcg|jb+SQLGQP>TyHQ@R$l_@mybhtj~}H;wp%`3)ESu~&O=C` zhFQ)e(7$Kvj0x9Z&17O3UD_MvTDAT#IKal?NETxWUYZJ;LcM-{*tV+k@&<9sO-vje zdD99rG~(Si(lLlJAZm2wY-}QGJ|&RjV?;jgF82kavK{>f^uBCYdK4{?GM7rp&C7X` z5-}?DwSrMMi`-DLz0eGFqY~RYCmvEctoggp%J2|pXz0>LCU1iZR?b#!hRpP>EV$h- zM|~kv{dSrFKu8zx;niV-R$6I7D7x9m!bTJBsjYS-Bmll zQxMY0j0${nQR}NN)S4iZ^t&U;R)R+Lzu%Bz&5E2B8`ybU1jnm5llQv~J}1Hw#&vz; z1Svf(2b?jqNA@MIsdexzFdtsV6wJXho9> zm}V)b@YQC+Acx?|A*3VLh^%EMbAR!c7{&-8b)xhIA2q=w0)Ci>_}g*EFe2G*5pgr3z;HisiMqT6NVXcM%5WHgOa(Nu<<*NO&f|3Yszy;NDO(BY-uA6?&LH1oe*Ckk$0$ zo26!p$6~UBO**VzgaI;6gYSHMb@$`d6pdXzks0ydkFLJVF6W-x`oqN{d+OMhcPWI5 zKq$5Q{t!YE{DzbD#T>;#KV~Nf7x(CzO(HJpptZw`lL&?5-eRcJZ@iP2{#uk%TAQKD z*##cS=8JLCKOS0PDTlPjUimtI#^-{9Gb{{I4F_I0{bdfCN-+z9; zP8Yx7iHKdB@@f;_+WNs7;moHyE!z0kE~P!zKq4ZD1!ccfO&x%5hdxoc$4PZI+d`C} zrxHyIbrBhV>|0J!xlytb5lYUQadOY%%3mxWU>y@dboNI&*@F{NY_uAYNfScPcgbu5;OuSNg?~BPmP7 z+kVlgQg_QvO~8-9m>gq_S8%-kYw8?2ZRo`R8$8kQDEgGPn?b0K95hyIp75ar6n%WD zYS6Z#>p9L`o6ceEwGbhFF+F8M`coM&5r`I9Q4hf-QF^TW%xbW-Q9mv{nha(*Z;;oq z1P2mH4~p~_k5MtKb7VAx(v1G9x+a4CP!3ywNX5})nxq+i@;$$77#!>p*^!s?V)N8a z_(hnJ-~R^9Va)}ai#3;Pu9SVjm(3_rR|xY}tWa4jA}}qEG+QvQoQm{i#4hy2fq8I_ zg`!-zf)bLiml~s1Um!Vxu4EarzDi!NU8iDaba6P0`x>E!T0YRi5ODeds2f0Ol1DFx z3tAXh)m%D-EM^$~a(EFiCQL6pL|TWd=M_i&UeN{?5!u`ZjIhvmrDZ_MbCw$eu_y@zskx8~D;wd<4i zYT2J}QsF5CwfmG{E{xC^cV?-FARo+%nZ%yhI;?ln*ZpoyY71a z=+UFH?MO|X1HT4Mswj0q zCx@n1!_UD$_P9#z?o`cXb0^Q|Ek_ss`Q#N&Y+EX1P1Nv`!j zGNy+h2#iXfSwSXHqGg8;H*nJT=!779p7#^d_lSwZeJ*`Jr(+?)wq$#a-cQH3fMwOf zfb>LFA`&xbIT4rtQmWN~Voi&mprh!(L{~zN@z@({N+k7YxEiGpwpH5+bY>>R=tQk9 zY4r^CimzUH;f1j*iNZMZm{0F#tqdqiKVuk!Hg2>3AW|(IItE0d!1HYTW!MN ztCuk;bomB(f7#`@JyEQZ$3s|eWm@FYV-34>g%FcEZMYsgK+E&OpuOj#&_sCL|-##sx}Vl5Ee2Zl=lnHuWuo^NB5&=BeiB2f3v8kG4Hc#2^}oia-k!}V+eh@Ysq2!+~#4y zsZ>(mkF_&y293V!>ZlDahuY;g(ltoZk}k;Aww|;cHn>8gL!+x~EZGQcx%=)H?z!j1 zZ&vL)bfHIpv{v%`)V>k^%$?@}EfvLbb+mwrrZ)ieo89~0(P$_+r&VZXmCyiS!KudnnXy|u?VNZ3$!Nd|5 zo+50$Cu1c{hUS%!oInF;)quy3(}utV8qLX>BfsIsAz#f(Wbovu){^fTxp^&m7Ff<( zV~#qzxR=1`C{{0hsuOurOVh; zhCWxh1k*cWX0)?9m!p7}6&t7JC~-vqrOcR6(We}u_U%-(Uun(~6=uGIv`_j*NYVo{ zbf|<70FDOgYqdQ3eOQR06#R_2_ZQ!3^Jw*mgCw$zOJ4rL>QG-qs0xNQ!7Byc{0uz7 z&<60v;*LB<;ie6*r@VgNU9Vt;>xaNfs+G@+5Dy%L@FSW&8 zIs*j9ep?u{c`?yz1>De+S7UpQC8jhXWgZJy1`X4s0Ic<$I4514`R*aNM~@=(-K!i7 zj5M0$aQ{Vl)3Y7{2)X{7a27S>&R`B6cEzd+2~XRI-PPJ1vUYbmq*rb{TEn%!zD2RF zbsm%;_Am&cJScRp9kxg3f7v-ooY|TKeZcv0oG^;jZRaPWB54~ScO6u z>Z!E9vk}h>bz;sKDHBW-n}e5|>oeM5$FY)Z#U^(OVZ+iH-91h-MImAl>Xd%1j;8`w z6}vF*sF$;FOrWZ1+-~-0b4w3D$~>@j@Dz6I%A&M`DlG56_m&+{4Gq~odSRkjo9qq} zB$~d~)~hm<7W>;d#tv)DO~t5pX>H!s&>G=PeVsQdg|bCB#p48Pw2gKYmS@v$_sWg8 zwlSM-TC|jD?b;-*LA)MD(6W|14KbNC*PN#D_CW+>6i~T*JXMYoqWQ_xA-D*=tLCc8 z6jIHXGzo(9n*_=QBH-F^4@LXcKEN_vSWr?2@MmcmxT}0YM^m*5D`xUr9+euts8$q2 zPCwcMd0J|ve;hmZo9}-2Lq(Yv)d+I_`2vi~`Bzc1RI^^~@y^z*|6BA&prUo5TNwUV zrCcQp$Hvqg_JdM3z061I$V5Jz5i6-3!hVb2&FHG$y29uV0Lq zZI(?PO@O)XD*B@r<$XTTza)au1b;PCfNh>8$4G0sGA& z4dIORsr2DM{6cC^*q1e>Jz;vxfe~0FbYNr1;&~^SK2xqcVKuqLW%@(#byrhn=|=jK3H{ znzbl$Hhu96r9UeYI4w|% z5+(@FYPZ%BG@!x*Lf@j$)~%qcxt}CosV%P5I}DpUT3-JBy=`Le(bwoCdhgfd^rXH+ zzB$i>#@jEI6Xs$w!`?g+Dc}UAeIgpJ*E{+2MyuIOuh@HQ>mp{?ucW0QqS%P=)VQ24 z;~G--gmFzlb4q15tW1#3Do{J4VJ6O-Rz#m(x;bg?>EIH=lwbkO&?^-$QP7mLyfl6wsxuM()9X&UI#Hw; zHvzcBs;E`aVYFv-&%hW&-=clwHS`poZiO~UD1aIixy{9 zU6q4rY-Np}I$~8X%|cv#(Z?>DCY&Z6)A3l+nLhsNT{MBKX? zn+WmJlce81aBG*4S-tOaz}%axsl#yCKlw(3OV_o+sNcaO7+b4v?1#FhbN#jOTd=6A zWQbpa|0+L;OeYF6$SV2OUn8aQJ{ zqz@GJq!>5@B=cDZ%2C!c`^QQ8-r*G_X+S7a2(_%x2Aju2cP|=7^%gHEwvAB6&#k(+?+i1kAzR4IRjN{&Zysp0`u5M|M#%LIemwrfs8kV&>88q zSz!tn_@(5KSqA zSQ=&9eR{J;kCTn<@)hbmi3Jx;y2C_n6%nuiv310dndh5y7s1{>RA9a*c1%kZ=fJP35aIjgG0X`t@ zY8hbeEer`ho!xF^NFJ$|!)fr2^vNykV?fTs&=nJ%evBoxM>_`y`l9C}BBXU5IC_z^ zyRMm}Mq;gc2WRR!EWZ#mo!mqJ4e)W2W#g>{8Ygj5+Oc*N(YsA+ntPN!s6(k2L6dR} zreeT#U(%JHEm4ND>TtxliqC9yNgYS=AEvjQXm-oHOVuLmHe?yc8nfvf#$0(dcB5E} zQ=pLXcbnE)DH;nt1HQ6yJv@0`iV#fty3spoTS=|-haJ8~n$$u_PA9#h^_yTEZaMQ7 zc^L!oM@|QM@F)5He|ZI?G#5anq z1bc5tyMb|$R9EP6Ia<0=Y34S-f^U1P2V=0+z4}y%Nem_8KH*_r`c6v?L|Y9=TL%b+ zvTlSf=ljd12H}0slu|82XHVkn#+lX4pUq^&Y0~rDQ`K`yY{&fqp9l-^YL?c)NRw{z z1v|zNtqvYuN0TN1e&)t$GUtQWQH1Fi$S#%+z74NnDG^3W`nMZi3FpEnO^J8N$DT7A zTeB2VE9!LL4Cg`!gX`V|hr=Y9*PHKGvUeKSz4e2et1=jk>ZIr7A9bKCAsqDk3ar^6 zNc+q6fA`cm@FTE6-ZSl*9(mWCJG*OUi~r?Kb3(H{8yn^phK2>79{~P#XgQ-i4FrhS5UGl(V?BsYX;FRM*kXJ+jPV?p(2<9>fPTLS@D>-p3xqdO# z6S46O@B>(R}iNeiGF76x!XEB)&i3RXDr7kFI2wfEJuqMY;_05|tz+!r4!6I_$f z8@zNzbu1p#@wEJ(nn7`SR|01gc1A-NPCij;C4@DOYNOsVT-~_kV23V$aI=D%+O@m& zVt#v_C9Km6)y|e>kz4?yCev5S5oS%Y+OFJ^(Uw7szOsJ;Csl`vYJyXv@a?jFQ!oyB zO}FY%Pq^M+A|f`6Gg7+v3)i;715X1(WCy(P#;8XbzRJ$m83<#c2C5x$$Y*u3|3{7@ zN!cKzO8ZkfDxx$DEI$SB=>Vh-ZeGktXP%L2BxoBIXlH0hOlRSql9?9ZBlew_lbg3| zPg5P~5folTJ5cB`esUm{Zi(9DmtZD4w0dMTh6-dfrnICsDyg`Z30Zkw$8D&~HP`;_ z!3Vo|=@%tah_UHg!u|gU=i)dmy;o=;D2#u1dsW0{WOzbM-ctJJ6^pomC-%!XAhE}R zuv~F^_WMfsW$0r~o4lqYnzQ6pJ+58)UnDfy8EscptIPq^=E7`rs!Dq)H5m@};c89B z_oZ4f*P?4!(o&sk!?`;LYr1guWe6#mP~>7}%}#n~fNNAKV@n_?u71&iE#n;LoC8p4 zTWQQ0CUwOtFfK4A-EbY0av|)JZ=HQozej83EolqEqRB&+ygJycEm$OJ0~E=$_l7VU z6HjgK!o!3AZVnW+i*85ov5%&45-Z<0Oq zJUH(O&vrni_d^w{Y??P71gg;J&T9+k4B)9ST~Jgw{~sVl9SSESmOwPpf~=+}b1+qd zv4GSM+_Vqhe0DoGk}%3{dlLdCL<@#to4h@&U~yTOump@oO2}Y?!OKebQ*{p3EOwFL z@qtt@&gD3A$h<=d`&NR!pGX_vIQ%QToH3*%pkG{`7ob!LgW} z-U#>8c`Q3H9zz*an0#>kYDf%O+0r?5HToSogURoclHccha_mvB%p3vb34zM^JS#Ba zieQ+}3(28|I45_hPWhb~P-KL#X$6dFTKjPVzZHTcl~M#80B`S~5`7t4+! zoToWFshPH6m#T9^l$jHWS)8q}ayo+Y&cUpdFtLpsvxc?$ z_M-Ic$S|`7h#M0)ORzoFJwqgOeM>jN+}7GdFc0=GqNsq=Xf$o&DVsKxmfG{u-nI>W#>S9GJ5lm#Mck1D6N<)f{nXj;eb75{1lFx;#3_n@hc#c- z+*IcC&OwgL(ugy$KyG$ALgTzq^{S}Xoa!sAEZstJ1>Zbzg_)tXGnRL@f0G2M={=}~ z`4x>v`S(I!(>B+4ydA7#7!xWa`CrmuEeuB!_;4?kRg0z|=PA*w3um;BBoBZukG3b} zu>aFvPCm2K)kujv1lI@$xfD(V5_#$1!GjR;x=)moYu~2(>Po&KYH)pTdB~_kY`gH@ zFmJNfiscMOtg)solyzG8ro$`f1Qt#ZB&qLix7#g( z$!XBCy83SE`#0VQnj#sJMG@&Kzay+bg11>9dg1tKL6w*+bXF)q)k!e?py8}4Qp7PXzCCiA#YdDn zj4mFz(PiQ=hZzB1u!k8KusNCeO?z6j`7r2my4-gpJNUOSK_a9V!afxX+D3-P|XdS8~0B=d>few>C8P?B2cm z*7M|#GIk+7wF!O*`lm)aJwIuQW{te6$4~cxeG63*%5HL6yrvB4Ji*`vInkeB)bPO-;>)Ftg`!b4Pb|D6pxwoVV$w zR>R+s;|hmtYx^~)sJ*J`;oQ&+$i5<72h-h<{JfGf56u7^=7@6gD_|qKK$Bzs`YC7z zc-!Y^I%wEz&24jy$U#Q!$sw>Zums<*!(pe%$wvntVZPo~wKS{G#jKfH-y$8qgrbb0 z<&HL*HJG~^Vmg5pob@GkV|xwM_D?FJlaq{|Kih^B30Ab3`{zIZ39feu6j_he9$QsS zurR*zf~cW+Iaj;AJ{7P@_itapuRP?9Kff0O`YISgHe4G-3F1Gjwj4lh9V`=&g}j~&iTVrOI6L))`}>tqD4J3 zU(r{YbU|n+KYs-JY1UX@jV&9*8`mATZ!vJ%U*Pu|Wo6|QfVvCbM z1Rfi&jT;~lJ9JCJ;&&Fl@A8xwN*4!*8d^y4&;HtTJ3aQHlkbt<2Y?|LseZ65# ze{Bf3OHK09Ucd97IU2ECbE?e>7cmNEFZ~C8?C{4iyVp?wqSD^v{5v^z05Myv1aC;3v9Io6`h+qV*W`(mDqltwT*A^p3npW-gY zdA*M-t!qEEqX4m1Ye(14XB)XiH&9WyJ?%s}j8*KFYid3GC`@Pqsuv%WeZW^MdT15u z;KwtC)w5}-IA_z~%bCy@R^uN%UOi0`H^0>Z0jOPsonuMxy^%Q+n`9#IAqjVfQr6%V@ zXxS82y>~5s4?sanHlL4V3E3@7Zn8U)aVO8?1Or%b?a`wzZwv+>ej}D58G@0_K35=O zBx!`RfR85MTQ`mv?1r^XgW%z$D|6P%4bDyR2_&C5Y2XPrrEAI2Ay~K zP3%{#6M|#;FHBAfcb(ZUQMHSmCW?dT27o^3z&JeW(sjg4zIdR|-%8WgYtK*i?aAnL zf|U={U$nU;fi7hj-Nv;!lCt^sL3c}2hlOVV@5e>rcODTIjQr9=*BkTQVLd-soILtM zo~7FjRwLNU#^CA<&fvWC^}Wn3W;S>|Jk(2}+JK0=8i6n}>OlpJdbn*z3&kM14#jSC z$ouyd7*Z!9oh)08i;t_I%yCkEWf1eiWFN1Xjwep95jMR@GsCdU8yKnxa!McoV}=># zc401@uTU0&R>MmeT7ONcT{fj3!D%D(U8q_M?u$8{S=)>75J@oqZW^8@Cx?H+I5V-ai(O61s(DSt5a6fO7e!Qr~kc*olrm!{^(}|cp z3PMZBV9e%a#eS#C2ir5X5xxZan>s`C)EPdQa`H{{$r~yl=Kpc_7GQFfXB+Um$6dzV z-F9YnWOrxXHR~JMxO<2}f+d6m3sO8-k>U=;DMeeLKue&s6xu?)lv3JK`n8Zf`=4`W z){^jj|NqZ*&8$r(Gw*rd=eZxdZ_y(j7edWphe-j!AHj|W-s@G23 z!KOq&BhalAcYxU9d=Y0~y)&34Ngfi-MGivDekAPCamWL3HBJJ2kJ{#cNxNE7TU5#m zUWg3RG=HSFk>H>g;?)F+>nA=RKZ_1-(1eH;^q)YYX;N;Eli)t*L zFV~k#=h`%8AK&zbQ5N41J-d4WW3g@;>hK4a}!#`8A<)WYpa`qzX;L z@*1pzUs82QmD%h$9Br|(lbJ3Jg?w|zJ&s3wq7Qm_;^WePdQZn7!9S^M!3hLU&4-5C zw4XdfR|qIXz|BuPJ1EKmAw}VtXP$u$$nR-%2}Mn8h=)_!m_sL0tnhdhC+IoDQ2VXT zRk3t6CP#8l%JG2c=7s=K!Z73T`!J_TYtGW#t+`+Gu)P1L6+Wk?=KrrgECrp-)jTPg zp33~}{Lw6l0SB8n?yi4qlti zV$?R+l&LBb)Ll4PfML?^U|K3FfW|;T{47K*Aw)e2b()c%J0!Z7{*J-I7-{v2dpg-Jr!cj zG3di|1x%o(&z6ZM)87Wt9>z4{GkL9HZLu~$;Ta}d1V>xCKDqw-CvLh)Q|>PvhQ9}+ zs$R%NnoDQ+^OOVYo1WO4>ex~cc`01;&1TRkg@cv3Inng2^2x0Dy1eO?J>fA-Ctqik z17>X+RGoUlE1_4-UWXln2(hicgRaf{+bC^-dQ;)WpSUAnu?HbkSJ$*sI2`L?U9A1$ zR5ix@;U{ma(tY56{LKVyf#53fC~&#jDB8h<2ekl-)WP+2PK3oV={K-eK0*LbK(N0s z9sk=U=;n6<-*MnHma*zE9gTs$qet7{4txo%7Xoqlnl!tItC_wH>q7FbDR504G-dTf zJnlC4_&E9am0_I54bkd0jFdjwCZE5kGO9u+^*TTA;?3J)Ih;a;p)Fz7={4}YPM>l4 zH!m4t1+#vAOL1)A<{+23uGU9jX=QFaY>jU_yiLYW-pK{*c=q*m>Mr`oa%GOk7K}8WhY9r_i7a%{91deC1J7~e*o|NrO<3~{lXj5LP#HLhqPl51B{c1jr+NCP z$P{$b2H7;L8aP`LO1{IoaHfcHm$c`Lgdk-Vj)+8UO3Cj!-~}vg+$ld&hL)W>B|PlC zE0IT;^P4l(DF2}+W{*%O%4jQ$O{J}@WFKcSpGdqYn#szYB( zlXOH2!)^bH)RTk(s}ZG10HjIj(#1tj2%`gTz;A9LFq){)X|r*&zalVftEPG5ML_VL zadDL%CfLhsY07o;BhVEWibqt8@hFDuQojL9@--y?ZybB3rk>=%J4Tk%8_=eX-fSdV zSs+N1>511Nf|qOlPM1C18S_-uJ0~7FZN$W}O{~ocma|DD8o?Pxlpfw>U%AA^FiB$7 zmc>_}6X9zP?vO2ic8pG9jqNnE*{NuusCs~F)WLnm|cM|96uJ$&omrO4|2h{ z(W9tYmm2z^QV(tt6fjkt7kc(h)_`zT2kJ%Dz?Yh)&SB{}Nmu^!>6BrpB zsljQ&AboV}D(vUrJ5?xSj4Vs#DT;DFbV>GXRK$V?A(+T{)-4y;?627S4`1fx-qSu) z&$cjmF8ng|Zdq@qOVg*Qw4Bg`enJoP@hI9*o(NTUVOgOKFXv{#9L#Ytru`|*`6;Lg z)T5~EEV71uJ5o&~|Eue(YDs?LuQ7`^82k8a)e2s5i1m?rn1o+Hbm)bBKRk2@wC$60 zRVvheG@T>({l1Dk$sf2LyZumvAkg?j301MA8M-x7i^(C5G#Rj0=xl88NJ9rj-qX@W zD+HbQ$oC_nK#QKC?suB2dTzh{_80GvZ=juyqBRW^Emg}8zbkS}dV@IL(R_?9%`QWl zrpdp+cS2`sXpN~f9WvzsbhGE9s&IDVW0lGm+|&9v$xY}KIGcnvbweIP;W-_ixRg>5 z@bo z$u`J=$S?K2pL^<98H3efB!qUuuAFuZKy*Y0ur2^)^#GIVlTnLHphFP1vLY41Xo{#B z5o{*jffGSOXJ`%Qf&@ztyKa=jT9l#q`#yBl)jLh~MaS#8Mz!20wLST&4`;Qff7ioz z`^_lX@Tl}1MR!CDj=a&5PBtq|>a{Hd(~ypY35GzWKiC|V`nVuR`UXH<7<6PsmNU^U zqdTt6IkQfSjcwVu{}G*pKt2Mgmn7y-ZYdyh8KEtxYuA~sePRo^atUU* zXrGu+ilrCI53T(KDV^3eLXbBtT@0CR#14i6Nf%64V>xJUuV_98 zRe#Utk&o!R=z5;DKi=FgFTsoJl078phT&FPdYeipY+Eh6Uy$CGKCSQ7cAo1R<}GZh zx{{#$1+U6wW22_W&>b z13bHCoZJL52eKqb3L96NPztA|`<5<-^=`{^Y1PI&Jh=#d2$EC|XAL@szrr-F?L3vW zKW-Id93Xn2UrRnVqXi@~`T(+tW1PTO+b(~)mG63oDB1}jd57c!zmj|nc}6gp=z;DZ zH5c-rYEf5(>|Bf&Dx{$AXL2}$8$aBI6_uAl%o%Dtr6;09^b+*qsQ$(3pu^)&j3TRO+R>KIW9%K58qFjmv;nAEkHXp0 z>D}N85YHBI_D)!SK;V19x+|#BK?gQ|^WI9rYh$E8fPYajba1#oA}_M`wwflKTU*nj z^Dt~%z4XC&pjPWMuI_00;@bVhGdI?E;*|9EGX{%gd2=hU{N?w(TmtwEp45G43tXev^4kBs>}_m)jxsHQguvI zyLA!kX6-t_Rkxw^wc3?y9cP zr4PV)L&GVYz{O)veJrqLji%q0`=fu`MM6AJ@!u6<_|@;LcQ~eXAU$oE`Q5lEjSXWr5W?>E2I|} z-?-d-@s%;bA#Aw|V!QeoS_Xz2&#okB)U?Rgmra>Absl?Bp(Lkc$fy49ijc7T?mR^z z_I$<@$Pg64S)^YB@9aAO6}^d+LD;oGJ|D5kA^17jmv?E7D68=eG$e*8x^JI%F5UMx z5r4xoQ#P|s{7gVu(l47~qcu!7Etea=XR&L^ZXHx#0uQsa?ke?hUQHjy{us{TBzzT| z=iy6~hjl76KNB8iDBTUv4bfgM-%`)iNbh~e*GZBFzJ){imCg0q$B`Nx1RjJ7q8uJP zD18hC-Mp#73Un|ue>g0oLK0^;btvCZgmyt!ABNKjAIw;7$yCaRFg8a;roxBNHhV={ zHl|JT6A@ZARsT%2FqAh#E;v|`CkTvvFyf61S%VhmjrV@unFG90Tj^~6>~KppU~J0N z@Ahyud{E@mwW2lC9f2tMta>NM;UB^pIg(u?$FiEjOvF_36QyR}yjCcMttW63V_-_l z0G~>{s4my2M6$C_cc}bAiYS2$s*I0s@U# z)Jmzi(&>D6yj4$Wxjk=R{8$e)3gV#RUJpcJx+kvfAM+v%PJErO$H+e5idMDZ1Oj{U zThcWMLkQj%7H&whh}McO?%v%Zj-PYU8~^xnGrJI&rTt4MKHPENBdQNJO|FAqP;;to zl^xtOXYEcTl$}fcIG4L^T6S+rK&=3_MA~THg0WG(beEapIWj1!pds;wEyI8P5rjh+iRM1?6^~T@-YRzERQigU+@MO*26f|blNRZrBwd-A#aB3 z<6C(HW5P~_S1^oyX-0<7-#;%|i;I8U=av4w%rcE}ahp9)u+5eS-R73;MqQConT z+6~CK_@EI==@bF1Po+-94VKJQz+Pb(oj8oUK z(vNpDb_3ej4&0%t7*48sSqtFzp-DOJR6NBwb3JF}lna=1vc7?WvW)M_Cr=HymCAA8 z#M!!&B+d=9Gv||ywRB6hcTfn$J_t4uAxA@D+vC9JAP|gsq^4(G`j4&_0ym&Um1&ZV z@Gt(&W4kb44}!T&DD+`V#3XjJy&GZ)<1v z9@%C`DPcjPfOBMYQFX6{VofGuL9U}lAB&+lrrQ~bwP~$lQx(hV*pwr4b)(MZ6?X2b zy^vm5-?>OybM)w2W?#x{^HfQH9AA#;-L@4?eSo;(EaJ^~<)~!iKstutILcpN)k@$c z9G#k64?hL{T@#UUWUWRMnDWZyR0cFdu`^SGR0xUyJe^0vr}bU_Dcv8m`Ta4lJpiv2 z+$>;SziJznBe_dx?dkt8m~D97Cwruu(7V4lI=aH>B*kLog0-8BJ~FuF1Nr^!1qWgn zqxU+cV{1p2k+|CzI=9tm*D)(Ddupw*_Iaqz+t?X3CkN9{Ih)%;mK80UG7s&!@GH=t zHH7>L#>$MQ>id~19XHc6=qVL8;Mpn=>cIlj3XaF;#GYe zi*gB_^l5ri>k_+Op!6{3%xw7QU;-JGCa!0{&td{zVbNw2CSNkp9O#m-IP_)ye>#w)FWcW&8A)d=0x~w7FzT`E|yEOfACkUrP?H93hk$6up>=OQ%H1r->R$ zd@KQo;p?<8lIek4+Vs-TLJq|g6S#D#+CmL3z@!bO#kt+nOcFM}OD)Nr>aE#3>%b@mhTPa@;Xh6;|lp0&3S#={0mE$_ll*2JfMRmd1f~%(s?J zJeX6*Y{1HV8_FLF7 zpzyqO6u;-+E1(w%4zsm3H8W@5y^~5{&GQKItA;KFCEr`V05&f3NmD1WTl(G@NwyFnuRRl`*|VvDMbtnI=43HMw-7`Z>s))T#i+mJ#P-e+d+7_nMp3|kaDbtgmb4X8s70|n`9cDNrkS5V} zf9=w1$!mu5jMNY}`U9~$mMf}MiCxGNj?s^%YcTGqPf&aTMRE{$xUmCcrFU3+&fUsu z89LJ1GH9{T4%zZPa~E_6EQTo>;CJ|N8nJ(MkVbK$!l2hiTLN}(D6&S+VM7#YHQ$rT z%OLy@Bc~%{EX((0t5EilEzK*q>*2-ge?imphb6QwAjuh>SA z%P-y+LP%0<9t|pL6K^frd}ErTaKwEY_*xtD89avSh;iZ~QFM2==~>ZCHx7Dugeq8< z8!HaGUQ}Y(9 z;9o%h)KthweyiGLb3Aknn9%a!&ll7pT|t~m$N0QF2u!ON%wV!izEG-m)&ba9wkBl~ zZ1qUZqD~7bC*|Wy!W9_%tBaF)l>IZXz|!vlsMTU1Vc>AjUSxFJ(k#xK{BMBs(W;e1 zApsAsb=6Xotv|%YD(7|k>FyV*bP&PboV`r?6ana&-K4=FW36BK>S;Sq$A9XpZw`Ym zGj#-ci$XMacUl-}&nHE{X;%X;aMcFz+Q3-7rJ<3e5Q}m5IdxQa`=(f3WjIE#2z#!F zXA=e+X7WEWaa%4)(ph|d`V0$SdFIk902xQ)X_7+C_U(5bZWuVdq&2+|{vq@i8J8%W zb1m}9E}Vrh?`?xjyD+D=0$HdS4kT|kMy|oubY|A6AYa)wt%&49L{HP8rMEM zbY9eyXRVKH%%BYI-Fqt6@Orn}+|Y4R6O15cVlF#4TyMbIqW zyp-J+a??bU5S1-46-B>ZT~~?Wg!_w{M}}G`1~%Ut%MC1|@O?LhWh7)}4&1q0^$|81 zXT7EN;_TI&{r`uRcPhIuS0!33U)OTpVEUMpT#o!@Zn;sXoD^U`Ok626vH&F8q1K#! zu}R*0iKj-@ddNm(1gkgQ?K~Bh_iRzDkZOz_EzoO=mAj+nHEX z+s(ClIhr}-?pcH_%2wN47Oq6be}5>G34pFzak2E2*BQ~-yrS$ZL;%;CNuqi19XgH; z4vbcy3@P5Iu=G4ynI&kPw_FhQuqKBL;TZd1M0yWO7Gy_42hLd3o#lu)y(KXkV!G#z zKLIH5*!WUd@71nv?FK)YH(%O(RpwQH^2P@J z@wxw)Qqh_k7@TQNo$jeImgg&##Oujq?&mZhm&EQ-MezPxPbo5y6Tejj+g=)0Ghyow zs(P|V;DsWmyFOp5h{|rRtR}d3q^(FJ4+wNI%jp}NbtWCczod72M3Hxk034>0901Yx?>WpOBQ#>NC(EG(}avdfDs@&-DDhULjo17b=}4`ZlHiw~U%W zS6oa-?dQZA}LBZo>2tx+o=U7NS?KHH9JO<%5=zgW=2hly^b)+pKm66WjX)Jbl zyivf}B<;w6E1r06OH|+fEYb^5Kd;>-_jHXGp>1#DRWW;#C%0jO;@+ia5p^|Y>?16gHor*lS zVjiZfgA*v`A$T(GV0C}X>sdcz6=rA5m2NvD2UNih-PB8!o;y9SAYTCa1r+LFOeNAQ z+Y+^%OMYFbwN0m&mrct7(j&dM7u#oYy95}miDI#wgNFJ8EERjABW(BUBBsR9ztW@!(T9uM*Nd@pvk7@_Y0BO2+ zsg#|cnl}kmnhG;xhM*Kj24^vem0W_ypf-&wX(}Teqa3Z(pZz#32VUQQTzVs4WslqR zH>|U?)2`<3=aZeRIn)!OnN}v~518MEZ*QvNCK7Per)U z-%Nc5G_2$Oob9^G@$<-COKVuk6H@46-`E?r27>WR(Zw8L8BbN?+;hLB*V42V0oUXe zV)scZkaM+jf*>Rsdh&KXkMLIa#LrR-m|OBiQA-$ofDOp-_ojls0_V;x1d;OpoZx+>vDS%jHa z-HMazfHj2@IG^9{ZtNzghj*+!EkVi$%e@RphEQdgq0%v%v?wz~ys*!L2`KPH6IBG& zAD5l>i2OjVOcXFWWz`9mdooA+9V51tE5Rv($KfW!){>rZ%j7uxBbd^x(rlC?hch*r zf`Zu8n#mOVP&tO3E%WSy)XULhO=}@eu?rif3SUZ@7#7u}Z9wgJojtaA zH$bD=yaDPO*ipnOo!$Bf_DrrG<4%DZMHJpftiFz#7m@Dw;uIP4AzQ=#&~iC`Wpvfa zG|6AlHBal**4W^pj?V5xRyjRUGgw9&`3-b7ApN#E=yV~Bud=zaZdmqZ!g$oy*GW(^ z-2S7ow$hKE*SQQ^I_rR4`r_q5E}IXMbVg@~Qm5ov9DhOIoVOr^kpv^XpRc9q277(R z6&L6?zJ&gl?39K5#Q1hoRVMCpk7r9UT5j?N_;%=z8d8oX z^6GTnlvEIynJs~4N(L(RF5CcSt1eFSNyyd*-27t5MtiE_pc59%Ohw_qx|)m)S;w#z zZS<^Go2cWEE!%8%9yPT$O#Fk%g$1)d=LFCn*RIml3a6}y$nd8(&k|f#Mi7Eklz#;TK?7jybDU9^Gkl z?3)?;^{J6HI4(^E1*Tihbxuenx>Ur}laRO6+j2Yu$;KN3Z4~7pw61QFg0NGOfmX8( zzD?J%bTfV45Wic6&6l9iGR{UIlCx)?KZZxW7klRK1m4@*QDEwqzSucl>oGOkC?tBB zu0AXHf{BiBGsRdTW2T89FzwGCt}gI-8Epigc>Bq#)^{>{etWgj>~dNrgmb)Y(PhpS zOCU9G%rVjL^|L0?n4x(wQq6Zph_-nss8@Jkiv+Fm3}c$eWvJFQ4J7qzcF1aPR1KuG@e9m%25 zcEQFK_MJ}lugGZ0k=Geu+xHlsR0B))<(Gl*Bk6zSFYo^7UO75w)&UGzz(SA3a&p)# zUarhHJ6Y2MsK6)L1>^0j(l(6dt{SR!2)1;#xFeHp)@{FqWksEd9%?$PxvFRP?%h|M zD&HjPT9-X(wwc*N-)T9!>^#+d>8`_vimemYtv`IKv)NrmX=pzF6K> zl^K=XMmb(8x6MiR75J=99b{M*wG2>dqv@UL zd`|Es1B0GwhO$DHa?mxlB8gHMzoxB+VBhN>45KtgF05-IxXTs3_?}Kg2wU0{EwUrH z+>8@3p#Xw~tS=(!<`pK!3qIJ!@pd0dX`_|&PylZ%ilCEDV5t;MVWP6W3E4+qDdStC zrd{pVz>k}cDN7+u4N8|=G;}9x3s#h-N(VS_5|Xq-27M1hATmsVR(5HTrQF1&dP^wo zbWpb9Da0+^J4)?+@%k1@{F&Q&*k4E5DBQRBzH5o6pL7AO4z3XqWHALj#ZD}&%vFnS zzggt;W@vf;SW3PUU#CN!VlHg3HRP8Jkq-ioBCgd>)2r*!RTM>wmTx_FNxisw@+Uia z=|idpVtxMpJc!?B5><*%G3K?I^OGfi<)aUy^Q@UjO{_63^BJ z@JrCwD$7<$nNow61`wyJec9PMX`bmG&pDJTWugG(j>?Zf1Gnv?q`$@t!Egk`P-GQL zTat)-&IDK@_3T$;9$k|1y9g}KB0NF z^bNGY)wm&HB=ry>`a><3UzA9Ap8XB+_ygvDMJ)bM1h6c4F{}=&{Jx<}{x~nwIG~$) zF%ImN4jLH!{9RzQ-EQDA0uc*=%YODjr!esT3v>mt>olqY>l-84`x^Tekc@1{1C#UN zm!V0GP-bpeqQ(j{Els6&q-OeOSL9+~YBpkqz$`yY*sk=s3i3Zkr>DVq;F<38R8VjA zf-9JN=gnh%Uhq(Tx4g}E+eoAl>~z;rl*q-4?Sy!jAz9Ugv9~kf0g8mRm38QXJ`XKz z(GeupsAo~zY8BM}-VH9Jq<@^_W6cO&I6nYx~3h=t^@3AkHiTaPj$^F zap{q(Qw_;@7AFbr?)E`~db?#FL3Vc~f)>3XilMxPMB`eM{Ro{vpL*&kDxJ}XJ(=bg z-|oDCD^!VbT{<5q;hm<*#qdvLJ7m&$HC1yv>86pqI=~;8ma#);&Clk`beQV+>?~$| zRr@ZlRp*_-7hMP0#jAqLIo=|_ia)Y|HMXqf+b25O2VEYT3wQx{z(=#j8eK4Mh@0sb zfS0@b3U{7{G2yCSKoQXmw`@Xxb|JTdrNQJ?dxY)3el#Y-XmWmQMGM94AkMhcT<^72 zh}|`=g!JI}>cDV?p~bkM+R5l3-Uu#u_j~_{3y?#LUH#$h#FDo!Szr~000PWzl*8nB zNzuT0QTBC84_#I>rdgy}s@Z}ihfYbx1N{b*1X{fG$3}>k*x_dZ*GknG7OwJkeQ+4fuBN{?=)y zoyL9CV(?KmJ<{6oS}5y98O06;Ca;BGlK0binT>tZ%p7PA3+4RfSY?)BKe4;yWDJd- znM~yDu~EbEa}Pg}j+SOwT0h(l?tChZGpS%=C$QHk+`F6Z^|hCDPxX&JtMX<16nGk1 z2>#eFO0!<`f3AN0^d6{MA@)ZGZ@ufTuO9vUuDigkk3vq_OfJ&G+DKb7MFWyr+0{=| z{fQLH;M#+V`l*N0E^q0ksPzi}u8bkp%dT)aW8n-#-L1vOsC~Sg@V)N_?Zc?bTax=~(~cU8iM|Gs+Rh0?J-4i^Uzamc_pYu3~2^b@b_ zv*;NSU>4gsbWFrx0v-d41MLK1fZ#e_Itb2G?obyMLr3N%aE|s?Hgpioq^7-(q!eym z&7=lAWNp;i@4jVR9IL2j*md+ovkaesh=l3EdY;i_#n_pHlyYZl>@9re@^|$LGQ4 zCN>JwmjW8OKIroK5|5%IJj2~tXp%kKUtFHHRDpC9PT8qMt zVJks~fq6%v$8Ml8Aur_~L!=Hjg*vZs$1L7JvN9X9$bRzmOaWtVip48%Zr=+lY&IIe zG4LgoK<5vPDEtcK6HUD&)v~a*tEZaayw$gMu`p-<#ck3$zdOkWU4(&PxPPsf=VhvI z2Y+V$0ez4$R-`AsU*ym9S?RjeXY&63l!{};4if(~S0tHJ1|znJ);w6P@+6PIZ@@rC zt80|5G2Kbx0mYx5gg1GHv-S8Id?7G*>aE<^S8g@NPtMV;{_gwLDr#NeA_o~x@GMsH zrf)1*hAv_VqcxQ$_`MgvhXGV$K#}vz7>kELUTl_OPPIt%r5C?DvhopFpN@I^`j^zR!QLnwzb zON&hubFEXGJuBWRxDCr2N75F8Y0unYBk?5W4mbC%$O^*`TqJwqbQ8rRhBcOORl35J z=zU2#)+vWH@)ZP#Y$+6Cw<#`xE8xdu8#-U%paOG~n;z^0K{jk|zHP=}&hEN9{+r6e zbi&OZtL)-!o#~&~$S_4Z4`yDkX~J3QIo<-s8rtPwx%`!2sJfoy|N4G?E5=T2QJd?4 zaL0=AuS54-BZ0vn18^I4L;y>!&&#CEiItZQB1N9fq4&ofzkZyiPr*C;x<6@DA~kJ?AK zRCA_g{~VO}O|fYC;0C78s6wj>x#hqU;QFc1^Q5GzClcX#3^UQZDhKnLw?Qn#+E}-K zFS?ATb*tsQ!oU4sO$*LVd|x5^jsH!my4emH5~ls>+GER2$* zuuajGJkl{p67tbKa$RfD9}%>R8z;so!r~yz9OpJLgcu2mn>n(r83!r(X^|{N;Q}2{ z5|rG$P!GhjER%4WtVX7yX2HDp7Y{t3XqhQ`2|t4MG8$T}SzczEJ5_T)bDic5YpOWa%T;^uAgE^<0lG&T~=<1Mhx%jSlw`oeF8P;`f zcDqOp%(l*=ep>7-2Wn-!$FIvZ(hj$NLnO{{7I*VPM;ga<7!wjMZJ|xnxydZf{T_m{ zO?ErancD*IqRYocwb?Fl5QRIrkU(q>CVcB_hX)2O9=Tv(;Flw>&>>&{esc3J=mCs- zNPX)1iV0t~g*jzKjEXj4vU&{|(vk8zuNZE&;#X-G2Q-w1X2`9YlGG8Rw~ zXBY_ace^Wa0-^W#`XP>H2)hYk^x=FHPVs~>eZ{yUa3$oyi2oKWXS?<$HV6;()DOw>Z@;{F8{5c5~v2R^vS9**yz%3>#PSY zbONjI!YQe~qdH;dM3UhZen{`+S@2QW){n|ac!g$%eBJkJF4jDydGWYB&o^lKI$yUaTEk+#o&@mfzmxEra8(J zgMvv>dQA~9h=X29s?;J73HP?c`#PYmf{=2w*J}IlY!%_RBmH2VbRE%J5fE@8A^jp> zZ?G6G2o!W2V#7GU}rpRyGw z6&Zi9~^U_kUE2MWz7`z5i)(@xSR2+nK&Y4nuTyYl~3Zqki< zN)cI2YEh)7kGgEUk@BRVACWwfXzTWohi;X2og)2ua*$XBbRgHYq$=26LlRga*V2^H zno!Qv+T4dxr-YLziwV+O0egyfaYcuo=Q#aP4}hAho>{;C(8i5doO&v_0!MI|_}98& zku#Y$wl_Z4*hJoWG=j5Ce_8YAdD;J0X=*i%npR~l9-YmsuVXD9L!o&P);SO zU|+Go*{*E` z3r@X!kNbiPpeyK*UvKldMxq)uhqDoW2l}@rDI@tI%~~1DE8e~GCN)1Y*2s%cB?mn#!16b9Y+p%3&)YPnB zZ(^-Nd?jQrbjt=<2AphFSEPWGn0RqxYCvb_34U&xsF_V$J;>2EApCW(W~s%jTi3bj@~znfN*;FM3}Xvka{o1E zK6&=Nr^n|-&Hli~7B%NuF}YZdo0L6Pr&K)x#&WOb?4GETa0kxq`@zaRg@LIcbz0G5 zdhw^fgPuF(cF@6M7<*Ix&@t?bQ%39xReN&l=_=N~3|=O%)(aXHwyQ4}hKjUDSNC~f zs5dLM)O@`2Jsz_%K^G43DL4L#9;wt6Dey_~|9uW~yr zO#KqZ)gz!pq!p>{vDmJR-1*D!9}3(Gj$NK0#JhCg=C8ZYD@L+F~&TijiR0 zu(2hz^SbsQ{zf%+iWf*8$!5Tr~}P4 ziq^zDuqW6;5Ox@D5N`fqyhss72&os20^YrW*nhwjqB*^E+C6(*B4!7d|5TCnhTDFA z9XR_BKm92%9R#c^AJ+&g7%ck=fg$OkXOH6REt^>)8LdQ4XY7TQY#&{1N?BkM) zBaX1u=kW48%Xd=uSsB)#9la({l{oP9O*ct*-FM%88eG#pxf%W$jA~-?TC9|@_i37g zns3c804crICkR&0E#IOTJ_UP@=S{(<<>7o~mEW1s;4<@>^0MWo0D0X4vr5`3(kl_7 zWkb-Fcht3E+!wxYtpaYBfZ)uQA=*gaeh+c>wM>ge-kvE8S;AOpAk8<~7n*D&>j>re z^YRODpVn~D*+z2)uHw<_sGAP^uGFp7yKLa8&Q3ie(0qUTkr1zAYT{4iI{Xd>R7sBu zRW8A;3kBHg*Lrk59P)$Tbq*8c=|=<>Xr080>$YC{Os!b^&;|0gYB@aOL1@x@*q4o` zBF0d#&Rtc+Uk8o~ltI(kL<;8-BzgA_yTvtc-knUxdFSW{d?M>}$BPp$EpD}@JJ)1< z%?$~j6YCb?xm0tT=;rJ_&0u-K9E>n*ZbZdHcG-@;3Bww@93v_DyB&(cif($rk0U#f zqa1L;u9dG-!8F(SmP91y5ai`^bLuR5Jg*SL$x#w4Khscd@-fd_I>!OVUc^RD)l{#4emEancmI5fP z+nn@aH+20+@R8r4;MxxXb_;lP{Qz|gz#)PmV_O=HC?i|Vd)JI0dbb5Un*nzwbA)_$ zuJqcIqL=fWQ8$bVx`fbHzb-BIk2j2yU77GHyzH=LJhZmCD(xAT-Jcx2l8JYklH25) zG}%x%_0-nPsU?1ns>#dYhoDJ0BJs<(pju@CD+K~dg<*D=MWwM!&+VPoKP*ASY4t-m zH^)-XNtJ*bI@&t~+++SZl(A{eX1@*6;V`r-)yWv+?O~x=`W6|sJajhx(#9j_ToMwk zl;_6I;Tt1XpFaW4s%<5>|M^-u1rwH#-4{qelVYW%m~GKL#hYjoNfogB%-+ zvdi~S1ZKRdT6f;9+GJNM>ZQZyMkYR(D688}bdTbcyeefMy#{_77Uj>qP?|^s zY(~4d)T*NtMQ55pXY|0Q2J`+8o(hFrTWMH;{`lR(&-Ox=qLmA&=Ww|or*VStgm?d`YUmcD%Dl`qE*m>51l zj2~EZ-|}I|Y&ONO4DCDRJ3ZR2SBA5Ui65xg*_6xgQ3y3NUYH`ZqYZ5ovnQ3o36$Au zGX_|fDgXH6kIO!xb8;K}kh}&Jnw+LxGdR=1GK~N|;;eHrD;-oebf_GVV7LpEAfD2? z0h?x~gdpbQco)MsAQ{I=1Ezq8$scrNi2i8v02@fXQ`=1mRe9+rH=Os^rp_;y@Mat$ zVjF&W7Bc+A{&3C1Z|dBp+>y=T?RStk$ zWAnLgiUV+b&0SsfOWi0d8my~#oibt&jlFwsJRF)|Wp`VP;|a9~X*K*PjBBzo@?WBA zqkcmrWbOr?1I>IMe3sig-gn|<#H(%_wTHE|8hYTWY9RPyUk6-)NEEcZ6>0CI zL;{MV8)@49-)bePef3kR8jO?v_WmmdH(HFi@38AS)lC^kUHvHsQ&LgD(BmR^b&sA(a|_x?n)dj0bAgO!1(Vxzh3 z9;1h|vj4CIa1Lk9hY!1jhMSz0h~Hxm8&kD4s{Qp$E`(o&3E4U<)~r!`*e_GNgr{Jb z(m*k1%B&Qo2I2A|PEC`QyBopDC52+M<|wA+nNGCivo-$I{+}s1$tQ_Gv$dPG(PVSf zF=WD$qKRg-M=FZXc3?kTlM#LrFna=_k6~;Dj>~)OoSFuLo1hbFLbVAQ=P9!`UO?$> zTZl`SnQ$79LT~nltxk-{N*^?t>QGb%!5Myx@|s`*xnk_qeFq~biMqgIi2T*T1K)~b zEG-ysYj5opc4MpOB{5;wlGd~I?3ODI0yh;xE*}lU6w1$UZ|fx}X$17j2>R7%1FPfTrn$zPxO>tUMb625TDQKI@1k7lP{4ZqWzj)f) zh!X+?7qoA_WauH+FR|))Rv^D)OdA~=M4ZdF4`m4(>d+_qhL3KtE z@f9V9GY}8PX@O=%@bSd!tHw~BQ$JS7gMSZ<%C_7+*$KY^vvM@A^qHN|ouKw0qviU! zq9uL8k@XxmbYB|pQs-2Z(7rf!4*hJ~JVB1*9r-pUZ|Wbt61ZY!4kZNXvqfvqtO5Q0 zCW0^{Xd5rxOIOIi7g~5a3izwLLm1S$U_qPoJQ#yA=dEGh`Evm0`QBm|#WRkTonv-^ z(z=P&jq|GP;Yu7w#dCbVI<1wfO$wZ#OF1j<(`SQz{*<$uuOXmq4MS_Cjt@SN&t_7N z>ED8Wr*UhdnkG%RW|OKBd6F46V@AF9iW#p1f$3%o)2so8GEuke-4;fS{&9LZx^?M07YIBxCVPFy z;bYz~0xnu zl+VHWYT-kffiIPw15o{l-xWltgmlfk0L8#oqdr_A_VQK})>5%OVy@;KmcG%a;V1k! zjcA=t&7;FHQm zjC)lZ*yF3lF!|(F2|uP{ypr;6NQsRIQu9@CCq z)Sf1J#K2tA2Ih4oXaBoK3Lj<0k3C70tw5fD4l( z43<|3a^yh%esw=$wi{PBb$~5&5^wCmspig91uFZdFYV!Wy@~Be2)vHh4|o4yBorAu zcG-Pbk963~WJkSHH?_-H;3-Jb^ePM$KGgVMA z{KLyUQIr%OTW&NVISU3izi>MWsPUhN)o>syO@@_fJuo247{>H`V<*Z=FXlE4(2nuy z2U>e@ZenA7i^FXINDkQ6pC>S~J=wB<^{)%)*1`ot6R`%6z=2R{M*M~>pUHyC>O;roRxm12V336xct5Lwrm4V zJ>b=Cbn9Wd@p6^rviU1qnU?C%l=$q{z`ue%(r7hRO7AF`HsR1T$@QF8QO`GMU*gl= z6KEk_O+6`0tQ%iL_Y`4ph1A+Z@)LEXRwwK{_3QQUS{?7WQyCch<&$Q?M0!S{>)X01 z_Lskwq$SMwKmYWDY+hO{T|Tssq~%Qs6#0XDt8!tSB6Tly^pQ+ge!%LnE~_c&3v8P_ z0zVENReDEtqTUH9)3;@6dt~<2HF%CEjwxmC>%cM5bL7iYLk30%vo17NdB#nDURQ&x z5E&ErvBaq*R`7MaG{LA#iC~TNsMbtkY_P8bWI8=MhKy+`h)h?56~n_}f+MtrFXd&f zYsN@Kf2Ylzp#=+DQ0#?mas++~Mm0{Q?@>3yPzmh4Q;=m}voG3Zb=l~$ZM&+=wr$(& zvTfV8yKLLG?Nzt_-`@A0@5FsLXUBOvvDd?lj5%|R#GETL#)usGlgT%QGkG0y=#exZ za`Ol^D7H;hK#YD0v4&aAzW2U2s|xx_%1Auc|9fx;JfVn64)3aPaAzi1jOn+!dXMJ} zBqDZfok&H@w#9F?Nt59ulHLT(lmu*nn+;}KJ=)J1ZkoQ_M5pyPfq?^h z>(kO8Ig%6&p*r%<5+ebTtF;E+6EQPvtInm@lTWyx6$TGWDMj;zEy?ryt{?@S$IuBK zyP`%N0%o1*Q>)|#F3mL#E+$}!O=e@qs1DQJ`HHy68#8m5VR>E;vUYvO^p;eSwQvD4 zl96?2zPfDlL`@#cURx&1L>e(AHdzlIF(B#`^+#Q{%ZRUos)7kt?FbEUU^$K0me9?9 ztjeJ;;XIAJi?TZ8Rc;Gp#XVWL;aKD`NjqH(wdO|jfwP4#+1KSde}aLVic*Py)O&$f zNOX;r43`ED&NHf*jj8N0<&*DI#*`{QfMlO4j{iFUa612F4WT)zU(Y{C()P$%)Q@Sy zSt5t}gc)hEQX62enN9e)5+Tz;rh@t1M6JGO4*|Y;^tv~H9bQ)ldPoQ$C6NH*!K zj*;l=x~Nm;6*lq9u8614&PlGu6`>wRxQt--++zii0W-lK$S~}yq|FlEVxN8AGhU{o zBS8jUQqYec0;7u>ia>{P1Os8HGZF}h6ag7DKFEMTpP;mtxL5oFVYrGxd~?CQ#Cn6X zY*?BxI-?}vwlM10RngiYclP34Ok6cq+%QW6qrHB4m*_ zl?#1f&qk;7G&rW@IElF@9_O#L<7tDWtf?ppl0sx8ve8r%pcOi6Am2QHr~Cm=&zt1% z0JC4E;U+(}DoM)Mrj)TvCBIkL=y)}yIwya<8_Y}qTR$k@8!p#Y=QizpBvl_s;;tf9 zZ>LoK-H;LZo;`&t&udA<<*kEUl-st>qiJ@;23X=IkHW58o2YMiO=q;NO_^9cceUMY z6A-P>mf>C zVoZygTvda)^gP2A*=piF$ssiBxwIp}`h$BmkdC-_mXlCap?$ZAC~E;3s@ea5rIiW! ztW)On%)*n57Dolo-y1}=*nn~sU3~1WR(+zpA-|fp!;ICoM>fctraV7`;W?Tr8gVbX zESpm@6{RAkbcJPm$(Zgk7dfD&K%codcz6=hg43YZMV;|o+4C0i5@ZKQ#Ht|`Fd*$( zGo38i7!BfUway<$yI1RVYhR+X2$+$H%eyvt`lzu|2risZ*N`Nd48JFw1Q7u%ns3OG zo@IT!d0D!~_(lyV_BErARjL|#6|^mrqMhAlqP~Y!895+4ed~WtXUPa>Z`VrNt#8;MWE-pdR$;FdP?d3b1-VDM3VaboBAu}A(hk6~xmWw{ z*;#{7DI=*{0;)YWCqr2x1onC_(^+JiP}(?!oR-Go>s4bU+LI*Owp2vyXK_3^sx)kx zkJ6>0n`yvb8pYe$+r;ddKM7yR!n+vNlRYxR4=lz8HZ>Rc_O87AvU8Ahd7aM^jodZz z;?2S1AGVTh8{6D?ND_`rBi5|ENzgEb#{XPd1Z!uP-putS9uE*2l2vWmV9?G4K3r`M z1={;$r&3XsV1(`IHY^FI%xR2RsL4D=q>;5J0djaGQ^zS&zV6m6Q**?bj;#3B0u zxGql=No50Z(;j2X{VYWj#a#)OXUFMY;+4|p}m zwpP}P-23MW8Z5uzuqCMmA0^UPSjeSJu2xDG;3(O%ZZ&apl4#$c~*}ve?dJL^i zl)%WiES%aT+D<(^iAQ(aV%Rlgl4_P2%gYwv@t%=QLp0NTKda(Z?316Y!NZH&4n_Mm z4XpjIiYWJR25j%2G&&NM&GI$5D2{rIX~TK;EcDppy$MSi!%aEfL)B9fTvZ0;(5B3) zO#_oszA*&XNxJdhK>23lT76{JutfruE2zw&E1IicF`1Y)doR*$Hw-o&d|X^m zF^!Xawu}HvUxi0am^BQyg{{^Kbp)A2sYJvRTT+s&YdSQH?l61SW}umK|N3-^k2{6q z&{jx4*dnKIR|?WP%}IKaanUoCE-TB=m8nQ%Ir7QukSU|Su+cQTX{7O$u=_C>Eiu&7 zwWTkonU1Z&9htdVe)<~ld{FF=prZQM62;=&5#r()JQdkCnIBK_#55mh%j*uU=(#2o z)fb=nmCuvbQfdWos#kfswy_v9;EV2y7q2D3x5@Sm(l9>FZ~C6Id{oUt2Y4ULo;ase zYlp3cs|70f#Se7wg9;yg2YBJfgrC6=cspP#zsDaidKfooM}Io8DO#STd4K3Bqc#fXqQP#$r1lp!?RwOH{Ox#c@sx>{4$uvCexN^DkW&YxQ}7v#c-$&<>J}EeK*~Z_~me5cra-E zCTxD2iYdA&cmHo4)2`RxN{dwH_s;iNp!5AwqG7K`n&DiyAps6Ld@|du;Z&Om<|=b1ciQ*}u3qnaExGOz3uSF(h($b0ou$rB8>#imuFCi7DsJ+(HHM<_Pv*l;Z5(Xn$2i6LpGTf`#Py=6;|oK$7{x2E9;31Yp=H6cb!)pG^(EaW zmskGZ&Lp~RYKzyip16-*&8Jfq>W|kJ8kRDiT%9>KxMhcy=oU3L?)9{4H8r|s{ICtM zJ&#e$YuK{3k2c;VWhJK%tjEpM8N(V{H1N$&O*$4GbG#i{>pm9m!@nfC^0P|sJG6_d zy*G=mm;i8JB|L^*Uf0v_N3K{dxTmg9cx|Js_}Fefwe~l6lUW@H9X@O*T}2vFZkY`c04aXGrPGI-kiJ_3q3mNyz&9=Bay zp9ZPyC*z&N|Ys+Z|l*azA$}LW<$FyN)k|*3px%vwEJ#b_d@@t~|d+zTw<- zeebY`V=SVhaXzi;d~Y8Zj_X2Vx;l(B-yQ(Ncqi^VpSHL%E%2a7mm*tImiGPQ6b{uR znkr6^C&tZsw1OySLt1JXJERjNVo^s7Wcr=nKOHGFLb{_kf?RS<=Q0sIs=o#@`6&he zv_YOi(i|BGBddJ#inKD23Ft(%p^QS|9ib>7anYybIS(KlG0!1*GMLJt*`ps(kI3rL zhBz>FQ^>L2axO&2is8FQyFoYtj|xksM#L&P0Xsq|gy9Zn3OeUArB?8NAx|vkez}Lk zP>IuqB{PuaN~;fOfnLrhq(;bxy!<>uYPB(X5HF0sNtKKf-|Nd1jFEVwS0#u#V47GA zB`N|=DLNChUmdc z?i+=?es&J&Q=H{DAel52fYsGFbOPJVtH&ar*FLjT<{Z;vD{b4PC{pm1^+@Tzb~KVQ zK*S3LQ`P=SEa=6E2pK#C62_C@<@KyuCF)6xBPJsN=5>2}>lUdi{{AfEKZ}otfhitX zOyqAwjGJL40V<+c$bbYnZ0K($%B?L|O4dj>`c4L15U1wm$&Ho+3f%@~Z%fUH1t0wK zrbeADwSM~>sbKb;I7?2AngcC8Qc8ymj)d@bM?+kfJT5X4JipV|!iW)5Tn1_m6?aed ziER*hKj8!-Gz6_bNCF6MVgvs^j{!q-NY9A_hB~^K|40-uc0gZ=55({dLCj;v5#z*_?~sS6rtN&yDs3E$w(aK^y@<~UO=1QpPo5N>R*6t-_MeSD zUoD%BXw?Syx>$q!l+fHKBz_S7F(pQx3W(tan_u_)x7!9EU(Z!9;igk?d=3{mcy}9~ z`XPJ?NO#bu*wZ+@xS1T!CDWO7Ewq^&hDvV}>XtlWI($P1xEv2@2Y};cMd4FfcLV)3C9#;WN-Pu;6oi&#e6)3?knM zK8*NGEZ>q8@S*Ylg5>}BB?8~-aNe`hQyV}WM)_ouWsGb=wF zBp_Ca0+WVzwaxyFC-*lm-mXhqKx;S7^>WMS{^^IEwyy40&9L*Avo~j!Yr!gCX0$Ed z_KU9jqUmU@%{ez)su_w{qfWO`TFBKpd?@R{SYwzIsSTj`Ec6pz2@Wf z<!)G17qod=q|E<;f<9zpSeb;?+`(ydc*PzF% zr_!_kxh}=t%Uw;EE@a<>Co^Yh^OAn!(pIy5z3Mo`_tPXehD8@I#`jk>US0oF+xktb ze#^k`6)W}?0%%R19Q4|CyYx)tr|>i;%l=@)>`1$f+MEJZnFFT;yDd9fdlcGm1`;UM zLoL!f=0o!|Ze-6QLT;+=X8g`tG@JdIHi|FEuQnz9+Zd+Xf)|K{@A_>3hubBK2mOY~ zi>hqrizr@a?aOhZ(lt4NpwE%|L~#cnppT)nys|Qm$doq?nLRsPo{R|KC=;1~WD955 zyDk93wy2q#ar!<3wRfH3f4p}s3W4TpoArY6@GNp{rm!s6gz<1~Y7PBJdJ_VWZgqxL zuJQlcbEgGRX_Hu?v zy{!VL32j&(9gS)x)`Bn|YBJ+czbhvJlA+|Oh24emEp z?j0^)i-)%Ngr*ZbV2)YeypqnB_g)Fj7+)`yD4*hGH1j?52dObk@F`aX8lX?AW7M0N zvN1f5F9$UL*9N6S_-;?lNAQQ52=)-MX%mOyt^INaBWBFMMLFL{9KNG-l(}Er{Z;R# ziF_a*pVZ#KA7n@R!e0!fY(yU+0`DS6CHes22i;V8#Lsg=X@h6qd7Zddrs;zLzPS6S ziRm=luoK%zi^wit*w@o-v=I>B7y#nef<$)Pa-YVLED>p*uYnH92Qr#@w$yQPJJK6` z!5#6-dMS6+4=U!_T%w|!WG9L$L^QCiFHs0QMWVP1=pj8dc@avqsF;+ zB~RlA_=)LA0NkhJa*xUCIoHAKwQZZxBFxrsOZ5$KAuTt#7n%W|B?OtYRkw%m>7gZ|_%Fhk!m{!mX8& zZ=D>F^t!#>gZc?_-5L&XL~98LOy1&r(Y^FMz4!NN+`>m~)p!G#QY^sP%oQY;12emP z$+8Q4f48i?yT%_iz56byFZ24Qs#>2gNu)RWeC0i7oZ`-R!+zcq+f==mm~Rigv&HPK z3ND9W+~R>xRgly+uGtvkwa{*kJDq;mKXP^J)~r{>W>BPXYhTM8KMo&0-Z{)tetyFC z$69*-|JMc_|AlzT%+AWj@&86ohVMlK8zcSy6h9+7A-r%F+jzN6J#w-xyPk`ank|%7 zx-{aIH$2%|(Icr+E$H>I=NWkTAzAWVkpyKSK_Lt6=EX)1by~$C%W`MGCa%hXMxVpB z-I_WY0-ktm-LsFqr}2LXAfRzX>V7A^Y{4r?=c`s)8*P&`>JL4 z-WMP8XuH#Tvi$0n=pB)NZTGQyqu%1$g+Ct~f!`ZD`+BS0Vx05idK)WndRrL5WBwcqGAn?CS`y5kWnqdL*?#GU0ga;Rv)5Njv1D;2u_bY)v8HjyvDbv#LfN9-F`a&$p`C%9k)6P*`v?yu(kEZN%6SOH zG@b{3qE9%2U$$Pf?T#N9UJI-RT)W}!q{;ephM-yz);I5$LI}UagBslv`M0u{PJBBW2EJG__U72-4<>pSP(oG0)uY0|w*=aB|WU(l1*_@no?!qgqY zd`;v?{MCWA^H=Q@zGxb+)j0s>ySJ1nD+MW5)(eZ16fnk;GuWHm0NPJd-ToQp`V_F| zq&EtA_kzzg%U*ao0eQRYWL!A0oFX&-@u$kBy`NK5G1VHOX_X4Q@jf4yE z51iTu`By3`g;uoY5HM zLGX=@+;#%v!nqvyN%a11d8xsIcQ_Iu%iS@YCnYCa+&Mtb|1m?=mpwt`>HxT}N14$OKF@ zXy~1=%DdgqT{l2E*y(bMh&4JH8bZ*#tdqdTajz&IKyei2i3_@5Y)Kb-xAHrrP@h#E zbL@yy;1yd?_$Tt&aVlcLZ~xl+UBGLO>ovV`yVo1wio;WWM1W&2L*1wIdssiYEFet} zw~_T>Ty9V}5xLbb^sQ3>*uz}rnFS6`ItgsyA)!1;sJx}TSt$#<8m_&{Z3pA?O0Hyy zm>72plo}{Gxu;?zU~-Pf@qN&5H0i;i65KKw3P-!T7yFStI41lnk=o7Wp3q^OJXojg zs9bvQ+&8+Gnx=|9|5z^Kyd9PcYeg1Fb=bk6y-E__O+v0(r{M=vd*u1rJJ2*aUL{`t zqso18bTE3j5JCJra=wwwru(B;j#B&M)>G09ILaJr9Z(0zC-OuCtyz#x>ZD8m&!8W& z-8jxlntjzEuA zh{}BC4u2ft{Zb(CfMs)aOW_X&Tkj+$P0VFFx?~$M601UR7=BIzA9-cSH@i8pvZ2YO zT>}bDo*5jrW!_`;^1#2;3*{8p8HziBlet1LmO^$X^Buqv66O88>G9JshOb1M6Rx*d zj~A2~O=p*RL$cN9$Dm59gljI-Md>!12gK_ti$-V6pkEZjdbzyQg0lUJd94neGjF2< zG=sAgjK(=w{}ha5xLF69|LP)2Cs6BaU9y6LQUS>J9B~BTG;z{FnjCpT`a|KaCFu{G zG!b|=4DZpScjR80o4wE1iO$OE{W|k+GR3XL^VBnL{V4#Pq{WVTPj87edOD#dlXUo- zg6~;rFT2nL!ihC;s!u8H5s0j`G7XAo+#TbMxf4yRnYzD;CGBjz&VfJZ|Zc%Ku~Qas9vJhZtldquje$Q=Q=Zh3_&s z+n{5nDvr*%7H7nzK!@}c$JqHeX5@L`@Nb4h{PP|sr7#D#bW7Afu60x-6#xp&N*P5( z8O>^LJiXvQl2haxePf*L$GupP$z1Z+RDY_ZMKF8tVeEOw_Yl$00+qPM$hYdYrMd0e z_MbTZxvK(O3N>S81qJsr6yTJKAkq-5&R8C*O(o`*v(kTU^Y_r6QM{w*!E`ASZm8yR z8BK7-LDBKZl8o&@i&XCd7t*asfRq12(EXtY@w#UG2Ymjo*A}sBw>wuLLTxb3u#X=A z$e^BiK!ZM}besD!&O2#X0yd>0CKxl!Y3&)(l%t7%8^o48M#Rcr54iG!+bSf+Oi_`o za=PV@UBO!`iY4)7($b2V(kG%ee>7^q#XWXF`5Wl@WaHZeZL-cHlDMPjUjLl_4vsRDUZKDYEg2j zxJVr}yG*zQzSTX%-cIC^E}i`p`X!^|5lK}SF&{u;W*45AlNRz_jc^CZwc65@4fwQ@ zc)ZT$^Dp7Asd|mW_3`iW#1?l)Q)G{^xeYn7>Zq@9{T+j+Xj|a+ZDIXCJ$56@#OXp)-CUaEkM`J4SZ^?)xuW>>JRsG@)n_9~QcO?;!d8sR0`N~syW&jb z{#q*D=wPR-5jx(wQN-FL!NfblR21c%01F-~wSSRYG}c#afF{z7QNo zcJyuZv!hEB#_WqsvWFz3dzSme5M8@KbxL3jehe#rUS#PRLMlqvAVri(Gn)=#RP;|7 zBH51auBko+d6nZ1CS`0A^Yl;TmD#n)yM4!YaP`>%1@)@_NRgDpL4dgHSb*o{rje08Ex{11q@pw%q*oC#9Vb)o=8l+W)uS8z-`OKMx^qPo1~NBNq#k2 z#IvJH@rv^ZVA5=-ns)L|OrP*R?cj;_Fiw5-(&I(g-aBfYA-lkS$e)`Uk*9P>=Mre? z1o9WeIbJ*JN6l0UZmpS0>GEC$)Eaeo0!Fv@cT~0qXl1jifM4{hBOK>= z(Y;NiAyqwjFv3k8QG_sYReVw+Lr-IU6zj1vqQeSS#!o_GDbn%Zu>*gC z&MG@prz;2rnsUySV)huy>`Y+`Pe8!N4+sjF2`qv+y1OT{Rm2wWWu(6Sd87U@S^NhH zbvTpLe4dHZkCa;v?r(+%hI$*P6+Af4PWlnA2orDKUFh$q_ue7R4WCG4XH)Lk+07Z0 zIrBf8%t8y*HMXM?GjYS6V9_l3$*#+dm=tlftDlBdefTwXsBHfFwi&fX?ywXvCm}^1 zwhQ>J$6VMQM|0FaS9M)t=$RXn>KTn$BD@BI_fbSFL2xJGvTbbQ1hBa&l8)6h-n)~~ z-7+ozR%N)7#Wx1d3z;h3pS86+PYGxvEpZ?T5IniTDZ0Co9dt+%Xe!I=k|ZV7{Ux@b zpZME?O8jEH7gu4@KFS%I#D}G%xKSCY<|1-NMNt>|wsYzZnG_*Sk^{@hqEq3uF<;Uk z*-N~YOH})4`7$R%RO#F;=0w*GTpvAURjJe_^2_~m zUl0Yb^n$Vlox|x%p~iNlorX=(b#3vj)b?lc$j6%ji3{ro$Ixe)kS73mV~cF?M>SFz zsugi<5S!E*E{pGF$*3Ct9YD=f4(3(|AAi?*ATley3Z)ZtyWH&G22JD~IlBN*9 z{mv4utsQ!9h(v%bF+26EavI#oz_rT|OIO8x>s|c@Z;ANTm%CtKKvX<89U?L>NehF> zOfr3LV^#2KDU;z#I9>}{v&o$*ZuWc(?3ne(-Nw)+sPvSh*19)KD^d+<3>3dPg>|7t zt}iRdB>Xs^_#`%eRIgnfr{}PIa9|8?bC*D!cLl5J)=SCBT|C&}-6EX&F|!m|3Tx|( zG^^kEN{NXfoF`g_pZHpY5P_yT@Z7_iC46C~jFf<`te%`my?2wOwKahnnyL6aeKuYL zpa)x(+_P$5^e3O6Wz=}54>8HkA8$!IOD#-pq_VuoIsK}nWC{p=x~z^2iuS|wVd6%s zch4&9uI%={E>)R6OuAGUV3Gf3*_1}l1)_s!+)X)nilKYUEsy2dEdCcn!0 zIQC@zEFUz4zOs08AR+fa$FC`SxT`UujZZ`8zFzFpt=&Xy(0rK^IykE>oA7%2=$%?W z`CbzXfvFRMzd@GkudV``G(-qa4yRh+WOh*wXHZdkY}PpmARbY(;o2`*2`-6*R`1n- zz#}^u71AZf8X(~`*JCQjqK&yYOGdAs@QcXIdD?d(s#Sc6A98Vf`!F4!W2Bu0x@#z0MXe~2t%n+hlBNbjRuXvqTq?tut=X`dp?B9MO?(X5XO=v- zW`Lk=hv`T{he*RXovQ8oX!n;--`+TZImNLN7{5#Zr4{+WlH{m^bypBF3m=O=7wsf8 zz!?}R_~7Sf0h+gHk(5KL={|qpnIx!7GON5knou7Lh9)E>GtSv|N3KkV`6C^M@MqL2 zwevVPPkw{#K201ykEK(62dhsofG5(ibO=)og!vH*$+%WuZvR@}3|oeI^tjvZg~aXS zsIHq{SWmweO=~~NzCCPWv#v53bu}mjrt0&tGEhATn+>wKG_sB+@iMf&e9#INmdLxb zt&KsK90A8z*{iLOyGr{c03C{ouL@_Mj9<&W#Nnf8(1K)zL!G7PgltLB>NgRE5pQIR zuWB|~(q=irJm?rZKAzvKSb4jK%2N^u++rXl{$EMNgx06YS467Hyo{M5Zd(huLi5{! z!x!Z@+&YE?OXrX`_H;4c@726J(=B`Z8@P*Gwyc`tco5jyvAoXE53!>Te1vxt&Vl@) zM|r`*BHN`!rVSpNIfKJ2VR~%mFf76Sm%jMj(u)m-!>D0r6DE7TU-X~To-nUdj(9); z=m(vbJk-qRqes>H@hXm!Xd|DxLVSYZv9B!z86|B$1F6s0l$|TjROLDF=#$=m99r#X zWCfW*&YXr52}mgg=grvb&jx;QBtB2#@7^Hq*h&-%JF*HQFKV5`W@1au$aq>Pp3K0x zVhls>KEWRO+uEhL!=lKNHDTd2%HaP<<4-)EdTNw+$?62`!5+bN@FHrpd=OUZL>RKa zscG~4PS}g-`y`~ZzpEZ{mVV3}S;syP5hAphrRTct)@kchY?wJfUBR(+&LcY4@U?$E ze8%O26H%0A7RCCty~fn(RZVJhcw<-Z`aHZZTdf(1P2JvP9k7!X1X8{^M2e;-s}G9A zzc&JW0NR)_%Utj~pK*_(t1SO6sZ64DgM|PjNp++?Xn|V?(WqxhSXuW&qYwjRu^X1# zHGV)9!wc*4=9esdr+NRg9EWdt zUAh77D;*@-W>o=V12+3LLPqaV1ei?j?19mhJ41~aaxJM2M~27KCO__wukXwlxF`nG zF1XU!jniGilowjZwik5GXz4*zq+#xF<{F73dc!jJ70GYfU$%PenR8g2Pq3y;_p|L* zAd~q@y9=)cyH9~9wIfJ}u(Vrm5x2)nQM(#3uZ`7cHyOQP`@SZIM!IVv^gneaS`a2? zi{X;BSwra4g*SIhxS~UTMr3+K4z2T=O%^A%YRxCPE z9?Ej~vOKEhZ+zNGp3)p&i%YOp$Hs6O3_bSbuy zDTv-ox2$&6f=1wLS)Cx0^VA+-V~BIyNaV)pKNNd}md+-}oqJdkz5VC|H{yCtCA{2h#$(murCgu^0&V_u8 z$9U>LHD=4+c!z#T%4a)otX0L={$M14<#n@!)CEz^Mj0;-|F_Xtx-E6Fq((Zv z5oNLYu~tj^F9x*HAB`pvW3nSdI(;l7nhAmRxXlrr1AGUJY8_{R0}Fm-YH?+ua;=I; z((-!7GpJ%Z*AXu!E_7m+_d^S9qK86(C`rPs$KIrSecls@GoZScE!7Vic1zy2N>UBo zJuGaifJ~zYUc^(QO;QHUo-3IJBt3@Uj=W||^G(YuxO9d+Cx~a<6cmPUNE>->y6!Py z{X%;77ZlmUbEf@}_qVCbE82(0tj*2O@#mS(S@6t!VdL@OU()NWEp-_-t4D(z$=3m= z+PP6nMVI$Zg!=UJ5f+}He3ZUHe2zYVdq$UfsZ6-bD4~r7c+44~1K^}pnf_eNt}tiAMh&B@lp@TtW@a6Yb8_$ugA3@zCWr6t5&u9GSMRwy(v zIZdmQ!5)k~*Ow0UrQrO@KG43waj`rm0!lKpPQ*Ho&#)XaO}DI)?(qs|afP8N78<%` z)xW_KmBlkP6cdig#Fb_IE=i3$>mc<|1)Bk?*#pbJt{lzMlWT4*d7`bGPJT%xUYryJ zP5G};55~t5ab;0d34yXv9gD~EoE<-PFT~fJQs4&D;#{H$`+0cr54B& z4gfJ&f<5#ZRLvb~EKzYxNdH6X52$q7uPB99PB8T@FB-4$g&L4bHhqBXjx}r)3If;g z!WukdDFLzchT`1$RaC+{*vMYSVgI)%gyXlYp5VP*Z_G1_NC56Rs(KV*HJ|7G_Tr1L5f82HB732_5&q2nm zQkqo|SyZtob-)g!#I1grPH!?xPY9VCg3ZOBakyKUP*p{G?l$39W+fY8{jJP~)L=JY znkAhuRrWEIFcQ-rmOpdgE$b~SMnS7!YdjUhrP|6?X-W`g>y9^;vMrF$Fbfw*AIaA< z)g~Jbm4+GoIF2-Pg0n?ut?o3Q$K=%2GZuX2S0tl|?tw55;Cn6jeF*UvZHF;0=Qoo} z4DX83M~xTL;I{f4jcPVc;2*4(5OjH!AIVtg0j`fdz}D-V?VJ*|g`Kh}2^_2P^LS3D z78)Fgct>czf>Zi7?nPhcZ3qRed99gyhN?b(zj|JiPuANbmC}Sf38V|ej&WY}Ws81A z2jUCT3Jx@AC_D5nw-uj70Vqa0taPjtxEZ9I)*%&8<>X$%%nu6~LFIv!+k}%s$k3_a zk@;RqHw5Do-wm#Rjjr2?bl(YjI$piY?&CAwWm6uK0!;p*M?|BRJlI54sB@YxuF~h@ z6{BhF+r7AKv=K(qp{Rkh5P0>`)mu){x0Be~R{il$PqbP#>BDNQFb&ji#xcgIPQL7L z8FywIeuVh~f@NqW6|Cnx1ktUTJ+!Ir>8jY=t1v(1ldEL2VBv)o0vA$+f9VGm8g~^( zO519nW=v;;*aRsM=qSYGH7P|TDuTm!J_<`($vFAqUtrIFD&ubx@-r=W>P5ek;OB!v zHfg2@(WS6j)DIFbMuhH*In#SkJW-JIJYe*2Gv1Qf>0gImcFZ(_`#^gaa)c+z238y8 zH`v$JIylMGcNvUWLmrrcuI}Kc)oG-fG$W6=Uo1$`)BB=vR5y!5-TMNx%b@h{Cs_Vc+JF*Py5Js<#+!QAGzIXv>{ZgZCEXy?C-C#c6 zjYlA4^nW7xw^{!1kH$!~_);m;uu*gZxa>^ z6$@~9#R4#xJ*QpDyg*UaoeTWBu<-t6zZHz7fuCwAKq-08Ucn6}BGYMBe+m3{#S?J9QaxPp%uls9IaJ%<1>5%kHcS`{wXIUm4S){9pgc5Gq}6;Iux0y|b=#zvD51A6d)(|$Wrv9QS79wyktT(24xt)!)RC&eI5 za5C0j&X?u82iYdTqye^aqaJ&hyTnW-iy#!adVg4L)tQfxS z6*&KsaiJP599H>MoVR`RPsebF^QMP2`F^dqIZ5hG_Zt^)e$n#EJ%};=hYn|&pQ2(Z zi|K?S5s998o~b6j;%0dy1ng*1r&BY|RDp-GTWI+^>h>`{ju_k+j27%rcY(j6NDddb zzq;=}_mJ21D!-t@1axvK18Wm2MwP5XrMz1(XKVH*!_j{rj);oCzsK!0M6wlTxFFlA|n?SB6kgNA#CPf*+{(~bYK3CTg^Z>bOC&Eykk ze}15$TUncX=(J7wEpexSTEIR%9(T|!WPnlZyotq-UuwLt>14OkLcm`oAzm@5Ie^6y zx#$lH|A$IBITE82jD!zluadTYkd*X@KOTqJ5zv2EL4P?;>R6ZV?$Oq?S)V(|rY6POK1Ea6zi}4W z#YZgPkLg^HQ3S5eag7Fc^<5!q>M~@`sZpuAcQe>SyZiubl_2h@0kY@kdQXM1I$1t` z=EAm>#fbD}5x|vUh(mm#Y99d5))H-XU*9ww0>kIpLVmGTq}B7(EFHB?g^ zJ01Rw8`{SY%oEs3NRS5AHgumf4x-51Ov45HPS;up>E{7VEt>;(Q5>z?XI-b;if!uY z@($7`y&6HEb@~9!QN@ih_9pV|F{LdO$CPIdKi#elVBjn33B(m)tRrZ(hu1>=@>)hsW6vI<7*93l;Cfw2sx1$Ivn)1f zgXi&S0frO}&Ge?l?@Xm$EJIL}`RxqQjx(4Xcb$@RMBhCeN)?M*;dm^6QU zIGnpFT#uanJf&9V`MZo#YyK%zrFW7Stu_}dzgtqkz}r1RkALzgf_-A)r}bef z>lV$){IL^lKt)9PsHGHBaLHE27hcP~BF04evxdofCXiTdy0(K&Pl3?f9O|c0lIq&r zMCLS`FlMwX3X`{^`c?6fOEA~s!VqN4Q>2~Mh%qasN^*PeJlp2ON||B|?;H7xxHioo zqsZZ4ZPk*&g0YG@i8}MdCB(hvIVxSN)GF9{yR`w!lh99m|N1VQU^J!)LQCCkGRR9;RD;e1eruf66 zSZovOtv~}vgSl_H7w-4D>Bo#sf%GHKM7kwd>NGP6;~TOb(__skB&i~@Z%Swhio za7k?he`xyHD}@Zm-N+myF&{-Cs{R`8FEMJ3jAG*WZPfar3GgzS0z~J&;JrJwhDB;t zZC3_jfmz_mx%4;|+?LEp|(An80ns%VRT?DN-_-F`wvf1bZSMVy)a9J=#@8hi%aGi_-LywAJcO%4Ge9 z7TM~m*`9J-%1Gowv73&ZpC@Sqrq)OA*V zV2ULPxqqax1t5jF>cW+eDs|3Pvz) z^JOb&>41VY6z!24;2-7XXjvP>rcN_07+&y7Y|--K7E#~m4&mBp&Tog|e1Y0V`p0(( ztFA-M@)OggKn%0<1TmT|Blve___DtC?}oeSQ03q7aMLdl9@O9Kz~k$#?O8!Z5tNE~ z-f_`Jzrmp#?W2-H0wws#hZ1 zsAkgcZd&u(ll|Qv2YQVnrw;Gu#5IzMn4{ph#)d`@K!E*OM5$xu%8Lo@)^dbijtx9A zH^I({>(W_uunxH3u(ZolzN#v~lyXJ&N=M7wzN^&5u#V7nP}71RGXb{AvZ4Xyz3kk_ z2embt%1UGS_I!CTaDz&6>gBZq2_xmpylzrznWh?9^|SshElj* z?3C7>e%p((Pk~nc{J-^$;cL!dyCzJe6q;E3CiBCg~kg%KB7N}q)j_a0}%h?0e zEI(4|Q~+&FG^AWr>L8LROSxnV_R9G{?a382sg(bqx7=5hjT()STYy zH~FLzaF#fa6#^jGWK5Vk_`-*ieo0LcPT+)!t9VJyS-ka4rDx!6VV5heGt+q`=ZIuR z4%*77Gny)4CaKjI;|cROs8Dq5sE?>0(up1Gu^)(eIIY(>wo3Haa*0SiFQzLiDr(NXkCGqvm{Ic8mu(0;da(l9IZv0$I)vT)d@;)z zA3_wD+SKF1qg(G7Xu2NlX+684+6}o&M8h4^cU0$a8$!C?Z{5gHna9GFqY0J?VhCgjrlo!6#oHxR($1tVI7k9x6YQlXt`=1` z#Xc*`9KxKc6$*=aOWFQsVGxhph___{G5O~+ojJ* z+S*Tw9(+UtMo(jc$lJ${9tO|oYeR`2E7vdg3t5;`po2J0apdT&ul#(uMdpZp2hUk^ zXKB#mtGkuJ^)co2o>rF3$cM*5Rgx(5i!USW4Jid9cDq@!pC7TH%-=mo5Rrf)AF07! zRl{I3f7RuU8|KL`Swk$>GW-(J`}Dhaonw|l5S+;UKw~F^zmcLOZ>_w&$rjPy_4x~C z|BQkD7hA(hAdzsO&Q48l%`!pUUm)W+f;(19c>Hc?up&UvL>^rxSbl*coKfFPCIzp+ zj`zBB{7$N|WM+Vo{~XdtVQOIqcI@*0D5PW0#PDw|mp;0bQQ?m>ZiB)^q`~w`^N%U| z=`-s#KvNdI=UbTYiqmky=6xuDbD zAvGMC6KY{BvL@Qxe}B7(5!~^beZL}pvAWTm17{wf){S68r{R#=Tf2Aev;}L6f$jWT2km^{e8wRy{+TB3MyNV$CD`(v6v8QAiV$_l6p1-U*kblxbr*Qfn>I z+kaTsOi8pUw0USSCb9`?S@B?2o_FTTZ@uuukhJi*?>v>x#kPdA0fJ|BSNza^vS;)N zcs5*bj@s*DcIDh9RB6)>rd}$eX*lmbJ{hVxf(KFgcOuW zG$TlvORmS>LO(--<%wLi+M2$!*FY`<%A9T{M^jn&fP;cJN}n+kS{r@h}S!_LH9 zl>4hG>|%clTo7=~nNPDE7blxL82)b= zXl~^nK24v=rYFQw5SSo}wBI<_9hSN7<2lG<>=O$i9#`j+=torjTRG$k?Iiu;T`ioi_)qRvsP+^`;(-rjV_DZ6~B}3?yAC$MUfMULtl5VJxjGQrbWkWk9DYfP4ug8MF$mK11 zj#PPW*k8|bIt(j_&6dER*8tInH+FcWRxSxAKBAsELZzgT1_c<{t-J-2%3Byi+B^oA zXbiZ=kAM`k`**|H7|k%+mvYUJHvG=7*=VveEhw)JEzhAj8%vX%t=KihsAJ5XT^fJP zCunQ@KM%zjhUDgX?KQGTbl<7hcrO~rxHKt}cljT?ez!{--+4$B1)Z+O7TOxlUNUFS zmf_(^JhB0M1v4N?$nDp_+5@x6mspIaI&tVL&pLg1qMVp4QPkH;BL;FBUrv^#U??!P zccco+C+A_N#uGD>Bh%o>Pp~n8m2WMon}~joauC%01kK6d*)p4Y3!4Yu1hr>6J(M<_ z6A#@@9a~2)idx~~?7*-AjS<7{t%vC6Q~Nn*gaRT#Xm2(u8@J4(x3`4)*~RfdB+r?x z#bM)0J&Ji!Alc^UnYz>0a`$yH=%HLV>%@6V815UR*?NxGn+EE&R`UP^E)C(V3+GAU z-9o(~U>i-n|BWYxHA^2l#wCf^>Y^U!4cY~P&Jmiq?37;wIy3X^k9&ta!kmVhv=oU2 zYJ-%hkFB}CZBMtx<&;F+eCFcV;Fcclw%d&Ay0|8Wuvak+;zRn7jpg|)n9wVUqZ|-Z z=$`aKU{lszlRzhbX&PmEs>L^Bq7lrJA-{y%%X<;QXQy2hi2Xt({>nQst7qrw)c((# z=fZ5uT?L{brz7N-w*@e?!2nTr*O6d}g3eWObri~AbNAZoDawLk z1xa*3u`+6v8NUya^i)6Rtoc**C`(=|67Q)o2~j4jV*Wfj74Hu zl3}I)N(B?5*Fo!Pqx;v&hy7X!Qq;M9Hx;vCUy-74`G%k)DO-f4AH!T^o!Ex`03*se z?ObFYvKYApIgTI!qYF%WU)513x3bSbGU@CrLCcW-GP3cJn9raJ37y5>>j zMbRwFV!{JnR~;)FG>LP-9lLk$zB1c(=9%3oa1}hOp@(EnYz)^?3?cqL1UUsS^VwCB z)?g9)2VXCC2!~!)Mu18AsLWBFWlau6tPsIBASL7mY0)kGAczW&N-6)TbDRfb0ugsXMtpbccuP7v$5X8;;NQg`oJYuetZm^xmO#cMXLdQD^;Uz zDPL)mosvuN3r1Zmz0#5X?RD2Z5{xh$rI}}w9g zA^4`oB|wZT>GaXe9VWYW&3SMkJ4Qo{#5VD}eWXiyRstnO)^5uap9W;JOPo=X{zkt~_rYg}*ZN1Cou^F?M|8{V)8jVvb z;YF^9a7oP=tFyYdM)C}AEqMlmbrdTWB$Upf7Pn>*R;#1tW8DK>Cdsbz+uds$i;Iqe z;VTwhiq8i(i$n96&J5`&Pu=|`&_dA`%-X9*iMXFAr47_a&X~4F)4`jidH68Tiyd`E zLbz*Trjg*D$ApL;P@57a2;j~JuZAP+t4{@SjxZHMDk8;OqlpB?zy#o|k8=rG!bS6~ zyO&}Hn*rawXVVo8blbMqPJY-B)$ddOvUl&^dA_~M9}gWmG?Y99hE|?a6nnSLgUcMr zkUYURYzgxgryvQbmCEi+E4g*QiQuI`Q;!|zD5kHehvJSjcT?Qv1v*J@<9d%mMth_@ zGqwSH5&d7J8Cij>g&g(f^6F-D)ee!`95hW2=T$dX|Hu7k0pm0nn4X%gF6q>~EHS;? zefH`>UnxzEIep!ntnd8|J@4;{vvy$e^z19Jg8TfjWIZW9bDq)7n&MbSPvB!K*t*lL zqh+YD9sFVa4bA|M)51-@O?Q%4F7FL!4G>>^94|3`BU}GYO?Okx_?qQbBMGo1y6C)I zBPISHCWehTiXD!syI!yC3pwD!a=c5Vn9fw57SVIya-|#VI3A^?4_^*ZEY4q6Xs4Ox zV#?_CcEx|TZVa=Tb?e&ZTzc&)s_!T2QgwA`Skz>v7CM*Ii0zAe4gxwx)-=;PPAiWb zUsLK>-q{{E`DAT!NIABFtYAa|!c1nA#VCBTbQS5c}pOEu)AS=rq z#F;xOsHv?KWO*k=*&5Ag;45rdM5*fRBlRDK%m`|7Plf()l;SaO%)i{*CyF-5<#*K7 zbZ`Of)k{7^uUn$m;cl~k9k}-REzG%hEO*Y0A;LK)Ya4?nDG2JVR^*emEb5<;4g?pd>+@|R<9W~;qAoB~Lk zfKgjL6|J}#d;t*XVxBhMhOLr$=iS97R#JWkq+%B(e({5d#ph3eN4BmQp#|Gs<(tZ% zz^|UemI;jIPTu3(yh%7w5B9xT^u|aCAbtj-;kqEnGsfMWJsf{?_prKMmQ=3kf3cZh zG3uFnNS(X^XagAw#!^O^7KPx^QU3C0)!vS4Vmwwg>TCiqe>ZJM4`$B`%73W4gKZ>n=fXsEi2Uc{(;GisX7Wv>V0&!sc8 z=CbZokrMB|kv*$XM=@+fFPF+%%Dhz$hj?yeSZ5GK|DM_+$-M=ggqK&l+ZY-`zD7y; zKQ^fYG(G^%Wg~^<(uK;L_8yyy*L$r$8x4lQJl;yD47K>JOi{uE-g}%kSG@ zTM^c~Mc=ae%Iuv7`rS(y4bfV(tTFGLvG3L7`HJpB&D@Oa$xRPUVKXrDS47`w2}b`F z4W#-`QK!-MQ0%- zE6&$RIMxUDxH}olhGCkdH0o*JloIuTaYq(E(rn$JyXC)#$F4+E;1EVL?ySpRuw4Vw zP|qlQMhcv-bw$#XD`p>e z`eKHhUe9XHmqP=cfcW8ZHcnoDWQ%m<_YsWk! zI`!fC(k(ACl{>+qMu3GE&}@)3YNBHroX+-v`Ez}OK?cenjl+-MNgchZSNqF|)$6VK zj077cKY^mn{WL>pfBgPXGARW8l9rd;$9}o-CvTtXlLv46{;O<~Tz`%`LTU^nNLh>Z zO{~Cv2J`9;hfwp>{FiKs@_lO-#{8eIcHU~SU&KzNv_(^ z{+b*&6=3-(Jrljy5mh&VHaLkpoH^u#%5-fts&2mng&wX z7p`ld#A7G8i=r~iq;(cwT+MyOR<}bwJGw}6Wtcf??hzt`PMZ$kDKrcHVfkPh=4^g( zeGAL|a%9+#3pl;HrJLrjY42mX2VkAEF5AEbei#?+B4@L*Bgzi>8k@bkpqn^To45J|@j4j6a5_9}6xscn`gVr7 z)=+FtTJ>E;xX#-q)?oWE3UQbBEFyp?ua0qQBkPRGGe|73SvN2Hx~T%pqU8Mt%>rg| z*6!S!8Ro+d|Izc%uFf6g^%oj>t7e_DP>kp>fX@Y&)b&%88^as4%0h4#JTA7OMuzQ2 zEnANdH#0n|_Z(bGZRrqN_7zO-h_c9U6lHD79(z5Mi9x(e61us=xe>ilSp7%UXKfRQ zupgpt!|xI(Pu$I9KM)@R6XmVaqw>9`f~rzwByzKGWVy*>n7j+fl%|SG1_%K^jPY;+xYY)_kQ-z`m-N_nWLtOuT%qMN54Sas18e@Eh^G4o6Qn(kvBxZfz(ax;$K)Xd-P(h_!;?^2K0S z9zYr8BjsP4G<$&SgGb&OQjcbP`&PFtRb*dyQm{{9wX{tZVh1TR>YZeLqU5F#xY!XAeH zo+#@o7eG8UD}hL;x}m>ZT-3xnYhd6@(pj)Fy6me8ovNTh&DuRQX}+)Db2M%Jk8v<6 znpk3&zd*5ik>Wh{BEZJbbivBX=-I;J>zI`*+1|?5c;vXJUJxV{%L&9?jRcP|*vEH0 z+4uZ0>UUdTuZWv1*kX>>1A7LhCFL8rm(I#j7%d&&o9D462?glZ6-6Jxi^1AhEy>c- z$riW6B|04P3UTvEzcZ4&}ezdFM~t zE=dVnAK((?!bA|lb*Z7$MGFHfNKqeVil?oekJc!ANx-0xJ zwEM-#Wys~obu;?$U*1Ab&SHL3d+5J%J4KS3-b+7I0;_H#&rfS%)wb32R#R=cd1a(b zl}(Ce5t^vpc?&I^=Zm96R(Wk_gyu1w-8XFRmn<$i)_sxP(Za)n&K`FRgp|t-dW>x+6WPEXeMWWrvqvHNfwB1`rPxe{^nh$Nd+pKLaJZRV)=$yz7EE?I-A85eDMj z8l(r=3nQn3cdIwmnbe!6HY%!(?x}9MnrX4h8mb%nXARht{v#>%d8W6oRL?u7H0r9w z36C56?dAvQ2hQqBM~z;{$&$5s-YB`6I|^6ssMTPM;jyun=83LvUuDNxt~TUP8hr6! zHr&K;&I{jNq28S*z6&1{c>dy4juig;*?co8DDPlWLPMhT3CQz;@o%7ibq;3W#asQ# zy9evE13P;+05(Pzn*uv?}`52aiYe5^8*Dlqt zQL?dJ-E+-!G_=zE;r2d?KW&XH=mgWkbGRs>-Y+{iaXR)q#9QY~#l}@II&MO~g}egI zKeNp;wH7dHw;}M%((b1NXc}Zs?z1n?_@wLd<;Z}CuX;@Od`;>d4S;FJ zZoje?GmFP$C)+eR?0LUS#!2C-rrecRV9)LidfSdt=P%E30U5tmJzNk|w`qfy>!AVmWWS`s;uQ${6117u5{rXeh@<4=m-?^?4Q z8#Sc-o6j_um3_y+vJ<6atw&LV@>jqqe^p)%6pPB@1VcNci?&g_FRNt?rlcdxFgg_Q zQ5n$L)2NFFNxhw{iMV50A=>5bZf)Tfoy`fn&Yr66^!74)k4uulZnZTDd3P%tBAdEN zs@0t&NQ??c8OFw$EhvL)xoitTTPWTcCK!Uvn>03!m9gp~+9WFgQWrO;QS`pn)(vyZ zKObkaUA48Fa*+u5W{f6L#@&(PMi}@juBovZah4Far4td2WYAOe*pjW0CVmS-BH@X> z*xTseAO~$KGpOar>KU`UU%$Ozk}ORQq*P80Owv~@I|JIyw`Pcd+#Gn)kqp?)O)og zcgr^J4L96lY{2x6A=J=oXn{CDAcPW1Ae01BNCM=Ayu4Hbq(C0J`tP05j3nFS{rbK^ z2Fb>{bI#st@4fa~8=JduW`BD>K}*%se(AMQdMzGJ6BNRXcMjsrqnt18uqF~;zWz!} zE>xQ&aGd|Nr5~fEg!E#{4j z(JSqfByEne;Eiw}Mw(&5psUGiKuZzZn%4_TN?bUyFlbc6Y^@!v{lz~t6p30P0ZLqT zh*=ECJq;+1QiHl|7fz$X&!4y_8MkuYIHyu6>-T^7Os8jf*rye`ha6vyE+~;4rluFej`0C+~C@r1->G&p25Q}*6J-K5j8PG@M1Sry2s!b!vt*7DbgA75+KD>1Hp>-L4P(V*JU-u&N7 zI+NcRSYU!FR4t03I6+8Ret+u}Qiw=IJHy^Dr!Bv7%z9Vvpj0;+lE< z4E-D>vCL_N#MBiom_Ivq90;T>DF88HuZhPXr4@EZi-0yxLRXI$KOltj#WtT?r3JSWGHE zNNzivqR}g*+7_bhAvx8h&|#L)UMYWoW(2+XY5bfW-T{WU?KrEMg%SEl6v_^`(@uoM zgxV#%UTfD}EA;N@4meEi_gSJ)9Xa{0|Q z4DP@$yj)SzSn^Ky!9Rl;>9lv0b_~Z9ixkT$6P7utt6Z$1`13i9sr86IutZ0O2|YVocZd)3gq2(1>ev0eQ&~ z>Ryk&|NVju(8aFB4HvGzBnQrNf35J)Lw~tT`uNL34?%q^Rx*!&-;dLT-<;~8Swv6| z7DkmSB2ETtagyNQi#EW#hZ~A01*NREbi7q>wki9s$sdN;Md_Z=-}KZO629N<*s(+W z?C{~krJmy${8Q=d%3I?diY+n*tM(j~^T&c;F^LKTZ>6U@S0QxnTzQ5*gw3ddK(pgS zwLD=dl`lZm9?@F}DgxJQQ6#aA+gKOT53}bP`>TvQrkZ5f8+Gxh0TM8!gZ3H7W z2CQChXK;W$nCM`wb?QugUOmd#c}sH1Tfpq23HGVX@;?x7P2{f~UnZ5!8s~Tn+@`X| zoknL={NB0|MC&lFZEOGH$VK#b|16!Twq@NBgu(dh^Q|N`t&ilQLQCBAK=&{uZ{3Uk zt|?MCUFN`4#IW(cuGRPYP(f98*=|{XLV;l1qnWdJX(K zj7o^iuLw&>H?L?@3`tSqu;LQMb&A_c-0~US>g=qvd*#rbh82gFbkODsJI^X3mL5K; zog?QVkM2*?5HA1uaOqETVHH{!I{R1A&o-3LqHL~3!v*5_@YHR9@YPY22}|+fKj