From 30e780554a5dad9d67764b40c829f5f46c0926c4 Mon Sep 17 00:00:00 2001 From: Gaurav Sharma Date: Thu, 10 Sep 2026 23:42:38 +0530 Subject: [PATCH 01/18] CHORE: Add paired profiler benchmarks and PR regression reports Replace historical CI benchmark comparisons with twenty shared workloads covering every profiler scenario and the existing large-query and insert benchmarks. Reuse profiling-enabled candidate builds for correctness tests with recording off, then compare base and candidate in fresh benchmark processes on the same agent and database. Keep default-build coverage and release build configuration separate. Publish bounded per-leg measurements and an advisory, commit-bound PR comment with slowdown signals, phase deltas and call-count changes. Run the comment publisher only from trusted base code and mark missing or invalid measurements incomplete. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/scripts/post_profiler_comment.py | 228 +++++++++++++++ .github/workflows/pr-profiler-report.yml | 34 +++ benchmarks/README.md | 63 ++++- benchmarks/profiler_ci.py | 259 +++++++++++++++++ benchmarks/profiler_report.py | 278 ++++++++++++++++++ benchmarks/profiler_workloads.py | 148 ++++++++++ eng/pipelines/pr-validation-pipeline.yml | 196 ++++++------- profiler/README.md | 5 +- tests/test_036_profiler_ci.py | 342 +++++++++++++++++++++++ 9 files changed, 1434 insertions(+), 119 deletions(-) create mode 100644 .github/scripts/post_profiler_comment.py create mode 100644 .github/workflows/pr-profiler-report.yml create mode 100644 benchmarks/profiler_ci.py create mode 100644 benchmarks/profiler_report.py create mode 100644 benchmarks/profiler_workloads.py create mode 100644 tests/test_036_profiler_ci.py diff --git a/.github/scripts/post_profiler_comment.py b/.github/scripts/post_profiler_comment.py new file mode 100644 index 000000000..cbc802bbc --- /dev/null +++ b/.github/scripts/post_profiler_comment.py @@ -0,0 +1,228 @@ +"""Read public ADO artifacts as data and update a SHA-bound PR performance comment.""" + +import argparse +import io +import json +import os +from pathlib import Path, PurePosixPath +import re +import stat +import sys +import time +from urllib.error import URLError +from urllib.parse import urlencode, urlparse +from urllib.request import HTTPRedirectHandler, Request, build_opener +import zipfile + +sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "benchmarks")) +from profiler_report import LEGS, MARKER, MAX_BYTES, render, validate + +ADO = "https://dev.azure.com/sqlclientdrivers/public/_apis/build" +REPOSITORY = "microsoft/mssql-python" + + +def allowed_url(url): + parsed = urlparse(url) + host = parsed.hostname or "" + if ( + parsed.scheme != "https" + or parsed.username + or parsed.password + or parsed.port not in (None, 443) + ): + return False + return host in ("api.github.com", "dev.azure.com", "sqlclientdrivers.visualstudio.com") or ( + host.endswith(".vsblob.vsassets.io") + or host.endswith(".blob.core.windows.net") + or host.endswith(".artifacts.visualstudio.com") + ) + + +class SafeRedirect(HTTPRedirectHandler): + def redirect_request(self, req, fp, code, msg, headers, newurl): + if not allowed_url(newurl): + raise ValueError("Artifact redirect outside permitted hosts") + redirected = super().redirect_request(req, fp, code, msg, headers, newurl) + if urlparse(req.full_url).hostname != urlparse(newurl).hostname: + redirected.remove_header("Authorization") + return redirected + + +def fetch(url, token=None, method=None, data=None, limit=4 * 1024 * 1024): + if not allowed_url(url): + raise ValueError("URL outside permitted hosts") + headers = {"Accept": "application/json", "User-Agent": "mssql-python-profiler-ci"} + if token: + if urlparse(url).hostname != "api.github.com": + raise ValueError("GitHub credentials must not be sent to artifact hosts") + headers["Authorization"] = "Bearer " + token + payload = None if data is None else json.dumps(data).encode() + if payload is not None: + headers["Content-Type"] = "application/json" + request = Request(url, headers=headers, method=method, data=payload) + with build_opener(SafeRedirect()).open(request, timeout=30) as response: + body = response.read(limit + 1) + if len(body) > limit: + raise ValueError("Response exceeds size limit") + return body + + +def api(url, **kwargs): + return json.loads(fetch(url, **kwargs).decode("utf-8-sig")) + + +def github(path, **kwargs): + return api( + f"https://api.github.com/repos/{REPOSITORY}/{path}", token=os.environ["GH_TOKEN"], **kwargs + ) + + +def artifact_report(raw): + """Read exactly one bounded JSON member; never extract or execute artifact files.""" + with zipfile.ZipFile(io.BytesIO(raw)) as archive: + members = archive.infolist() + if len(members) > 200 or sum(member.file_size for member in members) > 64 * 1024 * 1024: + raise ValueError("Oversized artifact") + reports = [ + member for member in members if PurePosixPath(member.filename).name == "report.json" + ] + if len(reports) != 1: + raise ValueError("Expected exactly one report.json") + member = reports[0] + path = PurePosixPath(member.filename) + if ( + path.is_absolute() + or ".." in path.parts + or "\\" in member.filename + or stat.S_ISLNK(member.external_attr >> 16) + or member.file_size > MAX_BYTES + ): + raise ValueError("Invalid report member") + if member.flag_bits & 1: + raise ValueError("Encrypted performance artifacts are unsupported") + return json.loads(archive.read(member).decode("utf-8")) + + +def publish(pr_number, head, body): + pr = github(f"pulls/{pr_number}") + if pr["state"] != "open" or pr["head"]["sha"] != head: + print("Not publishing stale performance results") + return + page = 1 + comment = None + while True: + comments = github(f"issues/{pr_number}/comments?per_page=100&page={page}") + comment = next( + ( + c + for c in comments + if c["user"]["login"] == "github-actions[bot]" and c["body"].startswith(MARKER) + ), + comment, + ) + if len(comments) < 100: + break + page += 1 + if comment: + if github(f"pulls/{pr_number}")["head"]["sha"] != head: + return + github(f"issues/comments/{comment['id']}", method="PATCH", data={"body": body}) + else: + if github(f"pulls/{pr_number}")["head"]["sha"] != head: + return + github(f"issues/{pr_number}/comments", method="POST", data={"body": body}) + + +def find_build(builds, number, head): + return next( + ( + build + for build in builds + if build.get("definition", {}).get("id") == 2128 + and build.get("repository", {}).get("id", "").lower() == REPOSITORY + and build.get("sourceBranch") == f"refs/pull/{number}/merge" + and build.get("triggerInfo", {}).get("pr.sourceSha") == head + and build.get("triggerInfo", {}).get("pr.number") == str(number) + ), + None, + ) + + +def run(number, head, wait_minutes): + publish( + number, + head, + f"{MARKER}\n## Profiler performance report\n" + f"Awaiting paired ADO measurements for head `{head}`. No regression verdict yet.", + ) + deadline = time.monotonic() + wait_minutes * 60 + build = None + while time.monotonic() < deadline: + pr = github(f"pulls/{number}") + if pr["state"] != "open" or pr["head"]["sha"] != head: + return + query = urlencode( + { + "definitions": 2128, + "branchName": f"refs/pull/{number}/merge", + "queryOrder": "queueTimeDescending", + "$top": 50, + "api-version": "7.1", + } + ) + build = find_build(api(f"{ADO}/builds?{query}")["value"], number, head) + if build and build["status"] == "completed": + break + time.sleep(30) + if build is None: + publish( + number, + head, + f"{MARKER}\n## Profiler performance report\n" + f"No matching ADO run became available for `{head}`. Results are incomplete.", + ) + return + build_id = build["id"] + source = build["sourceVersion"] + if type(build_id) is not int or build_id <= 0 or not re.fullmatch(r"[0-9a-f]{40}", source): + raise ValueError("Invalid ADO build identity") + # Authenticate the merge topology through GitHub, not the artifact's claims. + commit = github(f"git/commits/{source}") + if len(commit["parents"]) != 2 or commit["parents"][1]["sha"] != head: + raise ValueError("ADO merge does not match current PR head") + base = commit["parents"][0]["sha"] + artifacts = api(f"{ADO}/builds/{build_id}/artifacts?api-version=7.1")["value"] + reports, issues = [], [] + for leg in LEGS: + matching = [item for item in artifacts if item["name"] == "profiler-" + leg] + if len(matching) != 1: + issues.append(leg + " (missing)") + continue + try: + raw = fetch(matching[0]["resource"]["downloadUrl"], limit=32 * 1024 * 1024) + report = validate(artifact_report(raw), build_id, head, source, base) + if report["leg"] != leg: + raise ValueError("Artifact leg mismatch") + reports.append(report) + except (ValueError, KeyError, TypeError, URLError, zipfile.BadZipFile): + # Invalid data is visibly incomplete, never converted to a success verdict. + issues.append(leg + " (invalid artifact)") + if len({r["suite_hash"] for r in reports}) > 1: + reports = [] + issues.append("workload versions differ across legs") + publish(number, head, render(reports, head, build_id, issues)) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--pr", type=int, required=True) + parser.add_argument("--head", required=True) + parser.add_argument("--wait-minutes", type=int, default=95) + args = parser.parse_args() + if ( + args.pr <= 0 + or not re.fullmatch(r"[0-9a-f]{40}", args.head) + or not 1 <= args.wait_minutes <= 95 + ): + parser.error("Invalid PR, head SHA or wait limit") + run(args.pr, args.head, args.wait_minutes) diff --git a/.github/workflows/pr-profiler-report.yml b/.github/workflows/pr-profiler-report.yml new file mode 100644 index 000000000..7f65555e7 --- /dev/null +++ b/.github/workflows/pr-profiler-report.yml @@ -0,0 +1,34 @@ +name: PR Profiler Report + +# Privileged reporting only. No PR checkout, builds, or artifact execution here. +on: + pull_request_target: + branches: [main] + types: [opened, synchronize, reopened, ready_for_review] + +permissions: + contents: read + pull-requests: write + +concurrency: + group: profiler-report-${{ github.event.pull_request.number }} + cancel-in-progress: true + +jobs: + report: + runs-on: ubuntu-latest + timeout-minutes: 100 + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.base.sha }} + persist-credentials: false + - uses: actions/setup-python@v5 + with: + python-version: "3.13" + - name: Publish validated paired benchmark results + env: + GH_TOKEN: ${{ github.token }} + PR_NUMBER: ${{ github.event.pull_request.number }} + PR_HEAD: ${{ github.event.pull_request.head.sha }} + run: python .github/scripts/post_profiler_comment.py --pr "$PR_NUMBER" --head "$PR_HEAD" diff --git a/benchmarks/README.md b/benchmarks/README.md index ce0480057..5ef5284ba 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -7,8 +7,67 @@ This directory contains benchmark scripts for testing the performance of various ### 1. `bench_mssql.py` - Richbench Framework Benchmarks Comprehensive benchmarks using the richbench framework for detailed performance analysis. -### 2. `perf-benchmarking.py` - Real-World Query Benchmarks -Standalone script that tests real-world queries against AdventureWorks2022 database with statistical analysis. +### 2. `profiler_ci.py` - PR Regression Comparisons +CI compares profiling-enabled base and candidate revisions on the same agent and +SQL Server. It replaces the historical pyodbc-normalized CI comparison; the old +`perf-benchmarking.py` remains available for local driver-versus-pyodbc analysis. + +The 20-workload registry retains all 10 scenarios from `profiler/scenarios.py`, +the four AdventureWorks queries (including 1.2M rows), both legacy 100K-row +insertmanyvalues variants, two additional fetch batch sizes, and repeated positional +and named-parameter execution. + +```bash +# Requires build dependencies, pyarrow, and an AdventureWorks2022 connection. +python benchmarks/profiler_ci.py --base main --candidate HEAD \ + --leg Linux-SQL2022 --output profiler-results +python benchmarks/profiler_report.py profiler-results/report.json +``` + +CI uses the PR-merge commit's first parent as the exact base snapshot. The five +benchmark legs build the candidate with `ENABLE_PROFILING=1`, run pytest with +recording disabled by default, then reuse that binary for the benchmark. Existing +profiler-specific tests explicitly enable and clean up recording; ordinary driver +tests do not. A pre-test check requires the expected native configuration and +recording OFF. LocalDB, other Linux legs and the release pipelines still build the +default configuration. Windows profiling artifacts are named separately from +`ddbc_bindings` and are not release wheels. + +Only the base requires a second build, in a temporary git archive with profiling +enabled. Local invocations build both archives unless `--reuse-candidate` is supplied; +that option requires the requested candidate to be the current checkout HEAD. +Both sides execute the same version of the workload suite. Each pass runs in a +fresh interpreter. Five measured +pairs follow one discarded warmup pair, alternating base/PR order. A local subset is +available through `--scenarios`; it is not accepted as a complete CI report. + +The comparison is **advisory**, not a new merge gate. A regression signal requires +over 20% paired-median slowdown, at least 1 ms additional median wall time, and at +least 80% of pairs exceeding the relative threshold. Disagreement is reported as +noisy. Paired runs reduce agent-to-agent noise; they do not eliminate server +contention. Enabled profiler overhead is part of both measurements, so these are +not production-wheel latency estimates. + +Per-phase inclusive duration deltas and call-count changes help locate regressions; +they are not summed into wall-clock totals. Raw pairs and build/worker logs are +published on PR and main runs as `profiler--` artifacts. +The existing five benchmark legs are covered: Windows and macOS on SQL2022/2025, +and Linux Ubuntu on SQL2022. ARM, RHEL, Alpine, LocalDB, and Azure SQL are not +implicitly compared against other platforms. + +`PR Profiler Report` creates or updates one comment per PR. Its privileged job +checks out only the trusted base revision, never executes PR/artifact code, validates +bounded JSON against the ADO build and GitHub merge/head identities, and ignores +stale heads. Missing, skipped, malformed, or failed runs are shown as incomplete, +not as a clean performance verdict. As a new base-branch reporting workflow, it +starts reporting automatically after this infrastructure has merged; it does not +grant fork-authored workflow code write credentials. + +The first main comparison after introduction may lack profiling support on its +parent; that run is incomplete rather than falling back to an uninstrumented base. +Subsequent comparisons use the new artifact format and do not consume old +`perf-baseline-*` artifacts. For a fresh measurement after an ADO-only retry, +re-run the GitHub reporting workflow as well. ## Why Benchmarks? - To measure the efficiency of `pyodbc` and `mssql_python` in handling database operations. diff --git a/benchmarks/profiler_ci.py b/benchmarks/profiler_ci.py new file mode 100644 index 000000000..2f0e28fe0 --- /dev/null +++ b/benchmarks/profiler_ci.py @@ -0,0 +1,259 @@ +"""Build and measure base/candidate in isolated directories on the same CI agent.""" + +import argparse +import contextlib +import hashlib +import importlib.util +import io +import json +import os +from pathlib import Path +import platform +import re +import subprocess +import sys +import tarfile +import tempfile + +ROOT = Path(__file__).resolve().parents[1] +SHA = re.compile(r"[0-9a-f]{40}") +LEGS = ("Windows-SQL2022", "Windows-SQL2025", "macOS-SQL2022", "macOS-SQL2025", "Linux-SQL2022") + + +def git(*args): + return subprocess.check_output(["git", "-C", str(ROOT), *args], text=True).strip() + + +def resolve_revisions(base, candidate): + candidate = git("rev-parse", "--verify", "--end-of-options", f"{candidate}^{{commit}}") + # ADO validates refs/pull/N/merge. Its first parent is the exact target snapshot, + # not whichever main build happened to finish most recently. + base = git( + "rev-parse", "--verify", "--end-of-options", f"{base or candidate + '^1'}^{{commit}}" + ) + return base, candidate + + +def checkout(revision, path): + with tempfile.TemporaryFile() as archive: + subprocess.run(["git", "-C", str(ROOT), "archive", revision], stdout=archive, check=True) + archive.seek(0) + with tarfile.open(fileobj=archive) as tar: + tar.extractall(path, filter="data") + + +def build(path, log): + env = dict(os.environ, ENABLE_PROFILING="1") + # build scripts find Python via PATH; keep the controller's interpreter. + env["PATH"] = str(Path(sys.executable).parent) + os.pathsep + env["PATH"] + command = ["cmd", "/c", "build.bat"] if os.name == "nt" else ["bash", "build.sh"] + with log.open("w", encoding="utf-8") as output: + subprocess.run( + command, + cwd=path / "mssql_python/pybind", + env=env, + stdout=output, + stderr=subprocess.STDOUT, + timeout=900, + check=True, + ) + + +def check_build(source_root, profiling): + sys.path.insert(0, str(source_root)) + import mssql_python + import mssql_python_odbc + from mssql_python import ddbc_bindings, perf_timer + + for module in (mssql_python, ddbc_bindings, mssql_python_odbc): + if not Path(module.__file__).resolve().is_relative_to(source_root.resolve()): + raise RuntimeError("Imported driver outside the selected checkout") + if hasattr(ddbc_bindings, "profiling") != profiling: + raise RuntimeError("Native profiling build configuration mismatch") + if perf_timer.is_enabled() or (profiling and ddbc_bindings.profiling.is_enabled()): + raise RuntimeError("Profiling must default to recording OFF") + + +def load_suite(): + # Load only the common profiler package by path. Keep the revision checkout + # first on sys.path so lazy provider imports cannot select candidate binaries. + spec = importlib.util.spec_from_file_location( + "profiler", + ROOT / "profiler/__init__.py", + submodule_search_locations=[str(ROOT / "profiler")], + ) + module = importlib.util.module_from_spec(spec) + sys.modules["profiler"] = module + spec.loader.exec_module(module) + import profiler.core as core + import profiler_workloads as workloads + + return core, workloads + + +def worker(args): + # Import the chosen driver FIRST, then the SAME workload/controller for both + # revisions. Never mix two native extensions into one interpreter. + check_build(args.source_root, profiling=True) + core, workloads = load_suite() + + cases = workloads.registry() + chosen = args.scenarios or list(cases) + if set(chosen) - set(cases): + raise ValueError("Unknown benchmark scenario") + core.SCENARIOS = cases + # The runner owns enable/disable/cleanup, just as in the documented CLI. + with core.Profiler() as profiler: + with contextlib.redirect_stdout(io.StringIO()): + results = profiler.run(*chosen) + profiler._ensure_connection() + with profiler._conn.cursor() as cursor: + cursor.execute("SELECT CAST(SERVERPROPERTY('ProductVersion') AS VARCHAR(80))") + sql_version = cursor.fetchone()[0] + environment = dict( + os=platform.system(), + architecture=platform.machine().lower(), + python=platform.python_version(), + sql_version=sql_version, + ) + output = {} + for name, result in zip(chosen, results): + if result["cpp"] is None or result["py"] is None: + raise RuntimeError(f"Scenario {name} was skipped") + if not result["cpp"]: + raise RuntimeError(f"Scenario {name} has no native samples") + # Only generated workload counts and instrumentation, never query data. + output[name] = {key: result[key] for key in ("wall_ms", "cpp", "py")} + output[name]["work"] = result.get("detail", "Connection: 1").split(" (")[0] + args.output.write_text( + json.dumps(dict(environment=environment, scenarios=output), allow_nan=False), + encoding="utf-8", + ) + + +def measure(path, output, scenarios): + command = [ + sys.executable, + str(Path(__file__).resolve()), + "--worker", + "--source-root", + str(path), + "--output", + str(output), + ] + if scenarios: + command += ["--scenarios", *scenarios] + with output.with_suffix(".log").open("w", encoding="utf-8") as log: + subprocess.run(command, stdout=log, stderr=subprocess.STDOUT, timeout=240, check=True) + return json.loads(output.read_text(encoding="utf-8")) + + +def run(args): + base, candidate = resolve_revisions(args.base, args.candidate) + args.output.mkdir(parents=True, exist_ok=True) + report_path = args.output / "report.json" + report_path.unlink(missing_ok=True) + suite = hashlib.sha256() + for file in [ + Path(__file__), + ROOT / "benchmarks/profiler_workloads.py", + *sorted((ROOT / "profiler").glob("*.py")), + ]: + suite.update(file.name.encode()) + suite.update(file.read_bytes().replace(b"\r\n", b"\n")) + head = os.environ.get("SYSTEM_PULLREQUEST_SOURCECOMMITID", candidate) + if not SHA.fullmatch(head): + head = candidate + report = dict( + schema_version=1, + status="incomplete", + leg=args.leg, + base_commit=base, + source_commit=candidate, + head_commit=head, + build_id=int(os.environ.get("BUILD_BUILDID", "0")), + suite_hash=suite.hexdigest(), + samples=args.samples, + warmups=args.warmups, + pairs=[], + ) + report_path.write_text(json.dumps(report), encoding="utf-8") + # CI reuses the profiling build already exercised by pytest. The base always + # has its own checkout and process. Local runs can build both sides instead. + with tempfile.TemporaryDirectory(prefix="profiler-ci-") as directory: + paths = {side: Path(directory) / side for side in ("base", "candidate")} + for side, revision in (("base", base), ("candidate", candidate)): + if side == "candidate" and args.reuse_candidate: + if candidate != git("rev-parse", "HEAD"): + raise ValueError("--reuse-candidate requires candidate to be checkout HEAD") + paths[side] = ROOT + subprocess.run( + [sys.executable, str(Path(__file__).resolve()), "--check-build", "on"], + check=True, + timeout=60, + ) + continue + checkout(revision, paths[side]) + print(f"Building profiling {side}: {revision}", flush=True) + build(paths[side], args.output / f"build-{side}.log") + for sample in range(args.warmups + args.samples): + pair = {} + order = ("base", "candidate") if sample % 2 == 0 else ("candidate", "base") + for side in order: + print(f"Measuring pair {sample + 1}: {side}", flush=True) + pair[side] = measure( + paths[side], + args.output / f"{side}-{sample}.json", + args.scenarios, + ) + if pair["base"]["environment"] != pair["candidate"]["environment"]: + raise RuntimeError("Base and candidate environments differ") + if sample >= args.warmups: + report["pairs"].append(pair) + report_path.write_text(json.dumps(report, allow_nan=False), encoding="utf-8") + report["status"] = "complete" + report_path.write_text(json.dumps(report, allow_nan=False), encoding="utf-8") + print(f"Paired profiler report: {report_path}", flush=True) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--base", help="Exact base revision; defaults to candidate first parent") + parser.add_argument("--candidate", default="HEAD") + parser.add_argument("--leg", choices=LEGS) + parser.add_argument("--output", type=Path) + parser.add_argument("--samples", type=int, default=5) + parser.add_argument("--warmups", type=int, default=1) + parser.add_argument("--scenarios", nargs="+", help="Local subset; CI runs the full registry") + parser.add_argument("--worker", action="store_true", help=argparse.SUPPRESS) + parser.add_argument("--source-root", type=Path, help=argparse.SUPPRESS) + parser.add_argument( + "--reuse-candidate", + action="store_true", + help="Reuse checkout HEAD's profiling build after pytest", + ) + parser.add_argument( + "--check-build", + choices=("on", "off"), + help="Verify native compile configuration and recording OFF, then exit", + ) + args = parser.parse_args() + if args.check_build: + check_build(ROOT, profiling=args.check_build == "on") + elif args.worker: + if args.source_root is None or args.output is None: + parser.error("--worker requires --source-root and --output") + worker(args) + else: + if ( + not args.output + or not args.leg + or not 3 <= args.samples <= 15 + or not 1 <= args.warmups <= 3 + ): + parser.error("Choose a leg, 3-15 measured pairs and 1-3 warmup pairs") + run(args) + + +if __name__ == "__main__": + main() diff --git a/benchmarks/profiler_report.py b/benchmarks/profiler_report.py new file mode 100644 index 000000000..3725466eb --- /dev/null +++ b/benchmarks/profiler_report.py @@ -0,0 +1,278 @@ +"""Validate bounded profiler data and render an advisory, per-platform comparison.""" + +import argparse +import html +import json +import math +from pathlib import Path +import re +import statistics + +LEGS = ("Windows-SQL2022", "Windows-SQL2025", "macOS-SQL2022", "macOS-SQL2025", "Linux-SQL2022") +CASES = ( + "connect", + "select", + "insert", + "executemany", + "fetchall", + "fetchone", + "fetchmany", + "commit_rollback", + "arrow", + "insertmanyvalues", + "fetchmany_100", + "fetchmany_10000", + "prepared_qmark", + "prepared_named", + "legacy_insertmany", + "setinputsizes", + "join_aggregation", + "large_fetch", + "fetch_1_2m", + "cte", +) +MAX_BYTES = 8 * 1024 * 1024 +MARKER = "" +THRESHOLD = 0.20 +MIN_DELTA_MS = 1.0 + + +def number(value, maximum=1e12): + if type(value) not in (float, int) or not 0 <= value <= maximum: + raise ValueError("Invalid performance measurement") + return value + + +def text(value, limit=160): + if not isinstance(value, str) or not value or len(value) > limit: + raise ValueError("Invalid performance label") + if any(ord(char) < 32 or ord(char) == 127 for char in value): + raise ValueError("Control character in performance label") + return value + + +def validate(report, build_id=None, head=None, source=None, base=None): + if not isinstance(report, dict) or report.get("schema_version") != 1: + raise ValueError("Unsupported report schema") + if report.get("leg") not in LEGS or report.get("status") not in ("complete", "incomplete"): + raise ValueError("Invalid report status or leg") + for key, expected in ( + ("build_id", build_id), + ("head_commit", head), + ("source_commit", source), + ("base_commit", base), + ): + if expected is not None and report.get(key) != expected: + raise ValueError(f"Report provenance mismatch: {key}") + for key in ("head_commit", "source_commit", "base_commit"): + if not re.fullmatch(r"[0-9a-f]{40}", report.get(key, "")): + raise ValueError("Invalid commit identity") + if not re.fullmatch(r"[0-9a-f]{64}", report.get("suite_hash", "")): + raise ValueError("Invalid workload identity") + samples = report.get("samples") + if type(samples) is not int or not 3 <= samples <= 15: + raise ValueError("Insufficient or excessive samples") + if type(report.get("warmups")) is not int or not 1 <= report["warmups"] <= 3: + raise ValueError("Invalid warmup count") + pairs = report.get("pairs") + if not isinstance(pairs, list) or len(pairs) > samples: + raise ValueError("Invalid sample pairs") + if report["status"] == "incomplete": + return report + if len(pairs) != samples: + raise ValueError("Incomplete sample pairs") + environment = None + work = {} + for pair in pairs: + if not isinstance(pair, dict) or set(pair) != {"base", "candidate"}: + raise ValueError("Invalid paired sample") + for side in ("base", "candidate"): + sample = pair[side] + env = sample["environment"] + if not isinstance(env, dict) or set(env) != { + "os", + "architecture", + "python", + "sql_version", + }: + raise ValueError("Invalid environment") + for value in env.values(): + text(value) + expected_os, sql = report["leg"].split("-") + if env["os"] != {"macOS": "Darwin"}.get(expected_os, expected_os): + raise ValueError("Artifact platform does not match its leg") + if not env["sql_version"].startswith({"SQL2022": "16.", "SQL2025": "17."}[sql]): + raise ValueError("Artifact SQL version does not match its leg") + if environment is not None and environment != env: + raise ValueError("Environment changed between measurements") + environment = env + if set(sample["scenarios"]) != set(CASES): + raise ValueError("Scenario set incomplete or changed") + for name, scenario in sample["scenarios"].items(): + number(scenario["wall_ms"]) + if scenario["wall_ms"] <= 0: + raise ValueError("Zero workload time") + identity = text(scenario["work"]) + if name in work and work[name] != identity: + raise ValueError(f"Workload changed for {name}") + work[name] = identity + for layer in ("cpp", "py"): + stats = scenario[layer] + if ( + not isinstance(stats, dict) + or len(stats) > 300 + or (layer == "cpp" and not stats) + ): + raise ValueError("Missing or oversized profiling data") + for label, counter in stats.items(): + text(label) + if not label.startswith("ddbc::" if layer == "cpp" else "py::"): + raise ValueError("Invalid phase prefix") + calls = counter["calls"] + if type(calls) is not int or not 1 <= calls <= 100_000_000: + raise ValueError("Invalid call count") + for field in ("total_us", "min_us", "max_us"): + number(counter[field]) + if not counter["min_us"] <= counter["max_us"] <= counter["total_us"]: + raise ValueError("Inconsistent phase totals") + return report + + +def comparisons(report): + """Do not add inclusive phase totals together or treat them as wall-clock time.""" + output = [] + for name in CASES: + base = [pair["base"]["scenarios"][name] for pair in report["pairs"]] + candidate = [pair["candidate"]["scenarios"][name] for pair in report["pairs"]] + ratios = [new["wall_ms"] / old["wall_ms"] for old, new in zip(base, candidate)] + old = statistics.median(s["wall_ms"] for s in base) + new = statistics.median(s["wall_ms"] for s in candidate) + ratio = statistics.median(ratios) + # Requiring 80% of paired samples to agree avoids flagging one noisy pass. + agrees = sum(r > 1 + THRESHOLD for r in ratios) >= math.ceil(len(ratios) * 0.8) + status = ( + "regression" + if ratio > 1 + THRESHOLD and new - old >= MIN_DELTA_MS and agrees + else ("noisy" if ratio > 1 + THRESHOLD and new - old >= MIN_DELTA_MS else "ok") + ) + phases = [] + changed_counts = [] + for layer in ("cpp", "py"): + labels = set().union(*(s[layer] for s in base + candidate)) + for label in labels: + before = [s[layer].get(label) for s in base] + after = [s[layer].get(label) for s in candidate] + if not all(before) or not all(after): + changed_counts.append(f"{label} (added, removed, or intermittent)") + continue + before_calls = statistics.median(s["calls"] for s in before) + after_calls = statistics.median(s["calls"] for s in after) + if before_calls != after_calls: + changed_counts.append(f"{label} ({before_calls:g} -> {after_calls:g} calls)") + delta = ( + statistics.median(s["total_us"] for s in after) + - statistics.median(s["total_us"] for s in before) + ) / 1000 + if delta > 0: + phases.append((delta, label)) + output.append( + dict( + name=name, + base_ms=old, + candidate_ms=new, + change_pct=(ratio - 1) * 100, + status=status, + phases=sorted(phases, reverse=True)[:3], + counts=sorted(changed_counts)[:3], + ) + ) + return output + + +def escape(value): + value = html.escape(value, quote=True) + for char in "\\|`[]()*_~@": + value = value.replace(char, f"&#{ord(char)};") + return value + + +def render(reports, head, build_id, issues=()): + url = f"https://dev.azure.com/sqlclientdrivers/public/_build/results?buildId={build_id}" + lines = [ + MARKER, + "## Profiler performance report", + f"Head `{head}` | [ADO build {build_id}]({url})", + "", + "Advisory base vs PR-merge comparison. Both revisions are profiling-enabled, " + "measured on the same agent/database with alternating order and discarded warmups.", + "Flags require >20% paired median slowdown, >=1 ms added time, and 80% of pairs agreeing. " + "These are signals to investigate, not production-wheel latency guarantees.", + "", + ] + by_leg = {r["leg"]: r for r in reports} + for leg in LEGS: + report = by_leg.get(leg) + if report is None or report["status"] != "complete": + lines.append(f"**{leg}: incomplete/unavailable. No regression verdict.**") + continue + rows = comparisons(report) + env = report["pairs"][0]["base"]["environment"] + flags = [r for r in rows if r["status"] == "regression"] + noisy = sum(r["status"] == "noisy" for r in rows) + lines += [ + "", + f"### {leg}", + f"Base `{report['base_commit'][:12]}` -> merge `{report['source_commit'][:12]}`; " + f"Python {escape(env['python'])}, {escape(env['architecture'])}, " + f"SQL {escape(env['sql_version'])}; {report['samples']} pairs.", + f"**{len(flags)} regression signals, {noisy} noisy comparisons.**", + "
All scenarios and phase diagnostics", + "", + "| Scenario | Base ms | PR ms | Paired change | Result |", + "|---|---:|---:|---:|---|", + ] + for row in rows: + lines.append( + f"| {row['name']} | {row['base_ms']:.3f} | {row['candidate_ms']:.3f} | " + f"{row['change_pct']:+.1f}% | {row['status']} |" + ) + for row in rows: + if row["status"] != "ok" or row["counts"]: + detail = "; ".join( + f"{escape(label)} +{delta:.3f} ms" for delta, label in row["phases"] + ) + counts = "; ".join(escape(label) for label in row["counts"]) + lines.append( + f"\n**{row['name']}**: {detail or 'no positive phase delta'}." + + (f" Call changes: {counts}." if counts else "") + ) + lines += [ + "", + "Phase times are inclusive diagnostics, not additive wall-clock components.", + "
", + ] + if issues: + lines += [ + "", + "Some artifacts were missing or rejected: " + ", ".join(escape(x) for x in issues), + ] + lines += [ + "", + "Raw samples and build logs are attached to the ADO run as `profiler-*` artifacts.", + ] + body = "\n".join(lines) + if len(body) > 60000: + raise ValueError("Performance comment exceeds its size budget") + return body + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("reports", nargs="+", type=Path) + args = parser.parse_args() + reports = [validate(json.loads(path.read_text(encoding="utf-8"))) for path in args.reports] + print(render(reports, reports[0]["head_commit"], reports[0]["build_id"])) + + +if __name__ == "__main__": + main() diff --git a/benchmarks/profiler_workloads.py b/benchmarks/profiler_workloads.py new file mode 100644 index 000000000..b33240978 --- /dev/null +++ b/benchmarks/profiler_workloads.py @@ -0,0 +1,148 @@ +"""Fixed workloads shared by the base and candidate profiler builds.""" + +from functools import partial +import time + +from profiler import scenarios + +# Preserve the AdventureWorks workloads from the previous CI benchmark. +QUERIES = { + "join_aggregation": """ + SELECT p.ProductID, p.Name AS ProductName, pc.Name AS Category, + psc.Name AS Subcategory, COUNT(sod.SalesOrderDetailID) AS TotalOrders, + SUM(sod.OrderQty) AS TotalQuantity, SUM(sod.LineTotal) AS TotalRevenue, + AVG(sod.UnitPrice) AS AvgPrice + FROM Sales.SalesOrderDetail sod + INNER JOIN Production.Product p ON sod.ProductID = p.ProductID + INNER JOIN Production.ProductSubcategory psc ON p.ProductSubcategoryID = psc.ProductSubcategoryID + INNER JOIN Production.ProductCategory pc ON psc.ProductCategoryID = pc.ProductCategoryID + GROUP BY p.ProductID, p.Name, pc.Name, psc.Name + HAVING SUM(sod.LineTotal) > 10000 ORDER BY TotalRevenue DESC + """, + "large_fetch": """ + SELECT soh.SalesOrderID, soh.OrderDate, soh.DueDate, soh.ShipDate, soh.Status, + soh.SubTotal, soh.TaxAmt, soh.Freight, soh.TotalDue, c.CustomerID, + p.FirstName, p.LastName, a.AddressLine1, a.City, + sp.Name AS StateProvince, cr.Name AS Country + FROM Sales.SalesOrderHeader soh + INNER JOIN Sales.Customer c ON soh.CustomerID = c.CustomerID + INNER JOIN Person.Person p ON c.PersonID = p.BusinessEntityID + INNER JOIN Person.BusinessEntityAddress bea ON p.BusinessEntityID = bea.BusinessEntityID + INNER JOIN Person.Address a ON bea.AddressID = a.AddressID + INNER JOIN Person.StateProvince sp ON a.StateProvinceID = sp.StateProvinceID + INNER JOIN Person.CountryRegion cr ON sp.CountryRegionCode = cr.CountryRegionCode + WHERE soh.OrderDate >= '2013-01-01' + """, + "fetch_1_2m": """ + SELECT sod.SalesOrderID, sod.SalesOrderDetailID, sod.ProductID, + sod.OrderQty, sod.UnitPrice, sod.LineTotal, + p.Name AS ProductName, p.ProductNumber, p.Color, p.ListPrice, + n1.number AS RowMultiplier1 + FROM Sales.SalesOrderDetail sod + CROSS JOIN (SELECT TOP 10 ROW_NUMBER() OVER (ORDER BY (SELECT NULL)) AS number + FROM Sales.SalesOrderDetail) n1 + INNER JOIN Production.Product p ON sod.ProductID = p.ProductID + """, + "cte": """ + WITH SalesSummary AS ( + SELECT soh.SalesPersonID, YEAR(soh.OrderDate) AS OrderYear, + SUM(soh.TotalDue) AS YearlyTotal + FROM Sales.SalesOrderHeader soh WHERE soh.SalesPersonID IS NOT NULL + GROUP BY soh.SalesPersonID, YEAR(soh.OrderDate) + ), RankedSales AS ( + SELECT SalesPersonID, OrderYear, YearlyTotal, + RANK() OVER (PARTITION BY OrderYear ORDER BY YearlyTotal DESC) AS SalesRank + FROM SalesSummary + ) + SELECT rs.SalesPersonID, p.FirstName, p.LastName, + rs.OrderYear, rs.YearlyTotal, rs.SalesRank + FROM RankedSales rs INNER JOIN Person.Person p ON rs.SalesPersonID = p.BusinessEntityID + WHERE rs.SalesRank <= 10 ORDER BY rs.OrderYear DESC, rs.SalesRank + """, +} + + +def query(conn, ctx, sql): + with conn.cursor() as cursor: + ctx.enable() + try: + start = time.perf_counter() + cursor.execute(sql) + rows = cursor.fetchall() + wall_ms = (time.perf_counter() - start) * 1000 + cpp, py = ctx.collect() + return dict( + title="AdventureWorks query", + wall_ms=wall_ms, + cpp=cpp, + py=py, + detail=f"Rows: {len(rows)}", + ) + finally: + ctx.disable() + + +def parameter_execution(conn, ctx, named=False): + with conn.cursor() as cursor: + ctx.enable() + try: + start = time.perf_counter() + for value in range(100): + if named: + cursor.execute("SELECT %(value)s", {"value": value}) + else: + cursor.execute("SELECT ?", (value,)) + assert cursor.fetchone()[0] == value + wall_ms = (time.perf_counter() - start) * 1000 + cpp, py = ctx.collect() + return dict( + title="Parameterized execution", wall_ms=wall_ms, cpp=cpp, py=py, detail="Rows: 100" + ) + finally: + ctx.disable() + + +def legacy_insertmany(conn, ctx, input_sizes=False): + from mssql_python import SQL_INTEGER, SQL_VARCHAR + + sql = "INSERT INTO #ci_insert VALUES " + ",".join(["(?,?)"] * 1000) + batches = [ + [value for i in range(start, start + 1000) for value in (i, f"value_{i}")] + for start in range(0, 100_000, 1000) + ] + with conn.cursor() as cursor: + try: + cursor.execute( + "DROP TABLE IF EXISTS #ci_insert; " + "CREATE TABLE #ci_insert (id INT, val VARCHAR(100))" + ) + sizes = [(SQL_INTEGER, 0, 0), (SQL_VARCHAR, 100, 0)] * 1000 + ctx.enable() + start = time.perf_counter() + for params in batches: + if input_sizes: + cursor.setinputsizes(sizes) + cursor.execute(sql, params) + wall_ms = (time.perf_counter() - start) * 1000 + cpp, py = ctx.collect() + return dict( + title="Batched insert", wall_ms=wall_ms, cpp=cpp, py=py, detail="Rows: 100000" + ) + finally: + ctx.disable() + conn.rollback() + + +def registry(): + """Keep every PR #552 scenario, including its existing timing boundaries.""" + result = dict(scenarios.SCENARIOS) + result.update( + fetchmany_100=(partial(scenarios.fetchmany, batch_size=100), True), + fetchmany_10000=(partial(scenarios.fetchmany, batch_size=10000), True), + prepared_qmark=(parameter_execution, False), + prepared_named=(partial(parameter_execution, named=True), False), + legacy_insertmany=(legacy_insertmany, False), + setinputsizes=(partial(legacy_insertmany, input_sizes=True), False), + ) + result.update((name, (partial(query, sql=sql), False)) for name, sql in QUERIES.items()) + return result diff --git a/eng/pipelines/pr-validation-pipeline.yml b/eng/pipelines/pr-validation-pipeline.yml index 66db771d7..c1c270475 100644 --- a/eng/pipelines/pr-validation-pipeline.yml +++ b/eng/pipelines/pr-validation-pipeline.yml @@ -47,6 +47,7 @@ jobs: - job: pytestonwindows displayName: 'Windows x64' + timeoutInMinutes: 90 pool: vmImage: 'windows-latest' @@ -55,14 +56,25 @@ jobs: SQLServer2022: sqlVersion: 'SQL2022' pythonVersion: '3.13' + profilerBuild: '1' + profilerCheck: 'on' + bindingArtifact: 'ddbc_bindings-profiling-SQL2022' SQLServer2025: sqlVersion: 'SQL2025' pythonVersion: '3.14' + profilerBuild: '1' + profilerCheck: 'on' + bindingArtifact: 'ddbc_bindings-profiling-SQL2025' LocalDB_Python314: sqlVersion: 'LocalDB' pythonVersion: '3.14' + profilerBuild: '0' + profilerCheck: 'off' + bindingArtifact: 'ddbc_bindings' steps: + - checkout: self + fetchDepth: 0 - task: UsePythonVersion@0 inputs: versionSpec: '$(pythonVersion)' @@ -229,6 +241,11 @@ jobs: cd mssql_python\pybind build.bat x64 displayName: 'Build .pyd file' + env: + ENABLE_PROFILING: $(profilerBuild) + + - script: python benchmarks/profiler_ci.py --check-build $(profilerCheck) + displayName: 'Verify native configuration and recording OFF before pytest' - template: steps/install-mssql-py-core.yml parameters: @@ -296,33 +313,6 @@ jobs: env: DB_PASSWORD: $(DB_PASSWORD) - # Download baseline from latest main run (for PR comparison) - - task: DownloadPipelineArtifact@2 - inputs: - source: specific - project: $(System.TeamProjectId) - pipeline: $(System.DefinitionId) - runVersion: latestFromBranch - runBranch: refs/heads/main - artifact: 'perf-baseline-$(sqlVersion)' - path: $(Build.SourcesDirectory) - displayName: 'Download baseline from main' - condition: and(succeeded(), ne(variables['Build.SourceBranch'], 'refs/heads/main'), or(eq(variables['sqlVersion'], 'SQL2022'), eq(variables['sqlVersion'], 'SQL2025'))) - continueOnError: true - - # Rename downloaded baseline so the script finds it (artifact may be in a subfolder) - - powershell: | - $found = Get-ChildItem -Path "$(Build.SourcesDirectory)" -Filter "benchmark_results.json" -Recurse -File -ErrorAction SilentlyContinue | Select-Object -First 1 - if ($null -ne $found) { - Copy-Item $found.FullName "benchmark_baseline.json" -Force - Write-Host "Baseline file ready: benchmark_baseline.json (from $($found.FullName))" - } else { - Write-Host "No baseline file downloaded (first run or artifact missing)" - } - displayName: 'Prepare baseline file' - condition: and(succeeded(), ne(variables['Build.SourceBranch'], 'refs/heads/main'), or(eq(variables['sqlVersion'], 'SQL2022'), eq(variables['sqlVersion'], 'SQL2025'))) - continueOnError: true - # Run performance benchmarks on SQL Server 2022 - powershell: | Write-Host "Checking and installing ODBC Driver 18 for SQL Server..." @@ -406,24 +396,23 @@ jobs: exit 1 } - Write-Host "`nInstalling pyodbc..." - pip install pyodbc - - Write-Host "`nRunning performance benchmarks..." - python benchmarks/perf-benchmarking.py --baseline benchmark_baseline.json --json benchmark_results.json - displayName: 'Run performance benchmarks on SQL Server 2022/2025' + python benchmarks/profiler_ci.py --reuse-candidate --leg "Windows-$(sqlVersion)" --output profiler-results + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + displayName: 'Compare profiling builds on SQL Server 2022/2025' condition: and(succeeded(), or(eq(variables['sqlVersion'], 'SQL2022'), eq(variables['sqlVersion'], 'SQL2025'))) continueOnError: true + timeoutInMinutes: 40 env: + SYSTEM_PULLREQUEST_SOURCECOMMITID: $(System.PullRequest.SourceCommitId) DB_CONNECTION_STRING: 'Server=localhost;Database=AdventureWorks2022;Uid=sa;Pwd=$(DB_PASSWORD);TrustServerCertificate=yes' - # Publish benchmark results as artifact (consumed as baseline by future PR runs) + # Publish on PRs as well as main, including partial reports and failure logs. - task: PublishPipelineArtifact@1 inputs: - targetPath: benchmark_results.json - artifact: 'perf-baseline-$(sqlVersion)' - displayName: 'Publish benchmark baseline' - condition: and(succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/main'), or(eq(variables['sqlVersion'], 'SQL2022'), eq(variables['sqlVersion'], 'SQL2025'))) + targetPath: profiler-results + artifact: 'profiler-Windows-$(sqlVersion)' + displayName: 'Publish paired profiler measurements' + condition: and(succeededOrFailed(), or(eq(variables['sqlVersion'], 'SQL2022'), eq(variables['sqlVersion'], 'SQL2025'))) continueOnError: true - task: CopyFiles@2 @@ -443,7 +432,7 @@ jobs: - task: PublishBuildArtifacts@1 inputs: PathtoPublish: '$(Build.ArtifactStagingDirectory)' - ArtifactName: 'ddbc_bindings' + ArtifactName: '$(bindingArtifact)' publishLocation: 'Container' displayName: 'Publish build artifacts' @@ -488,6 +477,9 @@ jobs: pythonVersion: '3.14' steps: + - checkout: self + fetchDepth: 0 + - task: UsePythonVersion@0 inputs: versionSpec: '$(pythonVersion)' @@ -588,7 +580,7 @@ jobs: pip install -r requirements.txt echo "Building pybind bindings (.so) (overlapped with container setup)..." - ( cd mssql_python/pybind && ./build.sh ) + ( cd mssql_python/pybind && ENABLE_PROFILING=1 ./build.sh ) echo "Waiting for container setup (Colima + SQL Server) to finish..." if ! wait "$SQL_PID"; then @@ -601,6 +593,9 @@ jobs: env: DB_PASSWORD: $(DB_PASSWORD) + - script: python benchmarks/profiler_ci.py --check-build on + displayName: 'Verify native configuration and recording OFF before pytest' + - template: steps/install-mssql-py-core.yml parameters: platform: unix @@ -677,60 +672,35 @@ jobs: env: DB_PASSWORD: $(DB_PASSWORD) - # Download macOS baseline from latest main run (for PR comparison) - - task: DownloadPipelineArtifact@2 - inputs: - source: specific - project: $(System.TeamProjectId) - pipeline: $(System.DefinitionId) - runVersion: latestFromBranch - runBranch: refs/heads/main - artifact: 'perf-baseline-macOS-$(sqlVersion)' - path: $(Build.SourcesDirectory) - displayName: 'Download macOS baseline from main' - condition: and(succeeded(), ne(variables['Build.SourceBranch'], 'refs/heads/main'), or(eq(variables['sqlVersion'], 'SQL2022'), eq(variables['sqlVersion'], 'SQL2025'))) - continueOnError: true - - - script: | - found=$(find "$(Build.SourcesDirectory)" -name benchmark_results.json -type f 2>/dev/null | head -1) - if [ -n "$found" ]; then - cp "$found" benchmark_baseline.json - echo "Baseline file ready: benchmark_baseline.json (from $found)" - else - echo "No baseline file downloaded (first run or artifact missing)" - fi - displayName: 'Prepare macOS baseline file' - condition: and(succeeded(), ne(variables['Build.SourceBranch'], 'refs/heads/main'), or(eq(variables['sqlVersion'], 'SQL2022'), eq(variables['sqlVersion'], 'SQL2025'))) - continueOnError: true - # Run performance benchmarks on macOS - script: | - echo "Installing ODBC Driver 18 for pyodbc..." + set -euo pipefail + echo "Restoring build dependencies for isolated profiling builds..." brew tap microsoft/mssql-release https://github.com/Microsoft/homebrew-mssql-release # Newer Homebrew refuses to load formulae from third-party taps unless the tap is trusted brew trust microsoft/mssql-release || echo "brew trust failed; attempting install anyway" - HOMEBREW_ACCEPT_EULA=Y brew install msodbcsql18 || echo "ODBC Driver 18 install failed — pyodbc benchmarks will be skipped" - pip install pyodbc - echo "Running performance benchmarks..." - python benchmarks/perf-benchmarking.py --baseline benchmark_baseline.json --json benchmark_results.json - displayName: 'Run performance benchmarks on macOS $(sqlVersion)' + HOMEBREW_ACCEPT_EULA=Y brew install msodbcsql18 + python benchmarks/profiler_ci.py --reuse-candidate --leg "macOS-$(sqlVersion)" --output profiler-results + displayName: 'Compare profiling builds on macOS $(sqlVersion)' condition: and(succeeded(), or(eq(variables['sqlVersion'], 'SQL2022'), eq(variables['sqlVersion'], 'SQL2025'))) - timeoutInMinutes: 20 + timeoutInMinutes: 40 continueOnError: true env: + SYSTEM_PULLREQUEST_SOURCECOMMITID: $(System.PullRequest.SourceCommitId) DB_CONNECTION_STRING: 'Server=tcp:127.0.0.1,1433;Database=AdventureWorks2022;Uid=SA;Pwd=$(DB_PASSWORD);TrustServerCertificate=yes' - # Publish benchmark results as artifact on main merges + # Both revisions and samples travel together; no historical baseline lookup. - task: PublishPipelineArtifact@1 inputs: - targetPath: benchmark_results.json - artifact: 'perf-baseline-macOS-$(sqlVersion)' - displayName: 'Publish macOS benchmark baseline' - condition: and(succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/main'), or(eq(variables['sqlVersion'], 'SQL2022'), eq(variables['sqlVersion'], 'SQL2025'))) + targetPath: profiler-results + artifact: 'profiler-macOS-$(sqlVersion)' + displayName: 'Publish paired profiler measurements' + condition: and(succeededOrFailed(), or(eq(variables['sqlVersion'], 'SQL2022'), eq(variables['sqlVersion'], 'SQL2025'))) continueOnError: true - job: PytestOnLinux displayName: 'Linux x86_64' + timeoutInMinutes: 90 pool: vmImage: 'ubuntu-latest' @@ -764,6 +734,9 @@ jobs: useAzureSQL: 'false' steps: + - checkout: self + fetchDepth: 0 + - script: | # Create a Docker container for testing docker run -d --name test-container-$(distroName) \ @@ -900,11 +873,20 @@ jobs: - script: | # Build pybind bindings in the container - docker exec test-container-$(distroName) bash -c " + PROFILER_BUILD=0 + PROFILER_CHECK=off + if [ "$(distroName)" = "Ubuntu" ]; then + PROFILER_BUILD=1 + PROFILER_CHECK=on + fi + docker exec -e ENABLE_PROFILING="$PROFILER_BUILD" test-container-$(distroName) bash -c " + set -e source /opt/venv/bin/activate cd mssql_python/pybind chmod +x build.sh ./build.sh + cd ../.. + python benchmarks/profiler_ci.py --check-build $PROFILER_CHECK " displayName: 'Build pybind bindings (.so) in $(distroName) container' @@ -1007,56 +989,40 @@ jobs: echo "Running performance benchmarks on Ubuntu with SQL Server IP: $SQLSERVER_IP" docker exec \ + -e BUILD_BUILDID="$(Build.BuildId)" \ + -e SYSTEM_PULLREQUEST_SOURCECOMMITID="$(System.PullRequest.SourceCommitId)" \ -e DB_CONNECTION_STRING="Server=$SQLSERVER_IP;Database=AdventureWorks2022;Uid=SA;Pwd=$(DB_PASSWORD);TrustServerCertificate=yes" \ test-container-$(distroName) bash -c " + set -euo pipefail source /opt/venv/bin/activate - - echo 'Reinstalling ODBC Driver for benchmarking...' export DEBIAN_FRONTEND=noninteractive - - # Remove duplicate repository sources if they exist - rm -f /etc/apt/sources.list.d/microsoft-prod.list - - # Add Microsoft repository - curl -sSL https://packages.microsoft.com/keys/microsoft.asc | apt-key add - - curl -sSL https://packages.microsoft.com/config/ubuntu/24.04/prod.list > /etc/apt/sources.list.d/mssql-release.list - - # Update package lists + # The Microsoft repository was configured by the earlier build step. + # Restore the ODBC headers and library link removed before pytest. apt-get update -qq - - # Install unixodbc and its dependencies first (provides libodbcinst.so.2 needed by msodbcsql18) - echo 'Installing unixODBC dependencies...' - apt-get install -y --no-install-recommends unixodbc unixodbc-dev libodbc1 odbcinst odbcinst1debian2 - - # Verify libodbcinst.so.2 is available - ldconfig - ls -la /usr/lib/x86_64-linux-gnu/libodbcinst.so.2 || echo 'Warning: libodbcinst.so.2 not found' - - # Install ODBC Driver 18 - echo 'Installing msodbcsql18...' - ACCEPT_EULA=Y apt-get install -y msodbcsql18 - - # Verify ODBC driver installation - odbcinst -q -d -n 'ODBC Driver 18 for SQL Server' || echo 'Warning: ODBC Driver 18 not registered' - - echo 'Installing pyodbc for benchmarking...' - pip install pyodbc - echo 'Running performance benchmarks on $(distroName)' - if [ -f benchmark_baseline.json ]; then - python benchmarks/perf-benchmarking.py --baseline benchmark_baseline.json --json benchmark_results.json || echo 'Performance benchmark failed or database not available' - else - python benchmarks/perf-benchmarking.py --json benchmark_results.json || echo 'Performance benchmark failed or database not available' - fi + ACCEPT_EULA=Y apt-get install -y --no-install-recommends git unixodbc unixodbc-dev libodbc2 libodbcinst2 odbcinst msodbcsql18 + apt-get install -y --reinstall libodbcinst2 + git config --global --add safe.directory /workspace + odbcinst -q -d -n 'ODBC Driver 18 for SQL Server' + python benchmarks/profiler_ci.py --reuse-candidate --leg Linux-SQL2022 --output profiler-results " else echo "Skipping performance benchmarks on $(distroName) (only runs on Ubuntu with local SQL Server)" fi - displayName: 'Run performance benchmarks in $(distroName) container' + displayName: 'Compare profiling builds in $(distroName) container' condition: and(succeeded(), eq(variables['distroName'], 'Ubuntu'), eq(variables['useAzureSQL'], 'false')) continueOnError: true + timeoutInMinutes: 40 env: DB_PASSWORD: $(DB_PASSWORD) + - task: PublishPipelineArtifact@1 + inputs: + targetPath: profiler-results + artifact: profiler-Linux-SQL2022 + displayName: 'Publish paired profiler measurements' + condition: and(succeededOrFailed(), eq(variables['distroName'], 'Ubuntu'), eq(variables['useAzureSQL'], 'false')) + continueOnError: true + - script: | # Copy test results from container to host docker cp test-container-$(distroName):/workspace/test-results-$(distroName).xml $(Build.SourcesDirectory)/ diff --git a/profiler/README.md b/profiler/README.md index 3521ed9b3..753f665da 100644 --- a/profiler/README.md +++ b/profiler/README.md @@ -21,8 +21,9 @@ context manager whose end-to-end cost is within run-to-run noise. Runtime-instrumentation tests remain part of the driver test suite. Tests that require the dev-only `profiler/` package skip when it is absent from an installed -wheel. Broader profiler testing and profiling-enabled CI builds are deferred to -follow-up work. +wheel. The [paired CI benchmark guide](../benchmarks/README.md) describes isolated +profiling builds, scenario coverage and advisory PR regression comments. Broader +profiler testing remains follow-up work. Use controlled diagnostic workloads with one owner of the process-wide profiling state: enable, run the workload, wait for worker threads to finish, then collect. diff --git a/tests/test_036_profiler_ci.py b/tests/test_036_profiler_ci.py new file mode 100644 index 000000000..a7d7f0191 --- /dev/null +++ b/tests/test_036_profiler_ci.py @@ -0,0 +1,342 @@ +"""Contract tests for paired performance comparisons and data-only PR reporting.""" + +import copy +import importlib.util +import io +import json +from pathlib import Path +import sys +from types import SimpleNamespace +import zipfile + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +if not (ROOT / ".github/scripts/post_profiler_comment.py").is_file(): + pytest.skip("CI reporting tools are not installed in driver wheels", allow_module_level=True) + + +def load(name, path): + spec = importlib.util.spec_from_file_location(name, ROOT / path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +reporting = load("profiler_report", "benchmarks/profiler_report.py") +controller = load("profiler_ci", "benchmarks/profiler_ci.py") +sys.modules["profiler_report"] = reporting +publisher = load("post_profiler_comment", ".github/scripts/post_profiler_comment.py") + + +@pytest.fixture +def report(): + def sample(scale): + counter = dict(calls=1, total_us=1000, min_us=1000, max_us=1000) + return dict( + environment=dict( + os="Linux", architecture="x86_64", python="3.13.7", sql_version="16.0" + ), + scenarios={ + name: dict( + wall_ms=10 * scale, work="Rows: 100", cpp={"ddbc::query": counter}, py={} + ) + for name in reporting.CASES + }, + ) + + return dict( + schema_version=1, + status="complete", + leg="Linux-SQL2022", + base_commit="a" * 40, + source_commit="b" * 40, + head_commit="c" * 40, + suite_hash="d" * 64, + build_id=42, + samples=5, + warmups=1, + pairs=[dict(base=sample(1), candidate=sample(1.3)) for _ in range(5)], + ) + + +def test_consistent_slowdown_is_advisory_regression(report): + reporting.validate(report, 42, "c" * 40, "b" * 40, "a" * 40) + rows = reporting.comparisons(report) + assert all( + row["status"] == "regression" and row["change_pct"] == pytest.approx(30) for row in rows + ) + body = reporting.render([report], "c" * 40, 42) + assert "20 regression signals" in body + assert "incomplete/unavailable" in body # missing platforms never read as green + + +def test_noisy_slowdown_and_submillisecond_change_are_not_regressions(report): + for pair in report["pairs"][:2]: + pair["candidate"]["scenarios"]["select"]["wall_ms"] = 8 + report["pairs"][0]["candidate"]["scenarios"]["connect"]["wall_ms"] = 1000 + assert reporting.comparisons(report)[1]["status"] == "noisy" + for pair in report["pairs"]: + pair["base"]["scenarios"]["insert"]["wall_ms"] = 0.1 + pair["candidate"]["scenarios"]["insert"]["wall_ms"] = 0.2 + assert reporting.comparisons(report)[2]["status"] == "ok" + + +def test_phase_call_changes_are_reported_without_summing_nested_totals(report): + for pair in report["pairs"]: + pair["candidate"]["scenarios"]["select"]["cpp"] = { + "ddbc::query": dict(calls=2, total_us=5000, min_us=2000, max_us=3000), + } + row = reporting.comparisons(report)[1] + assert row["counts"] == ["ddbc::query (1 -> 2 calls)"] + assert row["phases"] == [(4.0, "ddbc::query")] + + +@pytest.mark.parametrize("case", ["nan", "missing", "environment", "work", "few", "prefix", "zero"]) +def test_reject_invalid_or_incomparable_data(report, case): + sample = report["pairs"][0]["candidate"] + if case == "nan": + sample["scenarios"]["select"]["wall_ms"] = float("nan") + elif case == "missing": + del sample["scenarios"]["select"] + elif case == "environment": + sample["environment"]["sql_version"] = "other" + elif case == "work": + sample["scenarios"]["select"]["work"] = "Rows: 200" + elif case == "few": + report["pairs"].pop() + elif case == "zero": + sample["scenarios"]["select"]["wall_ms"] = 0 + elif case == "prefix": + sample["scenarios"]["select"]["cpp"] = { + "not-native": sample["scenarios"]["select"]["cpp"]["ddbc::query"] + } + with pytest.raises(ValueError): + reporting.validate(report) + + +def test_reject_wrong_commit_and_preserve_incomplete_status(report): + with pytest.raises(ValueError, match="provenance"): + reporting.validate(report, head="e" * 40) + report["status"] = "incomplete" + report["pairs"] = [] + reporting.validate(report) + assert "No regression verdict" in reporting.render([report], "c" * 40, 42) + + +def zip_data(entries): + out = io.BytesIO() + with zipfile.ZipFile(out, "w") as archive: + for name, data in entries: + archive.writestr(name, data) + return out.getvalue() + + +def test_artifact_read_never_extracts_paths(report): + raw = json.dumps(report) + assert ( + publisher.artifact_report(zip_data([("profiler-Linux-SQL2022/report.json", raw)])) == report + ) + for entries in [ + [("../report.json", raw)], + [("/report.json", raw)], + [("a/report.json", raw), ("b/report.json", raw)], + [("logs.txt", "no report")], + ]: + with pytest.raises(ValueError): + publisher.artifact_report(zip_data(entries)) + + +def test_untrusted_labels_cannot_inject_links_mentions_or_markdown(): + assert reporting.escape("[click](https://example.com) @everyone | `code`") == ( + "[click](https://example.com) @everyone | `code`" + ) + assert not publisher.allowed_url("https://example.com/artifact") + assert not publisher.allowed_url("http://dev.azure.com/artifact") + assert not publisher.allowed_url("https://dev.azure.com@evil.example/artifact") + assert publisher.allowed_url("https://dev.azure.com/sqlclientdrivers/public/") + assert publisher.allowed_url( + "https://artprodcus3.artifacts.visualstudio.com/A1/_apis/artifact/" + ) + assert not publisher.allowed_url("https://artifacts.visualstudio.com.evil.example/artifact") + + +def test_build_selection_requires_exact_pr_head(): + build = dict( + id=42, + definition={"id": 2128}, + repository={"id": "microsoft/mssql-python"}, + sourceBranch="refs/pull/123/merge", + triggerInfo={"pr.number": "123", "pr.sourceSha": "c" * 40}, + ) + assert publisher.find_build([build], 123, "c" * 40) is build + for key in ("pr.number", "pr.sourceSha"): + bad = copy.deepcopy(build) + bad["triggerInfo"][key] = "different" + assert publisher.find_build([bad], 123, "c" * 40) is None + + +def test_publisher_does_not_post_stale_head(monkeypatch): + calls = [] + + def api(path, **kwargs): + calls.append((path, kwargs)) + return {"state": "open", "head": {"sha": "new-head"}} + + monkeypatch.setattr(publisher, "github", api) + publisher.publish(123, "old-head", "anything") + assert len(calls) == 1 and calls[0][1] == {} + + +def test_revisions_use_exact_first_parent(monkeypatch): + calls = [] + + def git(*args): + calls.append(args) + return "b" * 40 if len(calls) == 1 else "a" * 40 + + monkeypatch.setattr(controller, "git", git) + assert controller.resolve_revisions(None, "HEAD") == ("a" * 40, "b" * 40) + assert calls[1][-1] == "b" * 40 + "^1^{commit}" + + +def test_build_check_rejects_foreign_provider_and_enabled_recording(tmp_path, monkeypatch): + native = SimpleNamespace( + __file__=str(tmp_path / "binding.so"), profiling=SimpleNamespace(is_enabled=lambda: False) + ) + timer = SimpleNamespace(is_enabled=lambda: False) + package = SimpleNamespace( + __file__=str(tmp_path / "mssql_python/__init__.py"), ddbc_bindings=native, perf_timer=timer + ) + provider = SimpleNamespace(__file__=str(tmp_path / "mssql_python_odbc/__init__.py")) + monkeypatch.setitem(sys.modules, "mssql_python", package) + monkeypatch.setitem(sys.modules, "mssql_python_odbc", provider) + monkeypatch.setattr(sys, "path", sys.path[:]) + controller.check_build(tmp_path, True) + provider.__file__ = str(tmp_path.parent / "candidate-provider/__init__.py") + with pytest.raises(RuntimeError, match="outside"): + controller.check_build(tmp_path, True) + provider.__file__ = str(tmp_path / "provider/__init__.py") + timer.is_enabled = lambda: True + with pytest.raises(RuntimeError, match="recording OFF"): + controller.check_build(tmp_path, True) + + +def test_head_moving_while_listing_comments_prevents_publish(monkeypatch): + calls = [] + reads = 0 + + def api(path, **kwargs): + nonlocal reads + calls.append((path, kwargs)) + assert not kwargs, "No write allowed after head moved" + if path.startswith("pulls/"): + reads += 1 + return {"state": "open", "head": {"sha": "old" if reads == 1 else "new"}} + return [] + + monkeypatch.setattr(publisher, "github", api) + publisher.publish(1, "old", "data") + assert reads == 2 and len(calls) == 3 + + +@pytest.mark.parametrize("corrupt", [False, True]) +def test_publisher_renders_validated_artifact_and_marks_missing_legs(report, monkeypatch, corrupt): + posted = [] + build = dict( + id=42, + status="completed", + definition={"id": 2128}, + repository={"id": "microsoft/mssql-python"}, + sourceBranch="refs/pull/123/merge", + sourceVersion="b" * 40, + triggerInfo={"pr.number": "123", "pr.sourceSha": "c" * 40}, + ) + monkeypatch.setattr(publisher, "publish", lambda number, head, body: posted.append(body)) + monkeypatch.setattr( + publisher, + "github", + lambda path: ( + {"state": "open", "head": {"sha": "c" * 40}} + if path.startswith("pulls/") + else {"parents": [{"sha": "a" * 40}, {"sha": "c" * 40}]} + ), + ) + monkeypatch.setattr( + publisher, + "api", + lambda url: ( + { + "value": [ + { + "name": "profiler-Linux-SQL2022", + "resource": {"downloadUrl": "https://dev.azure.com/artifact"}, + } + ] + } + if "/artifacts?" in url + else {"value": [build]} + ), + ) + raw = b"invalid ZIP" if corrupt else zip_data([("report.json", json.dumps(report))]) + monkeypatch.setattr(publisher, "fetch", lambda *args, **kwargs: raw) + publisher.run(123, "c" * 40, 1) + assert len(posted) == 2 + assert posted[0].startswith(reporting.MARKER) + assert "Windows-SQL2022: incomplete/unavailable" in posted[1] + if corrupt: + assert reporting.escape("Linux-SQL2022 (invalid artifact)") in posted[1] + assert "regression signals" not in posted[1] + else: + assert "20 regression signals" in posted[1] + + +def test_artifact_symlink_and_oversized_json_are_rejected(): + symlink = zipfile.ZipInfo("report.json") + symlink.create_system = 3 + symlink.external_attr = 0o120777 << 16 + with pytest.raises(ValueError, match="Invalid report"): + publisher.artifact_report(zip_data([(symlink, "{}")])) + with pytest.raises(ValueError, match="Invalid report"): + publisher.artifact_report(zip_data([("report.json", " " * (reporting.MAX_BYTES + 1))])) + + +@pytest.mark.parametrize("environment", [{"os": "Windows"}, {"sql_version": "17.0"}]) +def test_report_leg_must_match_measured_environment(report, environment): + for pair in report["pairs"]: + for sample in pair.values(): + sample["environment"].update(environment) + with pytest.raises(ValueError, match="leg"): + reporting.validate(report) + + +def test_ci_reuses_profiling_builds_without_changing_release_defaults(): + pipeline = (ROOT / "eng/pipelines/pr-validation-pipeline.yml").read_text(encoding="utf-8") + assert "benchmarks/perf-benchmarking.py" not in pipeline + assert pipeline.count("python benchmarks/profiler_ci.py --reuse-candidate") == 3 + assert "profilerBuild: '0'" in pipeline # LocalDB still exercises the normal build + assert "ddbc_bindings-profiling-SQL2022" in pipeline + assert "ddbc_bindings-profiling-SQL2025" in pipeline + assert "ENABLE_PROFILING=1 ./build.sh" in pipeline + for release in (ROOT / "OneBranchPipelines").rglob("*.yml"): + assert "ENABLE_PROFILING" not in release.read_text(encoding="utf-8") + windows = pipeline.split("- job: pytestonwindows\n", 1)[1].split("\n- job:", 1)[0] + assert "profilerCheck: 'off'" in windows.split("LocalDB_Python314:", 1)[1].split("steps:", 1)[0] + assert windows.split("steps:", 1)[0].count("profilerCheck: 'on'") == 2 + linux = pipeline.split("- job: PytestOnLinux\n", 1)[1].split("\n- job:", 1)[0] + benchmark = linux.split("# Run performance benchmarks on Ubuntu", 1)[1] + assert '-e BUILD_BUILDID="$(Build.BuildId)"' in benchmark + assert "git config --global --add safe.directory /workspace" in benchmark + assert "apt-get install -y --reinstall libodbcinst2" in benchmark + assert "libodbc1 " not in benchmark and "odbcinst1debian2" not in benchmark + + +def test_comment_workflow_executes_only_trusted_base_code(): + workflow = (ROOT / ".github/workflows/pr-profiler-report.yml").read_text(encoding="utf-8") + assert "pull_request_target:" in workflow + assert "ref: ${{ github.event.pull_request.base.sha }}" in workflow + assert "persist-credentials: false" in workflow + assert ( + "head.ref" not in workflow + and "head.sha }}" not in workflow.split("ref:", 1)[1].split("persist", 1)[0] + ) From 665e659f298e81362c73563a9ccd326117c5e98a Mon Sep 17 00:00:00 2001 From: Gaurav Sharma Date: Fri, 11 Sep 2026 10:46:54 +0530 Subject: [PATCH 02/18] FIX: Repair hosted profiler setup and CI wait budgets Restore libodbcinst2 before installing the ODBC driver so its post-install script can run after pytest cleanup. Log and checkpoint each benchmark scenario, capture periodic worker stacks, and replace the four-minute whole-suite limit with a ten-minute worker limit under a shared 35-minute budget. Partial workers remain incomplete rather than producing a regression verdict; the hosted macOS stall still requires confirmation on the next run. Match coverage discovery to the exact PR head and wait through queued ADO coverage jobs with bounded requests, retries and completion grace. Accept valid coverage artifacts even when an independent matrix leg fails. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/workflows/pr-code-coverage.yml | 215 +++++++++++-------- benchmarks/README.md | 7 + benchmarks/profiler_ci.py | 72 +++++-- eng/pipelines/pr-validation-pipeline.yml | 2 +- tests/test_036_profiler_ci.py | 70 +++++++ tests/test_pr_code_coverage_workflow.py | 252 +++++++++++++++++++++++ 6 files changed, 511 insertions(+), 107 deletions(-) create mode 100644 tests/test_pr_code_coverage_workflow.py diff --git a/.github/workflows/pr-code-coverage.yml b/.github/workflows/pr-code-coverage.yml index 66c7d110b..096861a64 100644 --- a/.github/workflows/pr-code-coverage.yml +++ b/.github/workflows/pr-code-coverage.yml @@ -11,6 +11,7 @@ permissions: jobs: coverage-report: runs-on: ubuntu-latest + timeout-minutes: 145 permissions: pull-requests: write contents: read @@ -34,30 +35,44 @@ jobs: git show-ref --verify refs/remotes/origin/main || echo "Warning: origin/main not found" - name: Wait for ADO build to start + env: + PR_NUMBER: ${{ github.event.pull_request.number }} + PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }} run: | - PR_NUMBER=${{ github.event.pull_request.number }} - API_URL="https://dev.azure.com/sqlclientdrivers/public/_apis/build/builds?definitions=2128&queryOrder=queueTimeDescending&%24top=10&api-version=7.1-preview.7" + PR_BRANCH="refs/pull/$PR_NUMBER/merge" + API_URL="https://dev.azure.com/sqlclientdrivers/public/_apis/build/builds?definitions=2128&branchName=refs%2Fpull%2F${PR_NUMBER}%2Fmerge&queryOrder=queueTimeDescending&%24top=100&api-version=7.1-preview.7" + DEADLINE=$((SECONDS + 15 * 60)) + API_FAILURES=0 + BUILD_ID="" - echo "Waiting for Azure DevOps build to start for PR #$PR_NUMBER ..." + echo "Waiting up to 15 minutes for Azure DevOps build for PR #$PR_NUMBER at $PR_HEAD_SHA..." - for i in {1..30}; do - echo "Attempt $i/30: Checking if build has started..." - - # Fetch API response with error handling - API_RESPONSE=$(curl -s "$API_URL") - - # Check if response is valid JSON - if ! echo "$API_RESPONSE" | jq . >/dev/null 2>&1; then - echo "❌ Invalid JSON response from Azure DevOps API" - echo "Response received: $API_RESPONSE" - echo "This usually indicates the Azure DevOps pipeline has failed or API is unavailable" - exit 1 + while (( SECONDS < DEADLINE )); do + REQUEST_TIMEOUT=$((DEADLINE - SECONDS)) + if (( REQUEST_TIMEOUT <= 0 )); then break; fi + if (( REQUEST_TIMEOUT > 30 )); then REQUEST_TIMEOUT=30; fi + if API_RESPONSE=$(curl --fail --silent --show-error --connect-timeout 10 --max-time "$REQUEST_TIMEOUT" "$API_URL") && + jq -e 'type == "object" and (.value | type == "array")' <<< "$API_RESPONSE" >/dev/null 2>&1; then + API_FAILURES=0 + else + API_FAILURES=$((API_FAILURES + 1)) + echo "⚠️ Build API HTTP/JSON error ($API_FAILURES/5)" + if (( API_FAILURES >= 5 )); then + echo "❌ Azure DevOps build API unavailable after 5 consecutive failures" + exit 1 + fi + API_RESPONSE='{"value":[]}' fi - - # Parse build info safely - BUILD_INFO=$(echo "$API_RESPONSE" | jq -c --arg PR "$PR_NUMBER" '[.value[]? | select(.triggerInfo["pr.number"]?==$PR)] | .[0] // empty' 2>/dev/null) - - if [[ -n "$BUILD_INFO" && "$BUILD_INFO" != "null" && "$BUILD_INFO" != "empty" ]]; then + + # The merge ref is shared across revisions; match the actual PR head as well. + BUILD_INFO=$(jq -c --arg PR "$PR_NUMBER" --arg SHA "$PR_HEAD_SHA" --arg BRANCH "$PR_BRANCH" ' + [.value[]? | select( + .definition.id == 2128 and .sourceBranch == $BRANCH and + (.triggerInfo["pr.number"] | tostring) == $PR and + .triggerInfo["pr.sourceSha"] == $SHA + )] | .[0] // empty' <<< "$API_RESPONSE") + + if [[ -n "$BUILD_INFO" ]]; then STATUS=$(echo "$BUILD_INFO" | jq -r '.status // "unknown"') RESULT=$(echo "$BUILD_INFO" | jq -r '.result // "unknown"') BUILD_ID=$(echo "$BUILD_INFO" | jq -r '.id // "unknown"') @@ -74,83 +89,103 @@ jobs: echo "✅ Found build: ID=$BUILD_ID, Status=$STATUS, Result=$RESULT" echo "🔗 Build URL: $WEB_URL" - echo "ADO_URL=$WEB_URL" >> $GITHUB_ENV - echo "BUILD_ID=$BUILD_ID" >> $GITHUB_ENV - - # Check if build has failed early - if [[ "$STATUS" == "completed" && "$RESULT" == "failed" ]]; then - echo "❌ Azure DevOps build $BUILD_ID failed early" - echo "This coverage workflow cannot proceed when the main build fails." - exit 1 - fi - - echo "🚀 Build has started, proceeding to poll for coverage artifacts..." - break - else - echo "⏳ No build found for PR #$PR_NUMBER yet... (attempt $i/30)" - fi + echo "ADO_URL=$WEB_URL" >> "$GITHUB_ENV" + echo "BUILD_ID=$BUILD_ID" >> "$GITHUB_ENV" - if [[ $i -eq 30 ]]; then - echo "❌ Timeout: No build found for PR #$PR_NUMBER after 30 attempts" - echo "This may indicate the Azure DevOps pipeline was not triggered" - exit 1 + # A failed matrix leg does not invalidate a successful coverage artifact. + echo "🚀 Build found, proceeding to poll for coverage artifacts..." + break fi - sleep 10 + echo "⏳ No matching build found for PR #$PR_NUMBER at $PR_HEAD_SHA yet..." + SLEEP_SECONDS=$((DEADLINE - SECONDS)) + if (( SLEEP_SECONDS > 30 )); then SLEEP_SECONDS=30; fi + if (( SLEEP_SECONDS > 0 )); then sleep "$SLEEP_SECONDS"; fi done + if [[ -z "$BUILD_ID" ]]; then + echo "❌ Timeout: No build found for PR #$PR_NUMBER at $PR_HEAD_SHA within 15 minutes" + exit 1 + fi + - name: Download and parse coverage report run: | - BUILD_ID=${{ env.BUILD_ID }} + BUILD_URL="https://dev.azure.com/SqlClientDrivers/public/_apis/build/builds/$BUILD_ID?api-version=7.1-preview.7" ARTIFACTS_URL="https://dev.azure.com/SqlClientDrivers/public/_apis/build/builds/$BUILD_ID/artifacts?api-version=7.1-preview.5" - - echo "📥 Polling for coverage artifacts for build $BUILD_ID..." - - # Poll for coverage artifacts with retry logic + # Coverage may start after other queued jobs, each with a 90-minute timeout. + DEADLINE=$((SECONDS + 120 * 60)) + COMPLETED_AT=-1 + ARTIFACT_FAILURES=0 + BUILD_FAILURES=0 COVERAGE_ARTIFACT="" - for i in {1..60}; do - echo "Attempt $i/60: Checking for coverage artifacts..." - - # Fetch artifacts with error handling - ARTIFACTS_RESPONSE=$(curl -s "$ARTIFACTS_URL") - - # Check if response is valid JSON - if ! echo "$ARTIFACTS_RESPONSE" | jq . >/dev/null 2>&1; then - echo "⚠️ Invalid JSON response from artifacts API (attempt $i/60)" - if [[ $i -eq 60 ]]; then - echo "❌ Persistent API issues after 60 attempts" - echo "Response received: $ARTIFACTS_RESPONSE" + echo "📥 Waiting up to 120 minutes for coverage artifacts for build $BUILD_ID..." + + while (( SECONDS < DEADLINE )); do + REQUEST_TIMEOUT=$((DEADLINE - SECONDS)) + if (( REQUEST_TIMEOUT <= 0 )); then break; fi + if (( REQUEST_TIMEOUT > 30 )); then REQUEST_TIMEOUT=30; fi + if ARTIFACTS_RESPONSE=$(curl --fail --silent --show-error --connect-timeout 10 --max-time "$REQUEST_TIMEOUT" "$ARTIFACTS_URL") && + jq -e 'type == "object" and (.value | type == "array")' <<< "$ARTIFACTS_RESPONSE" >/dev/null 2>&1; then + ARTIFACT_FAILURES=0 + COVERAGE_ARTIFACT=$(jq -r ' + [.value[]? | select(.name | test("Code Coverage Report")) | + .resource.downloadUrl | select(type == "string" and length > 0)] | + .[0] // empty' <<< "$ARTIFACTS_RESPONSE") + if [[ -n "$COVERAGE_ARTIFACT" ]]; then + echo "✅ Found coverage artifact!" + break + fi + else + ARTIFACT_FAILURES=$((ARTIFACT_FAILURES + 1)) + echo "⚠️ Artifacts API HTTP/JSON error ($ARTIFACT_FAILURES/5)" + if (( ARTIFACT_FAILURES >= 5 )); then + echo "❌ Azure DevOps artifacts API unavailable after 5 consecutive failures" exit 1 fi - sleep 30 - continue fi - - # Show available artifacts for debugging - echo "🔍 Available artifacts:" - echo "$ARTIFACTS_RESPONSE" | jq -r '.value[]?.name // "No artifacts found"' - - # Find the coverage report artifact - COVERAGE_ARTIFACT=$(echo "$ARTIFACTS_RESPONSE" | jq -r '.value[]? | select(.name | test("Code Coverage Report")) | .resource.downloadUrl // empty' 2>/dev/null) - - if [[ -n "$COVERAGE_ARTIFACT" && "$COVERAGE_ARTIFACT" != "null" && "$COVERAGE_ARTIFACT" != "empty" ]]; then - echo "✅ Found coverage artifact on attempt $i!" - break + + # Inspect lifecycle, not aggregate result: independent matrix jobs can fail. + REQUEST_TIMEOUT=$((DEADLINE - SECONDS)) + if (( REQUEST_TIMEOUT <= 0 )); then break; fi + if (( REQUEST_TIMEOUT > 30 )); then REQUEST_TIMEOUT=30; fi + if BUILD_RESPONSE=$(curl --fail --silent --show-error --connect-timeout 10 --max-time "$REQUEST_TIMEOUT" "$BUILD_URL") && + jq -e --arg ID "$BUILD_ID" '(.id | tostring) == $ID and + (.status | type == "string")' <<< "$BUILD_RESPONSE" >/dev/null 2>&1; then + BUILD_FAILURES=0 + STATUS=$(jq -r '.status' <<< "$BUILD_RESPONSE") + RESULT=$(jq -r '.result // "unknown"' <<< "$BUILD_RESPONSE") + if [[ "$STATUS" == "completed" ]] && (( COMPLETED_AT < 0 )); then + COMPLETED_AT=$SECONDS + echo "Build completed ($RESULT); allowing 2 minutes for artifact propagation..." + fi else - echo "⏳ Coverage report not ready yet (attempt $i/60)..." - if [[ $i -eq 60 ]]; then - echo "❌ Timeout: Coverage report artifact not found after 60 attempts" - echo "Available artifacts:" - echo "$ARTIFACTS_RESPONSE" | jq -r '.value[]?.name // "No artifacts found"' + BUILD_FAILURES=$((BUILD_FAILURES + 1)) + echo "⚠️ Build lifecycle API HTTP/JSON error ($BUILD_FAILURES/5)" + if (( BUILD_FAILURES >= 5 )); then + echo "❌ Azure DevOps build lifecycle API unavailable after 5 consecutive failures" exit 1 fi - sleep 30 fi + + if (( COMPLETED_AT >= 0 && SECONDS - COMPLETED_AT >= 120 )); then + echo "❌ Build $BUILD_ID completed but coverage artifact is still unavailable after propagation grace" + exit 1 + fi + echo "⏳ Coverage report not ready yet..." + SLEEP_SECONDS=$((DEADLINE - SECONDS)) + if (( SLEEP_SECONDS > 30 )); then SLEEP_SECONDS=30; fi + if (( SLEEP_SECONDS > 0 )); then sleep "$SLEEP_SECONDS"; fi done - + + if [[ -z "$COVERAGE_ARTIFACT" ]]; then + echo "❌ Timeout: Coverage report artifact not found within 120 minutes" + exit 1 + fi + if [[ -n "$COVERAGE_ARTIFACT" && "$COVERAGE_ARTIFACT" != "null" && "$COVERAGE_ARTIFACT" != "empty" ]]; then echo "📊 Downloading coverage report..." - if ! curl -L "$COVERAGE_ARTIFACT" -o coverage-report.zip --fail --silent; then + if ! curl -L "$COVERAGE_ARTIFACT" -o coverage-report.zip --fail --silent --show-error \ + --connect-timeout 10 --max-time 60 --retry 2 --retry-delay 5 --retry-max-time 180; then echo "❌ Failed to download coverage report from Azure DevOps" echo "This indicates the coverage artifacts may not be available or accessible" exit 1 @@ -252,15 +287,18 @@ jobs: echo "📥 Fetching artifacts for build $BUILD_ID to find coverage files..." - # Fetch artifacts with error handling - ARTIFACTS_RESPONSE=$(curl -s "$ARTIFACTS_URL") - - # Check if response is valid JSON - if ! echo "$ARTIFACTS_RESPONSE" | jq . >/dev/null 2>&1; then - echo "❌ Invalid JSON response from artifacts API" - echo "Response received: $ARTIFACTS_RESPONSE" - exit 1 - fi + for i in {1..5}; do + if ARTIFACTS_RESPONSE=$(curl --fail --silent --show-error --connect-timeout 10 --max-time 30 "$ARTIFACTS_URL") && + jq -e 'type == "object" and (.value | type == "array")' <<< "$ARTIFACTS_RESPONSE" >/dev/null 2>&1; then + break + fi + echo "⚠️ Artifacts API HTTP/JSON error ($i/5)" + if [[ $i -eq 5 ]]; then + echo "❌ Azure DevOps artifacts API unavailable after 5 attempts" + exit 1 + fi + sleep 5 + done echo "🔍 Available artifacts:" echo "$ARTIFACTS_RESPONSE" | jq -r '.value[]?.name // "No artifacts found"' @@ -270,7 +308,8 @@ jobs: if [[ -n "$COVERAGE_XML_ARTIFACT" && "$COVERAGE_XML_ARTIFACT" != "null" && "$COVERAGE_XML_ARTIFACT" != "empty" ]]; then echo "📊 Downloading coverage artifact from: $COVERAGE_XML_ARTIFACT" - if ! curl -L "$COVERAGE_XML_ARTIFACT" -o coverage-artifacts.zip --fail --silent; then + if ! curl -L "$COVERAGE_XML_ARTIFACT" -o coverage-artifacts.zip --fail --silent --show-error \ + --connect-timeout 10 --max-time 60 --retry 2 --retry-delay 5 --retry-max-time 180; then echo "❌ Failed to download coverage artifacts" exit 1 fi diff --git a/benchmarks/README.md b/benchmarks/README.md index 5ef5284ba..85a1a736a 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -41,6 +41,13 @@ fresh interpreter. Five measured pairs follow one discarded warmup pair, alternating base/PR order. A local subset is available through `--scenarios`; it is not accepted as a complete CI report. +Each worker has a ten-minute limit for the entire workload suite, with a shared +35-minute build/measurement budget inside the CI step's 40-minute limit. Worker +logs identify each starting/completed scenario and emit stack traces every minute. +Per-worker JSON checkpoints retain finished scenarios and identify the active one +if the process fails or is killed. Partial workers never count as measured pairs; +incomplete reports do not produce a regression verdict. + The comparison is **advisory**, not a new merge gate. A regression signal requires over 20% paired-median slowdown, at least 1 ms additional median wall time, and at least 80% of pairs exceeding the relative threshold. Disagreement is reported as diff --git a/benchmarks/profiler_ci.py b/benchmarks/profiler_ci.py index 2f0e28fe0..64724a61a 100644 --- a/benchmarks/profiler_ci.py +++ b/benchmarks/profiler_ci.py @@ -2,6 +2,7 @@ import argparse import contextlib +import faulthandler import hashlib import importlib.util import io @@ -14,10 +15,14 @@ import sys import tarfile import tempfile +import time ROOT = Path(__file__).resolve().parents[1] SHA = re.compile(r"[0-9a-f]{40}") LEGS = ("Windows-SQL2022", "Windows-SQL2025", "macOS-SQL2022", "macOS-SQL2025", "Linux-SQL2022") +# Leave five minutes of the CI step's 40-minute budget for artifact publication. +BENCHMARK_TIMEOUT = 35 * 60 +WORKER_TIMEOUT = 10 * 60 def git(*args): @@ -42,7 +47,7 @@ def checkout(revision, path): tar.extractall(path, filter="data") -def build(path, log): +def build(path, log, timeout=900): env = dict(os.environ, ENABLE_PROFILING="1") # build scripts find Python via PATH; keep the controller's interpreter. env["PATH"] = str(Path(sys.executable).parent) + os.pathsep + env["PATH"] @@ -54,7 +59,7 @@ def build(path, log): env=env, stdout=output, stderr=subprocess.STDOUT, - timeout=900, + timeout=timeout, check=True, ) @@ -104,8 +109,30 @@ def worker(args): core.SCENARIOS = cases # The runner owns enable/disable/cleanup, just as in the documented CLI. with core.Profiler() as profiler: - with contextlib.redirect_stdout(io.StringIO()): - results = profiler.run(*chosen) + output = {} + for name in chosen: + print(f"Starting scenario: {name}", flush=True) + args.output.write_text( + json.dumps(dict(status="running", active_scenario=name, scenarios=output)), + encoding="utf-8", + ) + # Keep phase tables out of logs, but never hide which workload stalled. + with contextlib.redirect_stdout(io.StringIO()): + result = profiler.run(name)[0] + if result["cpp"] is None or result["py"] is None: + raise RuntimeError(f"Scenario {name} was skipped") + if not result["cpp"]: + raise RuntimeError(f"Scenario {name} has no native samples") + output[name] = {key: result[key] for key in ("wall_ms", "cpp", "py")} + output[name]["work"] = result.get("detail", "Connection: 1").split(" (")[0] + args.output.write_text( + json.dumps( + dict(status="running", active_scenario=None, scenarios=output), allow_nan=False + ), + encoding="utf-8", + ) + print(f"Completed scenario: {name} ({result['wall_ms']:.3f} ms)", flush=True) + print("Collecting server metadata", flush=True) profiler._ensure_connection() with profiler._conn.cursor() as cursor: cursor.execute("SELECT CAST(SERVERPROPERTY('ProductVersion') AS VARCHAR(80))") @@ -116,24 +143,16 @@ def worker(args): python=platform.python_version(), sql_version=sql_version, ) - output = {} - for name, result in zip(chosen, results): - if result["cpp"] is None or result["py"] is None: - raise RuntimeError(f"Scenario {name} was skipped") - if not result["cpp"]: - raise RuntimeError(f"Scenario {name} has no native samples") - # Only generated workload counts and instrumentation, never query data. - output[name] = {key: result[key] for key in ("wall_ms", "cpp", "py")} - output[name]["work"] = result.get("detail", "Connection: 1").split(" (")[0] args.output.write_text( json.dumps(dict(environment=environment, scenarios=output), allow_nan=False), encoding="utf-8", ) -def measure(path, output, scenarios): +def measure(path, output, scenarios, timeout=WORKER_TIMEOUT): command = [ sys.executable, + "-u", str(Path(__file__).resolve()), "--worker", "--source-root", @@ -143,11 +162,19 @@ def measure(path, output, scenarios): ] if scenarios: command += ["--scenarios", *scenarios] + output.unlink(missing_ok=True) with output.with_suffix(".log").open("w", encoding="utf-8") as log: - subprocess.run(command, stdout=log, stderr=subprocess.STDOUT, timeout=240, check=True) + subprocess.run(command, stdout=log, stderr=subprocess.STDOUT, timeout=timeout, check=True) return json.loads(output.read_text(encoding="utf-8")) +def remaining(deadline, limit): + seconds = deadline - time.monotonic() + if seconds <= 0: + raise TimeoutError("Profiler CI exhausted its overall build/measurement budget") + return min(seconds, limit) + + def run(args): base, candidate = resolve_revisions(args.base, args.candidate) args.output.mkdir(parents=True, exist_ok=True) @@ -178,6 +205,7 @@ def run(args): pairs=[], ) report_path.write_text(json.dumps(report), encoding="utf-8") + deadline = time.monotonic() + BENCHMARK_TIMEOUT # CI reuses the profiling build already exercised by pytest. The base always # has its own checkout and process. Local runs can build both sides instead. with tempfile.TemporaryDirectory(prefix="profiler-ci-") as directory: @@ -190,12 +218,12 @@ def run(args): subprocess.run( [sys.executable, str(Path(__file__).resolve()), "--check-build", "on"], check=True, - timeout=60, + timeout=remaining(deadline, 60), ) continue checkout(revision, paths[side]) print(f"Building profiling {side}: {revision}", flush=True) - build(paths[side], args.output / f"build-{side}.log") + build(paths[side], args.output / f"build-{side}.log", remaining(deadline, 900)) for sample in range(args.warmups + args.samples): pair = {} order = ("base", "candidate") if sample % 2 == 0 else ("candidate", "base") @@ -205,6 +233,7 @@ def run(args): paths[side], args.output / f"{side}-{sample}.json", args.scenarios, + remaining(deadline, WORKER_TIMEOUT), ) if pair["base"]["environment"] != pair["candidate"]["environment"]: raise RuntimeError("Base and candidate environments differ") @@ -243,7 +272,14 @@ def main(): elif args.worker: if args.source_root is None or args.output is None: parser.error("--worker requires --source-root and --output") - worker(args) + # Dumps contain stack locations, not locals or connection strings. The + # parent still kills/reaps the worker at its deadline if it cannot finish. + faulthandler.enable() + faulthandler.dump_traceback_later(60, repeat=True) + try: + worker(args) + finally: + faulthandler.cancel_dump_traceback_later() else: if ( not args.output diff --git a/eng/pipelines/pr-validation-pipeline.yml b/eng/pipelines/pr-validation-pipeline.yml index c1c270475..7ce613e1e 100644 --- a/eng/pipelines/pr-validation-pipeline.yml +++ b/eng/pipelines/pr-validation-pipeline.yml @@ -999,8 +999,8 @@ jobs: # The Microsoft repository was configured by the earlier build step. # Restore the ODBC headers and library link removed before pytest. apt-get update -qq - ACCEPT_EULA=Y apt-get install -y --no-install-recommends git unixodbc unixodbc-dev libodbc2 libodbcinst2 odbcinst msodbcsql18 apt-get install -y --reinstall libodbcinst2 + ACCEPT_EULA=Y apt-get install -y --no-install-recommends git unixodbc unixodbc-dev libodbc2 libodbcinst2 odbcinst msodbcsql18 git config --global --add safe.directory /workspace odbcinst -q -d -n 'ODBC Driver 18 for SQL Server' python benchmarks/profiler_ci.py --reuse-candidate --leg Linux-SQL2022 --output profiler-results diff --git a/tests/test_036_profiler_ci.py b/tests/test_036_profiler_ci.py index a7d7f0191..9c0b6b25d 100644 --- a/tests/test_036_profiler_ci.py +++ b/tests/test_036_profiler_ci.py @@ -5,8 +5,10 @@ import io import json from pathlib import Path +import subprocess import sys from types import SimpleNamespace +from unittest.mock import MagicMock import zipfile import pytest @@ -200,6 +202,71 @@ def git(*args): assert calls[1][-1] == "b" * 40 + "^1^{commit}" +@pytest.mark.parametrize("fail", [False, True]) +def test_worker_checkpoints_completed_and_active_scenarios(tmp_path, monkeypatch, capsys, fail): + output = tmp_path / "base-0.json" + result = dict(wall_ms=10.0, cpp={"ddbc::run": {}}, py={}, detail="Rows: 10") + profiler = MagicMock() + profiler.__enter__.return_value = profiler + profiler._conn.cursor.return_value.__enter__.return_value.fetchone.return_value = ("16.0",) + + def run(name): + partial = json.loads(output.read_text()) + assert partial["active_scenario"] == name + if name == "second": + assert set(partial["scenarios"]) == {"first"} + if fail: + raise RuntimeError("workload failure") + return [result] + + profiler.run.side_effect = run + core = SimpleNamespace(Profiler=lambda: profiler) + workloads = SimpleNamespace(registry=lambda: {"first": None, "second": None}) + monkeypatch.setattr(controller, "check_build", lambda *a, **kw: None) + monkeypatch.setattr(controller, "load_suite", lambda: (core, workloads)) + args = SimpleNamespace(source_root=tmp_path, scenarios=None, output=output) + if fail: + with pytest.raises(RuntimeError, match="workload failure"): + controller.worker(args) + assert json.loads(output.read_text())["active_scenario"] == "second" + else: + controller.worker(args) + final = json.loads(output.read_text()) + assert final["environment"]["sql_version"] == "16.0" + assert set(final["scenarios"]) == {"first", "second"} + assert "active_scenario" not in final + assert "Starting scenario: second" in capsys.readouterr().out + profiler.__exit__.assert_called_once() + + +def test_measure_timeout_retains_partial_results_and_log(tmp_path, monkeypatch): + output = tmp_path / "base-0.json" + output.write_text('{"stale": true}') + + def timeout(command, **kwargs): + assert not output.exists() + assert command[1] == "-u" + assert kwargs["timeout"] == 3 + output.write_text('{"status":"running","active_scenario":"fetchone"}') + kwargs["stdout"].write("Starting scenario: fetchone\n") + raise subprocess.TimeoutExpired(command, 3) + + monkeypatch.setattr(controller.subprocess, "run", timeout) + with pytest.raises(subprocess.TimeoutExpired): + controller.measure(tmp_path, output, ["fetchone"], timeout=3) + assert json.loads(output.read_text())["active_scenario"] == "fetchone" + assert "Starting scenario: fetchone" in output.with_suffix(".log").read_text() + + +def test_overall_budget_caps_build_and_worker_time(monkeypatch): + monkeypatch.setattr(controller.time, "monotonic", lambda: 100) + assert controller.remaining(110, controller.WORKER_TIMEOUT) == 10 + assert controller.remaining(1000, 60) == 60 + with pytest.raises(TimeoutError, match="overall"): + controller.remaining(100, controller.WORKER_TIMEOUT) + assert controller.BENCHMARK_TIMEOUT < 40 * 60 + + def test_build_check_rejects_foreign_provider_and_enabled_recording(tmp_path, monkeypatch): native = SimpleNamespace( __file__=str(tmp_path / "binding.so"), profiling=SimpleNamespace(is_enabled=lambda: False) @@ -328,6 +395,9 @@ def test_ci_reuses_profiling_builds_without_changing_release_defaults(): assert '-e BUILD_BUILDID="$(Build.BuildId)"' in benchmark assert "git config --global --add safe.directory /workspace" in benchmark assert "apt-get install -y --reinstall libodbcinst2" in benchmark + assert benchmark.index("apt-get install -y --reinstall libodbcinst2") < benchmark.index( + "ACCEPT_EULA=Y apt-get install" + ) assert "libodbc1 " not in benchmark and "odbcinst1debian2" not in benchmark diff --git a/tests/test_pr_code_coverage_workflow.py b/tests/test_pr_code_coverage_workflow.py new file mode 100644 index 000000000..3141aedd0 --- /dev/null +++ b/tests/test_pr_code_coverage_workflow.py @@ -0,0 +1,252 @@ +"""Run the workflow's polling shell with local curl fixtures and an accelerated clock.""" + +import json +import os +from pathlib import Path +import shutil +import subprocess +import sys +import textwrap + +import pytest + +WORKFLOW = Path(__file__).resolve().parents[1] / ".github/workflows/pr-code-coverage.yml" +pytestmark = pytest.mark.skipif( + not WORKFLOW.is_file() + or sys.platform == "win32" + or not shutil.which("bash") + or not shutil.which("jq"), + reason="requires a source checkout, Bash and jq (as on the coverage runner)", +) +SHA = "a" * 40 +ARTIFACT_URL = "https://dev.azure.com/SqlClientDrivers/public/coverage.zip" +EMPTY = {"value": []} +ARTIFACT = { + "value": [{"name": "Code Coverage Report_1", "resource": {"downloadUrl": ARTIFACT_URL}}] +} + + +def _build(build_id=174262, sha=SHA, pr="779", branch="refs/pull/779/merge", definition=2128): + return { + "id": build_id, + "definition": {"id": definition}, + "sourceBranch": branch, + "triggerInfo": {"pr.number": pr, "pr.sourceSha": sha}, + "status": "inProgress", + "_links": { + "web": { + "href": ( + "https://dev.azure.com/SqlClientDrivers/public/_build/results" + f"?buildId={build_id}" + ) + } + }, + } + + +def _script(step): + section = WORKFLOW.read_text(encoding="utf-8").split(f" - name: {step}\n", 1)[1] + section = section.split("\n - name:", 1)[0] + return textwrap.dedent(section.split(" run: |\n", 1)[1]) + + +def _run(tmp_path, script, fixtures): + for kind, responses in fixtures.items(): + (tmp_path / f"{kind}.count").write_text(str(len(responses)), encoding="utf-8") + (tmp_path / f"{kind}.next").write_text("0", encoding="utf-8") + for index, response in enumerate(responses): + code, body = response if isinstance(response, tuple) else (0, response) + body = body if isinstance(body, str) else json.dumps(body) + (tmp_path / f"{kind}.{index}.body").write_text(body, encoding="utf-8") + (tmp_path / f"{kind}.{index}.code").write_text(str(code), encoding="utf-8") + + prefix = r""" +SECONDS=0 +trap 'printf "%s\n" "$SECONDS" > "$FIXTURE_DIR/elapsed"' EXIT +sleep() { + printf "%s\n" "$1" >> "$FIXTURE_DIR/sleeps" + SECONDS=$((SECONDS + $1)) +} +curl() { + local url="${@: -1}" kind index count code + printf "%s\n" "$*" >> "$FIXTURE_DIR/requests" + case "$url" in + *"/artifacts?"*) kind=artifacts ;; + *"/builds?"*) kind=builds ;; + *"/builds/174262?"*) kind=build ;; + *) echo "Unexpected URL: $url" >&2; return 99 ;; + esac + if [[ ! -f "$FIXTURE_DIR/$kind.count" ]]; then + echo "Unexpected request: $kind" >&2 + return 99 + fi + index=$(< "$FIXTURE_DIR/$kind.next") + count=$(< "$FIXTURE_DIR/$kind.count") + printf "%s\n" "$((index + 1))" > "$FIXTURE_DIR/$kind.next" + if (( index >= count )); then index=$((count - 1)); fi + code=$(< "$FIXTURE_DIR/$kind.$index.code") + cat "$FIXTURE_DIR/$kind.$index.body" + return "$code" +} +""" + env = { + **os.environ, + "FIXTURE_DIR": str(tmp_path), + "GITHUB_ENV": str(tmp_path / "github-env"), + "PR_NUMBER": "779", + "PR_HEAD_SHA": SHA, + "BUILD_ID": "174262", + } + result = subprocess.run( + ["bash", "--noprofile", "--norc", "-eo", "pipefail", "-c", prefix + script], + env=env, + cwd=tmp_path, + capture_output=True, + text=True, + timeout=45, + ) + requests = (tmp_path / "requests").read_text(encoding="utf-8").splitlines() + for request in requests: + assert "--fail" in request + assert "--connect-timeout 10" in request + assert "--max-time " in request + timeout = int(request.split("--max-time ", 1)[1].split()[0]) + assert 0 < timeout <= 30 + assert "Unexpected request" not in result.stderr + assert "Unexpected URL" not in result.stderr + return result + + +def _poll(tmp_path, artifacts, builds): + script = _script("Download and parse coverage report") + # Only execute discovery; downloaded report contents are never executed by these tests. + script = script.split('\nif [[ -n "$COVERAGE_ARTIFACT" &&', 1)[0] + script += '\nprintf "COVERAGE_ARTIFACT=%s\\n" "$COVERAGE_ARTIFACT"\n' + return _run(tmp_path, script, {"artifacts": artifacts, "build": builds}) + + +def test_selects_exact_head_pr_branch_and_definition_even_when_build_failed(tmp_path): + matching = {**_build(), "status": "completed", "result": "failed"} + builds = [ + _build(174267, sha="b" * 40), + _build(174266, pr="780"), + _build(174265, branch="refs/heads/main"), + _build(174264, definition=9999), + {**_build(174263, sha=None), "sourceVersion": SHA}, + matching, + _build(174261), + ] + result = _run(tmp_path, _script("Wait for ADO build to start"), {"builds": [{"value": builds}]}) + assert result.returncode == 0, result.stdout + result.stderr + exported = (tmp_path / "github-env").read_text(encoding="utf-8") + assert "BUILD_ID=174262\n" in exported + assert f"ADO_URL={matching['_links']['web']['href']}\n" in exported + request = (tmp_path / "requests").read_text(encoding="utf-8") + assert "definitions=2128&branchName=refs%2Fpull%2F779%2Fmerge" in request + assert "queryOrder=queueTimeDescending" in request + + +def test_ignores_old_head_until_exact_build_appears_and_retries_bad_responses(tmp_path): + result = _run( + tmp_path, + _script("Wait for ADO build to start"), + { + "builds": [ + {"value": [_build(174261, sha="b" * 40)]}, + (22, "HTTP 503"), + "gateway error", + {"value": {}}, + {"value": [_build()]}, + ] + }, + ) + assert result.returncode == 0, result.stdout + result.stderr + assert "BUILD_ID=174262\n" in (tmp_path / "github-env").read_text(encoding="utf-8") + + +def test_accepts_late_artifact_after_failed_aggregate_completes(tmp_path): + result = _poll( + tmp_path, + [EMPTY] * 92 + [ARTIFACT], + [_build()] * 91 + [{**_build(), "status": "completed", "result": "failed"}], + ) + assert result.returncode == 0, result.stdout + result.stderr + assert f"COVERAGE_ARTIFACT={ARTIFACT_URL}" in result.stdout + assert "Build completed (failed)" in result.stdout + assert int((tmp_path / "elapsed").read_text()) > 45 * 60 + + +@pytest.mark.parametrize("result", ["succeeded", "failed", "canceled"]) +def test_completed_without_artifact_stops_after_short_grace(tmp_path, result): + completed = {**_build(), "status": "completed", "result": result} + run = _poll(tmp_path, [EMPTY], [completed]) + assert run.returncode != 0 + assert "after propagation grace" in run.stdout + assert 120 <= int((tmp_path / "elapsed").read_text()) < 180 + + +def test_immediately_available_artifact_needs_no_lifecycle_request(tmp_path): + result = _poll(tmp_path, [ARTIFACT], []) + assert result.returncode == 0, result.stdout + result.stderr + assert f"COVERAGE_ARTIFACT={ARTIFACT_URL}" in result.stdout + assert (tmp_path / "build.next").read_text() == "0" + + +def test_artifact_and_lifecycle_http_json_errors_are_retried(tmp_path): + result = _poll( + tmp_path, + [(22, "HTTP 502"), "{invalid", {"value": None}, EMPTY, ARTIFACT], + [(28, ""), "not json", {"status": "completed"}, _build()], + ) + assert result.returncode == 0, result.stdout + result.stderr + assert f"COVERAGE_ARTIFACT={ARTIFACT_URL}" in result.stdout + + +def test_xml_artifact_refresh_retries_http_and_json_errors(tmp_path): + script = _script("Download coverage XML from ADO") + script = script.replace("BUILD_ID=${{ env.BUILD_ID }}\n", "") + script = script.split('\necho "🔍 Available artifacts:"', 1)[0] + result = _run( + tmp_path, + script, + {"artifacts": [(22, "HTTP 503"), "invalid JSON", ARTIFACT]}, + ) + assert result.returncode == 0, result.stdout + result.stderr + assert (tmp_path / "artifacts.next").read_text().strip() == "3" + + +@pytest.mark.parametrize("failing_api", ["builds", "artifacts", "build"]) +def test_persistent_api_errors_have_finite_retries(tmp_path, failing_api): + if failing_api == "builds": + result = _run(tmp_path, _script("Wait for ADO build to start"), {"builds": ["not JSON"]}) + else: + result = _poll( + tmp_path, + ["not JSON"] if failing_api == "artifacts" else [EMPTY], + ["not JSON"] if failing_api == "build" else [_build()], + ) + assert result.returncode != 0 + assert "5 consecutive failures" in result.stdout + assert int((tmp_path / f"{failing_api}.next").read_text()) == 5 + assert int((tmp_path / "elapsed").read_text()) < 180 + + +@pytest.mark.parametrize("step,budget", [("build", 15 * 60), ("artifact", 120 * 60)]) +def test_missing_build_or_queued_coverage_obeys_wall_clock_budget(tmp_path, step, budget): + if step == "build": + result = _run( + tmp_path, + _script("Wait for ADO build to start"), + {"builds": [{"value": [_build(174261, sha="b" * 40)]}]}, + ) + else: + result = _poll(tmp_path, [EMPTY], [{**_build(), "status": "notStarted"}]) + assert result.returncode != 0 + assert "Timeout:" in result.stdout + assert budget <= int((tmp_path / "elapsed").read_text()) < budget + 30 + + +def test_job_budget_leaves_time_for_downloads_and_publishing(): + workflow = WORKFLOW.read_text(encoding="utf-8") + assert " timeout-minutes: 145\n" in workflow + assert "PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }}" in workflow From b505deb448c7f7c6000762899e3747d45a160445 Mon Sep 17 00:00:00 2001 From: Gaurav Sharma Date: Wed, 16 Sep 2026 00:21:57 +0530 Subject: [PATCH 03/18] FIX: Complete profiler CI samples and isolate incomplete reports Allow all twelve profiler passes to finish within coordinated CI and reporting budgets without reducing workload coverage or measured samples. Reject malformed report containers per leg and finalize unfinished ADO runs as incomplete at the publisher deadline. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/scripts/post_profiler_comment.py | 11 +- .github/workflows/pr-code-coverage.yml | 10 +- .github/workflows/pr-profiler-report.yml | 2 +- benchmarks/README.md | 14 +- benchmarks/profiler_ci.py | 5 +- benchmarks/profiler_report.py | 13 +- eng/pipelines/pr-validation-pipeline.yml | 18 ++- tests/test_036_profiler_ci.py | 160 +++++++++++++++++++++-- tests/test_pr_code_coverage_workflow.py | 10 +- 9 files changed, 203 insertions(+), 40 deletions(-) diff --git a/.github/scripts/post_profiler_comment.py b/.github/scripts/post_profiler_comment.py index cbc802bbc..5f5764f7b 100644 --- a/.github/scripts/post_profiler_comment.py +++ b/.github/scripts/post_profiler_comment.py @@ -19,6 +19,8 @@ ADO = "https://dev.azure.com/sqlclientdrivers/public/_apis/build" REPOSITORY = "microsoft/mssql-python" +# Allow a 150-minute ADO job plus queueing; the workflow reserves publication time. +WAIT_MINUTES = 210 def allowed_url(url): @@ -174,12 +176,13 @@ def run(number, head, wait_minutes): if build and build["status"] == "completed": break time.sleep(30) - if build is None: + if build is None or build.get("status") != "completed": publish( number, head, f"{MARKER}\n## Profiler performance report\n" - f"No matching ADO run became available for `{head}`. Results are incomplete.", + f"No matching ADO run completed within the {wait_minutes}-minute wait for `{head}`. " + "Results are incomplete. No regression verdict.", ) return build_id = build["id"] @@ -217,12 +220,12 @@ def run(number, head, wait_minutes): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--pr", type=int, required=True) parser.add_argument("--head", required=True) - parser.add_argument("--wait-minutes", type=int, default=95) + parser.add_argument("--wait-minutes", type=int, default=WAIT_MINUTES) args = parser.parse_args() if ( args.pr <= 0 or not re.fullmatch(r"[0-9a-f]{40}", args.head) - or not 1 <= args.wait_minutes <= 95 + or not 1 <= args.wait_minutes <= WAIT_MINUTES ): parser.error("Invalid PR, head SHA or wait limit") run(args.pr, args.head, args.wait_minutes) diff --git a/.github/workflows/pr-code-coverage.yml b/.github/workflows/pr-code-coverage.yml index 096861a64..88d37a058 100644 --- a/.github/workflows/pr-code-coverage.yml +++ b/.github/workflows/pr-code-coverage.yml @@ -11,7 +11,7 @@ permissions: jobs: coverage-report: runs-on: ubuntu-latest - timeout-minutes: 145 + timeout-minutes: 235 permissions: pull-requests: write contents: read @@ -112,13 +112,13 @@ jobs: run: | BUILD_URL="https://dev.azure.com/SqlClientDrivers/public/_apis/build/builds/$BUILD_ID?api-version=7.1-preview.7" ARTIFACTS_URL="https://dev.azure.com/SqlClientDrivers/public/_apis/build/builds/$BUILD_ID/artifacts?api-version=7.1-preview.5" - # Coverage may start after other queued jobs, each with a 90-minute timeout. - DEADLINE=$((SECONDS + 120 * 60)) + # Coverage may queue behind 150-minute benchmark jobs before its own run. + DEADLINE=$((SECONDS + 210 * 60)) COMPLETED_AT=-1 ARTIFACT_FAILURES=0 BUILD_FAILURES=0 COVERAGE_ARTIFACT="" - echo "📥 Waiting up to 120 minutes for coverage artifacts for build $BUILD_ID..." + echo "📥 Waiting up to 210 minutes for coverage artifacts for build $BUILD_ID..." while (( SECONDS < DEADLINE )); do REQUEST_TIMEOUT=$((DEADLINE - SECONDS)) @@ -178,7 +178,7 @@ jobs: done if [[ -z "$COVERAGE_ARTIFACT" ]]; then - echo "❌ Timeout: Coverage report artifact not found within 120 minutes" + echo "❌ Timeout: Coverage report artifact not found within 210 minutes" exit 1 fi diff --git a/.github/workflows/pr-profiler-report.yml b/.github/workflows/pr-profiler-report.yml index 7f65555e7..6b4503192 100644 --- a/.github/workflows/pr-profiler-report.yml +++ b/.github/workflows/pr-profiler-report.yml @@ -17,7 +17,7 @@ concurrency: jobs: report: runs-on: ubuntu-latest - timeout-minutes: 100 + timeout-minutes: 220 steps: - uses: actions/checkout@v4 with: diff --git a/benchmarks/README.md b/benchmarks/README.md index 85a1a736a..cd530f3b1 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -42,8 +42,11 @@ pairs follow one discarded warmup pair, alternating base/PR order. A local subse available through `--scenarios`; it is not accepted as a complete CI report. Each worker has a ten-minute limit for the entire workload suite, with a shared -35-minute build/measurement budget inside the CI step's 40-minute limit. Worker -logs identify each starting/completed scenario and emit stack traces every minute. +80-minute build/measurement budget inside the CI step's 90-minute limit. This fits +all twelve passes at five minutes each, a 15-minute base build and a one-minute +preflight, with headroom. The 150-minute jobs also allow for setup, pytest and +artifact publication. These are upper bounds, not mandatory wait times. +Worker logs identify each starting/completed scenario and emit stack traces every minute. Per-worker JSON checkpoints retain finished scenarios and identify the active one if the process fails or is killed. Partial workers never count as measured pairs; incomplete reports do not produce a regression verdict. @@ -70,6 +73,13 @@ not as a clean performance verdict. As a new base-branch reporting workflow, it starts reporting automatically after this infrastructure has merged; it does not grant fork-authored workflow code write credentials. +The publisher waits up to 210 minutes for the matching ADO run to complete, +including queueing, inside a 220-minute workflow. A still-queued or running build +at that deadline produces a final incomplete comment, not a lingering `Awaiting` +message. A malformed report invalidates only its own leg; valid legs still render. +Coverage polling also allows 210 minutes for its artifact, plus 15 minutes for +build discovery and ten for processing/publication in its 235-minute workflow. + The first main comparison after introduction may lack profiling support on its parent; that run is incomplete rather than falling back to an uninstrumented base. Subsequent comparisons use the new artifact format and do not consume old diff --git a/benchmarks/profiler_ci.py b/benchmarks/profiler_ci.py index 64724a61a..e38c72c83 100644 --- a/benchmarks/profiler_ci.py +++ b/benchmarks/profiler_ci.py @@ -20,8 +20,9 @@ ROOT = Path(__file__).resolve().parents[1] SHA = re.compile(r"[0-9a-f]{40}") LEGS = ("Windows-SQL2022", "Windows-SQL2025", "macOS-SQL2022", "macOS-SQL2025", "Linux-SQL2022") -# Leave five minutes of the CI step's 40-minute budget for artifact publication. -BENCHMARK_TIMEOUT = 35 * 60 +# Twelve five-minute passes plus a 15-minute base build and preflight need 76 +# minutes. Leave headroom here and ten more minutes for CI-step dependency setup. +BENCHMARK_TIMEOUT = 80 * 60 WORKER_TIMEOUT = 10 * 60 diff --git a/benchmarks/profiler_report.py b/benchmarks/profiler_report.py index 3725466eb..17fe8bd8f 100644 --- a/benchmarks/profiler_report.py +++ b/benchmarks/profiler_report.py @@ -88,6 +88,8 @@ def validate(report, build_id=None, head=None, source=None, base=None): raise ValueError("Invalid paired sample") for side in ("base", "candidate"): sample = pair[side] + if not isinstance(sample, dict): + raise ValueError("Invalid sample") env = sample["environment"] if not isinstance(env, dict) or set(env) != { "os", @@ -106,9 +108,14 @@ def validate(report, build_id=None, head=None, source=None, base=None): if environment is not None and environment != env: raise ValueError("Environment changed between measurements") environment = env - if set(sample["scenarios"]) != set(CASES): + scenarios = sample["scenarios"] + if not isinstance(scenarios, dict): + raise ValueError("Invalid scenarios object") + if set(scenarios) != set(CASES): raise ValueError("Scenario set incomplete or changed") - for name, scenario in sample["scenarios"].items(): + for name, scenario in scenarios.items(): + if not isinstance(scenario, dict): + raise ValueError("Invalid scenario") number(scenario["wall_ms"]) if scenario["wall_ms"] <= 0: raise ValueError("Zero workload time") @@ -128,6 +135,8 @@ def validate(report, build_id=None, head=None, source=None, base=None): text(label) if not label.startswith("ddbc::" if layer == "cpp" else "py::"): raise ValueError("Invalid phase prefix") + if not isinstance(counter, dict): + raise ValueError("Invalid phase counter") calls = counter["calls"] if type(calls) is not int or not 1 <= calls <= 100_000_000: raise ValueError("Invalid call count") diff --git a/eng/pipelines/pr-validation-pipeline.yml b/eng/pipelines/pr-validation-pipeline.yml index 7ce613e1e..29e2ea8a9 100644 --- a/eng/pipelines/pr-validation-pipeline.yml +++ b/eng/pipelines/pr-validation-pipeline.yml @@ -47,7 +47,7 @@ jobs: - job: pytestonwindows displayName: 'Windows x64' - timeoutInMinutes: 90 + timeoutInMinutes: 150 pool: vmImage: 'windows-latest' @@ -401,7 +401,7 @@ jobs: displayName: 'Compare profiling builds on SQL Server 2022/2025' condition: and(succeeded(), or(eq(variables['sqlVersion'], 'SQL2022'), eq(variables['sqlVersion'], 'SQL2025'))) continueOnError: true - timeoutInMinutes: 40 + timeoutInMinutes: 90 env: SYSTEM_PULLREQUEST_SOURCECOMMITID: $(System.PullRequest.SourceCommitId) DB_CONNECTION_STRING: 'Server=localhost;Database=AdventureWorks2022;Uid=sa;Pwd=$(DB_PASSWORD);TrustServerCertificate=yes' @@ -450,11 +450,9 @@ jobs: - job: PytestOnMacOS displayName: 'macOS x86_64' - # Colima + SQL Server container setup averages ~12.5 min but has a long tail - # (setup has been observed at 17-39 min). The ADO default job timeout is 60 min, - # which the setup tail plus tests plus the 20-min benchmark step can exceed, - # getting the job killed mid-step. Give the job enough headroom for the tail. - timeoutInMinutes: 90 + # Reserve 60 minutes outside the 90-minute benchmark step for Colima/SQL + # setup, pytest, fixture restore and artifact publication. + timeoutInMinutes: 150 pool: vmImage: 'macos-latest' @@ -683,7 +681,7 @@ jobs: python benchmarks/profiler_ci.py --reuse-candidate --leg "macOS-$(sqlVersion)" --output profiler-results displayName: 'Compare profiling builds on macOS $(sqlVersion)' condition: and(succeeded(), or(eq(variables['sqlVersion'], 'SQL2022'), eq(variables['sqlVersion'], 'SQL2025'))) - timeoutInMinutes: 40 + timeoutInMinutes: 90 continueOnError: true env: SYSTEM_PULLREQUEST_SOURCECOMMITID: $(System.PullRequest.SourceCommitId) @@ -700,7 +698,7 @@ jobs: - job: PytestOnLinux displayName: 'Linux x86_64' - timeoutInMinutes: 90 + timeoutInMinutes: 150 pool: vmImage: 'ubuntu-latest' @@ -1011,7 +1009,7 @@ jobs: displayName: 'Compare profiling builds in $(distroName) container' condition: and(succeeded(), eq(variables['distroName'], 'Ubuntu'), eq(variables['useAzureSQL'], 'false')) continueOnError: true - timeoutInMinutes: 40 + timeoutInMinutes: 90 env: DB_PASSWORD: $(DB_PASSWORD) diff --git a/tests/test_036_profiler_ci.py b/tests/test_036_profiler_ci.py index 9c0b6b25d..d2d3cec22 100644 --- a/tests/test_036_profiler_ci.py +++ b/tests/test_036_profiler_ci.py @@ -5,6 +5,7 @@ import io import json from pathlib import Path +import re import subprocess import sys from types import SimpleNamespace @@ -126,6 +127,28 @@ def test_reject_wrong_commit_and_preserve_incomplete_status(report): assert "No regression verdict" in reporting.render([report], "c" * 40, 42) +@pytest.mark.parametrize( + "path", + [ + ("pairs", 0, "candidate"), + ("pairs", 0, "candidate", "environment"), + ("pairs", 0, "candidate", "scenarios"), + ("pairs", 0, "candidate", "scenarios", "select"), + ("pairs", 0, "candidate", "scenarios", "select", "cpp"), + ("pairs", 0, "candidate", "scenarios", "select", "cpp", "ddbc::query"), + ], +) +@pytest.mark.parametrize("as_list", [False, True]) +def test_reject_non_object_sample_containers(report, path, as_list): + parent = report + for key in path[:-1]: + parent = parent[key] + key = path[-1] + parent[key] = list(parent[key]) if as_list else None + with pytest.raises(ValueError): + reporting.validate(report) + + def zip_data(entries): out = io.BytesIO() with zipfile.ZipFile(out, "w") as archive: @@ -264,7 +287,73 @@ def test_overall_budget_caps_build_and_worker_time(monkeypatch): assert controller.remaining(1000, 60) == 60 with pytest.raises(TimeoutError, match="overall"): controller.remaining(100, controller.WORKER_TIMEOUT) - assert controller.BENCHMARK_TIMEOUT < 40 * 60 + + +def test_full_sample_budget_fits_slow_hosted_workers(report, tmp_path, monkeypatch): + # Run 174385 completed workers in 169-285s. Budget twelve five-minute + # passes plus the full base-build/preflight allowance, not just measured pairs. + clock = [0] + measured = [] + monkeypatch.setattr(controller.time, "monotonic", lambda: clock[0]) + monkeypatch.setattr(controller, "resolve_revisions", lambda *a: ("a" * 40, "b" * 40)) + monkeypatch.setattr(controller, "git", lambda *a: "b" * 40) + monkeypatch.setattr(controller, "checkout", lambda *a: None) + monkeypatch.setenv("BUILD_BUILDID", "42") + monkeypatch.setenv("SYSTEM_PULLREQUEST_SOURCECOMMITID", "c" * 40) + + def build(path, log, timeout): + assert timeout >= 900 + clock[0] += 900 + + def preflight(command, **kwargs): + assert "--check-build" in command and kwargs["timeout"] == 60 + clock[0] += 60 + + def measure(path, output, scenarios, timeout): + assert scenarios is None + if timeout < 300: + raise subprocess.TimeoutExpired("hosted worker replay", timeout) + clock[0] += 300 + measured.append(output.name) + return copy.deepcopy(report["pairs"][0]["base"]) + + monkeypatch.setattr(controller, "build", build) + monkeypatch.setattr(controller.subprocess, "run", preflight) + monkeypatch.setattr(controller, "measure", measure) + args = SimpleNamespace( + base=None, + candidate="HEAD", + output=tmp_path, + leg=report["leg"], + samples=5, + warmups=1, + reuse_candidate=True, + scenarios=None, + ) + controller.run(args) + result = reporting.validate(json.loads((tmp_path / "report.json").read_text())) + assert result["status"] == "complete" and len(result["pairs"]) == 5 + assert measured == [ + f"{side}-{index}.json" + for index in range(6) + for side in (("base", "candidate") if index % 2 == 0 else ("candidate", "base")) + ] + assert clock[0] == 76 * 60 + assert clock[0] < controller.BENCHMARK_TIMEOUT + + +def test_ci_deadlines_include_setup_queueing_and_publication(): + pipeline = (ROOT / "eng/pipelines/pr-validation-pipeline.yml").read_text(encoding="utf-8") + for job in ("pytestonwindows", "PytestOnMacOS", "PytestOnLinux"): + section = pipeline.split(f"- job: {job}\n", 1)[1].split("\n- job:", 1)[0] + job_minutes = int(re.search(r"^ timeoutInMinutes: (\d+)$", section, re.M)[1]) + step_minutes = int(re.search(r"^ timeoutInMinutes: (\d+)$", section, re.M)[1]) + assert step_minutes * 60 >= controller.BENCHMARK_TIMEOUT + 10 * 60 + assert job_minutes >= step_minutes + 60 + assert publisher.WAIT_MINUTES >= job_minutes + 60 + workflow = (ROOT / ".github/workflows/pr-profiler-report.yml").read_text(encoding="utf-8") + workflow_minutes = int(re.search(r"timeout-minutes: (\d+)", workflow)[1]) + assert workflow_minutes >= publisher.WAIT_MINUTES + 10 def test_build_check_rejects_foreign_provider_and_enabled_recording(tmp_path, monkeypatch): @@ -307,12 +396,26 @@ def api(path, **kwargs): assert reads == 2 and len(calls) == 3 -@pytest.mark.parametrize("corrupt", [False, True]) +@pytest.mark.parametrize("corrupt", [None, "zip", "scenarios"]) def test_publisher_renders_validated_artifact_and_marks_missing_legs(report, monkeypatch, corrupt): posted = [] + windows = copy.deepcopy(report) + windows["leg"] = "Windows-SQL2022" + for pair in windows["pairs"]: + for sample in pair.values(): + sample["environment"]["os"] = "Windows" + if corrupt == "scenarios": + report["pairs"][0]["candidate"]["scenarios"] = list(reporting.CASES) + data = { + "Windows-SQL2022": zip_data([("report.json", json.dumps(windows))]), + "Linux-SQL2022": ( + b"invalid ZIP" if corrupt == "zip" else zip_data([("report.json", json.dumps(report))]) + ), + } build = dict( id=42, status="completed", + result="failed", definition={"id": 2128}, repository={"id": "microsoft/mssql-python"}, sourceBranch="refs/pull/123/merge", @@ -336,26 +439,65 @@ def test_publisher_renders_validated_artifact_and_marks_missing_legs(report, mon { "value": [ { - "name": "profiler-Linux-SQL2022", - "resource": {"downloadUrl": "https://dev.azure.com/artifact"}, + "name": "profiler-" + leg, + "resource": {"downloadUrl": "https://dev.azure.com/" + leg}, } + for leg in data ] } if "/artifacts?" in url else {"value": [build]} ), ) - raw = b"invalid ZIP" if corrupt else zip_data([("report.json", json.dumps(report))]) - monkeypatch.setattr(publisher, "fetch", lambda *args, **kwargs: raw) + monkeypatch.setattr(publisher, "fetch", lambda url, **kw: data[url.rsplit("/", 1)[-1]]) publisher.run(123, "c" * 40, 1) assert len(posted) == 2 assert posted[0].startswith(reporting.MARKER) - assert "Windows-SQL2022: incomplete/unavailable" in posted[1] + assert "### Windows-SQL2022" in posted[1] + assert "macOS-SQL2022: incomplete/unavailable" in posted[1] if corrupt: assert reporting.escape("Linux-SQL2022 (invalid artifact)") in posted[1] - assert "regression signals" not in posted[1] + assert "Linux-SQL2022: incomplete/unavailable" in posted[1] + assert posted[1].count("20 regression signals") == 1 else: - assert "20 regression signals" in posted[1] + assert posted[1].count("20 regression signals") == 2 + + +@pytest.mark.parametrize("status", [None, "notStarted", "inProgress"]) +def test_publisher_deadline_finishes_without_reading_unfinished_build_metadata(monkeypatch, status): + posted = [] + clock = [0] + build = dict( + id=42, + status=status, + sourceVersion=None, + definition={"id": 2128}, + repository={"id": "microsoft/mssql-python"}, + sourceBranch="refs/pull/123/merge", + triggerInfo={"pr.number": "123", "pr.sourceSha": "c" * 40}, + ) + + def github(path): + assert path == "pulls/123", "Unfinished builds must not query merge topology" + return {"state": "open", "head": {"sha": "c" * 40}} + + def api(url): + assert "/builds?" in url, "Unfinished builds must not query artifacts" + return {"value": [] if status is None else [build]} + + def sleep(seconds): + clock[0] += seconds + + monkeypatch.setattr(publisher.time, "monotonic", lambda: clock[0]) + monkeypatch.setattr(publisher.time, "sleep", sleep) + monkeypatch.setattr(publisher, "github", github) + monkeypatch.setattr(publisher, "api", api) + monkeypatch.setattr(publisher, "publish", lambda number, head, body: posted.append(body)) + publisher.run(123, "c" * 40, 1) + assert clock[0] == 60 and len(posted) == 2 + assert "Awaiting" in posted[0] and "Awaiting" not in posted[1] + assert "1-minute wait" in posted[1] and "incomplete" in posted[1] + assert "No regression verdict" in posted[1] def test_artifact_symlink_and_oversized_json_are_rejected(): diff --git a/tests/test_pr_code_coverage_workflow.py b/tests/test_pr_code_coverage_workflow.py index 3141aedd0..5f01bac8c 100644 --- a/tests/test_pr_code_coverage_workflow.py +++ b/tests/test_pr_code_coverage_workflow.py @@ -167,13 +167,13 @@ def test_ignores_old_head_until_exact_build_appears_and_retries_bad_responses(tm def test_accepts_late_artifact_after_failed_aggregate_completes(tmp_path): result = _poll( tmp_path, - [EMPTY] * 92 + [ARTIFACT], - [_build()] * 91 + [{**_build(), "status": "completed", "result": "failed"}], + [EMPTY] * 302 + [ARTIFACT], + [_build()] * 301 + [{**_build(), "status": "completed", "result": "failed"}], ) assert result.returncode == 0, result.stdout + result.stderr assert f"COVERAGE_ARTIFACT={ARTIFACT_URL}" in result.stdout assert "Build completed (failed)" in result.stdout - assert int((tmp_path / "elapsed").read_text()) > 45 * 60 + assert int((tmp_path / "elapsed").read_text()) > 150 * 60 @pytest.mark.parametrize("result", ["succeeded", "failed", "canceled"]) @@ -231,7 +231,7 @@ def test_persistent_api_errors_have_finite_retries(tmp_path, failing_api): assert int((tmp_path / "elapsed").read_text()) < 180 -@pytest.mark.parametrize("step,budget", [("build", 15 * 60), ("artifact", 120 * 60)]) +@pytest.mark.parametrize("step,budget", [("build", 15 * 60), ("artifact", 210 * 60)]) def test_missing_build_or_queued_coverage_obeys_wall_clock_budget(tmp_path, step, budget): if step == "build": result = _run( @@ -248,5 +248,5 @@ def test_missing_build_or_queued_coverage_obeys_wall_clock_budget(tmp_path, step def test_job_budget_leaves_time_for_downloads_and_publishing(): workflow = WORKFLOW.read_text(encoding="utf-8") - assert " timeout-minutes: 145\n" in workflow + assert " timeout-minutes: 235\n" in workflow assert "PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }}" in workflow From f5779f4223a4fc7133b4564dede3618bb8e15915 Mon Sep 17 00:00:00 2001 From: Gaurav Sharma Date: Wed, 16 Sep 2026 09:26:10 +0530 Subject: [PATCH 04/18] FIX: Stabilize profiler CI reporting and macOS tests Accelerate long-horizon workflow fixtures so macOS validates the polling budget without hundreds of real shell iterations. Recover coverage reporting onto newer exact-head ADO runs, pin privileged actions, align benchmark deadlines with worker limits, and keep subset reports incomplete. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/scripts/post_profiler_comment.py | 4 +- .github/workflows/pr-code-coverage.yml | 52 ++++++++++++++++++-- .github/workflows/pr-profiler-report.yml | 6 +-- benchmarks/README.md | 19 ++++---- benchmarks/profiler_ci.py | 15 +++--- eng/pipelines/pr-validation-pipeline.yml | 14 +++--- tests/test_036_profiler_ci.py | 16 +++++-- tests/test_pr_code_coverage_workflow.py | 61 +++++++++++++++++++----- 8 files changed, 139 insertions(+), 48 deletions(-) diff --git a/.github/scripts/post_profiler_comment.py b/.github/scripts/post_profiler_comment.py index 5f5764f7b..85c6524db 100644 --- a/.github/scripts/post_profiler_comment.py +++ b/.github/scripts/post_profiler_comment.py @@ -19,8 +19,8 @@ ADO = "https://dev.azure.com/sqlclientdrivers/public/_apis/build" REPOSITORY = "microsoft/mssql-python" -# Allow a 150-minute ADO job plus queueing; the workflow reserves publication time. -WAIT_MINUTES = 210 +# Allow a 160-minute ADO job plus queueing; the workflow reserves publication time. +WAIT_MINUTES = 220 def allowed_url(url): diff --git a/.github/workflows/pr-code-coverage.yml b/.github/workflows/pr-code-coverage.yml index e3c9478d4..a47d6734b 100644 --- a/.github/workflows/pr-code-coverage.yml +++ b/.github/workflows/pr-code-coverage.yml @@ -11,7 +11,7 @@ permissions: jobs: coverage-report: runs-on: ubuntu-latest - timeout-minutes: 235 + timeout-minutes: 245 permissions: pull-requests: write contents: read @@ -91,6 +91,8 @@ jobs: echo "🔗 Build URL: $WEB_URL" echo "ADO_URL=$WEB_URL" >> "$GITHUB_ENV" echo "BUILD_ID=$BUILD_ID" >> "$GITHUB_ENV" + echo "PR_NUMBER=$PR_NUMBER" >> "$GITHUB_ENV" + echo "PR_HEAD_SHA=$PR_HEAD_SHA" >> "$GITHUB_ENV" # A failed matrix leg does not invalidate a successful coverage artifact. echo "🚀 Build found, proceeding to poll for coverage artifacts..." @@ -112,13 +114,15 @@ jobs: run: | BUILD_URL="https://dev.azure.com/SqlClientDrivers/public/_apis/build/builds/$BUILD_ID?api-version=7.1-preview.7" ARTIFACTS_URL="https://dev.azure.com/SqlClientDrivers/public/_apis/build/builds/$BUILD_ID/artifacts?api-version=7.1-preview.5" - # Coverage may queue behind 150-minute benchmark jobs before its own run. - DEADLINE=$((SECONDS + 210 * 60)) + BUILDS_URL="https://dev.azure.com/sqlclientdrivers/public/_apis/build/builds?definitions=2128&branchName=refs%2Fpull%2F${PR_NUMBER}%2Fmerge&queryOrder=queueTimeDescending&%24top=100&api-version=7.1-preview.7" + PR_BRANCH="refs/pull/$PR_NUMBER/merge" + # Coverage may queue behind 160-minute benchmark jobs before its own run. + DEADLINE=$((SECONDS + 220 * 60)) COMPLETED_AT=-1 ARTIFACT_FAILURES=0 BUILD_FAILURES=0 COVERAGE_ARTIFACT="" - echo "📥 Waiting up to 210 minutes for coverage artifacts for build $BUILD_ID..." + echo "📥 Waiting up to 220 minutes for coverage artifacts for build $BUILD_ID..." while (( SECONDS < DEADLINE )); do REQUEST_TIMEOUT=$((DEADLINE - SECONDS)) @@ -154,6 +158,44 @@ jobs: BUILD_FAILURES=0 STATUS=$(jq -r '.status' <<< "$BUILD_RESPONSE") RESULT=$(jq -r '.result // "unknown"' <<< "$BUILD_RESPONSE") + if [[ "$STATUS" == "completed" && "$RESULT" == "canceled" ]]; then + if REPLACEMENTS=$(curl --fail --silent --show-error --connect-timeout 10 --max-time "$REQUEST_TIMEOUT" "$BUILDS_URL") && + REPLACEMENT=$(jq -ce --arg PR "$PR_NUMBER" --arg SHA "$PR_HEAD_SHA" \ + --arg BRANCH "$PR_BRANCH" --arg ID "$BUILD_ID" ' + [.value[]? | select( + .definition.id == 2128 and .sourceBranch == $BRANCH and + (.triggerInfo["pr.number"] | tostring) == $PR and + .triggerInfo["pr.sourceSha"] == $SHA and + .id > ($ID | tonumber) and + (.status != "completed" or .result != "canceled") + )] | .[0]' <<< "$REPLACEMENTS"); then + BUILD_ID=$(jq -r '.id' <<< "$REPLACEMENT") + [[ "$BUILD_ID" =~ ^[0-9]+$ ]] || { + echo "Invalid replacement Azure DevOps build ID" + exit 1 + } + BUILD_URL="https://dev.azure.com/SqlClientDrivers/public/_apis/build/builds/$BUILD_ID?api-version=7.1-preview.7" + ARTIFACTS_URL="https://dev.azure.com/SqlClientDrivers/public/_apis/build/builds/$BUILD_ID/artifacts?api-version=7.1-preview.5" + ADO_URL=$(jq -r '._links.web.href // empty' <<< "$REPLACEMENT") + if [[ -z "$ADO_URL" || ${#ADO_URL} -gt 500 || "$ADO_URL" == *$'\n'* || "$ADO_URL" == *$'\r'* ]]; then + echo "Invalid replacement Azure DevOps build URL" + exit 1 + fi + echo "BUILD_ID=$BUILD_ID" >> "$GITHUB_ENV" + echo "ADO_URL=$ADO_URL" >> "$GITHUB_ENV" + COMPLETED_AT=-1 + ARTIFACT_FAILURES=0 + BUILD_FAILURES=0 + echo "Selected ADO run was canceled; continuing with replacement build $BUILD_ID" + continue + fi + ARTIFACT_FAILURES=0 + echo "Canceled build $BUILD_ID has no replacement yet..." + SLEEP_SECONDS=$((DEADLINE - SECONDS)) + if (( SLEEP_SECONDS > 30 )); then SLEEP_SECONDS=30; fi + if (( SLEEP_SECONDS > 0 )); then sleep "$SLEEP_SECONDS"; fi + continue + fi if [[ "$STATUS" == "completed" ]] && (( COMPLETED_AT < 0 )); then COMPLETED_AT=$SECONDS echo "Build completed ($RESULT); allowing 2 minutes for artifact propagation..." @@ -178,7 +220,7 @@ jobs: done if [[ -z "$COVERAGE_ARTIFACT" ]]; then - echo "❌ Timeout: Coverage report artifact not found within 210 minutes" + echo "❌ Timeout: Coverage report artifact not found within 220 minutes" exit 1 fi diff --git a/.github/workflows/pr-profiler-report.yml b/.github/workflows/pr-profiler-report.yml index 6b4503192..f7b41f0eb 100644 --- a/.github/workflows/pr-profiler-report.yml +++ b/.github/workflows/pr-profiler-report.yml @@ -17,13 +17,13 @@ concurrency: jobs: report: runs-on: ubuntu-latest - timeout-minutes: 220 + timeout-minutes: 230 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 with: ref: ${{ github.event.pull_request.base.sha }} persist-credentials: false - - uses: actions/setup-python@v5 + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: "3.13" - name: Publish validated paired benchmark results diff --git a/benchmarks/README.md b/benchmarks/README.md index cd530f3b1..e7cd2f7ac 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -41,11 +41,12 @@ fresh interpreter. Five measured pairs follow one discarded warmup pair, alternating base/PR order. A local subset is available through `--scenarios`; it is not accepted as a complete CI report. -Each worker has a ten-minute limit for the entire workload suite, with a shared -80-minute build/measurement budget inside the CI step's 90-minute limit. This fits -all twelve passes at five minutes each, a 15-minute base build and a one-minute -preflight, with headroom. The 150-minute jobs also allow for setup, pytest and -artifact publication. These are upper bounds, not mandatory wait times. +Each worker has a six-minute limit for the entire workload suite, with a shared +90-minute build/measurement budget inside the CI step's 100-minute limit. This fits +all twelve permitted worker limits, a 15-minute base build and a one-minute +preflight, with headroom. Local runs receive 105 minutes because they build both +revisions. The 160-minute CI jobs also allow for setup, pytest and artifact +publication. These are upper bounds, not mandatory wait times. Worker logs identify each starting/completed scenario and emit stack traces every minute. Per-worker JSON checkpoints retain finished scenarios and identify the active one if the process fails or is killed. Partial workers never count as measured pairs; @@ -73,12 +74,12 @@ not as a clean performance verdict. As a new base-branch reporting workflow, it starts reporting automatically after this infrastructure has merged; it does not grant fork-authored workflow code write credentials. -The publisher waits up to 210 minutes for the matching ADO run to complete, -including queueing, inside a 220-minute workflow. A still-queued or running build +The publisher waits up to 220 minutes for the matching ADO run to complete, +including queueing, inside a 230-minute workflow. A still-queued or running build at that deadline produces a final incomplete comment, not a lingering `Awaiting` message. A malformed report invalidates only its own leg; valid legs still render. -Coverage polling also allows 210 minutes for its artifact, plus 15 minutes for -build discovery and ten for processing/publication in its 235-minute workflow. +Coverage polling also allows 220 minutes for its artifact, plus 15 minutes for +build discovery and ten for processing/publication in its 245-minute workflow. The first main comparison after introduction may lack profiling support on its parent; that run is incomplete rather than falling back to an uninstrumented base. diff --git a/benchmarks/profiler_ci.py b/benchmarks/profiler_ci.py index 3c2984b16..dbd49c430 100644 --- a/benchmarks/profiler_ci.py +++ b/benchmarks/profiler_ci.py @@ -20,10 +20,11 @@ ROOT = Path(__file__).resolve().parents[1] SHA = re.compile(r"[0-9a-f]{40}") LEGS = ("Windows-SQL2022", "Windows-SQL2025", "macOS-SQL2022", "macOS-SQL2025", "Linux-SQL2022") -# Twelve five-minute passes plus a 15-minute base build and preflight need 76 -# minutes. Leave headroom here and ten more minutes for CI-step dependency setup. -BENCHMARK_TIMEOUT = 80 * 60 -WORKER_TIMEOUT = 10 * 60 +# Twelve six-minute passes plus a 15-minute base build and preflight need 88 +# minutes. Local runs build both revisions and receive another 15 minutes. +BENCHMARK_TIMEOUT = 90 * 60 +LOCAL_BENCHMARK_TIMEOUT = 105 * 60 +WORKER_TIMEOUT = 6 * 60 def git(*args): @@ -221,7 +222,8 @@ def run(args): pairs=[], ) report_path.write_text(json.dumps(report), encoding="utf-8") - deadline = time.monotonic() + BENCHMARK_TIMEOUT + timeout = BENCHMARK_TIMEOUT if args.reuse_candidate else LOCAL_BENCHMARK_TIMEOUT + deadline = time.monotonic() + timeout # CI reuses the profiling build already exercised by pytest. The base always # has its own checkout and process. Local runs can build both sides instead. with tempfile.TemporaryDirectory(prefix="profiler-ci-") as directory: @@ -256,7 +258,8 @@ def run(args): if sample >= args.warmups: report["pairs"].append(pair) report_path.write_text(json.dumps(report, allow_nan=False), encoding="utf-8") - report["status"] = "complete" + if args.scenarios is None: + report["status"] = "complete" report_path.write_text(json.dumps(report, allow_nan=False), encoding="utf-8") print(f"Paired profiler report: {report_path}", flush=True) diff --git a/eng/pipelines/pr-validation-pipeline.yml b/eng/pipelines/pr-validation-pipeline.yml index b1d38334b..499dd25e9 100644 --- a/eng/pipelines/pr-validation-pipeline.yml +++ b/eng/pipelines/pr-validation-pipeline.yml @@ -51,7 +51,7 @@ jobs: - job: pytestonwindows displayName: 'Windows x64' - timeoutInMinutes: 150 + timeoutInMinutes: 160 pool: vmImage: 'windows-latest' @@ -405,7 +405,7 @@ jobs: displayName: 'Compare profiling builds on SQL Server 2022/2025' condition: and(succeeded(), or(eq(variables['sqlVersion'], 'SQL2022'), eq(variables['sqlVersion'], 'SQL2025'))) continueOnError: true - timeoutInMinutes: 90 + timeoutInMinutes: 100 env: SYSTEM_PULLREQUEST_SOURCECOMMITID: $(System.PullRequest.SourceCommitId) DB_CONNECTION_STRING: 'Server=localhost;Database=AdventureWorks2022;Uid=sa;Pwd=$(DB_PASSWORD);TrustServerCertificate=yes' @@ -454,9 +454,9 @@ jobs: - job: PytestOnMacOS displayName: 'macOS x86_64' - # Reserve 60 minutes outside the 90-minute benchmark step for Colima/SQL + # Reserve 60 minutes outside the 100-minute benchmark step for Colima/SQL # setup, pytest, fixture restore and artifact publication. - timeoutInMinutes: 150 + timeoutInMinutes: 160 pool: vmImage: 'macos-latest' @@ -657,7 +657,7 @@ jobs: python benchmarks/profiler_ci.py --reuse-candidate --leg "macOS-$(sqlVersion)" --output profiler-results displayName: 'Compare profiling builds on macOS $(sqlVersion)' condition: and(succeeded(), or(eq(variables['sqlVersion'], 'SQL2022'), eq(variables['sqlVersion'], 'SQL2025'))) - timeoutInMinutes: 90 + timeoutInMinutes: 100 continueOnError: true env: SYSTEM_PULLREQUEST_SOURCECOMMITID: $(System.PullRequest.SourceCommitId) @@ -683,7 +683,7 @@ jobs: - job: PytestOnLinux displayName: 'Linux x86_64' - timeoutInMinutes: 150 + timeoutInMinutes: 160 pool: vmImage: 'ubuntu-latest' @@ -952,7 +952,7 @@ jobs: displayName: 'Compare profiling builds in $(distroName) container' condition: and(succeeded(), eq(variables['distroName'], 'Ubuntu'), eq(variables['useAzureSQL'], 'false')) continueOnError: true - timeoutInMinutes: 90 + timeoutInMinutes: 100 env: DB_PASSWORD: $(DB_PASSWORD) BUILD_BUILDID: $(Build.BuildId) diff --git a/tests/test_036_profiler_ci.py b/tests/test_036_profiler_ci.py index d9864cb94..cd7edd22d 100644 --- a/tests/test_036_profiler_ci.py +++ b/tests/test_036_profiler_ci.py @@ -323,7 +323,10 @@ def test_overall_budget_caps_build_and_worker_time(monkeypatch): controller.remaining(100, controller.WORKER_TIMEOUT) -def test_full_sample_budget_fits_slow_hosted_workers(report, tmp_path, monkeypatch): +@pytest.mark.parametrize("scenarios,status", [(None, "complete"), (["select"], "incomplete")]) +def test_full_sample_budget_fits_slow_hosted_workers( + report, tmp_path, monkeypatch, scenarios, status +): # Run 174385 completed workers in 169-285s. Budget twelve five-minute # passes plus the full base-build/preflight allowance, not just measured pairs. clock = [0] @@ -344,7 +347,7 @@ def preflight(command, **kwargs): clock[0] += 60 def measure(path, output, scenarios, timeout): - assert scenarios is None + assert scenarios == args.scenarios if timeout < 300: raise subprocess.TimeoutExpired("hosted worker replay", timeout) clock[0] += 300 @@ -362,11 +365,11 @@ def measure(path, output, scenarios, timeout): samples=5, warmups=1, reuse_candidate=True, - scenarios=None, + scenarios=scenarios, ) controller.run(args) result = reporting.validate(json.loads((tmp_path / "report.json").read_text())) - assert result["status"] == "complete" and len(result["pairs"]) == 5 + assert result["status"] == status and len(result["pairs"]) == 5 assert measured == [ f"{side}-{index}.json" for index in range(6) @@ -389,6 +392,9 @@ def test_ci_deadlines_include_setup_queueing_and_publication(): workflow = (ROOT / ".github/workflows/pr-profiler-report.yml").read_text(encoding="utf-8") workflow_minutes = int(re.search(r"timeout-minutes: (\d+)", workflow)[1]) assert workflow_minutes >= publisher.WAIT_MINUTES + 10 + assert controller.LOCAL_BENCHMARK_TIMEOUT >= ( + 2 * 15 * 60 + 2 * (5 + 1) * controller.WORKER_TIMEOUT + ) def test_linux_profiler_step_does_not_put_database_password_on_command_line(): @@ -593,6 +599,8 @@ def test_comment_workflow_executes_only_trusted_base_code(): assert "pull_request_target:" in workflow assert "ref: ${{ github.event.pull_request.base.sha }}" in workflow assert "persist-credentials: false" in workflow + assert "actions/checkout@11d5960a326750d5838078e36cf38b85af677262" in workflow + assert "actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065" in workflow assert ( "head.ref" not in workflow and "head.sha }}" not in workflow.split("ref:", 1)[1].split("persist", 1)[0] diff --git a/tests/test_pr_code_coverage_workflow.py b/tests/test_pr_code_coverage_workflow.py index 5f01bac8c..e89fd66ef 100644 --- a/tests/test_pr_code_coverage_workflow.py +++ b/tests/test_pr_code_coverage_workflow.py @@ -50,7 +50,7 @@ def _script(step): return textwrap.dedent(section.split(" run: |\n", 1)[1]) -def _run(tmp_path, script, fixtures): +def _run(tmp_path, script, fixtures, clock_scale=1): for kind, responses in fixtures.items(): (tmp_path / f"{kind}.count").write_text(str(len(responses)), encoding="utf-8") (tmp_path / f"{kind}.next").write_text("0", encoding="utf-8") @@ -65,7 +65,7 @@ def _run(tmp_path, script, fixtures): trap 'printf "%s\n" "$SECONDS" > "$FIXTURE_DIR/elapsed"' EXIT sleep() { printf "%s\n" "$1" >> "$FIXTURE_DIR/sleeps" - SECONDS=$((SECONDS + $1)) + SECONDS=$((SECONDS + $1 * CLOCK_SCALE)) } curl() { local url="${@: -1}" kind index count code @@ -73,7 +73,7 @@ def _run(tmp_path, script, fixtures): case "$url" in *"/artifacts?"*) kind=artifacts ;; *"/builds?"*) kind=builds ;; - *"/builds/174262?"*) kind=build ;; + *"/builds/"*"?api-version="*) kind=build ;; *) echo "Unexpected URL: $url" >&2; return 99 ;; esac if [[ ! -f "$FIXTURE_DIR/$kind.count" ]]; then @@ -96,6 +96,7 @@ def _run(tmp_path, script, fixtures): "PR_NUMBER": "779", "PR_HEAD_SHA": SHA, "BUILD_ID": "174262", + "CLOCK_SCALE": str(clock_scale), } result = subprocess.run( ["bash", "--noprofile", "--norc", "-eo", "pipefail", "-c", prefix + script], @@ -117,12 +118,17 @@ def _run(tmp_path, script, fixtures): return result -def _poll(tmp_path, artifacts, builds): +def _poll(tmp_path, artifacts, builds, replacements=(EMPTY,), clock_scale=1): script = _script("Download and parse coverage report") # Only execute discovery; downloaded report contents are never executed by these tests. script = script.split('\nif [[ -n "$COVERAGE_ARTIFACT" &&', 1)[0] script += '\nprintf "COVERAGE_ARTIFACT=%s\\n" "$COVERAGE_ARTIFACT"\n' - return _run(tmp_path, script, {"artifacts": artifacts, "build": builds}) + return _run( + tmp_path, + script, + {"artifacts": artifacts, "build": builds, "builds": replacements}, + clock_scale, + ) def test_selects_exact_head_pr_branch_and_definition_even_when_build_failed(tmp_path): @@ -167,8 +173,9 @@ def test_ignores_old_head_until_exact_build_appears_and_retries_bad_responses(tm def test_accepts_late_artifact_after_failed_aggregate_completes(tmp_path): result = _poll( tmp_path, - [EMPTY] * 302 + [ARTIFACT], - [_build()] * 301 + [{**_build(), "status": "completed", "result": "failed"}], + [EMPTY] * 32 + [ARTIFACT], + [_build()] * 31 + [{**_build(), "status": "completed", "result": "failed"}], + clock_scale=10, ) assert result.returncode == 0, result.stdout + result.stderr assert f"COVERAGE_ARTIFACT={ARTIFACT_URL}" in result.stdout @@ -176,7 +183,7 @@ def test_accepts_late_artifact_after_failed_aggregate_completes(tmp_path): assert int((tmp_path / "elapsed").read_text()) > 150 * 60 -@pytest.mark.parametrize("result", ["succeeded", "failed", "canceled"]) +@pytest.mark.parametrize("result", ["succeeded", "failed"]) def test_completed_without_artifact_stops_after_short_grace(tmp_path, result): completed = {**_build(), "status": "completed", "result": result} run = _poll(tmp_path, [EMPTY], [completed]) @@ -185,6 +192,15 @@ def test_completed_without_artifact_stops_after_short_grace(tmp_path, result): assert 120 <= int((tmp_path / "elapsed").read_text()) < 180 +def test_canceled_without_replacement_obeys_wall_clock_budget(tmp_path): + canceled = {**_build(), "status": "completed", "result": "canceled"} + run = _poll(tmp_path, [(22, "not found")], [canceled], clock_scale=10) + assert run.returncode != 0 + assert "has no replacement yet" in run.stdout + assert "Timeout:" in run.stdout + assert 220 * 60 <= int((tmp_path / "elapsed").read_text()) < 225 * 60 + + def test_immediately_available_artifact_needs_no_lifecycle_request(tmp_path): result = _poll(tmp_path, [ARTIFACT], []) assert result.returncode == 0, result.stdout + result.stderr @@ -192,6 +208,22 @@ def test_immediately_available_artifact_needs_no_lifecycle_request(tmp_path): assert (tmp_path / "build.next").read_text() == "0" +def test_switches_from_canceled_run_to_newer_exact_head_build(tmp_path): + older = {**_build(174000), "status": "completed", "result": "succeeded"} + replacement = _build(175449) + result = _poll( + tmp_path, + [(22, "not found"), ARTIFACT], + [{**_build(), "status": "completed", "result": "canceled"}, replacement], + [{"value": [older, replacement]}], + ) + assert result.returncode == 0, result.stdout + result.stderr + assert "continuing with replacement build 175449" in result.stdout + exported = (tmp_path / "github-env").read_text(encoding="utf-8") + assert "BUILD_ID=175449\n" in exported + assert "buildId=175449\n" in exported + + def test_artifact_and_lifecycle_http_json_errors_are_retried(tmp_path): result = _poll( tmp_path, @@ -231,7 +263,7 @@ def test_persistent_api_errors_have_finite_retries(tmp_path, failing_api): assert int((tmp_path / "elapsed").read_text()) < 180 -@pytest.mark.parametrize("step,budget", [("build", 15 * 60), ("artifact", 210 * 60)]) +@pytest.mark.parametrize("step,budget", [("build", 15 * 60), ("artifact", 220 * 60)]) def test_missing_build_or_queued_coverage_obeys_wall_clock_budget(tmp_path, step, budget): if step == "build": result = _run( @@ -240,13 +272,18 @@ def test_missing_build_or_queued_coverage_obeys_wall_clock_budget(tmp_path, step {"builds": [{"value": [_build(174261, sha="b" * 40)]}]}, ) else: - result = _poll(tmp_path, [EMPTY], [{**_build(), "status": "notStarted"}]) + result = _poll( + tmp_path, + [EMPTY], + [{**_build(), "status": "notStarted"}], + clock_scale=10, + ) assert result.returncode != 0 assert "Timeout:" in result.stdout - assert budget <= int((tmp_path / "elapsed").read_text()) < budget + 30 + assert budget <= int((tmp_path / "elapsed").read_text()) < budget + 300 def test_job_budget_leaves_time_for_downloads_and_publishing(): workflow = WORKFLOW.read_text(encoding="utf-8") - assert " timeout-minutes: 235\n" in workflow + assert " timeout-minutes: 245\n" in workflow assert "PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }}" in workflow From 5ed1792fb0f55d8884ce56843bde9d2b7f79fcef Mon Sep 17 00:00:00 2001 From: Gaurav Sharma Date: Wed, 16 Sep 2026 10:04:32 +0530 Subject: [PATCH 05/18] FIX: Authenticate profiler reporting inputs Read coverage reports from fixed outputs without extracting untrusted archive paths into the checkout. Require authenticated source and base trees to use identical benchmark producers, and continue polling when an exact-head ADO run is canceled. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/scripts/extract_coverage_artifact.py | 70 ++++++++++ .github/scripts/post_profiler_comment.py | 32 ++++- .github/workflows/pr-code-coverage.yml | 47 ++----- benchmarks/profiler_ci.py | 13 +- benchmarks/profiler_report.py | 20 +++ tests/test_036_profiler_ci.py | 134 ++++++++++++++++++- 6 files changed, 258 insertions(+), 58 deletions(-) create mode 100644 .github/scripts/extract_coverage_artifact.py diff --git a/.github/scripts/extract_coverage_artifact.py b/.github/scripts/extract_coverage_artifact.py new file mode 100644 index 000000000..d31c96960 --- /dev/null +++ b/.github/scripts/extract_coverage_artifact.py @@ -0,0 +1,70 @@ +"""Copy one expected coverage report from a ZIP without extracting archive paths.""" + +import argparse +from pathlib import Path, PurePosixPath +import stat +import zipfile + +MAX_ARCHIVE_FILES = 10_000 +MAX_ARCHIVE_BYTES = 256 * 1024 * 1024 +MAX_REPORT_BYTES = 64 * 1024 * 1024 + + +def select(archive, kind): + members = archive.infolist() + if ( + len(members) > MAX_ARCHIVE_FILES + or sum(member.file_size for member in members) > MAX_ARCHIVE_BYTES + ): + raise ValueError("Coverage artifact exceeds size limits") + + candidates = [] + for member in members: + path = PurePosixPath(member.filename) + if ( + path.is_absolute() + or ".." in path.parts + or "\\" in member.filename + or member.flag_bits & 1 + or stat.S_ISLNK(member.external_attr >> 16) + or member.file_size > MAX_REPORT_BYTES + ): + continue + if kind == "html" and path.name == "index.html" and "Code Coverage Report" in str(path): + candidates.append((0, member)) + elif kind == "xml" and path.suffix.lower() == ".xml": + name = path.name.lower() + priority = ( + 0 + if str(path).endswith("unified-coverage/coverage.xml") + else (1 if name == "coverage.xml" else 2 if "coverage" in name else 3) + ) + candidates.append((priority, member)) + + if not candidates: + raise ValueError(f"No coverage {kind} report found") + priority = min(item[0] for item in candidates) + selected = [member for rank, member in candidates if rank == priority] + return selected + + +def copy_report(archive_path, output, kind): + with zipfile.ZipFile(archive_path) as archive: + selected = select(archive, kind) + data = archive.read(selected[0]) + if any(archive.read(member) != data for member in selected[1:]): + raise ValueError(f"Conflicting coverage {kind} reports") + Path(output).write_bytes(data) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("kind", choices=("html", "xml")) + parser.add_argument("archive", type=Path) + parser.add_argument("output", type=Path) + args = parser.parse_args() + copy_report(args.archive, args.output, args.kind) + + +if __name__ == "__main__": + main() diff --git a/.github/scripts/post_profiler_comment.py b/.github/scripts/post_profiler_comment.py index 85c6524db..30e167a0d 100644 --- a/.github/scripts/post_profiler_comment.py +++ b/.github/scripts/post_profiler_comment.py @@ -15,8 +15,9 @@ import zipfile sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "benchmarks")) -from profiler_report import LEGS, MARKER, MAX_BYTES, render, validate +from profiler_report import LEGS, MARKER, MAX_BYTES, render, suite_hash, suite_paths, validate +ROOT = Path(__file__).resolve().parents[2] ADO = "https://dev.azure.com/sqlclientdrivers/public/_apis/build" REPOSITORY = "microsoft/mssql-python" # Allow a 160-minute ADO job plus queueing; the workflow reserves publication time. @@ -150,6 +151,26 @@ def find_build(builds, number, head): ) +def suite_blobs(commit): + tree_sha = commit.get("tree", {}).get("sha") + if not re.fullmatch(r"[0-9a-f]{40}", tree_sha or ""): + raise ValueError("Invalid commit tree") + tree = github(f"git/trees/{tree_sha}?recursive=1") + if tree.get("truncated") is not False or not isinstance(tree.get("tree"), list): + raise ValueError("Incomplete commit tree") + expected = {path.relative_to(ROOT).as_posix() for path in suite_paths(ROOT)} + blobs = { + entry.get("path"): entry.get("sha") + for entry in tree["tree"] + if entry.get("type") == "blob" and entry.get("path") in expected + } + if set(blobs) != expected or any( + not re.fullmatch(r"[0-9a-f]{40}", sha or "") for sha in blobs.values() + ): + raise ValueError("Benchmark suite missing from commit tree") + return blobs + + def run(number, head, wait_minutes): publish( number, @@ -173,10 +194,10 @@ def run(number, head, wait_minutes): } ) build = find_build(api(f"{ADO}/builds?{query}")["value"], number, head) - if build and build["status"] == "completed": + if build and build["status"] == "completed" and build.get("result") != "canceled": break time.sleep(30) - if build is None or build.get("status") != "completed": + if build is None or build.get("status") != "completed" or build.get("result") == "canceled": publish( number, head, @@ -194,6 +215,7 @@ def run(number, head, wait_minutes): if len(commit["parents"]) != 2 or commit["parents"][1]["sha"] != head: raise ValueError("ADO merge does not match current PR head") base = commit["parents"][0]["sha"] + suite_unchanged = suite_blobs(commit) == suite_blobs(github(f"git/commits/{base}")) artifacts = api(f"{ADO}/builds/{build_id}/artifacts?api-version=7.1")["value"] reports, issues = [], [] for leg in LEGS: @@ -210,9 +232,9 @@ def run(number, head, wait_minutes): except (ValueError, KeyError, TypeError, URLError, zipfile.BadZipFile): # Invalid data is visibly incomplete, never converted to a success verdict. issues.append(leg + " (invalid artifact)") - if len({r["suite_hash"] for r in reports}) > 1: + if not suite_unchanged or any(report["suite_hash"] != suite_hash(ROOT) for report in reports): reports = [] - issues.append("workload versions differ across legs") + issues.append("workload version differs from trusted base") publish(number, head, render(reports, head, build_id, issues)) diff --git a/.github/workflows/pr-code-coverage.yml b/.github/workflows/pr-code-coverage.yml index a47d6734b..ee0982ee3 100644 --- a/.github/workflows/pr-code-coverage.yml +++ b/.github/workflows/pr-code-coverage.yml @@ -233,16 +233,12 @@ jobs: exit 1 fi - if ! unzip -o -q coverage-report.zip; then - echo "❌ Failed to extract coverage artifacts" - echo "Trying to extract with verbose output for debugging..." - unzip -l coverage-report.zip || echo "Failed to list archive contents" + INDEX_FILE="$RUNNER_TEMP/coverage-index.html" + if ! python .github/scripts/extract_coverage_artifact.py html coverage-report.zip "$INDEX_FILE"; then + echo "❌ Failed to read the coverage HTML artifact" exit 1 fi - - # Find the main index.html file - INDEX_FILE=$(find . -name "index.html" -path "*/Code Coverage Report*" | head -1) - + if [[ -f "$INDEX_FILE" ]]; then echo "🔍 Parsing coverage data from $INDEX_FILE..." @@ -309,8 +305,6 @@ jobs: echo "✅ Coverage data extracted successfully" else echo "❌ Could not find index.html in coverage report" - echo "Available files in the coverage report:" - find . -name "*.html" | head -10 || echo "No HTML files found" exit 1 fi else @@ -356,37 +350,12 @@ jobs: exit 1 fi - if ! unzip -o -q coverage-artifacts.zip; then - echo "❌ Failed to extract coverage artifacts" - echo "Trying to extract with verbose output for debugging..." - unzip -l coverage-artifacts.zip || echo "Failed to list archive contents" + COVERAGE_XML="$RUNNER_TEMP/coverage.xml" + if ! python .github/scripts/extract_coverage_artifact.py xml coverage-artifacts.zip "$COVERAGE_XML"; then + echo "❌ Failed to read the coverage XML artifact" exit 1 fi - - echo "🔍 Looking for coverage XML files in extracted artifacts..." - find . -name "*.xml" -type f | head -10 - - # Look for the main coverage.xml file in unified-coverage directory or any coverage XML - if [[ -f "unified-coverage/coverage.xml" ]]; then - echo "✅ Found unified coverage file at unified-coverage/coverage.xml" - cp "unified-coverage/coverage.xml" ./coverage.xml - elif [[ -f "coverage.xml" ]]; then - echo "✅ Found coverage.xml in root directory" - # Already in the right place - else - # Try to find any coverage XML file - COVERAGE_FILE=$(find . -name "*coverage*.xml" -type f | head -1) - if [[ -n "$COVERAGE_FILE" ]]; then - echo "✅ Found coverage file: $COVERAGE_FILE" - cp "$COVERAGE_FILE" ./coverage.xml - else - echo "❌ No coverage XML file found in artifacts" - echo "Available files:" - find . -name "*.xml" -type f - exit 1 - fi - fi - + cp "$COVERAGE_XML" ./coverage.xml echo "✅ Coverage XML file is ready at ./coverage.xml" ls -la ./coverage.xml else diff --git a/benchmarks/profiler_ci.py b/benchmarks/profiler_ci.py index dbd49c430..c9276f686 100644 --- a/benchmarks/profiler_ci.py +++ b/benchmarks/profiler_ci.py @@ -3,7 +3,6 @@ import argparse import contextlib import faulthandler -import hashlib import importlib.util import io import json @@ -17,6 +16,8 @@ import tempfile import time +from profiler_report import suite_hash + ROOT = Path(__file__).resolve().parents[1] SHA = re.compile(r"[0-9a-f]{40}") LEGS = ("Windows-SQL2022", "Windows-SQL2025", "macOS-SQL2022", "macOS-SQL2025", "Linux-SQL2022") @@ -197,14 +198,6 @@ def run(args): args.output.mkdir(parents=True, exist_ok=True) report_path = args.output / "report.json" report_path.unlink(missing_ok=True) - suite = hashlib.sha256() - for file in [ - Path(__file__), - ROOT / "benchmarks/profiler_workloads.py", - *sorted((ROOT / "profiler").glob("*.py")), - ]: - suite.update(file.name.encode()) - suite.update(file.read_bytes().replace(b"\r\n", b"\n")) head = os.environ.get("SYSTEM_PULLREQUEST_SOURCECOMMITID", candidate) if not SHA.fullmatch(head): head = candidate @@ -216,7 +209,7 @@ def run(args): source_commit=candidate, head_commit=head, build_id=int(os.environ.get("BUILD_BUILDID", "0")), - suite_hash=suite.hexdigest(), + suite_hash=suite_hash(ROOT), samples=args.samples, warmups=args.warmups, pairs=[], diff --git a/benchmarks/profiler_report.py b/benchmarks/profiler_report.py index 17fe8bd8f..da0aa3373 100644 --- a/benchmarks/profiler_report.py +++ b/benchmarks/profiler_report.py @@ -1,6 +1,7 @@ """Validate bounded profiler data and render an advisory, per-platform comparison.""" import argparse +import hashlib import html import json import math @@ -37,6 +38,25 @@ MIN_DELTA_MS = 1.0 +def suite_paths(root): + root = Path(root) + return [ + root / "eng/pipelines/pr-validation-pipeline.yml", + root / "benchmarks/profiler_ci.py", + root / "benchmarks/profiler_report.py", + root / "benchmarks/profiler_workloads.py", + *sorted((root / "profiler").glob("*.py")), + ] + + +def suite_hash(root): + digest = hashlib.sha256() + for file in suite_paths(root): + digest.update(file.name.encode()) + digest.update(file.read_bytes().replace(b"\r\n", b"\n")) + return digest.hexdigest() + + def number(value, maximum=1e12): if type(value) not in (float, int) or not 0 <= value <= maximum: raise ValueError("Invalid performance measurement") diff --git a/tests/test_036_profiler_ci.py b/tests/test_036_profiler_ci.py index cd7edd22d..09837b68d 100644 --- a/tests/test_036_profiler_ci.py +++ b/tests/test_036_profiler_ci.py @@ -28,9 +28,10 @@ def load(name, path): reporting = load("profiler_report", "benchmarks/profiler_report.py") -controller = load("profiler_ci", "benchmarks/profiler_ci.py") sys.modules["profiler_report"] = reporting +controller = load("profiler_ci", "benchmarks/profiler_ci.py") publisher = load("post_profiler_comment", ".github/scripts/post_profiler_comment.py") +extractor = load("extract_coverage_artifact", ".github/scripts/extract_coverage_artifact.py") @pytest.fixture @@ -158,6 +159,57 @@ def zip_data(entries): return out.getvalue() +@pytest.mark.parametrize( + "kind,member,data", + [ + ("html", "Code Coverage Report_1/index.html", b"coverage"), + ("xml", "unified-coverage/coverage.xml", b""), + ], +) +def test_coverage_artifact_reader_copies_only_expected_report(tmp_path, kind, member, data): + archive = tmp_path / "coverage.zip" + archive.write_bytes( + zip_data( + [ + (".github/actions/post-coverage-comment/action.yml", "malicious"), + ("../outside.txt", "escape"), + (member, data), + ] + ) + ) + output = tmp_path / f"report.{kind}" + extractor.copy_report(archive, output, kind) + assert output.read_bytes() == data + assert not (tmp_path.parent / "outside.txt").exists() + assert not (tmp_path / ".github").exists() + + +def test_coverage_artifact_reader_accepts_only_identical_duplicate_reports(tmp_path): + archive = tmp_path / "coverage.zip" + output = tmp_path / "coverage.xml" + archive.write_bytes( + zip_data( + [ + ("first/coverage.xml", ""), + ("second/coverage.xml", ""), + ] + ) + ) + extractor.copy_report(archive, output, "xml") + assert output.read_text() == "" + + archive.write_bytes( + zip_data( + [ + ("first/coverage.xml", ""), + ("second/coverage.xml", ""), + ] + ) + ) + with pytest.raises(ValueError, match="Conflicting"): + extractor.copy_report(archive, output, "xml") + + def test_artifact_read_never_extracts_paths(report): raw = json.dumps(report) assert ( @@ -257,6 +309,24 @@ def __exit__(self, *args): def test_report_cases_match_the_executed_workload_registry(): _, workloads = controller.load_suite() assert tuple(workloads.registry()) == reporting.CASES + assert ROOT / "benchmarks/profiler_report.py" in reporting.suite_paths(ROOT) + assert ROOT / "eng/pipelines/pr-validation-pipeline.yml" in reporting.suite_paths(ROOT) + + +def test_suite_blobs_require_complete_authenticated_tree(monkeypatch): + expected = [path.relative_to(ROOT).as_posix() for path in reporting.suite_paths(ROOT)] + tree = { + "truncated": False, + "tree": [ + {"path": path, "type": "blob", "sha": f"{index + 1:040x}"} + for index, path in enumerate(expected) + ], + } + monkeypatch.setattr(publisher, "github", lambda path: tree) + assert set(publisher.suite_blobs({"tree": {"sha": "a" * 40}})) == set(expected) + tree["tree"].pop() + with pytest.raises(ValueError, match="missing"): + publisher.suite_blobs({"tree": {"sha": "a" * 40}}) @pytest.mark.parametrize("fail", [False, True]) @@ -446,7 +516,7 @@ def api(path, **kwargs): assert reads == 2 and len(calls) == 3 -@pytest.mark.parametrize("corrupt", [None, "zip", "scenarios"]) +@pytest.mark.parametrize("corrupt", [None, "zip", "scenarios", "suite", "source"]) def test_publisher_renders_validated_artifact_and_marks_missing_legs(report, monkeypatch, corrupt): posted = [] windows = copy.deepcopy(report) @@ -456,6 +526,8 @@ def test_publisher_renders_validated_artifact_and_marks_missing_legs(report, mon sample["environment"]["os"] = "Windows" if corrupt == "scenarios": report["pairs"][0]["candidate"]["scenarios"] = list(reporting.CASES) + elif corrupt == "suite": + report["suite_hash"] = "e" * 64 data = { "Windows-SQL2022": zip_data([("report.json", json.dumps(windows))]), "Linux-SQL2022": ( @@ -473,6 +545,13 @@ def test_publisher_renders_validated_artifact_and_marks_missing_legs(report, mon triggerInfo={"pr.number": "123", "pr.sourceSha": "c" * 40}, ) monkeypatch.setattr(publisher, "publish", lambda number, head, body: posted.append(body)) + monkeypatch.setattr(publisher, "suite_hash", lambda root: "d" * 64) + suite_versions = iter(({"suite": "source"}, {"suite": "base"})) + monkeypatch.setattr( + publisher, + "suite_blobs", + lambda commit: next(suite_versions) if corrupt == "source" else {"suite": "same"}, + ) monkeypatch.setattr( publisher, "github", @@ -503,16 +582,60 @@ def test_publisher_renders_validated_artifact_and_marks_missing_legs(report, mon publisher.run(123, "c" * 40, 1) assert len(posted) == 2 assert posted[0].startswith(reporting.MARKER) - assert "### Windows-SQL2022" in posted[1] assert "macOS-SQL2022: incomplete/unavailable" in posted[1] - if corrupt: + if corrupt in ("suite", "source"): + assert "workload version differs from trusted base" in posted[1] + assert "regression signals" not in posted[1] + elif corrupt: + assert "### Windows-SQL2022" in posted[1] assert reporting.escape("Linux-SQL2022 (invalid artifact)") in posted[1] assert "Linux-SQL2022: incomplete/unavailable" in posted[1] assert posted[1].count("20 regression signals") == 1 else: + assert "### Windows-SQL2022" in posted[1] assert posted[1].count("20 regression signals") == 2 +def test_publisher_waits_for_newer_run_after_exact_head_build_is_canceled(report, monkeypatch): + canceled = dict( + id=41, + status="completed", + result="canceled", + definition={"id": 2128}, + repository={"id": "microsoft/mssql-python"}, + sourceBranch="refs/pull/123/merge", + sourceVersion="b" * 40, + triggerInfo={"pr.number": "123", "pr.sourceSha": "c" * 40}, + ) + replacement = {**canceled, "id": 42, "result": "failed"} + builds = iter(([canceled], [replacement])) + posted = [] + clock = [0] + + def api(url): + return {"value": next(builds)} if "/builds?" in url else {"value": []} + + def github(path): + return ( + {"state": "open", "head": {"sha": "c" * 40}} + if path.startswith("pulls/") + else {"parents": [{"sha": "a" * 40}, {"sha": "c" * 40}]} + ) + + monkeypatch.setattr(publisher, "api", api) + monkeypatch.setattr(publisher, "github", github) + monkeypatch.setattr(publisher, "publish", lambda number, head, body: posted.append(body)) + monkeypatch.setattr(publisher, "suite_blobs", lambda commit: {"suite": "same"}) + monkeypatch.setattr(publisher.time, "monotonic", lambda: clock[0]) + monkeypatch.setattr( + publisher.time, "sleep", lambda seconds: clock.__setitem__(0, clock[0] + seconds) + ) + publisher.run(123, "c" * 40, 1) + assert clock[0] == 30 + assert len(posted) == 2 + assert "buildId=42" in posted[1] + + @pytest.mark.parametrize("status", [None, "notStarted", "inProgress"]) def test_publisher_deadline_finishes_without_reading_unfinished_build_metadata(monkeypatch, status): posted = [] @@ -601,6 +724,9 @@ def test_comment_workflow_executes_only_trusted_base_code(): assert "persist-credentials: false" in workflow assert "actions/checkout@11d5960a326750d5838078e36cf38b85af677262" in workflow assert "actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065" in workflow + coverage = (ROOT / ".github/workflows/pr-code-coverage.yml").read_text(encoding="utf-8") + assert coverage.count("extract_coverage_artifact.py") == 2 + assert "unzip -o" not in coverage assert ( "head.ref" not in workflow and "head.sha }}" not in workflow.split("ref:", 1)[1].split("persist", 1)[0] From 8c8e3dd83534bf43bbd30f2dd96a7750fc0e076c Mon Sep 17 00:00:00 2001 From: Gaurav Sharma Date: Wed, 16 Sep 2026 10:49:36 +0530 Subject: [PATCH 06/18] REFACTOR: Organize PR performance reporting Move paired profiler benchmark measurement, workloads, and reporting into a source-only engineering package with direct module entrypoints. Present reviewer-facing results as an impact-first PR Performance Report while keeping stable workflow, artifact, and comment identities. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/scripts/post_profiler_comment.py | 24 +- .github/workflows/pr-profiler-report.yml | 2 +- benchmarks/README.md | 85 +----- eng/pipelines/pr-validation-pipeline.yml | 12 +- eng/profiler_benchmarks/README.md | 54 ++++ eng/profiler_benchmarks/__init__.py | 1 + .../profiler_benchmarks/controller.py | 19 +- .../profiler_benchmarks/report.py | 270 +++++++++++++++--- .../profiler_benchmarks/workloads.py | 2 +- profiler/README.md | 6 +- tests/test_036_profiler_ci.py | 115 ++++++-- 11 files changed, 422 insertions(+), 168 deletions(-) create mode 100644 eng/profiler_benchmarks/README.md create mode 100644 eng/profiler_benchmarks/__init__.py rename benchmarks/profiler_ci.py => eng/profiler_benchmarks/controller.py (96%) rename benchmarks/profiler_report.py => eng/profiler_benchmarks/report.py (52%) rename benchmarks/profiler_workloads.py => eng/profiler_benchmarks/workloads.py (98%) diff --git a/.github/scripts/post_profiler_comment.py b/.github/scripts/post_profiler_comment.py index 30e167a0d..475d43ca3 100644 --- a/.github/scripts/post_profiler_comment.py +++ b/.github/scripts/post_profiler_comment.py @@ -14,8 +14,16 @@ from urllib.request import HTTPRedirectHandler, Request, build_opener import zipfile -sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "benchmarks")) -from profiler_report import LEGS, MARKER, MAX_BYTES, render, suite_hash, suite_paths, validate +sys.path.insert(0, str(Path(__file__).resolve().parents[2])) +from eng.profiler_benchmarks.report import ( + LEGS, + MARKER, + MAX_BYTES, + render, + suite_hash, + suite_paths, + validate, +) ROOT = Path(__file__).resolve().parents[2] ADO = "https://dev.azure.com/sqlclientdrivers/public/_apis/build" @@ -175,8 +183,9 @@ def run(number, head, wait_minutes): publish( number, head, - f"{MARKER}\n## Profiler performance report\n" - f"Awaiting paired ADO measurements for head `{head}`. No regression verdict yet.", + f"{MARKER}\n## PR Performance Report\n\n" + "**Performance assessment pending.**\n\n" + f"Waiting for the matching performance run for head `{head}`.", ) deadline = time.monotonic() + wait_minutes * 60 build = None @@ -201,9 +210,10 @@ def run(number, head, wait_minutes): publish( number, head, - f"{MARKER}\n## Profiler performance report\n" - f"No matching ADO run completed within the {wait_minutes}-minute wait for `{head}`. " - "Results are incomplete. No regression verdict.", + f"{MARKER}\n## PR Performance Report\n\n" + "**Performance could not be assessed.**\n\n" + f"No matching performance run completed within the {wait_minutes}-minute wait " + f"for `{head}`. No result is available.", ) return build_id = build["id"] diff --git a/.github/workflows/pr-profiler-report.yml b/.github/workflows/pr-profiler-report.yml index f7b41f0eb..d14dd5770 100644 --- a/.github/workflows/pr-profiler-report.yml +++ b/.github/workflows/pr-profiler-report.yml @@ -1,4 +1,4 @@ -name: PR Profiler Report +name: PR Performance Report # Privileged reporting only. No PR checkout, builds, or artifact execution here. on: diff --git a/benchmarks/README.md b/benchmarks/README.md index e7cd2f7ac..07560c08a 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -7,85 +7,12 @@ This directory contains benchmark scripts for testing the performance of various ### 1. `bench_mssql.py` - Richbench Framework Benchmarks Comprehensive benchmarks using the richbench framework for detailed performance analysis. -### 2. `profiler_ci.py` - PR Regression Comparisons -CI compares profiling-enabled base and candidate revisions on the same agent and -SQL Server. It replaces the historical pyodbc-normalized CI comparison; the old -`perf-benchmarking.py` remains available for local driver-versus-pyodbc analysis. - -The 20-workload registry retains all 10 scenarios from `profiler/scenarios.py`, -the four AdventureWorks queries (including 1.2M rows), both legacy 100K-row -insertmanyvalues variants, two additional fetch batch sizes, and repeated positional -and named-parameter execution. - -```bash -# Requires build dependencies, pyarrow, and an AdventureWorks2022 connection. -python benchmarks/profiler_ci.py --base main --candidate HEAD \ - --leg Linux-SQL2022 --output profiler-results -python benchmarks/profiler_report.py profiler-results/report.json -``` - -CI uses the PR-merge commit's first parent as the exact base snapshot. The five -benchmark legs build the candidate with `ENABLE_PROFILING=1`, run pytest with -recording disabled by default, then reuse that binary for the benchmark. Existing -profiler-specific tests explicitly enable and clean up recording; ordinary driver -tests do not. A pre-test check requires the expected native configuration and -recording OFF. LocalDB, other Linux legs and the release pipelines still build the -default configuration. Windows profiling artifacts are named separately from -`ddbc_bindings` and are not release wheels. - -Only the base requires a second build, in a temporary git archive with profiling -enabled. Local invocations build both archives unless `--reuse-candidate` is supplied; -that option requires the requested candidate to be the current checkout HEAD. -Both sides execute the same version of the workload suite. Each pass runs in a -fresh interpreter. Five measured -pairs follow one discarded warmup pair, alternating base/PR order. A local subset is -available through `--scenarios`; it is not accepted as a complete CI report. - -Each worker has a six-minute limit for the entire workload suite, with a shared -90-minute build/measurement budget inside the CI step's 100-minute limit. This fits -all twelve permitted worker limits, a 15-minute base build and a one-minute -preflight, with headroom. Local runs receive 105 minutes because they build both -revisions. The 160-minute CI jobs also allow for setup, pytest and artifact -publication. These are upper bounds, not mandatory wait times. -Worker logs identify each starting/completed scenario and emit stack traces every minute. -Per-worker JSON checkpoints retain finished scenarios and identify the active one -if the process fails or is killed. Partial workers never count as measured pairs; -incomplete reports do not produce a regression verdict. - -The comparison is **advisory**, not a new merge gate. A regression signal requires -over 20% paired-median slowdown, at least 1 ms additional median wall time, and at -least 80% of pairs exceeding the relative threshold. Disagreement is reported as -noisy. Paired runs reduce agent-to-agent noise; they do not eliminate server -contention. Enabled profiler overhead is part of both measurements, so these are -not production-wheel latency estimates. - -Per-phase inclusive duration deltas and call-count changes help locate regressions; -they are not summed into wall-clock totals. Raw pairs and build/worker logs are -published on PR and main runs as `profiler--` artifacts. -The existing five benchmark legs are covered: Windows and macOS on SQL2022/2025, -and Linux Ubuntu on SQL2022. ARM, RHEL, Alpine, LocalDB, and Azure SQL are not -implicitly compared against other platforms. - -`PR Profiler Report` creates or updates one comment per PR. Its privileged job -checks out only the trusted base revision, never executes PR/artifact code, validates -bounded JSON against the ADO build and GitHub merge/head identities, and ignores -stale heads. Missing, skipped, malformed, or failed runs are shown as incomplete, -not as a clean performance verdict. As a new base-branch reporting workflow, it -starts reporting automatically after this infrastructure has merged; it does not -grant fork-authored workflow code write credentials. - -The publisher waits up to 220 minutes for the matching ADO run to complete, -including queueing, inside a 230-minute workflow. A still-queued or running build -at that deadline produces a final incomplete comment, not a lingering `Awaiting` -message. A malformed report invalidates only its own leg; valid legs still render. -Coverage polling also allows 220 minutes for its artifact, plus 15 minutes for -build discovery and ten for processing/publication in its 245-minute workflow. - -The first main comparison after introduction may lack profiling support on its -parent; that run is incomplete rather than falling back to an uninstrumented base. -Subsequent comparisons use the new artifact format and do not consume old -`perf-baseline-*` artifacts. For a fresh measurement after an ADO-only retry, -re-run the GitHub reporting workflow as well. +### 2. Profiler benchmark comparisons + +Profiler benchmarks are engineering infrastructure, separate from these standalone +scripts and from the runtime profiler. See +[`eng/profiler_benchmarks/README.md`](../eng/profiler_benchmarks/README.md). +Their reviewer-facing output is the impact-first **PR Performance Report**. ## Why Benchmarks? - To measure the efficiency of `pyodbc` and `mssql_python` in handling database operations. diff --git a/eng/pipelines/pr-validation-pipeline.yml b/eng/pipelines/pr-validation-pipeline.yml index 499dd25e9..8d3ec5de3 100644 --- a/eng/pipelines/pr-validation-pipeline.yml +++ b/eng/pipelines/pr-validation-pipeline.yml @@ -248,7 +248,7 @@ jobs: env: ENABLE_PROFILING: $(profilerBuild) - - script: python benchmarks/profiler_ci.py --check-build $(profilerCheck) + - script: python -m eng.profiler_benchmarks.controller --check-build $(profilerCheck) displayName: 'Verify native configuration and recording OFF before pytest' - template: steps/install-mssql-py-core.yml @@ -400,7 +400,7 @@ jobs: exit 1 } - python benchmarks/profiler_ci.py --reuse-candidate --leg "Windows-$(sqlVersion)" --output profiler-results + python -m eng.profiler_benchmarks.controller --reuse-candidate --leg "Windows-$(sqlVersion)" --output profiler-results if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } displayName: 'Compare profiling builds on SQL Server 2022/2025' condition: and(succeeded(), or(eq(variables['sqlVersion'], 'SQL2022'), eq(variables['sqlVersion'], 'SQL2025'))) @@ -567,7 +567,7 @@ jobs: env: DB_PASSWORD: $(DB_PASSWORD) - - script: python benchmarks/profiler_ci.py --check-build on + - script: python -m eng.profiler_benchmarks.controller --check-build on displayName: 'Verify native configuration and recording OFF before pytest' - template: steps/install-mssql-py-core.yml @@ -654,7 +654,7 @@ jobs: # Newer Homebrew refuses to load formulae from third-party taps unless the tap is trusted brew trust microsoft/mssql-release || echo "brew trust failed; attempting install anyway" HOMEBREW_ACCEPT_EULA=Y brew install msodbcsql18 - python benchmarks/profiler_ci.py --reuse-candidate --leg "macOS-$(sqlVersion)" --output profiler-results + python -m eng.profiler_benchmarks.controller --reuse-candidate --leg "macOS-$(sqlVersion)" --output profiler-results displayName: 'Compare profiling builds on macOS $(sqlVersion)' condition: and(succeeded(), or(eq(variables['sqlVersion'], 'SQL2022'), eq(variables['sqlVersion'], 'SQL2025'))) timeoutInMinutes: 100 @@ -825,7 +825,7 @@ jobs: chmod +x build.sh ./build.sh cd ../.. - python benchmarks/profiler_ci.py --check-build $PROFILER_CHECK + python -m eng.profiler_benchmarks.controller --check-build $PROFILER_CHECK " displayName: 'Build pybind bindings (.so) in $(distroName) container' @@ -944,7 +944,7 @@ jobs: ACCEPT_EULA=Y apt-get install -y --no-install-recommends git unixodbc unixodbc-dev libodbc2 libodbcinst2 odbcinst msodbcsql18 git config --global --add safe.directory /workspace odbcinst -q -d -n "ODBC Driver 18 for SQL Server" - python benchmarks/profiler_ci.py --reuse-candidate --leg Linux-SQL2022 --output profiler-results + python -m eng.profiler_benchmarks.controller --reuse-candidate --leg Linux-SQL2022 --output profiler-results ' else echo "Skipping performance benchmarks on $(distroName) (only runs on Ubuntu with local SQL Server)" diff --git a/eng/profiler_benchmarks/README.md b/eng/profiler_benchmarks/README.md new file mode 100644 index 000000000..e38243d91 --- /dev/null +++ b/eng/profiler_benchmarks/README.md @@ -0,0 +1,54 @@ +# Profiler Benchmarks + +This package compares profiling-enabled base and candidate revisions on the same +agent and SQL Server, then validates and renders advisory performance results. +It is separate from the runtime profiler in `profiler/` and the standalone scripts +in `benchmarks/`. + +The reviewer-facing experience is the **PR Performance Report**. It leads with +the performance impact, affected database tasks, and environment coverage. Phase +timings, all task measurements, commits, and methodology remain expandable evidence. + +## Local use + +```bash +# Requires build dependencies, pyarrow, and an AdventureWorks2022 connection. +python -m eng.profiler_benchmarks.controller --base main --candidate HEAD \ + --leg Linux-SQL2022 --output profiler-results +python -m eng.profiler_benchmarks.report profiler-results/report.json +``` + +The fixed 20-workload registry retains all 10 profiler scenarios, four +AdventureWorks queries, both legacy 100K-row insert variants, two fetch batch sizes, +and repeated positional and named-parameter execution. A local subset is available +through `--scenarios`; subset reports remain incomplete and cannot produce a verdict. + +## Measurement contract + +CI uses the PR-merge commit's first parent as the exact base. Selected legs build +the candidate with `ENABLE_PROFILING=1`, run pytest with recording disabled, and +reuse that binary. The base gets an isolated profiling build. Each side runs in a +fresh process, with five measured pairs after one discarded warmup pair and +alternating order. + +Workers have a six-minute limit. CI allows 90 minutes for measurement inside a +100-minute step and 160-minute job. Local runs receive 105 minutes because they +build both revisions. Partial workers never count as measured pairs, and incomplete +reports never produce a verdict. + +A regression signal requires over 20% paired-median slowdown, at least 1 ms added +median wall time, and at least 80% of pairs exceeding the relative threshold. +Per-phase inclusive deltas and call-count changes are diagnostics, not additive +wall-clock components. + +## Publication + +Five legs publish raw samples: Windows and macOS on SQL Server 2022/2025, and +Ubuntu on SQL Server 2022. The privileged GitHub publisher executes only trusted +base code, authenticates the ADO build and source/base suite trees, treats artifacts +as bounded data, and ignores stale heads. Missing, malformed, skipped, canceled, or +failed runs are incomplete. + +The publisher waits up to 220 minutes inside a 230-minute workflow. The first main +comparison after introduction may be incomplete because its parent lacks this +infrastructure. A fresh ADO-only retry also requires rerunning the GitHub publisher. diff --git a/eng/profiler_benchmarks/__init__.py b/eng/profiler_benchmarks/__init__.py new file mode 100644 index 000000000..a3a7b9735 --- /dev/null +++ b/eng/profiler_benchmarks/__init__.py @@ -0,0 +1 @@ +"""Paired profiler benchmark measurement and reporting.""" diff --git a/benchmarks/profiler_ci.py b/eng/profiler_benchmarks/controller.py similarity index 96% rename from benchmarks/profiler_ci.py rename to eng/profiler_benchmarks/controller.py index c9276f686..c26917bc3 100644 --- a/benchmarks/profiler_ci.py +++ b/eng/profiler_benchmarks/controller.py @@ -16,9 +16,10 @@ import tempfile import time -from profiler_report import suite_hash +from .report import suite_hash +from . import workloads -ROOT = Path(__file__).resolve().parents[1] +ROOT = Path(__file__).resolve().parents[2] SHA = re.compile(r"[0-9a-f]{40}") LEGS = ("Windows-SQL2022", "Windows-SQL2025", "macOS-SQL2022", "macOS-SQL2025", "Linux-SQL2022") # Twelve six-minute passes plus a 15-minute base build and preflight need 88 @@ -109,7 +110,6 @@ def load_suite(): sys.modules["profiler"] = module spec.loader.exec_module(module) import profiler.core as core - import profiler_workloads as workloads return core, workloads @@ -171,7 +171,8 @@ def measure(path, output, scenarios, timeout=WORKER_TIMEOUT): command = [ sys.executable, "-u", - str(Path(__file__).resolve()), + "-m", + "eng.profiler_benchmarks.controller", "--worker", "--source-root", str(path), @@ -189,7 +190,7 @@ def measure(path, output, scenarios, timeout=WORKER_TIMEOUT): def remaining(deadline, limit): seconds = deadline - time.monotonic() if seconds <= 0: - raise TimeoutError("Profiler CI exhausted its overall build/measurement budget") + raise TimeoutError("Profiler benchmarks exhausted their overall build/measurement budget") return min(seconds, limit) @@ -227,7 +228,13 @@ def run(args): raise ValueError("--reuse-candidate requires candidate to be checkout HEAD") paths[side] = ROOT subprocess.run( - [sys.executable, str(Path(__file__).resolve()), "--check-build", "on"], + [ + sys.executable, + "-m", + "eng.profiler_benchmarks.controller", + "--check-build", + "on", + ], check=True, timeout=remaining(deadline, 60), ) diff --git a/benchmarks/profiler_report.py b/eng/profiler_benchmarks/report.py similarity index 52% rename from benchmarks/profiler_report.py rename to eng/profiler_benchmarks/report.py index da0aa3373..91c1a0c58 100644 --- a/benchmarks/profiler_report.py +++ b/eng/profiler_benchmarks/report.py @@ -36,15 +36,38 @@ MARKER = "" THRESHOLD = 0.20 MIN_DELTA_MS = 1.0 +TASK_NAMES = { + "connect": "Connection opening", + "select": "SELECT queries", + "insert": "Row insertion", + "executemany": "Executemany inserts", + "fetchall": "Fetch-all queries", + "fetchone": "Row-by-row fetching", + "fetchmany": "Batched row fetching", + "commit_rollback": "Transaction commit and rollback", + "arrow": "Arrow row fetching", + "insertmanyvalues": "100,000-row insertion", + "fetchmany_100": "Row fetching in batches of 100", + "fetchmany_10000": "Row fetching in batches of 10,000", + "prepared_qmark": "Repeated positional queries", + "prepared_named": "Repeated named-parameter queries", + "legacy_insertmany": "Legacy 100,000-row insertion", + "setinputsizes": "Insertion with explicit input sizes", + "join_aggregation": "Joined aggregation queries", + "large_fetch": "Large joined-result fetching", + "fetch_1_2m": "1.2-million-row fetching", + "cte": "Common table expression queries", +} def suite_paths(root): root = Path(root) return [ root / "eng/pipelines/pr-validation-pipeline.yml", - root / "benchmarks/profiler_ci.py", - root / "benchmarks/profiler_report.py", - root / "benchmarks/profiler_workloads.py", + root / "eng/profiler_benchmarks/__init__.py", + root / "eng/profiler_benchmarks/controller.py", + root / "eng/profiler_benchmarks/report.py", + root / "eng/profiler_benchmarks/workloads.py", *sorted((root / "profiler").glob("*.py")), ] @@ -225,69 +248,224 @@ def escape(value): return value +def environment_name(leg): + operating_system, sql = leg.split("-") + return f"{operating_system} / SQL Server {sql.removeprefix('SQL')}" + + +def issue_reason(leg, issues): + prefix = leg + " (" + for issue in issues: + if issue.startswith(prefix) and issue.endswith(")"): + return issue[len(prefix) : -1] + global_issues = [issue for issue in issues if not any(issue.startswith(x + " (") for x in LEGS)] + return global_issues[0] if global_issues else "incomplete benchmark" + + def render(reports, head, build_id, issues=()): url = f"https://dev.azure.com/sqlclientdrivers/public/_build/results?buildId={build_id}" - lines = [ - MARKER, - "## Profiler performance report", - f"Head `{head}` | [ADO build {build_id}]({url})", - "", - "Advisory base vs PR-merge comparison. Both revisions are profiling-enabled, " - "measured on the same agent/database with alternating order and discarded warmups.", - "Flags require >20% paired median slowdown, >=1 ms added time, and 80% of pairs agreeing. " - "These are signals to investigate, not production-wheel latency guarantees.", + by_leg = {r["leg"]: r for r in reports} + completed = { + leg: (report, comparisons(report)) + for leg in LEGS + if (report := by_leg.get(leg)) is not None and report["status"] == "complete" + } + regressions = [ + (leg, row) + for leg, (_, rows) in completed.items() + for row in rows + if row["status"] == "regression" + ] + noisy = [ + (leg, row) + for leg, (_, rows) in completed.items() + for row in rows + if row["status"] == "noisy" + ] + missing = len(LEGS) - len(completed) + + if len(regressions) == 1: + leg, row = regressions[0] + opening = ( + f"This PR consistently slows {TASK_NAMES[row['name']].lower()} on " + f"{environment_name(leg)} by {row['change_pct']:.1f}%." + ) + elif regressions: + tasks = len({row["name"] for _, row in regressions}) + environments = len({leg for leg, _ in regressions}) + opening = ( + f"This PR has {len(regressions)} consistent slowdown signals across " + f"{tasks} database tasks and {environments} environments." + ) + elif noisy: + if len(noisy) == 1: + leg, row = noisy[0] + opening = ( + f"{TASK_NAMES[row['name']]} was slower on {environment_name(leg)}, " + "but the repeated comparisons were inconsistent." + ) + else: + tasks = len({row["name"] for _, row in noisy}) + environments = len({leg for leg, _ in noisy}) + opening = ( + f"No consistent slowdowns detected. {len(noisy)} inconsistent comparisons " + f"need review across {tasks} database tasks and {environments} environments." + ) + elif not completed: + opening = ( + "Performance could not be assessed because no environment produced a complete result." + ) + elif not missing: + opening = f"No consistent slowdowns detected across all {len(LEGS)} environments." + else: + completed_label = "environment" if len(completed) == 1 else "environments" + missing_label = "environment" if missing == 1 else "environments" + opening = ( + f"No consistent slowdowns in the {len(completed)} completed {completed_label}. " + f"No result is available for {missing} {missing_label}." + ) + + lines = [MARKER, "## PR Performance Report", "", f"**{opening}**", ""] + highlighted = regressions or noisy + if highlighted: + if not regressions: + lines += ["Inconsistent slowdowns to review:", ""] + lines += [ + "| Environment | Affected task | Before | After | Change |", + "|---|---|---:|---:|---:|", + ] + for leg, row in highlighted: + lines.append( + f"| {environment_name(leg)} | {TASK_NAMES[row['name']]} | " + f"{row['base_ms']:.3f} ms | {row['candidate_ms']:.3f} ms | " + f"{row['change_pct']:+.1f}% |" + ) + lines.append("") + if regressions: + lines.append( + "The largest recorded phase increases for these tasks are shown below. " + "Phase timings are supporting evidence, not root-cause proof." + ) + if noisy: + lines.append( + f"{len(noisy)} additional inconsistent slowdown" + f"{'s' if len(noisy) != 1 else ''} also need review." + ) + lines.append("") + + lines += [ + f"**Coverage:** {len(completed)} of {len(LEGS)} environments completed. " + "Advisory result; does not block merging.", "", + "| Environment | Status |", + "|---|---|", ] - by_leg = {r["leg"]: r for r in reports} for leg in LEGS: report = by_leg.get(leg) - if report is None or report["status"] != "complete": - lines.append(f"**{leg}: incomplete/unavailable. No regression verdict.**") + status = ( + "Completed" + if leg in completed + else f"No result available ({escape(issue_reason(leg, issues))})" + ) + lines.append(f"| {environment_name(leg)} | {status} |") + + lines += [ + "", + "
", + "Affected phases and call counts", + "", + "Phase times are inclusive diagnostics and must not be added together. " + "They identify where measured time changed, not why it changed.", + ] + diagnostics = 0 + for leg, (_, rows) in completed.items(): + relevant = [row for row in rows if row["status"] != "ok" or row["counts"]] + if not relevant: continue - rows = comparisons(report) - env = report["pairs"][0]["base"]["environment"] - flags = [r for r in rows if r["status"] == "regression"] - noisy = sum(r["status"] == "noisy" for r in rows) + lines += ["", f"### {environment_name(leg)}"] + for row in relevant: + diagnostics += 1 + phases = "; ".join(f"{escape(label)} +{delta:.3f} ms" for delta, label in row["phases"]) + counts = "; ".join(escape(label) for label in row["counts"]) + detail = phases or "no positive phase delta" + if counts: + detail += f". Call changes: {counts}" + lines.append(f"**{TASK_NAMES[row['name']]}:** {detail}.") + if not diagnostics: + lines += ["", "No affected phases or call-count changes were recorded."] + lines += [ + "", + "
", + "", + "
", + "All database tasks and timings", + ] + + for leg, (report, rows) in completed.items(): lines += [ "", - f"### {leg}", - f"Base `{report['base_commit'][:12]}` -> merge `{report['source_commit'][:12]}`; " - f"Python {escape(env['python'])}, {escape(env['architecture'])}, " - f"SQL {escape(env['sql_version'])}; {report['samples']} pairs.", - f"**{len(flags)} regression signals, {noisy} noisy comparisons.**", - "
All scenarios and phase diagnostics", - "", - "| Scenario | Base ms | PR ms | Paired change | Result |", + f"### {environment_name(leg)}", + "| Database task | Before | After | Paired change | Result |", "|---|---:|---:|---:|---|", ] for row in rows: + result = { + "regression": "consistent slowdown", + "noisy": "inconsistent slowdown", + "ok": "no signal", + }[row["status"]] lines.append( - f"| {row['name']} | {row['base_ms']:.3f} | {row['candidate_ms']:.3f} | " - f"{row['change_pct']:+.1f}% | {row['status']} |" + f"| {TASK_NAMES[row['name']]} | {row['base_ms']:.3f} ms | " + f"{row['candidate_ms']:.3f} ms | {row['change_pct']:+.1f}% | {result} |" ) - for row in rows: - if row["status"] != "ok" or row["counts"]: - detail = "; ".join( - f"{escape(label)} +{delta:.3f} ms" for delta, label in row["phases"] - ) - counts = "; ".join(escape(label) for label in row["counts"]) - lines.append( - f"\n**{row['name']}**: {detail or 'no positive phase delta'}." - + (f" Call changes: {counts}." if counts else "") - ) + lines += [ + "", + "
", + "", + "
", + "Build, commits and measurement details", + "", + ] + lines += [ + f"[ADO build {build_id}]({url})", + "", + f"PR head: `{head}`", + ] + if completed: + first = next(iter(completed.values()))[0] lines += [ + f"Base: `{first['base_commit']}`", + f"Measured merge: `{first['source_commit']}`", "", - "Phase times are inclusive diagnostics, not additive wall-clock components.", - "
", ] + for leg, (report, _) in completed.items(): + env = report["pairs"][0]["base"]["environment"] + lines.append( + f"- {environment_name(leg)}: Python {escape(env['python'])}, " + f"{escape(env['architecture'])}, SQL {escape(env['sql_version'])}; " + f"{report['samples']} paired comparisons and {report['warmups']} warmup." + ) + lines += [ + "", + "A consistent slowdown requires more than 20% median paired slowdown, at least " + "1 ms between the median runtimes, and at least 80% of pairs exceeding the " + "relative threshold. An inconsistent slowdown crosses the first two thresholds " + "without enough pair agreement.", + "", + "The displayed change is the median of paired before-and-after ratios. It is not " + "recalculated from the two displayed median runtimes.", + ] if issues: - lines += [ - "", - "Some artifacts were missing or rejected: " + ", ".join(escape(x) for x in issues), - ] + lines += ["", "Unavailable or rejected data: " + ", ".join(escape(x) for x in issues)] lines += [ "", - "Raw samples and build logs are attached to the ADO run as `profiler-*` artifacts.", + "Both revisions use profiling-enabled builds on the same agent and database, " + "with alternating order and discarded warmups. Results are diagnostic and do " + "not represent production-wheel latency.", + "", + "Raw samples and logs are attached to the ADO run as `profiler-*` artifacts.", + "", + "
", ] body = "\n".join(lines) if len(body) > 60000: diff --git a/benchmarks/profiler_workloads.py b/eng/profiler_benchmarks/workloads.py similarity index 98% rename from benchmarks/profiler_workloads.py rename to eng/profiler_benchmarks/workloads.py index b33240978..324600860 100644 --- a/benchmarks/profiler_workloads.py +++ b/eng/profiler_benchmarks/workloads.py @@ -1,4 +1,4 @@ -"""Fixed workloads shared by the base and candidate profiler builds.""" +"""Fixed workloads shared by the base and candidate profiler benchmark builds.""" from functools import partial import time diff --git a/profiler/README.md b/profiler/README.md index 753f665da..8c0662b14 100644 --- a/profiler/README.md +++ b/profiler/README.md @@ -21,9 +21,9 @@ context manager whose end-to-end cost is within run-to-run noise. Runtime-instrumentation tests remain part of the driver test suite. Tests that require the dev-only `profiler/` package skip when it is absent from an installed -wheel. The [paired CI benchmark guide](../benchmarks/README.md) describes isolated -profiling builds, scenario coverage and advisory PR regression comments. Broader -profiler testing remains follow-up work. +wheel. The [profiler benchmark guide](../eng/profiler_benchmarks/README.md) describes +isolated profiling builds, scenario coverage and advisory PR regression comments. +Broader profiler testing remains follow-up work. Use controlled diagnostic workloads with one owner of the process-wide profiling state: enable, run the workload, wait for worker threads to finish, then collect. diff --git a/tests/test_036_profiler_ci.py b/tests/test_036_profiler_ci.py index 09837b68d..8a3b7bd27 100644 --- a/tests/test_036_profiler_ci.py +++ b/tests/test_036_profiler_ci.py @@ -19,6 +19,9 @@ if not (ROOT / ".github/scripts/post_profiler_comment.py").is_file(): pytest.skip("CI reporting tools are not installed in driver wheels", allow_module_level=True) +from eng.profiler_benchmarks import controller +from eng.profiler_benchmarks import report as reporting + def load(name, path): spec = importlib.util.spec_from_file_location(name, ROOT / path) @@ -27,9 +30,6 @@ def load(name, path): return module -reporting = load("profiler_report", "benchmarks/profiler_report.py") -sys.modules["profiler_report"] = reporting -controller = load("profiler_ci", "benchmarks/profiler_ci.py") publisher = load("post_profiler_comment", ".github/scripts/post_profiler_comment.py") extractor = load("extract_coverage_artifact", ".github/scripts/extract_coverage_artifact.py") @@ -72,8 +72,12 @@ def test_consistent_slowdown_is_advisory_regression(report): row["status"] == "regression" and row["change_pct"] == pytest.approx(30) for row in rows ) body = reporting.render([report], "c" * 40, 42) - assert "20 regression signals" in body - assert "incomplete/unavailable" in body # missing platforms never read as green + assert "20 consistent slowdown signals" in body + assert "| Linux / SQL Server 2022 | Connection opening |" in body + assert "| macOS / SQL Server 2022 | No result available" in body + assert body.index("consistent slowdown signals") < body.index( + "Build, commits and measurement details" + ) def test_noisy_slowdown_and_submillisecond_change_are_not_regressions(report): @@ -126,7 +130,75 @@ def test_reject_wrong_commit_and_preserve_incomplete_status(report): report["status"] = "incomplete" report["pairs"] = [] reporting.validate(report) - assert "No regression verdict" in reporting.render([report], "c" * 40, 42) + assert "Performance could not be assessed" in reporting.render([report], "c" * 40, 42) + + +def set_leg(report, leg): + report = copy.deepcopy(report) + report["leg"] = leg + operating_system, sql = leg.split("-") + for pair in report["pairs"]: + for sample in pair.values(): + sample["environment"]["os"] = {"macOS": "Darwin"}.get( + operating_system, operating_system + ) + sample["environment"]["sql_version"] = "16.0" if sql == "SQL2022" else "17.0" + return report + + +def clear_slowdowns(report): + for pair in report["pairs"]: + for name in reporting.CASES: + pair["candidate"]["scenarios"][name]["wall_ms"] = pair["base"]["scenarios"][name][ + "wall_ms" + ] + return report + + +def test_impact_summary_handles_single_inconsistent_and_complete_clean_results(report): + clean = clear_slowdowns(copy.deepcopy(report)) + for pair, scale in zip(clean["pairs"], (1.3, 1.3, 1.3, 0.8, 0.8)): + pair["candidate"]["scenarios"]["fetchone"]["wall_ms"] *= scale + noisy = reporting.render([clean], "c" * 40, 42) + assert ( + "**Row-by-row fetching was slower on Linux / SQL Server 2022, " + "but the repeated comparisons were inconsistent.**" + ) in noisy + assert "Inconsistent slowdowns to review:" in noisy + + complete = [set_leg(clear_slowdowns(copy.deepcopy(report)), leg) for leg in reporting.LEGS] + clean_body = reporting.render(complete, "c" * 40, 42) + assert "**No consistent slowdowns detected across all 5 environments.**" in clean_body + assert "**Coverage:** 5 of 5 environments completed." in clean_body + + +def test_impact_summary_handles_single_regression_partial_and_no_results(report): + single = clear_slowdowns(copy.deepcopy(report)) + for pair in single["pairs"]: + pair["candidate"]["scenarios"]["fetchone"]["wall_ms"] *= 1.3 + body = reporting.render([single], "c" * 40, 42) + assert ( + "**This PR consistently slows row-by-row fetching on Linux / SQL Server 2022 " "by 30.0%.**" + ) in body + assert "Affected phases and call counts" in body + assert "All database tasks and timings" in body + assert "Build, commits and measurement details" in body + assert "median of paired before-and-after ratios" in body + + partial = reporting.render( + [clear_slowdowns(copy.deepcopy(report))], + "c" * 40, + 42, + ["Windows-SQL2022 (missing)"], + ) + assert "No consistent slowdowns in the 1 completed environment." in partial + assert "No result is available for 4 environments." in partial + assert "| Windows / SQL Server 2022 | No result available (missing) |" in partial + assert "pending" not in partial.lower() + + unavailable = reporting.render([], "c" * 40, 42, ["Linux-SQL2022 (invalid artifact)"]) + assert "Performance could not be assessed" in unavailable + assert "No consistent slowdowns" not in unavailable @pytest.mark.parametrize( @@ -309,7 +381,8 @@ def __exit__(self, *args): def test_report_cases_match_the_executed_workload_registry(): _, workloads = controller.load_suite() assert tuple(workloads.registry()) == reporting.CASES - assert ROOT / "benchmarks/profiler_report.py" in reporting.suite_paths(ROOT) + assert ROOT / "eng/profiler_benchmarks/__init__.py" in reporting.suite_paths(ROOT) + assert ROOT / "eng/profiler_benchmarks/report.py" in reporting.suite_paths(ROOT) assert ROOT / "eng/pipelines/pr-validation-pipeline.yml" in reporting.suite_paths(ROOT) @@ -413,6 +486,7 @@ def build(path, log, timeout): clock[0] += 900 def preflight(command, **kwargs): + assert command[1:3] == ["-m", "eng.profiler_benchmarks.controller"] assert "--check-build" in command and kwargs["timeout"] == 60 clock[0] += 60 @@ -454,7 +528,9 @@ def test_ci_deadlines_include_setup_queueing_and_publication(): for job in ("pytestonwindows", "PytestOnMacOS", "PytestOnLinux"): section = pipeline.split(f"- job: {job}\n", 1)[1].split("\n- job:", 1)[0] job_minutes = int(re.search(r"^ timeoutInMinutes: (\d+)$", section, re.M)[1]) - benchmark_step = section.split("python benchmarks/profiler_ci.py --reuse-candidate", 1)[1] + benchmark_step = section.split( + "python -m eng.profiler_benchmarks.controller --reuse-candidate", 1 + )[1] step_minutes = int(re.search(r"^ timeoutInMinutes: (\d+)$", benchmark_step, re.M)[1]) assert step_minutes * 60 >= controller.BENCHMARK_TIMEOUT + 10 * 60 assert job_minutes >= step_minutes + 60 @@ -582,18 +658,18 @@ def test_publisher_renders_validated_artifact_and_marks_missing_legs(report, mon publisher.run(123, "c" * 40, 1) assert len(posted) == 2 assert posted[0].startswith(reporting.MARKER) - assert "macOS-SQL2022: incomplete/unavailable" in posted[1] + assert "| macOS / SQL Server 2022 | No result available" in posted[1] if corrupt in ("suite", "source"): assert "workload version differs from trusted base" in posted[1] - assert "regression signals" not in posted[1] + assert "consistent slowdown signals" not in posted[1] elif corrupt: - assert "### Windows-SQL2022" in posted[1] + assert "### Windows / SQL Server 2022" in posted[1] assert reporting.escape("Linux-SQL2022 (invalid artifact)") in posted[1] - assert "Linux-SQL2022: incomplete/unavailable" in posted[1] - assert posted[1].count("20 regression signals") == 1 + assert "| Linux / SQL Server 2022 | No result available (invalid artifact) |" in posted[1] + assert posted[1].count("20 consistent slowdown signals") == 1 else: - assert "### Windows-SQL2022" in posted[1] - assert posted[1].count("20 regression signals") == 2 + assert "### Windows / SQL Server 2022" in posted[1] + assert posted[1].count("40 consistent slowdown signals") == 1 def test_publisher_waits_for_newer_run_after_exact_head_build_is_canceled(report, monkeypatch): @@ -668,9 +744,10 @@ def sleep(seconds): monkeypatch.setattr(publisher, "publish", lambda number, head, body: posted.append(body)) publisher.run(123, "c" * 40, 1) assert clock[0] == 60 and len(posted) == 2 - assert "Awaiting" in posted[0] and "Awaiting" not in posted[1] - assert "1-minute wait" in posted[1] and "incomplete" in posted[1] - assert "No regression verdict" in posted[1] + assert "Performance assessment pending" in posted[0] + assert "Performance assessment pending" not in posted[1] + assert "1-minute wait" in posted[1] + assert "Performance could not be assessed" in posted[1] def test_artifact_symlink_and_oversized_json_are_rejected(): @@ -695,7 +772,7 @@ def test_report_leg_must_match_measured_environment(report, environment): def test_ci_reuses_profiling_builds_without_changing_release_defaults(): pipeline = (ROOT / "eng/pipelines/pr-validation-pipeline.yml").read_text(encoding="utf-8") assert "benchmarks/perf-benchmarking.py" not in pipeline - assert pipeline.count("python benchmarks/profiler_ci.py --reuse-candidate") == 3 + assert pipeline.count("python -m eng.profiler_benchmarks.controller --reuse-candidate") == 3 assert "profilerBuild: '0'" in pipeline # LocalDB still exercises the normal build assert "ddbc_bindings-profiling-SQL2022" in pipeline assert "ddbc_bindings-profiling-SQL2025" in pipeline From 00da039ce36c1b91b968ac22d1fb381a1659480e Mon Sep 17 00:00:00 2001 From: Gaurav Sharma Date: Wed, 16 Sep 2026 11:32:10 +0530 Subject: [PATCH 07/18] FIX: Finalize and bound performance reports Replace pending reports with explicit unavailable results for malformed API data, moved suite files, invalid provenance, and exhausted retries. Bound coverage downloads, validate archive and API shapes, recheck current head and base before writes, and preserve canceled-run replacement safety. Remove duplicated constants, fixtures, scenario lists, and documentation without dropping behavior or security coverage. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/scripts/extract_coverage_artifact.py | 15 +- .github/scripts/post_profiler_comment.py | 190 +++++++++++--- .github/workflows/pr-code-coverage.yml | 15 +- eng/profiler_benchmarks/README.md | 45 +--- eng/profiler_benchmarks/controller.py | 3 +- eng/profiler_benchmarks/report.py | 31 +-- tests/test_036_profiler_ci.py | 258 +++++++++++++------ tests/test_pr_code_coverage_workflow.py | 8 +- 8 files changed, 371 insertions(+), 194 deletions(-) diff --git a/.github/scripts/extract_coverage_artifact.py b/.github/scripts/extract_coverage_artifact.py index d31c96960..ddfc7c15c 100644 --- a/.github/scripts/extract_coverage_artifact.py +++ b/.github/scripts/extract_coverage_artifact.py @@ -34,11 +34,14 @@ def select(archive, kind): candidates.append((0, member)) elif kind == "xml" and path.suffix.lower() == ".xml": name = path.name.lower() - priority = ( - 0 - if str(path).endswith("unified-coverage/coverage.xml") - else (1 if name == "coverage.xml" else 2 if "coverage" in name else 3) - ) + if str(path).endswith("unified-coverage/coverage.xml"): + priority = 0 + elif name == "coverage.xml": + priority = 1 + elif "coverage" in name: + priority = 2 + else: + continue candidates.append((priority, member)) if not candidates: @@ -49,6 +52,8 @@ def select(archive, kind): def copy_report(archive_path, output, kind): + if Path(archive_path).stat().st_size > MAX_ARCHIVE_BYTES: + raise ValueError("Coverage archive exceeds size limit") with zipfile.ZipFile(archive_path) as archive: selected = select(archive, kind) data = archive.read(selected[0]) diff --git a/.github/scripts/post_profiler_comment.py b/.github/scripts/post_profiler_comment.py index 475d43ca3..a9c2a2a58 100644 --- a/.github/scripts/post_profiler_comment.py +++ b/.github/scripts/post_profiler_comment.py @@ -28,6 +28,7 @@ ROOT = Path(__file__).resolve().parents[2] ADO = "https://dev.azure.com/sqlclientdrivers/public/_apis/build" REPOSITORY = "microsoft/mssql-python" +HEADER = f"{MARKER}\n## PR Performance Report\n\n" # Allow a 160-minute ADO job plus queueing; the workflow reserves publication time. WAIT_MINUTES = 220 @@ -114,9 +115,16 @@ def artifact_report(raw): return json.loads(archive.read(member).decode("utf-8")) -def publish(pr_number, head, body): - pr = github(f"pulls/{pr_number}") - if pr["state"] != "open" or pr["head"]["sha"] != head: +def publish(pr_number, head, body, base=None): + def current(): + pr = github(f"pulls/{pr_number}") + return ( + pr["state"] == "open" + and pr["head"]["sha"] == head + and (base is None or pr["base"]["sha"] == base) + ) + + if not current(): print("Not publishing stale performance results") return page = 1 @@ -135,11 +143,11 @@ def publish(pr_number, head, body): break page += 1 if comment: - if github(f"pulls/{pr_number}")["head"]["sha"] != head: + if not current(): return github(f"issues/comments/{comment['id']}", method="PATCH", data={"body": body}) else: - if github(f"pulls/{pr_number}")["head"]["sha"] != head: + if not current(): return github(f"issues/{pr_number}/comments", method="POST", data={"body": body}) @@ -149,7 +157,11 @@ def find_build(builds, number, head): ( build for build in builds - if build.get("definition", {}).get("id") == 2128 + if isinstance(build, dict) + and isinstance(build.get("definition"), dict) + and isinstance(build.get("repository"), dict) + and isinstance(build.get("triggerInfo"), dict) + and build.get("definition", {}).get("id") == 2128 and build.get("repository", {}).get("id", "").lower() == REPOSITORY and build.get("sourceBranch") == f"refs/pull/{number}/merge" and build.get("triggerInfo", {}).get("pr.sourceSha") == head @@ -159,6 +171,22 @@ def find_build(builds, number, head): ) +def build_items(response): + items = response.get("value") if isinstance(response, dict) else None + if not isinstance(items, list) or not all( + isinstance(build, dict) + and isinstance(build.get("id"), int) + and isinstance(build.get("status"), str) + and isinstance(build.get("definition"), dict) + and isinstance(build.get("repository"), dict) + and isinstance(build.get("triggerInfo"), dict) + and isinstance(build.get("sourceBranch"), str) + for build in items + ): + raise ValueError("Invalid build list") + return items + + def suite_blobs(commit): tree_sha = commit.get("tree", {}).get("sha") if not re.fullmatch(r"[0-9a-f]{40}", tree_sha or ""): @@ -179,54 +207,138 @@ def suite_blobs(commit): return blobs +def artifact_items(response): + items = response.get("value") if isinstance(response, dict) else None + if not isinstance(items, list) or not all( + isinstance(item, dict) + and isinstance(item.get("name"), str) + and isinstance(item.get("resource"), dict) + for item in items + ): + raise ValueError("Invalid artifact list") + return items + + +def unavailable(number, head, reason, base=None): + publish( + number, + head, + HEADER + "**Performance could not be assessed.**\n\n" + reason + " No result is available.", + base, + ) + + def run(number, head, wait_minutes): publish( number, head, - f"{MARKER}\n## PR Performance Report\n\n" - "**Performance assessment pending.**\n\n" - f"Waiting for the matching performance run for head `{head}`.", + HEADER + + "**Performance assessment pending.**\n\n" + + f"Waiting for the matching performance run for head `{head}`.", ) deadline = time.monotonic() + wait_minutes * 60 build = None + pr_base = None + failures = 0 while time.monotonic() < deadline: - pr = github(f"pulls/{number}") - if pr["state"] != "open" or pr["head"]["sha"] != head: - return - query = urlencode( - { - "definitions": 2128, - "branchName": f"refs/pull/{number}/merge", - "queryOrder": "queueTimeDescending", - "$top": 50, - "api-version": "7.1", - } - ) - build = find_build(api(f"{ADO}/builds?{query}")["value"], number, head) - if build and build["status"] == "completed" and build.get("result") != "canceled": + try: + pr = github(f"pulls/{number}") + if ( + not isinstance(pr, dict) + or not isinstance(pr.get("state"), str) + or not isinstance(pr.get("head"), dict) + or not isinstance(pr.get("base"), dict) + ): + raise ValueError + current_head = pr["head"].get("sha") + current_base = pr["base"].get("sha") + pr_base = current_base + query = urlencode( + { + "definitions": 2128, + "branchName": f"refs/pull/{number}/merge", + "queryOrder": "queueTimeDescending", + "$top": 50, + "api-version": "7.1", + } + ) + build = find_build(build_items(api(f"{ADO}/builds?{query}")), number, head) + if pr["state"] != "open" or current_head != head: + return + complete = ( + build is not None + and build.get("status") == "completed" + and build.get("result") != "canceled" + ) + except (ValueError, KeyError, TypeError, URLError, TimeoutError): + failures += 1 + if failures >= 5: + unavailable(number, head, "Performance data services failed repeatedly.", pr_base) + return + time.sleep(30) + continue + failures = 0 + if complete: break time.sleep(30) if build is None or build.get("status") != "completed" or build.get("result") == "canceled": - publish( + unavailable( number, head, - f"{MARKER}\n## PR Performance Report\n\n" - "**Performance could not be assessed.**\n\n" f"No matching performance run completed within the {wait_minutes}-minute wait " - f"for `{head}`. No result is available.", + f"for `{head}`.", + pr_base, ) return build_id = build["id"] - source = build["sourceVersion"] - if type(build_id) is not int or build_id <= 0 or not re.fullmatch(r"[0-9a-f]{40}", source): - raise ValueError("Invalid ADO build identity") - # Authenticate the merge topology through GitHub, not the artifact's claims. - commit = github(f"git/commits/{source}") - if len(commit["parents"]) != 2 or commit["parents"][1]["sha"] != head: - raise ValueError("ADO merge does not match current PR head") - base = commit["parents"][0]["sha"] - suite_unchanged = suite_blobs(commit) == suite_blobs(github(f"git/commits/{base}")) - artifacts = api(f"{ADO}/builds/{build_id}/artifacts?api-version=7.1")["value"] + source = build.get("sourceVersion") + try: + if ( + type(build_id) is not int + or build_id <= 0 + or not re.fullmatch(r"[0-9a-f]{40}", source) + or not re.fullmatch(r"[0-9a-f]{40}", pr_base or "") + ): + raise ValueError + # Authenticate both sides of the merge through the current GitHub PR. + commit = github(f"git/commits/{source}") + if len(commit["parents"]) != 2 or [parent["sha"] for parent in commit["parents"]] != [ + pr_base, + head, + ]: + raise ValueError + base = pr_base + except (ValueError, KeyError, TypeError, URLError, TimeoutError): + unavailable(number, head, "Build provenance validation failed.", pr_base) + return + try: + suite_unchanged = suite_blobs(commit) == suite_blobs(github(f"git/commits/{base}")) + except (ValueError, KeyError, TypeError, URLError, TimeoutError): + unavailable( + number, + head, + "Benchmark suite validation failed because a required file changed.", + pr_base, + ) + return + artifact_deadline = time.monotonic() + 120 + artifacts = None + failures = 0 + while time.monotonic() < artifact_deadline: + try: + artifacts = artifact_items(api(f"{ADO}/builds/{build_id}/artifacts?api-version=7.1")) + failures = 0 + if any(item.get("name", "").startswith("profiler-") for item in artifacts): + break + except (ValueError, KeyError, TypeError, URLError, TimeoutError): + failures += 1 + if failures >= 5: + artifacts = None + break + time.sleep(30) + if artifacts is None: + unavailable(number, head, "Performance artifacts remained unavailable.", pr_base) + return reports, issues = [], [] for leg in LEGS: matching = [item for item in artifacts if item["name"] == "profiler-" + leg] @@ -239,13 +351,13 @@ def run(number, head, wait_minutes): if report["leg"] != leg: raise ValueError("Artifact leg mismatch") reports.append(report) - except (ValueError, KeyError, TypeError, URLError, zipfile.BadZipFile): + except (ValueError, KeyError, TypeError, RecursionError, URLError, zipfile.BadZipFile): # Invalid data is visibly incomplete, never converted to a success verdict. issues.append(leg + " (invalid artifact)") if not suite_unchanged or any(report["suite_hash"] != suite_hash(ROOT) for report in reports): reports = [] issues.append("workload version differs from trusted base") - publish(number, head, render(reports, head, build_id, issues)) + publish(number, head, render(reports, head, build_id, issues), base) if __name__ == "__main__": diff --git a/.github/workflows/pr-code-coverage.yml b/.github/workflows/pr-code-coverage.yml index ee0982ee3..b6592a111 100644 --- a/.github/workflows/pr-code-coverage.yml +++ b/.github/workflows/pr-code-coverage.yml @@ -135,10 +135,6 @@ jobs: [.value[]? | select(.name | test("Code Coverage Report")) | .resource.downloadUrl | select(type == "string" and length > 0)] | .[0] // empty' <<< "$ARTIFACTS_RESPONSE") - if [[ -n "$COVERAGE_ARTIFACT" ]]; then - echo "✅ Found coverage artifact!" - break - fi else ARTIFACT_FAILURES=$((ARTIFACT_FAILURES + 1)) echo "⚠️ Artifacts API HTTP/JSON error ($ARTIFACT_FAILURES/5)" @@ -186,6 +182,7 @@ jobs: COMPLETED_AT=-1 ARTIFACT_FAILURES=0 BUILD_FAILURES=0 + COVERAGE_ARTIFACT="" echo "Selected ADO run was canceled; continuing with replacement build $BUILD_ID" continue fi @@ -196,6 +193,10 @@ jobs: if (( SLEEP_SECONDS > 0 )); then sleep "$SLEEP_SECONDS"; fi continue fi + if [[ -n "$COVERAGE_ARTIFACT" ]]; then + echo "✅ Found coverage artifact!" + break + fi if [[ "$STATUS" == "completed" ]] && (( COMPLETED_AT < 0 )); then COMPLETED_AT=$SECONDS echo "Build completed ($RESULT); allowing 2 minutes for artifact propagation..." @@ -227,7 +228,8 @@ jobs: if [[ -n "$COVERAGE_ARTIFACT" && "$COVERAGE_ARTIFACT" != "null" && "$COVERAGE_ARTIFACT" != "empty" ]]; then echo "📊 Downloading coverage report..." if ! curl -L "$COVERAGE_ARTIFACT" -o coverage-report.zip --fail --silent --show-error \ - --connect-timeout 10 --max-time 60 --retry 2 --retry-delay 5 --retry-max-time 180; then + --connect-timeout 10 --max-time 60 --max-filesize 268435456 \ + --retry 2 --retry-delay 5 --retry-max-time 180; then echo "❌ Failed to download coverage report from Azure DevOps" echo "This indicates the coverage artifacts may not be available or accessible" exit 1 @@ -345,7 +347,8 @@ jobs: if [[ -n "$COVERAGE_XML_ARTIFACT" && "$COVERAGE_XML_ARTIFACT" != "null" && "$COVERAGE_XML_ARTIFACT" != "empty" ]]; then echo "📊 Downloading coverage artifact from: $COVERAGE_XML_ARTIFACT" if ! curl -L "$COVERAGE_XML_ARTIFACT" -o coverage-artifacts.zip --fail --silent --show-error \ - --connect-timeout 10 --max-time 60 --retry 2 --retry-delay 5 --retry-max-time 180; then + --connect-timeout 10 --max-time 60 --max-filesize 268435456 \ + --retry 2 --retry-delay 5 --retry-max-time 180; then echo "❌ Failed to download coverage artifacts" exit 1 fi diff --git a/eng/profiler_benchmarks/README.md b/eng/profiler_benchmarks/README.md index e38243d91..905363ffb 100644 --- a/eng/profiler_benchmarks/README.md +++ b/eng/profiler_benchmarks/README.md @@ -1,13 +1,7 @@ # Profiler Benchmarks -This package compares profiling-enabled base and candidate revisions on the same -agent and SQL Server, then validates and renders advisory performance results. -It is separate from the runtime profiler in `profiler/` and the standalone scripts -in `benchmarks/`. - -The reviewer-facing experience is the **PR Performance Report**. It leads with -the performance impact, affected database tasks, and environment coverage. Phase -timings, all task measurements, commits, and methodology remain expandable evidence. +This source-only package powers the impact-first **PR Performance Report**. It is +separate from the runtime profiler in `profiler/` and standalone `benchmarks/`. ## Local use @@ -18,36 +12,25 @@ python -m eng.profiler_benchmarks.controller --base main --candidate HEAD \ python -m eng.profiler_benchmarks.report profiler-results/report.json ``` -The fixed 20-workload registry retains all 10 profiler scenarios, four -AdventureWorks queries, both legacy 100K-row insert variants, two fetch batch sizes, -and repeated positional and named-parameter execution. A local subset is available -through `--scenarios`; subset reports remain incomplete and cannot produce a verdict. +The fixed registry has 20 tasks. `--scenarios` runs a local subset, but subset +reports remain incomplete and cannot produce a verdict. ## Measurement contract -CI uses the PR-merge commit's first parent as the exact base. Selected legs build -the candidate with `ENABLE_PROFILING=1`, run pytest with recording disabled, and -reuse that binary. The base gets an isolated profiling build. Each side runs in a -fresh process, with five measured pairs after one discarded warmup pair and -alternating order. - -Workers have a six-minute limit. CI allows 90 minutes for measurement inside a -100-minute step and 160-minute job. Local runs receive 105 minutes because they -build both revisions. Partial workers never count as measured pairs, and incomplete -reports never produce a verdict. +CI uses the PR merge's first parent as the exact base. It reuses the +profiling-enabled candidate build after pytest and builds the base separately. +Fresh processes run five measured pairs after one warmup pair in alternating order. -A regression signal requires over 20% paired-median slowdown, at least 1 ms added -median wall time, and at least 80% of pairs exceeding the relative threshold. -Per-phase inclusive deltas and call-count changes are diagnostics, not additive -wall-clock components. +Workers have six minutes each. CI allows 90 minutes inside a 100-minute step and +160-minute job; local runs receive 105 minutes because they build both revisions. +Partial results never produce a verdict. ## Publication -Five legs publish raw samples: Windows and macOS on SQL Server 2022/2025, and -Ubuntu on SQL Server 2022. The privileged GitHub publisher executes only trusted -base code, authenticates the ADO build and source/base suite trees, treats artifacts -as bounded data, and ignores stale heads. Missing, malformed, skipped, canceled, or -failed runs are incomplete. +Five environments publish raw samples: Windows and macOS on SQL Server 2022/2025, +and Ubuntu on SQL Server 2022. The privileged publisher runs trusted base code, +authenticates benchmark producers, validates bounded artifacts, and ignores stale +heads. Missing, malformed, canceled, or failed results remain unavailable. The publisher waits up to 220 minutes inside a 230-minute workflow. The first main comparison after introduction may be incomplete because its parent lacks this diff --git a/eng/profiler_benchmarks/controller.py b/eng/profiler_benchmarks/controller.py index c26917bc3..ff9c344b1 100644 --- a/eng/profiler_benchmarks/controller.py +++ b/eng/profiler_benchmarks/controller.py @@ -16,12 +16,11 @@ import tempfile import time -from .report import suite_hash +from .report import LEGS, suite_hash from . import workloads ROOT = Path(__file__).resolve().parents[2] SHA = re.compile(r"[0-9a-f]{40}") -LEGS = ("Windows-SQL2022", "Windows-SQL2025", "macOS-SQL2022", "macOS-SQL2025", "Linux-SQL2022") # Twelve six-minute passes plus a 15-minute base build and preflight need 88 # minutes. Local runs build both revisions and receive another 15 minutes. BENCHMARK_TIMEOUT = 90 * 60 diff --git a/eng/profiler_benchmarks/report.py b/eng/profiler_benchmarks/report.py index 91c1a0c58..af2eaaf1c 100644 --- a/eng/profiler_benchmarks/report.py +++ b/eng/profiler_benchmarks/report.py @@ -10,32 +10,6 @@ import statistics LEGS = ("Windows-SQL2022", "Windows-SQL2025", "macOS-SQL2022", "macOS-SQL2025", "Linux-SQL2022") -CASES = ( - "connect", - "select", - "insert", - "executemany", - "fetchall", - "fetchone", - "fetchmany", - "commit_rollback", - "arrow", - "insertmanyvalues", - "fetchmany_100", - "fetchmany_10000", - "prepared_qmark", - "prepared_named", - "legacy_insertmany", - "setinputsizes", - "join_aggregation", - "large_fetch", - "fetch_1_2m", - "cte", -) -MAX_BYTES = 8 * 1024 * 1024 -MARKER = "" -THRESHOLD = 0.20 -MIN_DELTA_MS = 1.0 TASK_NAMES = { "connect": "Connection opening", "select": "SELECT queries", @@ -58,6 +32,11 @@ "fetch_1_2m": "1.2-million-row fetching", "cte": "Common table expression queries", } +CASES = tuple(TASK_NAMES) +MAX_BYTES = 8 * 1024 * 1024 +MARKER = "" +THRESHOLD = 0.20 +MIN_DELTA_MS = 1.0 def suite_paths(root): diff --git a/tests/test_036_profiler_ci.py b/tests/test_036_profiler_ci.py index 8a3b7bd27..9362e9c2a 100644 --- a/tests/test_036_profiler_ci.py +++ b/tests/test_036_profiler_ci.py @@ -11,6 +11,7 @@ import tarfile from types import SimpleNamespace from unittest.mock import MagicMock +from urllib.error import URLError import zipfile import pytest @@ -34,6 +35,29 @@ def load(name, path): extractor = load("extract_coverage_artifact", ".github/scripts/extract_coverage_artifact.py") +def ado_build(**values): + build = dict( + id=42, + status="completed", + result="failed", + definition={"id": 2128}, + repository={"id": "microsoft/mssql-python"}, + sourceBranch="refs/pull/123/merge", + sourceVersion="b" * 40, + triggerInfo={"pr.number": "123", "pr.sourceSha": "c" * 40}, + ) + build.update(values) + return build + + +def pr_topology(head="c" * 40, base="a" * 40, merge_base=None): + return lambda path: ( + {"state": "open", "head": {"sha": head}, "base": {"sha": base}} + if path.startswith("pulls/") + else {"parents": [{"sha": merge_base or base}, {"sha": head}]} + ) + + @pytest.fixture def report(): def sample(scale): @@ -256,30 +280,36 @@ def test_coverage_artifact_reader_copies_only_expected_report(tmp_path, kind, me assert not (tmp_path / ".github").exists() -def test_coverage_artifact_reader_accepts_only_identical_duplicate_reports(tmp_path): +@pytest.mark.parametrize("second,valid", [("", True), ("", False)]) +def test_coverage_artifact_reader_accepts_only_identical_duplicate_reports(tmp_path, second, valid): archive = tmp_path / "coverage.zip" output = tmp_path / "coverage.xml" archive.write_bytes( zip_data( [ ("first/coverage.xml", ""), - ("second/coverage.xml", ""), - ] - ) - ) - extractor.copy_report(archive, output, "xml") - assert output.read_text() == "" - - archive.write_bytes( - zip_data( - [ - ("first/coverage.xml", ""), - ("second/coverage.xml", ""), + ("second/coverage.xml", second), ] ) ) - with pytest.raises(ValueError, match="Conflicting"): + if valid: extractor.copy_report(archive, output, "xml") + assert output.read_text() == "" + else: + with pytest.raises(ValueError, match="Conflicting"): + extractor.copy_report(archive, output, "xml") + + +def test_coverage_artifact_reader_rejects_oversized_or_unrelated_archives(tmp_path): + archive = tmp_path / "coverage.zip" + archive.write_bytes(zip_data([("test-results.xml", "")])) + with pytest.raises(ValueError, match="No coverage xml"): + extractor.copy_report(archive, tmp_path / "coverage.xml", "xml") + with archive.open("wb") as stream: + stream.seek(extractor.MAX_ARCHIVE_BYTES) + stream.write(b"x") + with pytest.raises(ValueError, match="archive exceeds"): + extractor.copy_report(archive, tmp_path / "coverage.xml", "xml") def test_artifact_read_never_extracts_paths(report): @@ -312,13 +342,7 @@ def test_untrusted_labels_cannot_inject_links_mentions_or_markdown(): def test_build_selection_requires_exact_pr_head(): - build = dict( - id=42, - definition={"id": 2128}, - repository={"id": "microsoft/mssql-python"}, - sourceBranch="refs/pull/123/merge", - triggerInfo={"pr.number": "123", "pr.sourceSha": "c" * 40}, - ) + build = ado_build() assert publisher.find_build([build], 123, "c" * 40) is build for key in ("pr.number", "pr.sourceSha"): bad = copy.deepcopy(build) @@ -402,6 +426,26 @@ def test_suite_blobs_require_complete_authenticated_tree(monkeypatch): publisher.suite_blobs({"tree": {"sha": "a" * 40}}) +def test_publisher_finishes_unavailable_when_checked_suite_file_moves(monkeypatch): + posted = [] + build = ado_build() + monkeypatch.setattr( + publisher, "publish", lambda number, head, body, base=None: posted.append(body) + ) + monkeypatch.setattr(publisher, "github", pr_topology()) + monkeypatch.setattr(publisher, "api", lambda url: {"value": [build]}) + monkeypatch.setattr( + publisher, + "suite_blobs", + MagicMock(side_effect=ValueError("Benchmark suite missing from commit tree")), + ) + publisher.run(123, "c" * 40, 1) + assert len(posted) == 2 + assert "Performance assessment pending" in posted[0] + assert "Performance could not be assessed" in posted[1] + assert "required file changed" in posted[1] + + @pytest.mark.parametrize("fail", [False, True]) def test_worker_checkpoints_completed_and_active_scenarios(tmp_path, monkeypatch, capsys, fail): output = tmp_path / "base-0.json" @@ -592,7 +636,32 @@ def api(path, **kwargs): assert reads == 2 and len(calls) == 3 -@pytest.mark.parametrize("corrupt", [None, "zip", "scenarios", "suite", "source"]) +def test_base_moving_while_listing_comments_prevents_publish(monkeypatch): + calls = [] + reads = 0 + + def api(path, **kwargs): + nonlocal reads + calls.append((path, kwargs)) + assert not kwargs, "No write allowed after base moved" + if path.startswith("pulls/"): + reads += 1 + return { + "state": "open", + "head": {"sha": "head"}, + "base": {"sha": "base" if reads == 1 else "new-base"}, + } + return [] + + monkeypatch.setattr(publisher, "github", api) + publisher.publish(1, "head", "data", "base") + assert reads == 2 and len(calls) == 3 + + +@pytest.mark.parametrize( + "corrupt", + [None, "zip", "scenarios", "suite", "source", "base", "provenance", "recursion", "delayed"], +) def test_publisher_renders_validated_artifact_and_marks_missing_legs(report, monkeypatch, corrupt): posted = [] windows = copy.deepcopy(report) @@ -607,20 +676,36 @@ def test_publisher_renders_validated_artifact_and_marks_missing_legs(report, mon data = { "Windows-SQL2022": zip_data([("report.json", json.dumps(windows))]), "Linux-SQL2022": ( - b"invalid ZIP" if corrupt == "zip" else zip_data([("report.json", json.dumps(report))]) + b"invalid ZIP" + if corrupt == "zip" + else zip_data( + [ + ( + "report.json", + ( + "[" * 2000 + "0" + "]" * 2000 + if corrupt == "recursion" + else json.dumps(report) + ), + ) + ] + ) ), } - build = dict( - id=42, - status="completed", - result="failed", - definition={"id": 2128}, - repository={"id": "microsoft/mssql-python"}, - sourceBranch="refs/pull/123/merge", - sourceVersion="b" * 40, - triggerInfo={"pr.number": "123", "pr.sourceSha": "c" * 40}, + build = ado_build() + if corrupt == "provenance": + del build["sourceVersion"] + artifacts = [ + { + "name": "profiler-" + leg, + "resource": {"downloadUrl": "https://dev.azure.com/" + leg}, + } + for leg in data + ] + artifact_responses = iter(([], artifacts) if corrupt == "delayed" else (artifacts,)) + monkeypatch.setattr( + publisher, "publish", lambda number, head, body, base=None: posted.append(body) ) - monkeypatch.setattr(publisher, "publish", lambda number, head, body: posted.append(body)) monkeypatch.setattr(publisher, "suite_hash", lambda root: "d" * 64) suite_versions = iter(({"suite": "source"}, {"suite": "base"})) monkeypatch.setattr( @@ -631,38 +716,28 @@ def test_publisher_renders_validated_artifact_and_marks_missing_legs(report, mon monkeypatch.setattr( publisher, "github", - lambda path: ( - {"state": "open", "head": {"sha": "c" * 40}} - if path.startswith("pulls/") - else {"parents": [{"sha": "a" * 40}, {"sha": "c" * 40}]} - ), + pr_topology(base="e" * 40, merge_base="a" * 40) if corrupt == "base" else pr_topology(), ) monkeypatch.setattr( publisher, "api", lambda url: ( - { - "value": [ - { - "name": "profiler-" + leg, - "resource": {"downloadUrl": "https://dev.azure.com/" + leg}, - } - for leg in data - ] - } - if "/artifacts?" in url - else {"value": [build]} + {"value": next(artifact_responses)} if "/artifacts?" in url else {"value": [build]} ), ) monkeypatch.setattr(publisher, "fetch", lambda url, **kw: data[url.rsplit("/", 1)[-1]]) + monkeypatch.setattr(publisher.time, "sleep", lambda seconds: None) publisher.run(123, "c" * 40, 1) assert len(posted) == 2 assert posted[0].startswith(reporting.MARKER) + if corrupt in ("base", "provenance"): + assert "Build provenance validation failed" in posted[1] + return assert "| macOS / SQL Server 2022 | No result available" in posted[1] if corrupt in ("suite", "source"): assert "workload version differs from trusted base" in posted[1] assert "consistent slowdown signals" not in posted[1] - elif corrupt: + elif corrupt in ("zip", "scenarios", "recursion"): assert "### Windows / SQL Server 2022" in posted[1] assert reporting.escape("Linux-SQL2022 (invalid artifact)") in posted[1] assert "| Linux / SQL Server 2022 | No result available (invalid artifact) |" in posted[1] @@ -673,16 +748,7 @@ def test_publisher_renders_validated_artifact_and_marks_missing_legs(report, mon def test_publisher_waits_for_newer_run_after_exact_head_build_is_canceled(report, monkeypatch): - canceled = dict( - id=41, - status="completed", - result="canceled", - definition={"id": 2128}, - repository={"id": "microsoft/mssql-python"}, - sourceBranch="refs/pull/123/merge", - sourceVersion="b" * 40, - triggerInfo={"pr.number": "123", "pr.sourceSha": "c" * 40}, - ) + canceled = ado_build(id=41, result="canceled") replacement = {**canceled, "id": 42, "result": "failed"} builds = iter(([canceled], [replacement])) posted = [] @@ -691,23 +757,18 @@ def test_publisher_waits_for_newer_run_after_exact_head_build_is_canceled(report def api(url): return {"value": next(builds)} if "/builds?" in url else {"value": []} - def github(path): - return ( - {"state": "open", "head": {"sha": "c" * 40}} - if path.startswith("pulls/") - else {"parents": [{"sha": "a" * 40}, {"sha": "c" * 40}]} - ) - monkeypatch.setattr(publisher, "api", api) - monkeypatch.setattr(publisher, "github", github) - monkeypatch.setattr(publisher, "publish", lambda number, head, body: posted.append(body)) + monkeypatch.setattr(publisher, "github", pr_topology()) + monkeypatch.setattr( + publisher, "publish", lambda number, head, body, base=None: posted.append(body) + ) monkeypatch.setattr(publisher, "suite_blobs", lambda commit: {"suite": "same"}) monkeypatch.setattr(publisher.time, "monotonic", lambda: clock[0]) monkeypatch.setattr( publisher.time, "sleep", lambda seconds: clock.__setitem__(0, clock[0] + seconds) ) publisher.run(123, "c" * 40, 1) - assert clock[0] == 30 + assert clock[0] == 150 assert len(posted) == 2 assert "buildId=42" in posted[1] @@ -716,19 +777,11 @@ def github(path): def test_publisher_deadline_finishes_without_reading_unfinished_build_metadata(monkeypatch, status): posted = [] clock = [0] - build = dict( - id=42, - status=status, - sourceVersion=None, - definition={"id": 2128}, - repository={"id": "microsoft/mssql-python"}, - sourceBranch="refs/pull/123/merge", - triggerInfo={"pr.number": "123", "pr.sourceSha": "c" * 40}, - ) + build = ado_build(status=status, sourceVersion=None) def github(path): assert path == "pulls/123", "Unfinished builds must not query merge topology" - return {"state": "open", "head": {"sha": "c" * 40}} + return {"state": "open", "head": {"sha": "c" * 40}, "base": {"sha": "a" * 40}} def api(url): assert "/builds?" in url, "Unfinished builds must not query artifacts" @@ -741,7 +794,9 @@ def sleep(seconds): monkeypatch.setattr(publisher.time, "sleep", sleep) monkeypatch.setattr(publisher, "github", github) monkeypatch.setattr(publisher, "api", api) - monkeypatch.setattr(publisher, "publish", lambda number, head, body: posted.append(body)) + monkeypatch.setattr( + publisher, "publish", lambda number, head, body, base=None: posted.append(body) + ) publisher.run(123, "c" * 40, 1) assert clock[0] == 60 and len(posted) == 2 assert "Performance assessment pending" in posted[0] @@ -750,6 +805,46 @@ def sleep(seconds): assert "Performance could not be assessed" in posted[1] +def test_publisher_retries_transient_polling_failures_before_finalizing(monkeypatch): + posted = [] + clock = [0] + responses = iter((URLError("temporary"), ValueError("bad JSON"), {"value": []})) + monkeypatch.setattr( + publisher, "publish", lambda number, head, body, base=None: posted.append(body) + ) + monkeypatch.setattr(publisher, "github", pr_topology()) + monkeypatch.setattr(publisher, "api", lambda url: next(responses)) + monkeypatch.setattr(publisher.time, "monotonic", lambda: clock[0]) + monkeypatch.setattr( + publisher.time, "sleep", lambda seconds: clock.__setitem__(0, clock[0] + seconds) + ) + publisher.run(123, "c" * 40, 1) + assert clock[0] == 60 + assert len(posted) == 2 + assert "Performance could not be assessed" in posted[1] + + +def test_publisher_retries_malformed_pr_and_artifact_responses(monkeypatch): + posted = [] + clock = [0] + prs = iter(({}, pr_topology()("pulls/123"))) + monkeypatch.setattr( + publisher, "publish", lambda number, head, body, base=None: posted.append(body) + ) + monkeypatch.setattr(publisher, "github", lambda path: next(prs)) + monkeypatch.setattr(publisher, "api", lambda url: {"value": []}) + monkeypatch.setattr(publisher.time, "monotonic", lambda: clock[0]) + monkeypatch.setattr( + publisher.time, "sleep", lambda seconds: clock.__setitem__(0, clock[0] + seconds) + ) + publisher.run(123, "c" * 40, 1) + assert len(posted) == 2 and "Performance could not be assessed" in posted[1] + with pytest.raises(ValueError, match="artifact list"): + publisher.artifact_items({"value": [None]}) + with pytest.raises(ValueError, match="build list"): + publisher.build_items({"value": [{}]}) + + def test_artifact_symlink_and_oversized_json_are_rejected(): symlink = zipfile.ZipInfo("report.json") symlink.create_system = 3 @@ -803,6 +898,7 @@ def test_comment_workflow_executes_only_trusted_base_code(): assert "actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065" in workflow coverage = (ROOT / ".github/workflows/pr-code-coverage.yml").read_text(encoding="utf-8") assert coverage.count("extract_coverage_artifact.py") == 2 + assert coverage.count("--max-filesize 268435456") == 2 assert "unzip -o" not in coverage assert ( "head.ref" not in workflow diff --git a/tests/test_pr_code_coverage_workflow.py b/tests/test_pr_code_coverage_workflow.py index e89fd66ef..d49e6960c 100644 --- a/tests/test_pr_code_coverage_workflow.py +++ b/tests/test_pr_code_coverage_workflow.py @@ -201,11 +201,11 @@ def test_canceled_without_replacement_obeys_wall_clock_budget(tmp_path): assert 220 * 60 <= int((tmp_path / "elapsed").read_text()) < 225 * 60 -def test_immediately_available_artifact_needs_no_lifecycle_request(tmp_path): - result = _poll(tmp_path, [ARTIFACT], []) +def test_immediately_available_artifact_is_accepted_after_lifecycle_check(tmp_path): + result = _poll(tmp_path, [ARTIFACT], [_build()]) assert result.returncode == 0, result.stdout + result.stderr assert f"COVERAGE_ARTIFACT={ARTIFACT_URL}" in result.stdout - assert (tmp_path / "build.next").read_text() == "0" + assert (tmp_path / "build.next").read_text().strip() == "1" def test_switches_from_canceled_run_to_newer_exact_head_build(tmp_path): @@ -213,7 +213,7 @@ def test_switches_from_canceled_run_to_newer_exact_head_build(tmp_path): replacement = _build(175449) result = _poll( tmp_path, - [(22, "not found"), ARTIFACT], + [ARTIFACT, ARTIFACT], [{**_build(), "status": "completed", "result": "canceled"}, replacement], [{"value": [older, replacement]}], ) From e2997e00cac1daa0dae5377ba5a0765ec3b29b5d Mon Sep 17 00:00:00 2001 From: Gaurav Sharma Date: Wed, 16 Sep 2026 12:35:42 +0530 Subject: [PATCH 08/18] FIX: Isolate performance artifact timeouts Treat body-read timeouts as an unavailable platform so valid performance results still publish. Validate repository identities and wait through artifact propagation before finalizing the report. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/scripts/post_profiler_comment.py | 14 ++++++- tests/test_036_profiler_ci.py | 49 +++++++++++++++++++----- 2 files changed, 51 insertions(+), 12 deletions(-) diff --git a/.github/scripts/post_profiler_comment.py b/.github/scripts/post_profiler_comment.py index a9c2a2a58..08c1ba630 100644 --- a/.github/scripts/post_profiler_comment.py +++ b/.github/scripts/post_profiler_comment.py @@ -160,6 +160,7 @@ def find_build(builds, number, head): if isinstance(build, dict) and isinstance(build.get("definition"), dict) and isinstance(build.get("repository"), dict) + and isinstance(build["repository"].get("id"), str) and isinstance(build.get("triggerInfo"), dict) and build.get("definition", {}).get("id") == 2128 and build.get("repository", {}).get("id", "").lower() == REPOSITORY @@ -179,6 +180,7 @@ def build_items(response): and isinstance(build.get("status"), str) and isinstance(build.get("definition"), dict) and isinstance(build.get("repository"), dict) + and isinstance(build["repository"].get("id"), str) and isinstance(build.get("triggerInfo"), dict) and isinstance(build.get("sourceBranch"), str) for build in items @@ -328,7 +330,7 @@ def run(number, head, wait_minutes): try: artifacts = artifact_items(api(f"{ADO}/builds/{build_id}/artifacts?api-version=7.1")) failures = 0 - if any(item.get("name", "").startswith("profiler-") for item in artifacts): + if {"profiler-" + leg for leg in LEGS} <= {item["name"] for item in artifacts}: break except (ValueError, KeyError, TypeError, URLError, TimeoutError): failures += 1 @@ -351,7 +353,15 @@ def run(number, head, wait_minutes): if report["leg"] != leg: raise ValueError("Artifact leg mismatch") reports.append(report) - except (ValueError, KeyError, TypeError, RecursionError, URLError, zipfile.BadZipFile): + except ( + ValueError, + KeyError, + TypeError, + RecursionError, + TimeoutError, + URLError, + zipfile.BadZipFile, + ): # Invalid data is visibly incomplete, never converted to a success verdict. issues.append(leg + " (invalid artifact)") if not suite_unchanged or any(report["suite_hash"] != suite_hash(ROOT) for report in reports): diff --git a/tests/test_036_profiler_ci.py b/tests/test_036_profiler_ci.py index 9362e9c2a..1b6c8e4bd 100644 --- a/tests/test_036_profiler_ci.py +++ b/tests/test_036_profiler_ci.py @@ -660,7 +660,18 @@ def api(path, **kwargs): @pytest.mark.parametrize( "corrupt", - [None, "zip", "scenarios", "suite", "source", "base", "provenance", "recursion", "delayed"], + [ + None, + "zip", + "timeout", + "scenarios", + "suite", + "source", + "base", + "provenance", + "recursion", + "delayed", + ], ) def test_publisher_renders_validated_artifact_and_marks_missing_legs(report, monkeypatch, corrupt): posted = [] @@ -702,7 +713,17 @@ def test_publisher_renders_validated_artifact_and_marks_missing_legs(report, mon } for leg in data ] - artifact_responses = iter(([], artifacts) if corrupt == "delayed" else (artifacts,)) + artifact_responses = [[], artifacts] if corrupt == "delayed" else [artifacts] + clock = [0] + + def api(url): + if "/artifacts?" not in url: + return {"value": [build]} + response = ( + artifact_responses.pop(0) if len(artifact_responses) > 1 else artifact_responses[0] + ) + return {"value": response} + monkeypatch.setattr( publisher, "publish", lambda number, head, body, base=None: posted.append(body) ) @@ -718,15 +739,19 @@ def test_publisher_renders_validated_artifact_and_marks_missing_legs(report, mon "github", pr_topology(base="e" * 40, merge_base="a" * 40) if corrupt == "base" else pr_topology(), ) + monkeypatch.setattr(publisher, "api", api) + + def fetch(url, **kwargs): + leg = url.rsplit("/", 1)[-1] + if corrupt == "timeout" and leg == "Linux-SQL2022": + raise TimeoutError("timed out") + return data[leg] + + monkeypatch.setattr(publisher, "fetch", fetch) + monkeypatch.setattr(publisher.time, "monotonic", lambda: clock[0]) monkeypatch.setattr( - publisher, - "api", - lambda url: ( - {"value": next(artifact_responses)} if "/artifacts?" in url else {"value": [build]} - ), + publisher.time, "sleep", lambda seconds: clock.__setitem__(0, clock[0] + seconds) ) - monkeypatch.setattr(publisher, "fetch", lambda url, **kw: data[url.rsplit("/", 1)[-1]]) - monkeypatch.setattr(publisher.time, "sleep", lambda seconds: None) publisher.run(123, "c" * 40, 1) assert len(posted) == 2 assert posted[0].startswith(reporting.MARKER) @@ -737,7 +762,7 @@ def test_publisher_renders_validated_artifact_and_marks_missing_legs(report, mon if corrupt in ("suite", "source"): assert "workload version differs from trusted base" in posted[1] assert "consistent slowdown signals" not in posted[1] - elif corrupt in ("zip", "scenarios", "recursion"): + elif corrupt in ("zip", "timeout", "scenarios", "recursion"): assert "### Windows / SQL Server 2022" in posted[1] assert reporting.escape("Linux-SQL2022 (invalid artifact)") in posted[1] assert "| Linux / SQL Server 2022 | No result available (invalid artifact) |" in posted[1] @@ -843,6 +868,10 @@ def test_publisher_retries_malformed_pr_and_artifact_responses(monkeypatch): publisher.artifact_items({"value": [None]}) with pytest.raises(ValueError, match="build list"): publisher.build_items({"value": [{}]}) + malformed = ado_build() + malformed["repository"]["id"] = None + with pytest.raises(ValueError, match="build list"): + publisher.build_items({"value": [malformed]}) def test_artifact_symlink_and_oversized_json_are_rejected(): From 04c62ba20cb75be74d9ad9fe3ad8ec1ed1f3497f Mon Sep 17 00:00:00 2001 From: Copilot <223556219+Copilot@users.noreply.github.com> Date: Thu, 17 Sep 2026 10:57:02 +0530 Subject: [PATCH 09/18] FIX: Harden profiler CI orchestration Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/workflows/pr-code-coverage.yml | 11 +++- eng/profiler_benchmarks/controller.py | 37 ++++++++++- eng/profiler_benchmarks/report.py | 17 ++++- tests/test_036_profiler_ci.py | 82 +++++++++++++++++++++++++ tests/test_pr_code_coverage_workflow.py | 8 +++ 5 files changed, 148 insertions(+), 7 deletions(-) diff --git a/.github/workflows/pr-code-coverage.yml b/.github/workflows/pr-code-coverage.yml index b6592a111..a245ae2da 100644 --- a/.github/workflows/pr-code-coverage.yml +++ b/.github/workflows/pr-code-coverage.yml @@ -122,9 +122,11 @@ jobs: ARTIFACT_FAILURES=0 BUILD_FAILURES=0 COVERAGE_ARTIFACT="" + COVERAGE_ARTIFACT_APPROVED=false echo "📥 Waiting up to 220 minutes for coverage artifacts for build $BUILD_ID..." while (( SECONDS < DEADLINE )); do + COVERAGE_ARTIFACT="" REQUEST_TIMEOUT=$((DEADLINE - SECONDS)) if (( REQUEST_TIMEOUT <= 0 )); then break; fi if (( REQUEST_TIMEOUT > 30 )); then REQUEST_TIMEOUT=30; fi @@ -183,6 +185,7 @@ jobs: ARTIFACT_FAILURES=0 BUILD_FAILURES=0 COVERAGE_ARTIFACT="" + COVERAGE_ARTIFACT_APPROVED=false echo "Selected ADO run was canceled; continuing with replacement build $BUILD_ID" continue fi @@ -193,7 +196,11 @@ jobs: if (( SLEEP_SECONDS > 0 )); then sleep "$SLEEP_SECONDS"; fi continue fi - if [[ -n "$COVERAGE_ARTIFACT" ]]; then + if [[ -n "$COVERAGE_ARTIFACT" ]] && + { [[ "$STATUS" == "inProgress" ]] || + { [[ "$STATUS" == "completed" ]] && + [[ "$RESULT" =~ ^(succeeded|partiallySucceeded|failed)$ ]]; }; }; then + COVERAGE_ARTIFACT_APPROVED=true echo "✅ Found coverage artifact!" break fi @@ -220,7 +227,7 @@ jobs: if (( SLEEP_SECONDS > 0 )); then sleep "$SLEEP_SECONDS"; fi done - if [[ -z "$COVERAGE_ARTIFACT" ]]; then + if [[ "$COVERAGE_ARTIFACT_APPROVED" != true || -z "$COVERAGE_ARTIFACT" ]]; then echo "❌ Timeout: Coverage report artifact not found within 220 minutes" exit 1 fi diff --git a/eng/profiler_benchmarks/controller.py b/eng/profiler_benchmarks/controller.py index ff9c344b1..21e82badd 100644 --- a/eng/profiler_benchmarks/controller.py +++ b/eng/profiler_benchmarks/controller.py @@ -10,6 +10,7 @@ from pathlib import Path, PurePosixPath import platform import re +import signal import subprocess import sys import tarfile @@ -26,6 +27,7 @@ BENCHMARK_TIMEOUT = 90 * 60 LOCAL_BENCHMARK_TIMEOUT = 105 * 60 WORKER_TIMEOUT = 6 * 60 +WINDOWS = os.name == "nt" def git(*args): @@ -65,21 +67,50 @@ def checkout(revision, path): tar.extractall(path, members=members) +def terminate_process_tree(process): + if WINDOWS: + result = subprocess.run( + ["taskkill", "/PID", str(process.pid), "/T", "/F"], + capture_output=True, + text=True, + ) + if result.returncode: + if process.poll() is None: + process.kill() + process.wait() + raise RuntimeError(f"Failed to terminate build process tree: {result.stdout.strip()}") + process.wait(timeout=5) + return + + try: + os.killpg(process.pid, signal.SIGKILL) + except ProcessLookupError: + pass + process.wait(timeout=5) + + def build(path, log, timeout=900): env = dict(os.environ, ENABLE_PROFILING="1") # build scripts find Python via PATH; keep the controller's interpreter. env["PATH"] = str(Path(sys.executable).parent) + os.pathsep + env["PATH"] command = ["cmd", "/c", "build.bat"] if os.name == "nt" else ["bash", "build.sh"] with log.open("w", encoding="utf-8") as output: - subprocess.run( + process = subprocess.Popen( command, cwd=path / "mssql_python/pybind", env=env, stdout=output, stderr=subprocess.STDOUT, - timeout=timeout, - check=True, + start_new_session=not WINDOWS, + creationflags=subprocess.CREATE_NEW_PROCESS_GROUP if WINDOWS else 0, ) + try: + returncode = process.wait(timeout=timeout) + except subprocess.TimeoutExpired: + terminate_process_tree(process) + raise + if returncode: + raise subprocess.CalledProcessError(returncode, command) def check_build(source_root, profiling): diff --git a/eng/profiler_benchmarks/report.py b/eng/profiler_benchmarks/report.py index af2eaaf1c..8d5d11702 100644 --- a/eng/profiler_benchmarks/report.py +++ b/eng/profiler_benchmarks/report.py @@ -73,16 +73,19 @@ def text(value, limit=160): return value -def validate(report, build_id=None, head=None, source=None, base=None): +def validate(report, build_id=None, head=None, source=None, base=None, suite=None): if not isinstance(report, dict) or report.get("schema_version") != 1: raise ValueError("Unsupported report schema") if report.get("leg") not in LEGS or report.get("status") not in ("complete", "incomplete"): raise ValueError("Invalid report status or leg") + if type(report.get("build_id")) is not int or report["build_id"] < 0: + raise ValueError("Invalid build_id") for key, expected in ( ("build_id", build_id), ("head_commit", head), ("source_commit", source), ("base_commit", base), + ("suite_hash", suite), ): if expected is not None and report.get(key) != expected: raise ValueError(f"Report provenance mismatch: {key}") @@ -457,7 +460,17 @@ def main(): parser.add_argument("reports", nargs="+", type=Path) args = parser.parse_args() reports = [validate(json.loads(path.read_text(encoding="utf-8"))) for path in args.reports] - print(render(reports, reports[0]["head_commit"], reports[0]["build_id"])) + first = reports[0] + for report in reports[1:]: + validate( + report, + build_id=first["build_id"], + head=first["head_commit"], + source=first["source_commit"], + base=first["base_commit"], + suite=first["suite_hash"], + ) + print(render(reports, first["head_commit"], first["build_id"])) if __name__ == "__main__": diff --git a/tests/test_036_profiler_ci.py b/tests/test_036_profiler_ci.py index 1b6c8e4bd..86d3d98a8 100644 --- a/tests/test_036_profiler_ci.py +++ b/tests/test_036_profiler_ci.py @@ -4,11 +4,14 @@ import importlib.util import io import json +import os from pathlib import Path import re +import signal import subprocess import sys import tarfile +import time from types import SimpleNamespace from unittest.mock import MagicMock from urllib.error import URLError @@ -157,6 +160,13 @@ def test_reject_wrong_commit_and_preserve_incomplete_status(report): assert "Performance could not be assessed" in reporting.render([report], "c" * 40, 42) +@pytest.mark.parametrize("build_id", [None, True, -1]) +def test_reject_invalid_build_id(report, build_id): + report["build_id"] = build_id + with pytest.raises(ValueError, match="build_id"): + reporting.validate(report) + + def set_leg(report, leg): report = copy.deepcopy(report) report["leg"] = leg @@ -170,6 +180,28 @@ def set_leg(report, leg): return report +@pytest.mark.parametrize( + "key,value", + [ + ("build_id", 43), + ("head_commit", "e" * 40), + ("source_commit", "e" * 40), + ("base_commit", "e" * 40), + ("suite_hash", "e" * 64), + ], +) +def test_standalone_report_rejects_mixed_provenance(report, tmp_path, monkeypatch, key, value): + first = tmp_path / "linux.json" + second = tmp_path / "windows.json" + first.write_text(json.dumps(report), encoding="utf-8") + other = set_leg(report, "Windows-SQL2022") + other[key] = value + second.write_text(json.dumps(other), encoding="utf-8") + monkeypatch.setattr(sys, "argv", ["report", str(first), str(second)]) + with pytest.raises(ValueError, match=key): + reporting.main() + + def clear_slowdowns(report): for pair in report["pairs"]: for name in reporting.CASES: @@ -502,6 +534,56 @@ def timeout(command, **kwargs): assert "Starting scenario: fetchone" in output.with_suffix(".log").read_text() +@pytest.mark.skipif(os.name == "nt", reason="exercises POSIX process groups") +def test_build_timeout_terminates_descendants(tmp_path, monkeypatch): + pybind = tmp_path / "mssql_python/pybind" + pybind.mkdir(parents=True) + pid_file = tmp_path / "descendant.pid" + monkeypatch.setenv("DESCENDANT_PID", str(pid_file)) + (pybind / "build.sh").write_text( + "#!/usr/bin/env bash\n" + f'"{sys.executable}" -c "import time; time.sleep(60)" &\n' + 'echo "$!" > "$DESCENDANT_PID"\n' + "wait\n", + encoding="utf-8", + ) + + descendant = None + try: + with pytest.raises(subprocess.TimeoutExpired): + controller.build(tmp_path, tmp_path / "build.log", timeout=1) + descendant = int(pid_file.read_text(encoding="utf-8")) + deadline = time.monotonic() + 5 + while time.monotonic() < deadline: + try: + os.kill(descendant, 0) + except ProcessLookupError: + break + time.sleep(0.05) + else: + pytest.fail("build descendant survived timeout cleanup") + finally: + if descendant is not None: + try: + os.kill(descendant, signal.SIGKILL) + except ProcessLookupError: + pass + + +def test_windows_process_tree_cleanup_uses_taskkill(monkeypatch): + process = MagicMock(pid=123) + taskkill = MagicMock(return_value=subprocess.CompletedProcess([], 0, "")) + monkeypatch.setattr(controller, "WINDOWS", True) + monkeypatch.setattr(controller.subprocess, "run", taskkill) + controller.terminate_process_tree(process) + taskkill.assert_called_once_with( + ["taskkill", "/PID", "123", "/T", "/F"], + capture_output=True, + text=True, + ) + process.wait.assert_called_once_with(timeout=5) + + def test_overall_budget_caps_build_and_worker_time(monkeypatch): monkeypatch.setattr(controller.time, "monotonic", lambda: 100) assert controller.remaining(110, controller.WORKER_TIMEOUT) == 10 diff --git a/tests/test_pr_code_coverage_workflow.py b/tests/test_pr_code_coverage_workflow.py index d49e6960c..f5c3841d6 100644 --- a/tests/test_pr_code_coverage_workflow.py +++ b/tests/test_pr_code_coverage_workflow.py @@ -201,6 +201,14 @@ def test_canceled_without_replacement_obeys_wall_clock_budget(tmp_path): assert 220 * 60 <= int((tmp_path / "elapsed").read_text()) < 225 * 60 +def test_canceled_visible_artifact_without_replacement_is_rejected(tmp_path): + canceled = {**_build(), "status": "completed", "result": "canceled"} + run = _poll(tmp_path, [ARTIFACT], [canceled], clock_scale=10) + assert run.returncode != 0 + assert "has no replacement yet" in run.stdout + assert "Timeout:" in run.stdout + + def test_immediately_available_artifact_is_accepted_after_lifecycle_check(tmp_path): result = _poll(tmp_path, [ARTIFACT], [_build()]) assert result.returncode == 0, result.stdout + result.stderr From 43b266996e8d6519c6c60e7350f3be98aeca53dc Mon Sep 17 00:00:00 2001 From: Gaurav Sharma Date: Thu, 17 Sep 2026 12:56:38 +0530 Subject: [PATCH 10/18] REFACTOR: Deepen profiler report assessment Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/scripts/post_profiler_comment.py | 155 +++++++------------ .github/workflows/pr-code-coverage.yml | 25 +-- eng/profiler_benchmarks/report.py | 184 ++++++++++++++++++++++- tests/test_036_profiler_ci.py | 134 ++++++++++++++--- 4 files changed, 362 insertions(+), 136 deletions(-) diff --git a/.github/scripts/post_profiler_comment.py b/.github/scripts/post_profiler_comment.py index 08c1ba630..4c9d720a8 100644 --- a/.github/scripts/post_profiler_comment.py +++ b/.github/scripts/post_profiler_comment.py @@ -1,34 +1,24 @@ """Read public ADO artifacts as data and update a SHA-bound PR performance comment.""" import argparse -import io +from http.client import HTTPException import json import os -from pathlib import Path, PurePosixPath +from pathlib import Path import re -import stat import sys import time from urllib.error import URLError from urllib.parse import urlencode, urlparse from urllib.request import HTTPRedirectHandler, Request, build_opener -import zipfile sys.path.insert(0, str(Path(__file__).resolve().parents[2])) -from eng.profiler_benchmarks.report import ( - LEGS, - MARKER, - MAX_BYTES, - render, - suite_hash, - suite_paths, - validate, -) +from eng.profiler_benchmarks import report as reporting ROOT = Path(__file__).resolve().parents[2] ADO = "https://dev.azure.com/sqlclientdrivers/public/_apis/build" REPOSITORY = "microsoft/mssql-python" -HEADER = f"{MARKER}\n## PR Performance Report\n\n" +HEADER = f"{reporting.MARKER}\n## PR Performance Report\n\n" # Allow a 160-minute ADO job plus queueing; the workflow reserves publication time. WAIT_MINUTES = 220 @@ -72,8 +62,11 @@ def fetch(url, token=None, method=None, data=None, limit=4 * 1024 * 1024): if payload is not None: headers["Content-Type"] = "application/json" request = Request(url, headers=headers, method=method, data=payload) - with build_opener(SafeRedirect()).open(request, timeout=30) as response: - body = response.read(limit + 1) + try: + with build_opener(SafeRedirect()).open(request, timeout=30) as response: + body = response.read(limit + 1) + except HTTPException as error: + raise URLError("Incomplete HTTP response") from error if len(body) > limit: raise ValueError("Response exceeds size limit") return body @@ -89,32 +82,6 @@ def github(path, **kwargs): ) -def artifact_report(raw): - """Read exactly one bounded JSON member; never extract or execute artifact files.""" - with zipfile.ZipFile(io.BytesIO(raw)) as archive: - members = archive.infolist() - if len(members) > 200 or sum(member.file_size for member in members) > 64 * 1024 * 1024: - raise ValueError("Oversized artifact") - reports = [ - member for member in members if PurePosixPath(member.filename).name == "report.json" - ] - if len(reports) != 1: - raise ValueError("Expected exactly one report.json") - member = reports[0] - path = PurePosixPath(member.filename) - if ( - path.is_absolute() - or ".." in path.parts - or "\\" in member.filename - or stat.S_ISLNK(member.external_attr >> 16) - or member.file_size > MAX_BYTES - ): - raise ValueError("Invalid report member") - if member.flag_bits & 1: - raise ValueError("Encrypted performance artifacts are unsupported") - return json.loads(archive.read(member).decode("utf-8")) - - def publish(pr_number, head, body, base=None): def current(): pr = github(f"pulls/{pr_number}") @@ -135,7 +102,8 @@ def current(): ( c for c in comments - if c["user"]["login"] == "github-actions[bot]" and c["body"].startswith(MARKER) + if c["user"]["login"] == "github-actions[bot]" + and c["body"].startswith(reporting.MARKER) ), comment, ) @@ -189,26 +157,6 @@ def build_items(response): return items -def suite_blobs(commit): - tree_sha = commit.get("tree", {}).get("sha") - if not re.fullmatch(r"[0-9a-f]{40}", tree_sha or ""): - raise ValueError("Invalid commit tree") - tree = github(f"git/trees/{tree_sha}?recursive=1") - if tree.get("truncated") is not False or not isinstance(tree.get("tree"), list): - raise ValueError("Incomplete commit tree") - expected = {path.relative_to(ROOT).as_posix() for path in suite_paths(ROOT)} - blobs = { - entry.get("path"): entry.get("sha") - for entry in tree["tree"] - if entry.get("type") == "blob" and entry.get("path") in expected - } - if set(blobs) != expected or any( - not re.fullmatch(r"[0-9a-f]{40}", sha or "") for sha in blobs.values() - ): - raise ValueError("Benchmark suite missing from commit tree") - return blobs - - def artifact_items(response): items = response.get("value") if isinstance(response, dict) else None if not isinstance(items, list) or not all( @@ -302,27 +250,26 @@ def run(number, head, wait_minutes): or not re.fullmatch(r"[0-9a-f]{40}", pr_base or "") ): raise ValueError - # Authenticate both sides of the merge through the current GitHub PR. commit = github(f"git/commits/{source}") - if len(commit["parents"]) != 2 or [parent["sha"] for parent in commit["parents"]] != [ - pr_base, - head, - ]: - raise ValueError base = pr_base + base_commit = github(f"git/commits/{base}") + if not isinstance(commit, dict) or not isinstance(base_commit, dict): + raise ValueError + source_tree_info = commit.get("tree") + base_tree_info = base_commit.get("tree") + if not isinstance(source_tree_info, dict) or not isinstance(base_tree_info, dict): + raise ValueError + source_tree_sha = source_tree_info.get("sha") + base_tree_sha = base_tree_info.get("sha") + if not re.fullmatch(r"[0-9a-f]{40}", source_tree_sha or "") or not re.fullmatch( + r"[0-9a-f]{40}", base_tree_sha or "" + ): + raise ValueError + source_tree = github(f"git/trees/{source_tree_sha}?recursive=1") + base_tree = github(f"git/trees/{base_tree_sha}?recursive=1") except (ValueError, KeyError, TypeError, URLError, TimeoutError): unavailable(number, head, "Build provenance validation failed.", pr_base) return - try: - suite_unchanged = suite_blobs(commit) == suite_blobs(github(f"git/commits/{base}")) - except (ValueError, KeyError, TypeError, URLError, TimeoutError): - unavailable( - number, - head, - "Benchmark suite validation failed because a required file changed.", - pr_base, - ) - return artifact_deadline = time.monotonic() + 120 artifacts = None failures = 0 @@ -330,7 +277,9 @@ def run(number, head, wait_minutes): try: artifacts = artifact_items(api(f"{ADO}/builds/{build_id}/artifacts?api-version=7.1")) failures = 0 - if {"profiler-" + leg for leg in LEGS} <= {item["name"] for item in artifacts}: + if {"profiler-" + leg for leg in reporting.LEGS} <= { + item["name"] for item in artifacts + }: break except (ValueError, KeyError, TypeError, URLError, TimeoutError): failures += 1 @@ -341,33 +290,35 @@ def run(number, head, wait_minutes): if artifacts is None: unavailable(number, head, "Performance artifacts remained unavailable.", pr_base) return - reports, issues = [], [] - for leg in LEGS: + artifact_urls, issues = {}, [] + for leg in reporting.LEGS: matching = [item for item in artifacts if item["name"] == "profiler-" + leg] if len(matching) != 1: issues.append(leg + " (missing)") continue - try: - raw = fetch(matching[0]["resource"]["downloadUrl"], limit=32 * 1024 * 1024) - report = validate(artifact_report(raw), build_id, head, source, base) - if report["leg"] != leg: - raise ValueError("Artifact leg mismatch") - reports.append(report) - except ( - ValueError, - KeyError, - TypeError, - RecursionError, - TimeoutError, - URLError, - zipfile.BadZipFile, - ): - # Invalid data is visibly incomplete, never converted to a success verdict. + url = matching[0]["resource"].get("downloadUrl") + if not isinstance(url, str): issues.append(leg + " (invalid artifact)") - if not suite_unchanged or any(report["suite_hash"] != suite_hash(ROOT) for report in reports): - reports = [] - issues.append("workload version differs from trusted base") - publish(number, head, render(reports, head, build_id, issues), base) + continue + artifact_urls[leg] = url + + def load_artifact(url): + try: + return fetch(url, limit=32 * 1024 * 1024) + except (TimeoutError, URLError) as error: + raise ValueError("Artifact download failed") from error + + evidence = reporting.AssessmentEvidence( + build=build, + head=head, + base=base, + merge_commit=commit, + base_commit=base_commit, + source_tree=source_tree, + base_tree=base_tree, + trusted_root=ROOT, + ) + publish(number, head, reporting.assess(evidence, artifact_urls, load_artifact, issues), base) if __name__ == "__main__": diff --git a/.github/workflows/pr-code-coverage.yml b/.github/workflows/pr-code-coverage.yml index a245ae2da..0a733e9ef 100644 --- a/.github/workflows/pr-code-coverage.yml +++ b/.github/workflows/pr-code-coverage.yml @@ -365,9 +365,8 @@ jobs: echo "❌ Failed to read the coverage XML artifact" exit 1 fi - cp "$COVERAGE_XML" ./coverage.xml - echo "✅ Coverage XML file is ready at ./coverage.xml" - ls -la ./coverage.xml + echo "✅ Coverage XML file is ready at $COVERAGE_XML" + ls -la "$COVERAGE_XML" else echo "❌ Could not find coverage artifacts" echo "This indicates the Azure DevOps CodeCoverageReport job may not have run successfully" @@ -375,22 +374,24 @@ jobs: fi - name: Generate patch coverage report + env: + COVERAGE_XML: ${{ runner.temp }}/coverage.xml run: | # Install dependencies pip install diff-cover jq sudo apt-get update && sudo apt-get install -y libxml2-utils # Verify coverage.xml exists before proceeding - if [[ ! -f coverage.xml ]]; then + if [[ ! -f "$COVERAGE_XML" ]]; then echo "❌ coverage.xml not found in current directory" echo "Available files:" ls -la | head -20 exit 1 fi - echo "✅ coverage.xml found, size: $(wc -c < coverage.xml) bytes" + echo "✅ coverage.xml found, size: $(wc -c < "$COVERAGE_XML") bytes" echo "🔍 Coverage file preview (first 10 lines):" - head -10 coverage.xml + head -10 "$COVERAGE_XML" # Generate diff coverage report using the new command format echo "🚀 Generating patch coverage report..." @@ -414,27 +415,27 @@ jobs: # Debug: Check coverage.xml content for specific files echo "🔍 Coverage.xml analysis:" echo "Python files mentioned in coverage.xml:" - grep -o 'filename="[^"]*\.py"' coverage.xml | head -10 || echo "Could not extract filenames" + grep -o 'filename="[^"]*\.py"' "$COVERAGE_XML" | head -10 || echo "Could not extract filenames" echo "Sample coverage data:" - head -20 coverage.xml + head -20 "$COVERAGE_XML" # Use the new format for diff-cover commands echo "🚀 Running diff-cover..." - diff-cover coverage.xml \ + diff-cover "$COVERAGE_XML" \ --compare-branch=main \ --html-report patch-coverage.html \ --json-report patch-coverage.json \ --markdown-report patch-coverage.md || { echo "❌ diff-cover failed with exit code $?" echo "Checking if coverage.xml is valid XML..." - if ! xmllint --noout coverage.xml 2>/dev/null; then + if ! xmllint --noout "$COVERAGE_XML" 2>/dev/null; then echo "❌ coverage.xml is not valid XML" echo "First 50 lines of coverage.xml:" - head -50 coverage.xml + head -50 "$COVERAGE_XML" else echo "✅ coverage.xml is valid XML" echo "🔍 diff-cover verbose output:" - diff-cover coverage.xml --compare-branch=main --markdown-report debug-patch-coverage.md -v || echo "Verbose diff-cover also failed" + diff-cover "$COVERAGE_XML" --compare-branch=main --markdown-report debug-patch-coverage.md -v || echo "Verbose diff-cover also failed" fi # Don't exit here, let's see what files were created } diff --git a/eng/profiler_benchmarks/report.py b/eng/profiler_benchmarks/report.py index 8d5d11702..6fff6c9f2 100644 --- a/eng/profiler_benchmarks/report.py +++ b/eng/profiler_benchmarks/report.py @@ -1,13 +1,17 @@ """Validate bounded profiler data and render an advisory, per-platform comparison.""" import argparse +from dataclasses import dataclass import hashlib import html +import io import json import math -from pathlib import Path +from pathlib import Path, PurePosixPath import re +import stat import statistics +import zipfile LEGS = ("Windows-SQL2022", "Windows-SQL2025", "macOS-SQL2022", "macOS-SQL2025", "Linux-SQL2022") TASK_NAMES = { @@ -34,6 +38,8 @@ } CASES = tuple(TASK_NAMES) MAX_BYTES = 8 * 1024 * 1024 +MAX_COMMENT_CHARS = 60000 +MAX_DIAGNOSTIC_ROWS = 20 MARKER = "" THRESHOLD = 0.20 MIN_DELTA_MS = 1.0 @@ -47,6 +53,8 @@ def suite_paths(root): root / "eng/profiler_benchmarks/controller.py", root / "eng/profiler_benchmarks/report.py", root / "eng/profiler_benchmarks/workloads.py", + root / "eng/scripts/setup_sql_container.py", + root / "requirements.txt", *sorted((root / "profiler").glob("*.py")), ] @@ -59,6 +67,72 @@ def suite_hash(root): return digest.hexdigest() +@dataclass(frozen=True) +class AssessmentEvidence: + build: dict + head: str + base: str + merge_commit: dict + base_commit: dict + source_tree: dict + base_tree: dict + trusted_root: Path + + +def artifact_report(raw): + """Read exactly one bounded JSON member; never extract or execute artifact files.""" + with zipfile.ZipFile(io.BytesIO(raw)) as archive: + members = archive.infolist() + if len(members) > 200 or sum(member.file_size for member in members) > 64 * 1024 * 1024: + raise ValueError("Oversized artifact") + reports = [ + member for member in members if PurePosixPath(member.filename).name == "report.json" + ] + if len(reports) != 1: + raise ValueError("Expected exactly one report.json") + member = reports[0] + path = PurePosixPath(member.filename) + if ( + path.is_absolute() + or ".." in path.parts + or "\\" in member.filename + or stat.S_ISLNK(member.external_attr >> 16) + or member.file_size > MAX_BYTES + ): + raise ValueError("Invalid report member") + if member.flag_bits & 1: + raise ValueError("Encrypted performance artifacts are unsupported") + return json.loads(archive.read(member).decode("utf-8")) + + +def suite_blobs(tree, root): + if ( + not isinstance(tree, dict) + or tree.get("truncated") is not False + or not isinstance(tree.get("tree"), list) + or not all(isinstance(entry, dict) for entry in tree["tree"]) + ): + raise ValueError("Incomplete commit tree") + expected = {path.relative_to(root).as_posix() for path in suite_paths(root)} + blobs = { + entry.get("path"): entry.get("sha") + for entry in tree["tree"] + if entry.get("type") == "blob" and entry.get("path") in expected + } + if set(blobs) != expected or any( + not re.fullmatch(r"[0-9a-f]{40}", sha or "") for sha in blobs.values() + ): + raise ValueError("Benchmark suite missing from commit tree") + return blobs + + +def unavailable(reason): + return ( + f"{MARKER}\n## PR Performance Report\n\n" + f"**Performance could not be assessed.**\n\n{reason} No result is available." + ) + + def number(value, maximum=1e12): if type(value) not in (float, int) or not 0 <= value <= maximum: raise ValueError("Invalid performance measurement") @@ -172,6 +246,82 @@ def validate(report, build_id=None, head=None, source=None, base=None, suite=Non return report +def assess(evidence, artifact_urls, load_artifact, issues=()): + issues = list(issues) + try: + if ( + not isinstance(evidence.build, dict) + or not isinstance(evidence.merge_commit, dict) + or not isinstance(evidence.base_commit, dict) + or not isinstance(evidence.source_tree, dict) + or not isinstance(evidence.base_tree, dict) + ): + raise ValueError + build_id = evidence.build.get("id") + source = evidence.build.get("sourceVersion") + if ( + type(build_id) is not int + or build_id <= 0 + or not re.fullmatch(r"[0-9a-f]{40}", source or "") + or not re.fullmatch(r"[0-9a-f]{40}", evidence.head) + or not re.fullmatch(r"[0-9a-f]{40}", evidence.base) + or evidence.merge_commit.get("sha") != source + or evidence.base_commit.get("sha") != evidence.base + or [parent["sha"] for parent in evidence.merge_commit["parents"]] + != [evidence.base, evidence.head] + ): + raise ValueError + source_tree_sha = evidence.merge_commit["tree"]["sha"] + base_tree_sha = evidence.base_commit["tree"]["sha"] + if ( + not re.fullmatch(r"[0-9a-f]{40}", source_tree_sha) + or not re.fullmatch(r"[0-9a-f]{40}", base_tree_sha) + or evidence.source_tree.get("sha") != source_tree_sha + or evidence.base_tree.get("sha") != base_tree_sha + ): + raise ValueError + except (KeyError, TypeError, ValueError): + return unavailable("Build provenance validation failed.") + + try: + suite_unchanged = suite_blobs(evidence.source_tree, evidence.trusted_root) == suite_blobs( + evidence.base_tree, evidence.trusted_root + ) + trusted_suite = suite_hash(evidence.trusted_root) + except (KeyError, TypeError, ValueError): + return unavailable("Benchmark suite validation failed because a required file changed.") + + reports = [] + for leg, url in artifact_urls.items(): + try: + report = validate( + artifact_report(load_artifact(url)), + build_id, + evidence.head, + source, + evidence.base, + ) + if report["leg"] != leg: + raise ValueError("Artifact leg mismatch") + reports.append(report) + except ( + KeyError, + RecursionError, + TypeError, + ValueError, + zipfile.BadZipFile, + ): + issues.append(leg + " (invalid artifact)") + + if not suite_unchanged or any(report["suite_hash"] != trusted_suite for report in reports): + reports = [] + issues.append("workload version differs from trusted base") + try: + return render(reports, evidence.head, build_id, issues) + except ValueError: + return unavailable("Performance report rendering failed.") + + def comparisons(report): """Do not add inclusive phase totals together or treat them as wall-clock time.""" output = [] @@ -247,6 +397,8 @@ def issue_reason(leg, issues): def render(reports, head, build_id, issues=()): url = f"https://dev.azure.com/sqlclientdrivers/public/_build/results?buildId={build_id}" by_leg = {r["leg"]: r for r in reports} + if len(by_leg) != len(reports): + raise ValueError("Duplicate performance report leg") completed = { leg: (report, comparisons(report)) for leg in LEGS @@ -351,6 +503,7 @@ def render(reports, head, build_id, issues=()): ) lines.append(f"| {environment_name(leg)} | {status} |") + diagnostics_start = len(lines) lines += [ "", "
", @@ -360,12 +513,15 @@ def render(reports, head, build_id, issues=()): "They identify where measured time changed, not why it changed.", ] diagnostics = 0 + total_diagnostics = 0 for leg, (_, rows) in completed.items(): relevant = [row for row in rows if row["status"] != "ok" or row["counts"]] - if not relevant: + total_diagnostics += len(relevant) + visible = relevant[: max(0, MAX_DIAGNOSTIC_ROWS - diagnostics)] + if not visible: continue lines += ["", f"### {environment_name(leg)}"] - for row in relevant: + for row in visible: diagnostics += 1 phases = "; ".join(f"{escape(label)} +{delta:.3f} ms" for delta, label in row["phases"]) counts = "; ".join(escape(label) for label in row["counts"]) @@ -375,9 +531,18 @@ def render(reports, head, build_id, issues=()): lines.append(f"**{TASK_NAMES[row['name']]}:** {detail}.") if not diagnostics: lines += ["", "No affected phases or call-count changes were recorded."] + elif total_diagnostics > diagnostics: + lines += [ + "", + f"{total_diagnostics - diagnostics} additional diagnostic rows are available " + "in the raw ADO artifacts.", + ] lines += [ "", "
", + ] + diagnostics_end = len(lines) + lines += [ "", "
", "All database tasks and timings", @@ -450,7 +615,18 @@ def render(reports, head, build_id, issues=()): "
", ] body = "\n".join(lines) - if len(body) > 60000: + if len(body) > MAX_COMMENT_CHARS: + lines[diagnostics_start:diagnostics_end] = [ + "", + "
", + "Affected phases and call counts", + "", + f"{total_diagnostics} diagnostic rows are available in the raw ADO artifacts.", + "", + "
", + ] + body = "\n".join(lines) + if len(body) > MAX_COMMENT_CHARS: raise ValueError("Performance comment exceeds its size budget") return body diff --git a/tests/test_036_profiler_ci.py b/tests/test_036_profiler_ci.py index 86d3d98a8..782196e6a 100644 --- a/tests/test_036_profiler_ci.py +++ b/tests/test_036_profiler_ci.py @@ -1,6 +1,7 @@ """Contract tests for paired performance comparisons and data-only PR reporting.""" import copy +from http.client import IncompleteRead import importlib.util import io import json @@ -54,11 +55,34 @@ def ado_build(**values): def pr_topology(head="c" * 40, base="a" * 40, merge_base=None): - return lambda path: ( - {"state": "open", "head": {"sha": head}, "base": {"sha": base}} - if path.startswith("pulls/") - else {"parents": [{"sha": merge_base or base}, {"sha": head}]} - ) + def response(path): + if path.startswith("pulls/"): + return {"state": "open", "head": {"sha": head}, "base": {"sha": base}} + if path.startswith("git/commits/"): + commit_sha = path.removeprefix("git/commits/") + source = commit_sha == "b" * 40 + return { + "sha": commit_sha, + "parents": [{"sha": merge_base or base}, {"sha": head}] if source else [], + "tree": {"sha": ("d" if source else "e") * 40}, + } + if path.startswith("git/trees/"): + tree_sha = path.removeprefix("git/trees/").split("?", 1)[0] + return { + "sha": tree_sha, + "truncated": False, + "tree": [ + { + "path": file.relative_to(ROOT).as_posix(), + "type": "blob", + "sha": f"{index + 1:040x}", + } + for index, file in enumerate(reporting.suite_paths(ROOT)) + ], + } + raise AssertionError(f"Unexpected GitHub path: {path}") + + return response @pytest.fixture @@ -202,6 +226,57 @@ def test_standalone_report_rejects_mixed_provenance(report, tmp_path, monkeypatc reporting.main() +def test_render_rejects_duplicate_legs(report): + with pytest.raises(ValueError, match="Duplicate"): + reporting.render([report, copy.deepcopy(report)], "c" * 40, 42) + + +def test_render_bounds_schema_valid_diagnostics(report): + reports = [set_leg(copy.deepcopy(report), leg) for leg in reporting.LEGS] + labels = ["ddbc::" + str(index) + "_" * 152 for index in range(3)] + for item in reports: + for pair in item["pairs"]: + for name in reporting.CASES: + for side, calls in (("base", 1), ("candidate", 2)): + pair[side]["scenarios"][name]["cpp"] = { + label: dict(calls=calls, total_us=2000, min_us=1000, max_us=1000) + for label in labels + } + reporting.validate(item) + body = reporting.render(reports, "c" * 40, 42) + assert len(body) <= 60000 + assert "100 diagnostic rows are available in the raw ADO artifacts" in body + assert "All database tasks and timings" in body + assert "Build, commits and measurement details" in body + + +@pytest.mark.parametrize("invalid", ["source commit", "base commit", "source tree"]) +def test_assessment_binds_all_evidence_to_authenticated_commits(invalid): + evidence = reporting.AssessmentEvidence( + build=ado_build(), + head="c" * 40, + base="a" * 40, + merge_commit={ + "sha": "b" * 40, + "parents": [{"sha": "a" * 40}, {"sha": "c" * 40}], + "tree": {"sha": "d" * 40}, + }, + base_commit={"sha": "a" * 40, "tree": {"sha": "e" * 40}}, + source_tree={"sha": "d" * 40, "truncated": False, "tree": []}, + base_tree={"sha": "e" * 40, "truncated": False, "tree": []}, + trusted_root=ROOT, + ) + if invalid == "source commit": + evidence.merge_commit["sha"] = "f" * 40 + elif invalid == "base commit": + evidence.base_commit["sha"] = "f" * 40 + else: + evidence = reporting.AssessmentEvidence(**{**evidence.__dict__, "source_tree": []}) + body = reporting.assess(evidence, {}, lambda url: pytest.fail("must not download")) + assert "Performance could not be assessed" in body + assert "Build provenance validation failed" in body + + def clear_slowdowns(report): for pair in report["pairs"]: for name in reporting.CASES: @@ -347,7 +422,7 @@ def test_coverage_artifact_reader_rejects_oversized_or_unrelated_archives(tmp_pa def test_artifact_read_never_extracts_paths(report): raw = json.dumps(report) assert ( - publisher.artifact_report(zip_data([("profiler-Linux-SQL2022/report.json", raw)])) == report + reporting.artifact_report(zip_data([("profiler-Linux-SQL2022/report.json", raw)])) == report ) for entries in [ [("../report.json", raw)], @@ -356,7 +431,7 @@ def test_artifact_read_never_extracts_paths(report): [("logs.txt", "no report")], ]: with pytest.raises(ValueError): - publisher.artifact_report(zip_data(entries)) + reporting.artifact_report(zip_data(entries)) def test_untrusted_labels_cannot_inject_links_mentions_or_markdown(): @@ -373,6 +448,14 @@ def test_untrusted_labels_cannot_inject_links_mentions_or_markdown(): assert not publisher.allowed_url("https://artifacts.visualstudio.com.evil.example/artifact") +def test_incomplete_http_response_is_normalized_for_terminal_fallback(monkeypatch): + opener = MagicMock() + opener.open.side_effect = IncompleteRead(b"partial") + monkeypatch.setattr(publisher, "build_opener", lambda *args: opener) + with pytest.raises(URLError, match="Incomplete HTTP response"): + publisher.fetch("https://api.github.com/repos/microsoft/mssql-python") + + def test_build_selection_requires_exact_pr_head(): build = ado_build() assert publisher.find_build([build], 123, "c" * 40) is build @@ -440,9 +523,11 @@ def test_report_cases_match_the_executed_workload_registry(): assert ROOT / "eng/profiler_benchmarks/__init__.py" in reporting.suite_paths(ROOT) assert ROOT / "eng/profiler_benchmarks/report.py" in reporting.suite_paths(ROOT) assert ROOT / "eng/pipelines/pr-validation-pipeline.yml" in reporting.suite_paths(ROOT) + assert ROOT / "eng/scripts/setup_sql_container.py" in reporting.suite_paths(ROOT) + assert ROOT / "requirements.txt" in reporting.suite_paths(ROOT) -def test_suite_blobs_require_complete_authenticated_tree(monkeypatch): +def test_suite_blobs_require_complete_authenticated_tree(): expected = [path.relative_to(ROOT).as_posix() for path in reporting.suite_paths(ROOT)] tree = { "truncated": False, @@ -451,11 +536,13 @@ def test_suite_blobs_require_complete_authenticated_tree(monkeypatch): for index, path in enumerate(expected) ], } - monkeypatch.setattr(publisher, "github", lambda path: tree) - assert set(publisher.suite_blobs({"tree": {"sha": "a" * 40}})) == set(expected) + assert set(reporting.suite_blobs(tree, ROOT)) == set(expected) tree["tree"].pop() with pytest.raises(ValueError, match="missing"): - publisher.suite_blobs({"tree": {"sha": "a" * 40}}) + reporting.suite_blobs(tree, ROOT) + tree["tree"].append(None) + with pytest.raises(ValueError, match="Incomplete"): + reporting.suite_blobs(tree, ROOT) def test_publisher_finishes_unavailable_when_checked_suite_file_moves(monkeypatch): @@ -465,9 +552,17 @@ def test_publisher_finishes_unavailable_when_checked_suite_file_moves(monkeypatc publisher, "publish", lambda number, head, body, base=None: posted.append(body) ) monkeypatch.setattr(publisher, "github", pr_topology()) - monkeypatch.setattr(publisher, "api", lambda url: {"value": [build]}) + artifacts = [ + {"name": "profiler-" + leg, "resource": {"downloadUrl": "https://dev.azure.com/" + leg}} + for leg in reporting.LEGS + ] monkeypatch.setattr( publisher, + "api", + lambda url: {"value": artifacts if "/artifacts?" in url else [build]}, + ) + monkeypatch.setattr( + reporting, "suite_blobs", MagicMock(side_effect=ValueError("Benchmark suite missing from commit tree")), ) @@ -809,12 +904,12 @@ def api(url): monkeypatch.setattr( publisher, "publish", lambda number, head, body, base=None: posted.append(body) ) - monkeypatch.setattr(publisher, "suite_hash", lambda root: "d" * 64) + monkeypatch.setattr(reporting, "suite_hash", lambda root: "d" * 64) suite_versions = iter(({"suite": "source"}, {"suite": "base"})) monkeypatch.setattr( - publisher, + reporting, "suite_blobs", - lambda commit: next(suite_versions) if corrupt == "source" else {"suite": "same"}, + lambda *args: next(suite_versions) if corrupt == "source" else {"suite": "same"}, ) monkeypatch.setattr( publisher, @@ -869,7 +964,7 @@ def api(url): monkeypatch.setattr( publisher, "publish", lambda number, head, body, base=None: posted.append(body) ) - monkeypatch.setattr(publisher, "suite_blobs", lambda commit: {"suite": "same"}) + monkeypatch.setattr(reporting, "suite_blobs", lambda *args: {"suite": "same"}) monkeypatch.setattr(publisher.time, "monotonic", lambda: clock[0]) monkeypatch.setattr( publisher.time, "sleep", lambda seconds: clock.__setitem__(0, clock[0] + seconds) @@ -961,9 +1056,9 @@ def test_artifact_symlink_and_oversized_json_are_rejected(): symlink.create_system = 3 symlink.external_attr = 0o120777 << 16 with pytest.raises(ValueError, match="Invalid report"): - publisher.artifact_report(zip_data([(symlink, "{}")])) + reporting.artifact_report(zip_data([(symlink, "{}")])) with pytest.raises(ValueError, match="Invalid report"): - publisher.artifact_report(zip_data([("report.json", " " * (reporting.MAX_BYTES + 1))])) + reporting.artifact_report(zip_data([("report.json", " " * (reporting.MAX_BYTES + 1))])) @pytest.mark.parametrize("environment", [{"os": "Windows"}, {"sql_version": "17.0"}]) @@ -1011,6 +1106,9 @@ def test_comment_workflow_executes_only_trusted_base_code(): assert coverage.count("extract_coverage_artifact.py") == 2 assert coverage.count("--max-filesize 268435456") == 2 assert "unzip -o" not in coverage + assert 'cp "$COVERAGE_XML"' not in coverage + assert 'diff-cover "$COVERAGE_XML"' in coverage + assert "COVERAGE_XML: ${{ runner.temp }}/coverage.xml" in coverage assert ( "head.ref" not in workflow and "head.sha }}" not in workflow.split("ref:", 1)[1].split("persist", 1)[0] From a2dde75b483516077606d86dc29a2dbc985c3cb5 Mon Sep 17 00:00:00 2001 From: Gaurav Sharma Date: Thu, 17 Sep 2026 14:45:15 +0530 Subject: [PATCH 11/18] FIX: Complete profiler CI review hardening Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/scripts/post_profiler_comment.py | 19 +++++-- .github/workflows/pr-code-coverage.yml | 10 ++-- tests/test_036_profiler_ci.py | 64 ++++++++++++++++++++++++ 3 files changed, 86 insertions(+), 7 deletions(-) diff --git a/.github/scripts/post_profiler_comment.py b/.github/scripts/post_profiler_comment.py index 4c9d720a8..6fd2387db 100644 --- a/.github/scripts/post_profiler_comment.py +++ b/.github/scripts/post_profiler_comment.py @@ -120,6 +120,17 @@ def current(): github(f"issues/{pr_number}/comments", method="POST", data={"body": body}) +def publish_with_retry(pr_number, head, body, base=None, attempts=3): + for attempt in range(attempts): + try: + publish(pr_number, head, body, base) + return + except (KeyError, TypeError, ValueError, TimeoutError, URLError): + if attempt + 1 == attempts: + raise + time.sleep(5) + + def find_build(builds, number, head): return next( ( @@ -170,7 +181,7 @@ def artifact_items(response): def unavailable(number, head, reason, base=None): - publish( + publish_with_retry( number, head, HEADER + "**Performance could not be assessed.**\n\n" + reason + " No result is available.", @@ -179,7 +190,7 @@ def unavailable(number, head, reason, base=None): def run(number, head, wait_minutes): - publish( + publish_with_retry( number, head, HEADER @@ -318,7 +329,9 @@ def load_artifact(url): base_tree=base_tree, trusted_root=ROOT, ) - publish(number, head, reporting.assess(evidence, artifact_urls, load_artifact, issues), base) + publish_with_retry( + number, head, reporting.assess(evidence, artifact_urls, load_artifact, issues), base + ) if __name__ == "__main__": diff --git a/.github/workflows/pr-code-coverage.yml b/.github/workflows/pr-code-coverage.yml index 0a733e9ef..061843f34 100644 --- a/.github/workflows/pr-code-coverage.yml +++ b/.github/workflows/pr-code-coverage.yml @@ -234,7 +234,8 @@ jobs: if [[ -n "$COVERAGE_ARTIFACT" && "$COVERAGE_ARTIFACT" != "null" && "$COVERAGE_ARTIFACT" != "empty" ]]; then echo "📊 Downloading coverage report..." - if ! curl -L "$COVERAGE_ARTIFACT" -o coverage-report.zip --fail --silent --show-error \ + COVERAGE_ARCHIVE="$RUNNER_TEMP/coverage-report.zip" + if ! curl -L "$COVERAGE_ARTIFACT" -o "$COVERAGE_ARCHIVE" --fail --silent --show-error \ --connect-timeout 10 --max-time 60 --max-filesize 268435456 \ --retry 2 --retry-delay 5 --retry-max-time 180; then echo "❌ Failed to download coverage report from Azure DevOps" @@ -243,7 +244,7 @@ jobs: fi INDEX_FILE="$RUNNER_TEMP/coverage-index.html" - if ! python .github/scripts/extract_coverage_artifact.py html coverage-report.zip "$INDEX_FILE"; then + if ! python .github/scripts/extract_coverage_artifact.py html "$COVERAGE_ARCHIVE" "$INDEX_FILE"; then echo "❌ Failed to read the coverage HTML artifact" exit 1 fi @@ -353,7 +354,8 @@ jobs: if [[ -n "$COVERAGE_XML_ARTIFACT" && "$COVERAGE_XML_ARTIFACT" != "null" && "$COVERAGE_XML_ARTIFACT" != "empty" ]]; then echo "📊 Downloading coverage artifact from: $COVERAGE_XML_ARTIFACT" - if ! curl -L "$COVERAGE_XML_ARTIFACT" -o coverage-artifacts.zip --fail --silent --show-error \ + COVERAGE_XML_ARCHIVE="$RUNNER_TEMP/coverage-artifacts.zip" + if ! curl -L "$COVERAGE_XML_ARTIFACT" -o "$COVERAGE_XML_ARCHIVE" --fail --silent --show-error \ --connect-timeout 10 --max-time 60 --max-filesize 268435456 \ --retry 2 --retry-delay 5 --retry-max-time 180; then echo "❌ Failed to download coverage artifacts" @@ -361,7 +363,7 @@ jobs: fi COVERAGE_XML="$RUNNER_TEMP/coverage.xml" - if ! python .github/scripts/extract_coverage_artifact.py xml coverage-artifacts.zip "$COVERAGE_XML"; then + if ! python .github/scripts/extract_coverage_artifact.py xml "$COVERAGE_XML_ARCHIVE" "$COVERAGE_XML"; then echo "❌ Failed to read the coverage XML artifact" exit 1 fi diff --git a/tests/test_036_profiler_ci.py b/tests/test_036_profiler_ci.py index 782196e6a..5bbd72027 100644 --- a/tests/test_036_profiler_ci.py +++ b/tests/test_036_profiler_ci.py @@ -26,6 +26,7 @@ from eng.profiler_benchmarks import controller from eng.profiler_benchmarks import report as reporting +from eng.profiler_benchmarks import workloads as benchmark_workloads def load(name, path): @@ -477,6 +478,16 @@ def api(path, **kwargs): assert len(calls) == 1 and calls[0][1] == {} +def test_publisher_retries_transient_comment_failures(monkeypatch): + publish = MagicMock(side_effect=[URLError("temporary"), None]) + sleeps = [] + monkeypatch.setattr(publisher, "publish", publish) + monkeypatch.setattr(publisher.time, "sleep", sleeps.append) + publisher.publish_with_retry(123, "a" * 40, "body") + assert publish.call_count == 2 + assert sleeps == [5] + + def test_revisions_use_exact_first_parent(monkeypatch): calls = [] @@ -527,6 +538,52 @@ def test_report_cases_match_the_executed_workload_registry(): assert ROOT / "requirements.txt" in reporting.suite_paths(ROOT) +def test_query_workload_executes_and_collects(monkeypatch): + cursor = MagicMock() + cursor.fetchall.return_value = [(1,), (2,)] + connection = MagicMock() + connection.cursor.return_value.__enter__.return_value = cursor + context = MagicMock() + context.collect.return_value = ({"cpp": {}}, {"py": {}}) + monkeypatch.setattr(benchmark_workloads.time, "perf_counter", MagicMock(side_effect=[1, 1.1])) + result = benchmark_workloads.query(connection, context, "SELECT 1") + cursor.execute.assert_called_once_with("SELECT 1") + assert result["detail"] == "Rows: 2" + context.enable.assert_called_once() + context.disable.assert_called_once() + + +@pytest.mark.parametrize("named", [False, True]) +def test_parameter_workload_executes_both_binding_forms(named): + cursor = MagicMock() + cursor.fetchone.side_effect = [(value,) for value in range(100)] + connection = MagicMock() + connection.cursor.return_value.__enter__.return_value = cursor + context = MagicMock() + context.collect.return_value = ({}, {}) + result = benchmark_workloads.parameter_execution(connection, context, named=named) + expected = ("SELECT %(value)s", {"value": 0}) if named else ("SELECT ?", (0,)) + assert cursor.execute.call_args_list[0].args == expected + assert cursor.execute.call_count == 100 + assert result["detail"] == "Rows: 100" + context.disable.assert_called_once() + + +@pytest.mark.parametrize("input_sizes", [False, True]) +def test_legacy_insert_workload_executes_both_variants(input_sizes): + cursor = MagicMock() + connection = MagicMock() + connection.cursor.return_value.__enter__.return_value = cursor + context = MagicMock() + context.collect.return_value = ({}, {}) + result = benchmark_workloads.legacy_insertmany(connection, context, input_sizes=input_sizes) + assert cursor.execute.call_count == 101 + assert cursor.setinputsizes.call_count == (100 if input_sizes else 0) + assert result["detail"] == "Rows: 100000" + connection.rollback.assert_called_once() + context.disable.assert_called_once() + + def test_suite_blobs_require_complete_authenticated_tree(): expected = [path.relative_to(ROOT).as_posix() for path in reporting.suite_paths(ROOT)] tree = { @@ -654,6 +711,9 @@ def test_build_timeout_terminates_descendants(tmp_path, monkeypatch): os.kill(descendant, 0) except ProcessLookupError: break + stat = Path(f"/proc/{descendant}/stat") + if stat.is_file() and stat.read_text(encoding="utf-8").split()[2] == "Z": + break time.sleep(0.05) else: pytest.fail("build descendant survived timeout cleanup") @@ -1106,6 +1166,10 @@ def test_comment_workflow_executes_only_trusted_base_code(): assert coverage.count("extract_coverage_artifact.py") == 2 assert coverage.count("--max-filesize 268435456") == 2 assert "unzip -o" not in coverage + assert '-o "$COVERAGE_ARCHIVE"' in coverage + assert '-o "$COVERAGE_XML_ARCHIVE"' in coverage + assert "-o coverage-report.zip" not in coverage + assert "-o coverage-artifacts.zip" not in coverage assert 'cp "$COVERAGE_XML"' not in coverage assert 'diff-cover "$COVERAGE_XML"' in coverage assert "COVERAGE_XML: ${{ runner.temp }}/coverage.xml" in coverage From 31f34cb5ea1b02ca3accdf7f54a9759abf59ce84 Mon Sep 17 00:00:00 2001 From: Gaurav Sharma Date: Thu, 17 Sep 2026 16:10:29 +0530 Subject: [PATCH 12/18] FIX: Close profiler CI validation gaps Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/scripts/extract_coverage_artifact.py | 4 ++- .github/workflows/pr-code-coverage.yml | 6 +--- benchmarks/README.md | 6 +++- eng/profiler_benchmarks/README.md | 3 +- eng/profiler_benchmarks/report.py | 7 ++++ tests/test_036_profiler_ci.py | 35 ++++++++++++++++++++ 6 files changed, 53 insertions(+), 8 deletions(-) diff --git a/.github/scripts/extract_coverage_artifact.py b/.github/scripts/extract_coverage_artifact.py index ddfc7c15c..12a74b84f 100644 --- a/.github/scripts/extract_coverage_artifact.py +++ b/.github/scripts/extract_coverage_artifact.py @@ -22,7 +22,9 @@ def select(archive, kind): for member in members: path = PurePosixPath(member.filename) if ( - path.is_absolute() + member.is_dir() + or stat.S_ISDIR(member.external_attr >> 16) + or path.is_absolute() or ".." in path.parts or "\\" in member.filename or member.flag_bits & 1 diff --git a/.github/workflows/pr-code-coverage.yml b/.github/workflows/pr-code-coverage.yml index 061843f34..fb1413c2e 100644 --- a/.github/workflows/pr-code-coverage.yml +++ b/.github/workflows/pr-code-coverage.yml @@ -174,11 +174,7 @@ jobs: } BUILD_URL="https://dev.azure.com/SqlClientDrivers/public/_apis/build/builds/$BUILD_ID?api-version=7.1-preview.7" ARTIFACTS_URL="https://dev.azure.com/SqlClientDrivers/public/_apis/build/builds/$BUILD_ID/artifacts?api-version=7.1-preview.5" - ADO_URL=$(jq -r '._links.web.href // empty' <<< "$REPLACEMENT") - if [[ -z "$ADO_URL" || ${#ADO_URL} -gt 500 || "$ADO_URL" == *$'\n'* || "$ADO_URL" == *$'\r'* ]]; then - echo "Invalid replacement Azure DevOps build URL" - exit 1 - fi + ADO_URL="https://dev.azure.com/sqlclientdrivers/public/_build/results?buildId=$BUILD_ID" echo "BUILD_ID=$BUILD_ID" >> "$GITHUB_ENV" echo "ADO_URL=$ADO_URL" >> "$GITHUB_ENV" COMPLETED_AT=-1 diff --git a/benchmarks/README.md b/benchmarks/README.md index 07560c08a..4a79375bd 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -7,7 +7,11 @@ This directory contains benchmark scripts for testing the performance of various ### 1. `bench_mssql.py` - Richbench Framework Benchmarks Comprehensive benchmarks using the richbench framework for detailed performance analysis. -### 2. Profiler benchmark comparisons +### 2. `perf-benchmarking.py` - Real-World Query Benchmarks + +Direct `pyodbc` and `mssql_python` comparisons against AdventureWorks2022. + +### 3. Profiler benchmark comparisons Profiler benchmarks are engineering infrastructure, separate from these standalone scripts and from the runtime profiler. See diff --git a/eng/profiler_benchmarks/README.md b/eng/profiler_benchmarks/README.md index 905363ffb..490802036 100644 --- a/eng/profiler_benchmarks/README.md +++ b/eng/profiler_benchmarks/README.md @@ -30,7 +30,8 @@ Partial results never produce a verdict. Five environments publish raw samples: Windows and macOS on SQL Server 2022/2025, and Ubuntu on SQL Server 2022. The privileged publisher runs trusted base code, authenticates benchmark producers, validates bounded artifacts, and ignores stale -heads. Missing, malformed, canceled, or failed results remain unavailable. +heads. A failed aggregate build can still publish when its authenticated artifacts +validate. Missing, malformed, canceled, incomplete, or invalid data remains unavailable. The publisher waits up to 220 minutes inside a 230-minute workflow. The first main comparison after introduction may be incomplete because its parent lacks this diff --git a/eng/profiler_benchmarks/report.py b/eng/profiler_benchmarks/report.py index 6fff6c9f2..561cd32ac 100644 --- a/eng/profiler_benchmarks/report.py +++ b/eng/profiler_benchmarks/report.py @@ -148,6 +148,13 @@ def text(value, limit=160): def validate(report, build_id=None, head=None, source=None, base=None, suite=None): + try: + return _validate(report, build_id, head, source, base, suite) + except KeyError as error: + raise ValueError(f"Missing performance report field: {error.args[0]}") from error + + +def _validate(report, build_id=None, head=None, source=None, base=None, suite=None): if not isinstance(report, dict) or report.get("schema_version") != 1: raise ValueError("Unsupported report schema") if report.get("leg") not in LEGS or report.get("status") not in ("complete", "incomplete"): diff --git a/tests/test_036_profiler_ci.py b/tests/test_036_profiler_ci.py index 5bbd72027..fe096a30b 100644 --- a/tests/test_036_profiler_ci.py +++ b/tests/test_036_profiler_ci.py @@ -176,6 +176,19 @@ def test_reject_invalid_or_incomparable_data(report, case): reporting.validate(report) +@pytest.mark.parametrize("field", ["environment", "scenarios", "cpp", "calls"]) +def test_missing_report_fields_are_normalized_to_value_error(report, field): + sample = report["pairs"][0]["base"] + if field in ("environment", "scenarios"): + del sample[field] + elif field == "cpp": + del sample["scenarios"]["select"][field] + else: + del sample["scenarios"]["select"]["cpp"]["ddbc::query"][field] + with pytest.raises(ValueError, match="Missing performance report field"): + reporting.validate(report) + + def test_reject_wrong_commit_and_preserve_incomplete_status(report): with pytest.raises(ValueError, match="provenance"): reporting.validate(report, head="e" * 40) @@ -418,6 +431,15 @@ def test_coverage_artifact_reader_rejects_oversized_or_unrelated_archives(tmp_pa stream.write(b"x") with pytest.raises(ValueError, match="archive exceeds"): extractor.copy_report(archive, tmp_path / "coverage.xml", "xml") + archive.write_bytes(zip_data([("coverage.xml/", b"")])) + with pytest.raises(ValueError, match="No coverage xml"): + extractor.copy_report(archive, tmp_path / "coverage.xml", "xml") + directory = zipfile.ZipInfo("coverage.xml") + directory.create_system = 3 + directory.external_attr = 0o40755 << 16 + archive.write_bytes(zip_data([(directory, b"")])) + with pytest.raises(ValueError, match="No coverage xml"): + extractor.copy_report(archive, tmp_path / "coverage.xml", "xml") def test_artifact_read_never_extracts_paths(report): @@ -1155,6 +1177,14 @@ def test_ci_reuses_profiling_builds_without_changing_release_defaults(): assert "libodbc1 " not in benchmark and "odbcinst1debian2" not in benchmark +def test_profiler_documentation_preserves_standalone_benchmarks_and_failed_build_contract(): + benchmarks = (ROOT / "benchmarks/README.md").read_text(encoding="utf-8") + assert "perf-benchmarking.py" in benchmarks + assert "Profiler benchmark comparisons" in benchmarks + contract = (ROOT / "eng/profiler_benchmarks/README.md").read_text(encoding="utf-8") + assert "failed aggregate build can still publish" in contract + + def test_comment_workflow_executes_only_trusted_base_code(): workflow = (ROOT / ".github/workflows/pr-profiler-report.yml").read_text(encoding="utf-8") assert "pull_request_target:" in workflow @@ -1170,6 +1200,11 @@ def test_comment_workflow_executes_only_trusted_base_code(): assert '-o "$COVERAGE_XML_ARCHIVE"' in coverage assert "-o coverage-report.zip" not in coverage assert "-o coverage-artifacts.zip" not in coverage + assert coverage.count("._links.web.href") == 1 + assert ( + 'ADO_URL="https://dev.azure.com/sqlclientdrivers/public/_build/results?buildId=$BUILD_ID"' + in coverage + ) assert 'cp "$COVERAGE_XML"' not in coverage assert 'diff-cover "$COVERAGE_XML"' in coverage assert "COVERAGE_XML: ${{ runner.temp }}/coverage.xml" in coverage From c1b9ab81ee0298263a8456e202c6c76c8841cc24 Mon Sep 17 00:00:00 2001 From: Gaurav Sharma Date: Thu, 17 Sep 2026 17:18:25 +0530 Subject: [PATCH 13/18] FIX: Harden profiler artifact finalization Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/scripts/post_profiler_comment.py | 3 +- eng/profiler_benchmarks/report.py | 2 + tests/test_036_profiler_ci.py | 50 ++++++++++++++++++++++-- 3 files changed, 50 insertions(+), 5 deletions(-) diff --git a/.github/scripts/post_profiler_comment.py b/.github/scripts/post_profiler_comment.py index 6fd2387db..eba84383a 100644 --- a/.github/scripts/post_profiler_comment.py +++ b/.github/scripts/post_profiler_comment.py @@ -281,10 +281,9 @@ def run(number, head, wait_minutes): except (ValueError, KeyError, TypeError, URLError, TimeoutError): unavailable(number, head, "Build provenance validation failed.", pr_base) return - artifact_deadline = time.monotonic() + 120 artifacts = None failures = 0 - while time.monotonic() < artifact_deadline: + while time.monotonic() < deadline: try: artifacts = artifact_items(api(f"{ADO}/builds/{build_id}/artifacts?api-version=7.1")) failures = 0 diff --git a/eng/profiler_benchmarks/report.py b/eng/profiler_benchmarks/report.py index 561cd32ac..b5768861e 100644 --- a/eng/profiler_benchmarks/report.py +++ b/eng/profiler_benchmarks/report.py @@ -12,6 +12,7 @@ import stat import statistics import zipfile +import zlib LEGS = ("Windows-SQL2022", "Windows-SQL2025", "macOS-SQL2022", "macOS-SQL2025", "Linux-SQL2022") TASK_NAMES = { @@ -317,6 +318,7 @@ def assess(evidence, artifact_urls, load_artifact, issues=()): TypeError, ValueError, zipfile.BadZipFile, + zlib.error, ): issues.append(leg + " (invalid artifact)") diff --git a/tests/test_036_profiler_ci.py b/tests/test_036_profiler_ci.py index fe096a30b..88a46fbd7 100644 --- a/tests/test_036_profiler_ci.py +++ b/tests/test_036_profiler_ci.py @@ -17,6 +17,7 @@ from unittest.mock import MagicMock from urllib.error import URLError import zipfile +import zlib import pytest @@ -929,6 +930,7 @@ def api(path, **kwargs): "base", "provenance", "recursion", + "deflate", "delayed", ], ) @@ -1007,6 +1009,15 @@ def fetch(url, **kwargs): return data[leg] monkeypatch.setattr(publisher, "fetch", fetch) + if corrupt == "deflate": + artifact_report = reporting.artifact_report + + def corrupt_deflate(raw): + if raw == data["Linux-SQL2022"]: + raise zlib.error("corrupt deflate stream") + return artifact_report(raw) + + monkeypatch.setattr(reporting, "artifact_report", corrupt_deflate) monkeypatch.setattr(publisher.time, "monotonic", lambda: clock[0]) monkeypatch.setattr( publisher.time, "sleep", lambda seconds: clock.__setitem__(0, clock[0] + seconds) @@ -1021,7 +1032,7 @@ def fetch(url, **kwargs): if corrupt in ("suite", "source"): assert "workload version differs from trusted base" in posted[1] assert "consistent slowdown signals" not in posted[1] - elif corrupt in ("zip", "timeout", "scenarios", "recursion"): + elif corrupt in ("zip", "timeout", "scenarios", "recursion", "deflate"): assert "### Windows / SQL Server 2022" in posted[1] assert reporting.escape("Linux-SQL2022 (invalid artifact)") in posted[1] assert "| Linux / SQL Server 2022 | No result available (invalid artifact) |" in posted[1] @@ -1051,8 +1062,8 @@ def api(url): monkeypatch.setattr( publisher.time, "sleep", lambda seconds: clock.__setitem__(0, clock[0] + seconds) ) - publisher.run(123, "c" * 40, 1) - assert clock[0] == 150 + publisher.run(123, "c" * 40, 4) + assert clock[0] == 240 assert len(posted) == 2 assert "buildId=42" in posted[1] @@ -1133,6 +1144,39 @@ def test_publisher_retries_malformed_pr_and_artifact_responses(monkeypatch): publisher.build_items({"value": [malformed]}) +def test_artifact_polling_uses_remaining_publication_budget(monkeypatch): + posted = [] + clock = [0] + build = ado_build() + artifacts = [ + {"name": "profiler-" + leg, "resource": {"downloadUrl": "https://dev.azure.com/" + leg}} + for leg in reporting.LEGS + ] + artifact_responses = iter([[]] * 5 + [artifacts]) + + def api(url): + return {"value": next(artifact_responses)} if "/artifacts?" in url else {"value": [build]} + + monkeypatch.setattr(publisher, "api", api) + monkeypatch.setattr(publisher, "github", pr_topology()) + monkeypatch.setattr( + publisher, "publish", lambda number, head, body, base=None: posted.append(body) + ) + monkeypatch.setattr(reporting, "assess", lambda *args: "final report") + monkeypatch.setattr(publisher.time, "monotonic", lambda: clock[0]) + monkeypatch.setattr( + publisher.time, "sleep", lambda seconds: clock.__setitem__(0, clock[0] + seconds) + ) + publisher.run(123, "c" * 40, 4) + assert clock[0] == 150 + assert posted == [ + publisher.HEADER + + "**Performance assessment pending.**\n\n" + + f"Waiting for the matching performance run for head `{'c' * 40}`.", + "final report", + ] + + def test_artifact_symlink_and_oversized_json_are_rejected(): symlink = zipfile.ZipInfo("report.json") symlink.create_system = 3 From dd7a951469ec5f190c355b2dca2accdd4c6abc83 Mon Sep 17 00:00:00 2001 From: Gaurav Sharma Date: Thu, 17 Sep 2026 18:43:38 +0530 Subject: [PATCH 14/18] FIX: Scope profiler CI to pull requests Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- eng/pipelines/pr-validation-pipeline.yml | 14 +++++++------- eng/profiler_benchmarks/controller.py | 7 ++++--- tests/test_036_profiler_ci.py | 20 ++++++++++++++++++++ 3 files changed, 31 insertions(+), 10 deletions(-) diff --git a/eng/pipelines/pr-validation-pipeline.yml b/eng/pipelines/pr-validation-pipeline.yml index 86dd7a798..c8e861afe 100644 --- a/eng/pipelines/pr-validation-pipeline.yml +++ b/eng/pipelines/pr-validation-pipeline.yml @@ -409,20 +409,20 @@ jobs: python -m eng.profiler_benchmarks.controller --reuse-candidate --leg "Windows-$(sqlVersion)" --output profiler-results if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } displayName: 'Compare profiling builds on SQL Server 2022/2025' - condition: and(succeeded(), or(eq(variables['sqlVersion'], 'SQL2022'), eq(variables['sqlVersion'], 'SQL2025'))) + condition: and(succeeded(), eq(variables['Build.Reason'], 'PullRequest'), or(eq(variables['sqlVersion'], 'SQL2022'), eq(variables['sqlVersion'], 'SQL2025'))) continueOnError: true timeoutInMinutes: 100 env: SYSTEM_PULLREQUEST_SOURCECOMMITID: $(System.PullRequest.SourceCommitId) DB_CONNECTION_STRING: 'Server=localhost;Database=AdventureWorks2022;Uid=sa;Pwd=$(DB_PASSWORD);TrustServerCertificate=yes' - # Publish on PRs as well as main, including partial reports and failure logs. + # Publish partial reports and failure logs for PR assessment. - task: PublishPipelineArtifact@1 inputs: targetPath: profiler-results artifact: 'profiler-Windows-$(sqlVersion)' displayName: 'Publish paired profiler measurements' - condition: and(succeededOrFailed(), or(eq(variables['sqlVersion'], 'SQL2022'), eq(variables['sqlVersion'], 'SQL2025'))) + condition: and(succeededOrFailed(), eq(variables['Build.Reason'], 'PullRequest'), or(eq(variables['sqlVersion'], 'SQL2022'), eq(variables['sqlVersion'], 'SQL2025'))) continueOnError: true - task: CopyFiles@2 @@ -662,7 +662,7 @@ jobs: HOMEBREW_ACCEPT_EULA=Y brew install msodbcsql18 python -m eng.profiler_benchmarks.controller --reuse-candidate --leg "macOS-$(sqlVersion)" --output profiler-results displayName: 'Compare profiling builds on macOS $(sqlVersion)' - condition: and(succeeded(), or(eq(variables['sqlVersion'], 'SQL2022'), eq(variables['sqlVersion'], 'SQL2025'))) + condition: and(succeeded(), eq(variables['Build.Reason'], 'PullRequest'), or(eq(variables['sqlVersion'], 'SQL2022'), eq(variables['sqlVersion'], 'SQL2025'))) timeoutInMinutes: 100 continueOnError: true env: @@ -675,7 +675,7 @@ jobs: targetPath: profiler-results artifact: 'profiler-macOS-$(sqlVersion)' displayName: 'Publish paired profiler measurements' - condition: and(succeededOrFailed(), or(eq(variables['sqlVersion'], 'SQL2022'), eq(variables['sqlVersion'], 'SQL2025'))) + condition: and(succeededOrFailed(), eq(variables['Build.Reason'], 'PullRequest'), or(eq(variables['sqlVersion'], 'SQL2022'), eq(variables['sqlVersion'], 'SQL2025'))) continueOnError: true - script: | @@ -956,7 +956,7 @@ jobs: echo "Skipping performance benchmarks on $(distroName) (only runs on Ubuntu with local SQL Server)" fi displayName: 'Compare profiling builds in $(distroName) container' - condition: and(succeeded(), eq(variables['distroName'], 'Ubuntu'), eq(variables['useAzureSQL'], 'false')) + condition: and(succeeded(), eq(variables['Build.Reason'], 'PullRequest'), eq(variables['distroName'], 'Ubuntu'), eq(variables['useAzureSQL'], 'false')) continueOnError: true timeoutInMinutes: 100 env: @@ -969,7 +969,7 @@ jobs: targetPath: profiler-results artifact: profiler-Linux-SQL2022 displayName: 'Publish paired profiler measurements' - condition: and(succeededOrFailed(), eq(variables['distroName'], 'Ubuntu'), eq(variables['useAzureSQL'], 'false')) + condition: and(succeededOrFailed(), eq(variables['Build.Reason'], 'PullRequest'), eq(variables['distroName'], 'Ubuntu'), eq(variables['useAzureSQL'], 'false')) continueOnError: true - script: | diff --git a/eng/profiler_benchmarks/controller.py b/eng/profiler_benchmarks/controller.py index 21e82badd..97376d1c2 100644 --- a/eng/profiler_benchmarks/controller.py +++ b/eng/profiler_benchmarks/controller.py @@ -75,9 +75,10 @@ def terminate_process_tree(process): text=True, ) if result.returncode: - if process.poll() is None: - process.kill() - process.wait() + if process.poll() is not None: + return + process.kill() + process.wait() raise RuntimeError(f"Failed to terminate build process tree: {result.stdout.strip()}") process.wait(timeout=5) return diff --git a/tests/test_036_profiler_ci.py b/tests/test_036_profiler_ci.py index 88a46fbd7..06c3f6514 100644 --- a/tests/test_036_profiler_ci.py +++ b/tests/test_036_profiler_ci.py @@ -762,6 +762,19 @@ def test_windows_process_tree_cleanup_uses_taskkill(monkeypatch): process.wait.assert_called_once_with(timeout=5) +def test_windows_process_tree_cleanup_accepts_already_exited_process(monkeypatch): + process = MagicMock(pid=123) + process.poll.return_value = 0 + monkeypatch.setattr(controller, "WINDOWS", True) + monkeypatch.setattr( + controller.subprocess, + "run", + MagicMock(return_value=subprocess.CompletedProcess([], 128, "", "not found")), + ) + controller.terminate_process_tree(process) + process.kill.assert_not_called() + + def test_overall_budget_caps_build_and_worker_time(monkeypatch): monkeypatch.setattr(controller.time, "monotonic", lambda: 100) assert controller.remaining(110, controller.WORKER_TIMEOUT) == 10 @@ -1200,6 +1213,13 @@ def test_ci_reuses_profiling_builds_without_changing_release_defaults(): pipeline = (ROOT / "eng/pipelines/pr-validation-pipeline.yml").read_text(encoding="utf-8") assert "benchmarks/perf-benchmarking.py" not in pipeline assert pipeline.count("python -m eng.profiler_benchmarks.controller --reuse-candidate") == 3 + profiler_conditions = re.findall( + r"displayName: '(?:Compare profiling builds[^']*|Publish paired profiler measurements)'\n" + r" condition: ([^\n]+)", + pipeline, + ) + assert len(profiler_conditions) == 6 + assert all("eq(variables['Build.Reason'], 'PullRequest')" in condition for condition in profiler_conditions) assert "profilerBuild: '0'" in pipeline # LocalDB still exercises the normal build assert "ddbc_bindings-profiling-SQL2022" in pipeline assert "ddbc_bindings-profiling-SQL2025" in pipeline From 650c31e494f478123a5af0a831a2b91b8a02a272 Mon Sep 17 00:00:00 2001 From: Gaurav Sharma Date: Thu, 17 Sep 2026 18:51:56 +0530 Subject: [PATCH 15/18] FIX: Finalize profiler CI readiness Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- tests/test_036_profiler_ci.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/test_036_profiler_ci.py b/tests/test_036_profiler_ci.py index 06c3f6514..472d076d0 100644 --- a/tests/test_036_profiler_ci.py +++ b/tests/test_036_profiler_ci.py @@ -1219,7 +1219,10 @@ def test_ci_reuses_profiling_builds_without_changing_release_defaults(): pipeline, ) assert len(profiler_conditions) == 6 - assert all("eq(variables['Build.Reason'], 'PullRequest')" in condition for condition in profiler_conditions) + assert all( + "eq(variables['Build.Reason'], 'PullRequest')" in condition + for condition in profiler_conditions + ) assert "profilerBuild: '0'" in pipeline # LocalDB still exercises the normal build assert "ddbc_bindings-profiling-SQL2022" in pipeline assert "ddbc_bindings-profiling-SQL2025" in pipeline From f565b1b9ec5a121a5dd9f707c40ce39d59fc6bd9 Mon Sep 17 00:00:00 2001 From: Gaurav Sharma <223556219+Copilot@users.noreply.github.com> Date: Thu, 17 Sep 2026 19:21:19 +0530 Subject: [PATCH 16/18] FIX: Bound profiler CI to current PR runs Prevent stale coverage publication, keep profiling configuration off main, and reject unsupported terminal ADO results. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/scripts/post_profiler_comment.py | 21 +++++++++-- .github/workflows/pr-code-coverage.yml | 4 +++ eng/pipelines/pr-validation-pipeline.yml | 41 ++++++++++++++-------- tests/test_036_profiler_ci.py | 44 ++++++++++++++++++++---- tests/test_pr_code_coverage_workflow.py | 5 +++ 5 files changed, 93 insertions(+), 22 deletions(-) diff --git a/.github/scripts/post_profiler_comment.py b/.github/scripts/post_profiler_comment.py index eba84383a..155e0c9d7 100644 --- a/.github/scripts/post_profiler_comment.py +++ b/.github/scripts/post_profiler_comment.py @@ -21,6 +21,7 @@ HEADER = f"{reporting.MARKER}\n## PR Performance Report\n\n" # Allow a 160-minute ADO job plus queueing; the workflow reserves publication time. WAIT_MINUTES = 220 +COMPLETED_RESULTS = {"succeeded", "partiallySucceeded", "failed"} def allowed_url(url): @@ -226,10 +227,22 @@ def run(number, head, wait_minutes): build = find_build(build_items(api(f"{ADO}/builds?{query}")), number, head) if pr["state"] != "open" or current_head != head: return + if ( + build is not None + and build.get("status") == "completed" + and build.get("result") not in COMPLETED_RESULTS | {"canceled"} + ): + unavailable( + number, + head, + "Performance run completed with an unsupported result.", + pr_base, + ) + return complete = ( build is not None and build.get("status") == "completed" - and build.get("result") != "canceled" + and build.get("result") in COMPLETED_RESULTS ) except (ValueError, KeyError, TypeError, URLError, TimeoutError): failures += 1 @@ -242,7 +255,11 @@ def run(number, head, wait_minutes): if complete: break time.sleep(30) - if build is None or build.get("status") != "completed" or build.get("result") == "canceled": + if ( + build is None + or build.get("status") != "completed" + or build.get("result") not in COMPLETED_RESULTS + ): unavailable( number, head, diff --git a/.github/workflows/pr-code-coverage.yml b/.github/workflows/pr-code-coverage.yml index fb1413c2e..909950571 100644 --- a/.github/workflows/pr-code-coverage.yml +++ b/.github/workflows/pr-code-coverage.yml @@ -8,6 +8,10 @@ on: permissions: contents: read +concurrency: + group: pr-code-coverage-${{ github.event.pull_request.number }} + cancel-in-progress: true + jobs: coverage-report: runs-on: ubuntu-latest diff --git a/eng/pipelines/pr-validation-pipeline.yml b/eng/pipelines/pr-validation-pipeline.yml index c8e861afe..5f5854bf5 100644 --- a/eng/pipelines/pr-validation-pipeline.yml +++ b/eng/pipelines/pr-validation-pipeline.yml @@ -60,20 +60,17 @@ jobs: SQLServer2022: sqlVersion: 'SQL2022' pythonVersion: '3.13' - profilerBuild: '1' - profilerCheck: 'on' - bindingArtifact: 'ddbc_bindings-profiling-SQL2022' + profilerBuild: '0' + bindingArtifact: 'ddbc_bindings' SQLServer2025: sqlVersion: 'SQL2025' pythonVersion: '3.14' - profilerBuild: '1' - profilerCheck: 'on' - bindingArtifact: 'ddbc_bindings-profiling-SQL2025' + profilerBuild: '0' + bindingArtifact: 'ddbc_bindings' LocalDB_Python314: sqlVersion: 'LocalDB' pythonVersion: '3.14' profilerBuild: '0' - profilerCheck: 'off' bindingArtifact: 'ddbc_bindings' steps: @@ -86,6 +83,12 @@ jobs: githubToken: $(GITHUB_TOKEN) displayName: 'Use Python $(pythonVersion)' + - powershell: | + Write-Host "##vso[task.setvariable variable=profilerBuild]1" + Write-Host "##vso[task.setvariable variable=bindingArtifact]ddbc_bindings-profiling-$(sqlVersion)" + displayName: 'Configure PR profiling build' + condition: and(succeeded(), eq(variables['Build.Reason'], 'PullRequest'), ne(variables['sqlVersion'], 'LocalDB')) + - script: | python -m pip install --upgrade pip pip install -r requirements.txt @@ -254,8 +257,9 @@ jobs: env: ENABLE_PROFILING: $(profilerBuild) - - script: python -m eng.profiler_benchmarks.controller --check-build $(profilerCheck) + - script: python -m eng.profiler_benchmarks.controller --check-build on displayName: 'Verify native configuration and recording OFF before pytest' + condition: and(succeeded(), eq(variables['Build.Reason'], 'PullRequest'), ne(variables['sqlVersion'], 'LocalDB')) - template: steps/install-mssql-py-core.yml parameters: @@ -560,7 +564,11 @@ jobs: pip install -r requirements.txt echo "Building pybind bindings (.so) (overlapped with container setup)..." - ( cd mssql_python/pybind && ENABLE_PROFILING=1 ./build.sh ) + PROFILER_BUILD=0 + if [ "$(Build.Reason)" = "PullRequest" ]; then + PROFILER_BUILD=1 + fi + ( cd mssql_python/pybind && ENABLE_PROFILING="$PROFILER_BUILD" ./build.sh ) echo "Waiting for container setup (Colima + SQL Server) to finish..." SQL_STATUS=0 @@ -575,6 +583,7 @@ jobs: - script: python -m eng.profiler_benchmarks.controller --check-build on displayName: 'Verify native configuration and recording OFF before pytest' + condition: and(succeeded(), eq(variables['Build.Reason'], 'PullRequest')) - template: steps/install-mssql-py-core.yml parameters: @@ -819,10 +828,8 @@ jobs: - script: | # Build pybind bindings in the container PROFILER_BUILD=0 - PROFILER_CHECK=off - if [ "$(distroName)" = "Ubuntu" ]; then + if [ "$(Build.Reason)" = "PullRequest" ] && [ "$(distroName)" = "Ubuntu" ]; then PROFILER_BUILD=1 - PROFILER_CHECK=on fi docker exec -e ENABLE_PROFILING="$PROFILER_BUILD" test-container-$(distroName) bash -c " set -e @@ -830,9 +837,15 @@ jobs: cd mssql_python/pybind chmod +x build.sh ./build.sh - cd ../.. - python -m eng.profiler_benchmarks.controller --check-build $PROFILER_CHECK " + if [ "$PROFILER_BUILD" = "1" ]; then + docker exec test-container-$(distroName) bash -c " + set -e + source /opt/venv/bin/activate + cd /workspace + python -m eng.profiler_benchmarks.controller --check-build on + " + fi displayName: 'Build pybind bindings (.so) in $(distroName) container' - template: steps/install-mssql-py-core.yml diff --git a/tests/test_036_profiler_ci.py b/tests/test_036_profiler_ci.py index 472d076d0..ca3052e61 100644 --- a/tests/test_036_profiler_ci.py +++ b/tests/test_036_profiler_ci.py @@ -1081,6 +1081,25 @@ def api(url): assert "buildId=42" in posted[1] +@pytest.mark.parametrize("result", [None, "unknown"]) +def test_publisher_rejects_unsupported_completed_results(monkeypatch, result): + posted = [] + build = ado_build(result=result) + + def api(url): + assert "/builds?" in url, "Unsupported builds must not query artifacts" + return {"value": [build]} + + monkeypatch.setattr(publisher, "api", api) + monkeypatch.setattr(publisher, "github", pr_topology()) + monkeypatch.setattr( + publisher, "publish", lambda number, head, body, base=None: posted.append(body) + ) + publisher.run(123, "c" * 40, 1) + assert len(posted) == 2 + assert "unsupported result" in posted[1] + + @pytest.mark.parametrize("status", [None, "notStarted", "inProgress"]) def test_publisher_deadline_finishes_without_reading_unfinished_build_metadata(monkeypatch, status): posted = [] @@ -1223,16 +1242,29 @@ def test_ci_reuses_profiling_builds_without_changing_release_defaults(): "eq(variables['Build.Reason'], 'PullRequest')" in condition for condition in profiler_conditions ) - assert "profilerBuild: '0'" in pipeline # LocalDB still exercises the normal build - assert "ddbc_bindings-profiling-SQL2022" in pipeline - assert "ddbc_bindings-profiling-SQL2025" in pipeline - assert "ENABLE_PROFILING=1 ./build.sh" in pipeline + assert pipeline.count("profilerBuild: '0'") == 3 + assert pipeline.count("bindingArtifact: 'ddbc_bindings'") == 3 for release in (ROOT / "OneBranchPipelines").rglob("*.yml"): assert "ENABLE_PROFILING" not in release.read_text(encoding="utf-8") windows = pipeline.split("- job: pytestonwindows\n", 1)[1].split("\n- job:", 1)[0] - assert "profilerCheck: 'off'" in windows.split("LocalDB_Python314:", 1)[1].split("steps:", 1)[0] - assert windows.split("steps:", 1)[0].count("profilerCheck: 'on'") == 2 + assert "##vso[task.setvariable variable=profilerBuild]1" in windows + assert ( + "##vso[task.setvariable variable=bindingArtifact]" "ddbc_bindings-profiling-$(sqlVersion)" + ) in windows + assert ( + "condition: and(succeeded(), eq(variables['Build.Reason'], 'PullRequest'), " + "ne(variables['sqlVersion'], 'LocalDB'))" + ) in windows + macos = pipeline.split("- job: PytestOnMacOS\n", 1)[1].split("\n- job:", 1)[0] + assert 'if [ "$(Build.Reason)" = "PullRequest" ]; then' in macos + assert 'ENABLE_PROFILING="$PROFILER_BUILD" ./build.sh' in macos + assert "condition: and(succeeded(), eq(variables['Build.Reason'], 'PullRequest'))" in macos linux = pipeline.split("- job: PytestOnLinux\n", 1)[1].split("\n- job:", 1)[0] + assert ( + 'if [ "$(Build.Reason)" = "PullRequest" ] && [ "$(distroName)" = "Ubuntu" ]; then' in linux + ) + assert 'if [ "$PROFILER_BUILD" = "1" ]; then' in linux + assert "python -m eng.profiler_benchmarks.controller --check-build on" in linux benchmark = linux.split("# Run performance benchmarks on Ubuntu", 1)[1] assert "-e BUILD_BUILDID \\" in benchmark assert "BUILD_BUILDID: $(Build.BuildId)" in benchmark diff --git a/tests/test_pr_code_coverage_workflow.py b/tests/test_pr_code_coverage_workflow.py index f5c3841d6..9f116b6db 100644 --- a/tests/test_pr_code_coverage_workflow.py +++ b/tests/test_pr_code_coverage_workflow.py @@ -293,5 +293,10 @@ def test_missing_build_or_queued_coverage_obeys_wall_clock_budget(tmp_path, step def test_job_budget_leaves_time_for_downloads_and_publishing(): workflow = WORKFLOW.read_text(encoding="utf-8") + assert ( + "concurrency:\n" + " group: pr-code-coverage-${{ github.event.pull_request.number }}\n" + " cancel-in-progress: true\n" + ) in workflow assert " timeout-minutes: 245\n" in workflow assert "PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }}" in workflow From 0a960926b5f73b7947a80248106cf7b36e789061 Mon Sep 17 00:00:00 2001 From: Gaurav Sharma <223556219+Copilot@users.noreply.github.com> Date: Thu, 17 Sep 2026 19:32:17 +0530 Subject: [PATCH 17/18] FIX: Select Windows profiler tasks by reason Use mutually exclusive build and artifact tasks because Azure matrix variables are read-only. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- eng/pipelines/pr-validation-pipeline.yml | 34 +++++++++++++----------- tests/test_036_profiler_ci.py | 14 +++++----- 2 files changed, 27 insertions(+), 21 deletions(-) diff --git a/eng/pipelines/pr-validation-pipeline.yml b/eng/pipelines/pr-validation-pipeline.yml index 5f5854bf5..6e5038d89 100644 --- a/eng/pipelines/pr-validation-pipeline.yml +++ b/eng/pipelines/pr-validation-pipeline.yml @@ -60,18 +60,12 @@ jobs: SQLServer2022: sqlVersion: 'SQL2022' pythonVersion: '3.13' - profilerBuild: '0' - bindingArtifact: 'ddbc_bindings' SQLServer2025: sqlVersion: 'SQL2025' pythonVersion: '3.14' - profilerBuild: '0' - bindingArtifact: 'ddbc_bindings' LocalDB_Python314: sqlVersion: 'LocalDB' pythonVersion: '3.14' - profilerBuild: '0' - bindingArtifact: 'ddbc_bindings' steps: - checkout: self @@ -83,12 +77,6 @@ jobs: githubToken: $(GITHUB_TOKEN) displayName: 'Use Python $(pythonVersion)' - - powershell: | - Write-Host "##vso[task.setvariable variable=profilerBuild]1" - Write-Host "##vso[task.setvariable variable=bindingArtifact]ddbc_bindings-profiling-$(sqlVersion)" - displayName: 'Configure PR profiling build' - condition: and(succeeded(), eq(variables['Build.Reason'], 'PullRequest'), ne(variables['sqlVersion'], 'LocalDB')) - - script: | python -m pip install --upgrade pip pip install -r requirements.txt @@ -253,9 +241,16 @@ jobs: - script: | cd mssql_python\pybind build.bat x64 - displayName: 'Build .pyd file' + displayName: 'Build profiling .pyd file' + condition: and(succeeded(), eq(variables['Build.Reason'], 'PullRequest'), ne(variables['sqlVersion'], 'LocalDB')) env: - ENABLE_PROFILING: $(profilerBuild) + ENABLE_PROFILING: 1 + + - script: | + cd mssql_python\pybind + build.bat x64 + displayName: 'Build .pyd file' + condition: and(succeeded(), or(ne(variables['Build.Reason'], 'PullRequest'), eq(variables['sqlVersion'], 'LocalDB'))) - script: python -m eng.profiler_benchmarks.controller --check-build on displayName: 'Verify native configuration and recording OFF before pytest' @@ -446,9 +441,18 @@ jobs: - task: PublishBuildArtifacts@1 inputs: PathtoPublish: '$(Build.ArtifactStagingDirectory)' - ArtifactName: '$(bindingArtifact)' + ArtifactName: 'ddbc_bindings-profiling-$(sqlVersion)' + publishLocation: 'Container' + displayName: 'Publish profiling build artifacts' + condition: and(succeeded(), eq(variables['Build.Reason'], 'PullRequest'), ne(variables['sqlVersion'], 'LocalDB')) + + - task: PublishBuildArtifacts@1 + inputs: + PathtoPublish: '$(Build.ArtifactStagingDirectory)' + ArtifactName: 'ddbc_bindings' publishLocation: 'Container' displayName: 'Publish build artifacts' + condition: and(succeeded(), or(ne(variables['Build.Reason'], 'PullRequest'), eq(variables['sqlVersion'], 'LocalDB'))) - task: PublishTestResults@2 condition: succeededOrFailed() diff --git a/tests/test_036_profiler_ci.py b/tests/test_036_profiler_ci.py index ca3052e61..d76fbfab9 100644 --- a/tests/test_036_profiler_ci.py +++ b/tests/test_036_profiler_ci.py @@ -1242,19 +1242,21 @@ def test_ci_reuses_profiling_builds_without_changing_release_defaults(): "eq(variables['Build.Reason'], 'PullRequest')" in condition for condition in profiler_conditions ) - assert pipeline.count("profilerBuild: '0'") == 3 - assert pipeline.count("bindingArtifact: 'ddbc_bindings'") == 3 for release in (ROOT / "OneBranchPipelines").rglob("*.yml"): assert "ENABLE_PROFILING" not in release.read_text(encoding="utf-8") windows = pipeline.split("- job: pytestonwindows\n", 1)[1].split("\n- job:", 1)[0] - assert "##vso[task.setvariable variable=profilerBuild]1" in windows - assert ( - "##vso[task.setvariable variable=bindingArtifact]" "ddbc_bindings-profiling-$(sqlVersion)" - ) in windows + assert "##vso[task.setvariable" not in windows + assert "ENABLE_PROFILING: 1" in windows + assert "ArtifactName: 'ddbc_bindings-profiling-$(sqlVersion)'" in windows + assert "ArtifactName: 'ddbc_bindings'" in windows assert ( "condition: and(succeeded(), eq(variables['Build.Reason'], 'PullRequest'), " "ne(variables['sqlVersion'], 'LocalDB'))" ) in windows + assert ( + "condition: and(succeeded(), or(ne(variables['Build.Reason'], 'PullRequest'), " + "eq(variables['sqlVersion'], 'LocalDB')))" + ) in windows macos = pipeline.split("- job: PytestOnMacOS\n", 1)[1].split("\n- job:", 1)[0] assert 'if [ "$(Build.Reason)" = "PullRequest" ]; then' in macos assert 'ENABLE_PROFILING="$PROFILER_BUILD" ./build.sh' in macos From 1b44b272bd94aa29828224f40a85a6fcaa01aa65 Mon Sep 17 00:00:00 2001 From: Gaurav Sharma <223556219+Copilot@users.noreply.github.com> Date: Fri, 18 Sep 2026 10:24:12 +0530 Subject: [PATCH 18/18] FIX: Normalize interrupted profiler responses Route response-body connection failures through the existing retry and terminal fallback handling. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/scripts/post_profiler_comment.py | 2 +- tests/test_036_profiler_ci.py | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/scripts/post_profiler_comment.py b/.github/scripts/post_profiler_comment.py index 155e0c9d7..6b015f0cf 100644 --- a/.github/scripts/post_profiler_comment.py +++ b/.github/scripts/post_profiler_comment.py @@ -66,7 +66,7 @@ def fetch(url, token=None, method=None, data=None, limit=4 * 1024 * 1024): try: with build_opener(SafeRedirect()).open(request, timeout=30) as response: body = response.read(limit + 1) - except HTTPException as error: + except (HTTPException, ConnectionError) as error: raise URLError("Incomplete HTTP response") from error if len(body) > limit: raise ValueError("Response exceeds size limit") diff --git a/tests/test_036_profiler_ci.py b/tests/test_036_profiler_ci.py index d76fbfab9..c5223d106 100644 --- a/tests/test_036_profiler_ci.py +++ b/tests/test_036_profiler_ci.py @@ -472,9 +472,10 @@ def test_untrusted_labels_cannot_inject_links_mentions_or_markdown(): assert not publisher.allowed_url("https://artifacts.visualstudio.com.evil.example/artifact") -def test_incomplete_http_response_is_normalized_for_terminal_fallback(monkeypatch): +@pytest.mark.parametrize("error", [IncompleteRead(b"partial"), ConnectionResetError("reset")]) +def test_incomplete_http_response_is_normalized_for_terminal_fallback(monkeypatch, error): opener = MagicMock() - opener.open.side_effect = IncompleteRead(b"partial") + opener.open.return_value.__enter__.return_value.read.side_effect = error monkeypatch.setattr(publisher, "build_opener", lambda *args: opener) with pytest.raises(URLError, match="Incomplete HTTP response"): publisher.fetch("https://api.github.com/repos/microsoft/mssql-python")