From d94899a37b59761762a56df8bbb68fb7fcc83379 Mon Sep 17 00:00:00 2001 From: Gaurav Sharma <223556219+Copilot@users.noreply.github.com> Date: Thu, 10 Sep 2026 16:17:32 +0530 Subject: [PATCH 1/2] CHORE: align native build options and wheel binary validation Set explicit native build defaults and validate extracted wheel payloads before release. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../build-release-package-pipeline.yml | 33 +- .../jobs/consolidate-artifacts-job.yml | 2 +- .../jobs/consolidate-odbc-artifacts-job.yml | 2 +- .../jobs/scan-wheel-binaries-job.yml | 28 ++ .../official-release-pipeline.yml | 9 + .../stages/build-linux-single-stage.yml | 2 +- .../stages/build-macos-single-stage.yml | 2 +- .../steps/scan-wheel-binaries-step.yml | 44 +++ eng/scripts/scan_wheel_binaries.py | 215 ++++++++++ mssql_python/pybind/CMakeLists.txt | 7 + mssql_python/pybind/build.sh | 14 +- .../pybind/cmake/NativeBuildOptions.cmake | 34 ++ mssql_python/pybind/native_build_config.h | 5 + tests/test_037_native_build_checks.py | 371 ++++++++++++++++++ 14 files changed, 746 insertions(+), 22 deletions(-) create mode 100644 OneBranchPipelines/jobs/scan-wheel-binaries-job.yml create mode 100644 OneBranchPipelines/steps/scan-wheel-binaries-step.yml create mode 100644 eng/scripts/scan_wheel_binaries.py create mode 100644 mssql_python/pybind/cmake/NativeBuildOptions.cmake create mode 100644 mssql_python/pybind/native_build_config.h create mode 100644 tests/test_037_native_build_checks.py diff --git a/OneBranchPipelines/build-release-package-pipeline.yml b/OneBranchPipelines/build-release-package-pipeline.yml index b5b719cef..6268f8129 100644 --- a/OneBranchPipelines/build-release-package-pipeline.yml +++ b/OneBranchPipelines/build-release-package-pipeline.yml @@ -64,7 +64,7 @@ parameters: default: 'NonOfficial' # Enable/disable SDL security tasks (BinSkim, CredScan, PoliCheck, etc.) - # Set to false for faster builds during development + # NonOfficial builds may disable these for development; Official builds always run them. - name: runSdlTasks displayName: 'Run SDL Security Tasks' type: boolean @@ -281,7 +281,7 @@ extends: # Checks for known vulnerabilities in compiled artifacts # break:true = fail build if critical issues found armory: - enabled: ${{ parameters.runSdlTasks }} + enabled: ${{ or(eq(variables.effectiveOneBranchType, 'Official'), parameters.runSdlTasks) }} break: true # AsyncSdl - Asynchronous SDL tasks (run after build completion) @@ -297,10 +297,10 @@ extends: # - Control Flow Guard (CFG) # Scans: .pyd (Python), .dll/.exe (Windows), .so (Linux), .dylib (macOS) binskim: - enabled: ${{ parameters.runSdlTasks }} + enabled: ${{ or(eq(variables.effectiveOneBranchType, 'Official'), parameters.runSdlTasks) }} break: true # Fail build on critical BinSkim errors # Recursive scan of all binary file types - analyzeTarget: '$(Build.SourcesDirectory)/**/*.{pyd,dll,exe,so,dylib}' + analyzeTarget: '$(Build.SourcesDirectory)/**/*.{pyd,dll,exe,so,so.*,dylib,rll}' analyzeRecurse: true # SARIF output (Static Analysis Results Interchange Format) logFile: '$(Build.ArtifactStagingDirectory)/BinSkimResults.sarif' @@ -308,7 +308,7 @@ extends: # CodeInspector - Source code security analysis # Checks Python/C++ code for security anti-patterns codeinspector: - enabled: ${{ parameters.runSdlTasks }} + enabled: ${{ or(eq(variables.effectiveOneBranchType, 'Official'), parameters.runSdlTasks) }} logLevel: Error # CodeQL - Semantic code analysis (GitHub Advanced Security) @@ -319,7 +319,7 @@ extends: # - Integer overflows # security-extended suite = comprehensive security queries codeql: - enabled: ${{ parameters.runSdlTasks }} + enabled: ${{ or(eq(variables.effectiveOneBranchType, 'Official'), parameters.runSdlTasks) }} language: 'python,cpp' sourceRoot: '$(REPO_ROOT)' querySuite: security-extended @@ -328,7 +328,7 @@ extends: # Detects hardcoded credentials, API keys, passwords in code # Uses global baseline/suppression files configured above credscan: - enabled: ${{ parameters.runSdlTasks }} + enabled: ${{ or(eq(variables.effectiveOneBranchType, 'Official'), parameters.runSdlTasks) }} # ESLint - JavaScript/TypeScript linter # Disabled: Not applicable to Python/C++ project @@ -339,7 +339,7 @@ extends: # Scans code and documentation for inappropriate terms # Exclusion file contains approved exceptions (technical terms) policheck: - enabled: ${{ parameters.runSdlTasks }} + enabled: ${{ or(eq(variables.effectiveOneBranchType, 'Official'), parameters.runSdlTasks) }} break: true exclusionFile: '$(REPO_ROOT)/.config/PolicheckExclusions.xml' @@ -352,7 +352,7 @@ extends: # Uploads security scan results (SARIF files) to pipeline artifacts # Used for audit trail and compliance reporting publishLogs: - enabled: ${{ parameters.runSdlTasks }} + enabled: ${{ or(eq(variables.effectiveOneBranchType, 'Official'), parameters.runSdlTasks) }} # SBOM - Software Bill of Materials # Generates machine-readable list of all dependencies @@ -360,7 +360,7 @@ extends: # Format: SPDX or CycloneDX # Version automatically detected from wheel metadata (setup.py) sbom: - enabled: ${{ parameters.runSdlTasks }} + enabled: ${{ or(eq(variables.effectiveOneBranchType, 'Official'), parameters.runSdlTasks) }} # This pipeline always builds both packages; name the SBOM after the primary # mssql-python wheel set. packageName: 'mssql-python' @@ -369,7 +369,7 @@ extends: # Uploads scan results to Microsoft's TSA tool for tracking # Only enabled for Official builds (production compliance requirement) tsa: - enabled: ${{ and(eq(variables.effectiveOneBranchType, 'Official'), parameters.runSdlTasks) }} + enabled: ${{ eq(variables.effectiveOneBranchType, 'Official') }} configFile: '$(REPO_ROOT)/.config/tsaoptions.json' # ========================= @@ -486,7 +486,7 @@ extends: # - dist/bindings/Windows/*.{pyd,pdb} (Windows native extensions) # - dist/bindings/macOS/*.so (macOS universal2 binaries) # - dist/bindings/Linux/*.so (Linux native extensions) - # This stage also runs final BinSkim scan on all binaries + # ScanWheelBinaries scans the consolidated wheel payloads on a Windows host. - stage: Consolidate displayName: 'Consolidate All Artifacts' dependsOn: @@ -566,3 +566,12 @@ extends: # mssql-python build stages now install the external mssql-python-odbc wheel # (from ConsolidateOdbc) and run the full pytest suite against it — so the # external-package resolution is already validated end-to-end during the build. + + - ${{ if or(eq(variables.effectiveOneBranchType, 'Official'), parameters.runSdlTasks) }}: + - stage: ScanWheelBinaries + displayName: 'Validate binary scan coverage' + dependsOn: + - Consolidate + - ConsolidateOdbc + jobs: + - template: /OneBranchPipelines/jobs/scan-wheel-binaries-job.yml@self diff --git a/OneBranchPipelines/jobs/consolidate-artifacts-job.yml b/OneBranchPipelines/jobs/consolidate-artifacts-job.yml index 40cda12b0..8ff1809ac 100644 --- a/OneBranchPipelines/jobs/consolidate-artifacts-job.yml +++ b/OneBranchPipelines/jobs/consolidate-artifacts-job.yml @@ -19,7 +19,7 @@ jobs: vmImage: 'ubuntu-latest' variables: - # Disable BinSkim - consolidation job only downloads artifacts, no binary builds + # Wheel payloads are scanned by the downstream ScanWheelBinaries stage. - name: ob_sdl_binskim_enabled value: false - name: ob_outputDirectory diff --git a/OneBranchPipelines/jobs/consolidate-odbc-artifacts-job.yml b/OneBranchPipelines/jobs/consolidate-odbc-artifacts-job.yml index f0d8a0715..e0173b24a 100644 --- a/OneBranchPipelines/jobs/consolidate-odbc-artifacts-job.yml +++ b/OneBranchPipelines/jobs/consolidate-odbc-artifacts-job.yml @@ -22,7 +22,7 @@ jobs: vmImage: 'ubuntu-latest' variables: - # Consolidation only moves files; no binaries to scan. + # Wheel payloads are scanned by the downstream ScanWheelBinaries stage. - name: ob_sdl_binskim_enabled value: false - name: ob_outputDirectory diff --git a/OneBranchPipelines/jobs/scan-wheel-binaries-job.yml b/OneBranchPipelines/jobs/scan-wheel-binaries-job.yml new file mode 100644 index 000000000..c795ea7d8 --- /dev/null +++ b/OneBranchPipelines/jobs/scan-wheel-binaries-job.yml @@ -0,0 +1,28 @@ +jobs: + - job: ScanWheelBinaries + displayName: 'Scan all built wheel payloads' + pool: + type: windows + isCustom: true + name: Python-1ES-pool + demands: + - imageOverride -equals PYTHON-1ES-MMS2022 + variables: + ob_outputDirectory: '$(Build.ArtifactStagingDirectory)' + steps: + - checkout: self + fetchDepth: 1 + - task: DownloadPipelineArtifact@2 + displayName: 'Download both consolidated wheel packages' + inputs: + buildType: 'current' + itemPattern: | + drop_Consolidate_ConsolidateArtifacts/dist/*.whl + drop_Consolidate_ConsolidateArtifacts/symbols/** + drop_ConsolidateOdbc_ConsolidateArtifacts/dist/*.whl + targetPath: '$(Pipeline.Workspace)/wheels-to-scan' + - template: /OneBranchPipelines/steps/scan-wheel-binaries-step.yml@self + parameters: + wheelDirectory: '$(Pipeline.Workspace)/wheels-to-scan' + expectedWheelCount: 41 + symbolDirectory: '$(Pipeline.Workspace)/wheels-to-scan' diff --git a/OneBranchPipelines/official-release-pipeline.yml b/OneBranchPipelines/official-release-pipeline.yml index 3198908bc..3c8102d69 100644 --- a/OneBranchPipelines/official-release-pipeline.yml +++ b/OneBranchPipelines/official-release-pipeline.yml @@ -302,6 +302,15 @@ extends: Write-Host "`nAll wheels verified successfully!" + - template: /OneBranchPipelines/steps/scan-wheel-binaries-step.yml@self + parameters: + wheelDirectory: '$(Build.SourcesDirectory)/artifacts/dist' + symbolDirectory: '$(Build.SourcesDirectory)/artifacts/symbols' + ${{ if eq(parameters.releasePackage, 'mssql-python') }}: + expectedWheelCount: 34 + ${{ else }}: + expectedWheelCount: 7 + # Step 5: Publish Symbols (mssql-python only; mssql-python-odbc has no PDBs) - ${{ if and(eq(parameters.publishSymbols, true), eq(parameters.releasePackage, 'mssql-python')) }}: - template: /OneBranchPipelines/steps/symbol-publishing-step.yml@self diff --git a/OneBranchPipelines/stages/build-linux-single-stage.yml b/OneBranchPipelines/stages/build-linux-single-stage.yml index 30c10a349..a307cbd15 100644 --- a/OneBranchPipelines/stages/build-linux-single-stage.yml +++ b/OneBranchPipelines/stages/build-linux-single-stage.yml @@ -52,7 +52,7 @@ stages: timeoutInMinutes: 120 variables: - # Disable BinSkim for Linux - requires ICU libraries not available in manylinux/musllinux containers + # Wheel payloads are scanned on the Windows ScanWheelBinaries stage. - name: ob_sdl_binskim_enabled value: false # OneBranch output directory for artifacts (wheels, bindings, symbols) diff --git a/OneBranchPipelines/stages/build-macos-single-stage.yml b/OneBranchPipelines/stages/build-macos-single-stage.yml index 20962ce66..90c2ae401 100644 --- a/OneBranchPipelines/stages/build-macos-single-stage.yml +++ b/OneBranchPipelines/stages/build-macos-single-stage.yml @@ -54,7 +54,7 @@ stages: # Build Variables variables: - # Disable BinSkim (Windows-focused binary analyzer) - macOS uses Mach-O format, not PE + # Wheel payloads are scanned on the Windows ScanWheelBinaries stage. - name: ob_sdl_binskim_enabled value: false # OneBranch artifact output directory diff --git a/OneBranchPipelines/steps/scan-wheel-binaries-step.yml b/OneBranchPipelines/steps/scan-wheel-binaries-step.yml new file mode 100644 index 000000000..8d644a15a --- /dev/null +++ b/OneBranchPipelines/steps/scan-wheel-binaries-step.yml @@ -0,0 +1,44 @@ +parameters: + - name: wheelDirectory + type: string + - name: expectedWheelCount + type: number + - name: symbolDirectory + type: string + +steps: + - task: UsePythonVersion@0 + inputs: + versionSpec: '3.13' + architecture: 'x64' + displayName: 'Select Python for binary inventory' + + - task: PowerShell@2 + displayName: 'Scan built wheel payloads' + inputs: + targetType: 'inline' + script: | + $ErrorActionPreference = 'Stop' + $toolRoot = "$(Build.BinariesDirectory)/binskim-4.4.9.11-$(Build.BuildId)-$(System.JobAttempt)" + New-Item -ItemType Directory -Path $toolRoot | Out-Null + $package = "$toolRoot/package.zip" + Invoke-WebRequest -UseBasicParsing -Uri "https://www.nuget.org/api/v2/package/Microsoft.CodeAnalysis.BinSkim/4.4.9.11" -OutFile $package + $expected = "69678989cbc273b5b50fcf98fb0fd978e1e35a3f844acb24254b31f0ce90c447" + if ((Get-FileHash $package -Algorithm SHA256).Hash.ToLowerInvariant() -ne $expected) { + throw "BinSkim package digest mismatch" + } + Expand-Archive -Path $package -DestinationPath "$toolRoot/tool" + $scanner = "$toolRoot/tool/tools/net9.0/win-x64/BinSkim.exe" + & $scanner --version + if ($LASTEXITCODE -ne 0) { throw "BinSkim could not start" } + python "$(Build.SourcesDirectory)/eng/scripts/scan_wheel_binaries.py" ` + "${{ parameters.wheelDirectory }}" "$(Build.BinariesDirectory)/wheel-scan-$(Build.BuildId)-$(System.JobAttempt)" $scanner ` + --expected-wheels ${{ parameters.expectedWheelCount }} --symbols "${{ parameters.symbolDirectory }}" + if ($LASTEXITCODE -ne 0) { throw "Wheel binary validation failed" } + + - task: PublishPipelineArtifact@1 + displayName: 'Publish binary scan reports' + condition: succeededOrFailed() + inputs: + targetPath: '$(Build.BinariesDirectory)/wheel-scan-$(Build.BuildId)-$(System.JobAttempt)/reports' + artifact: 'wheel-binary-scan-$(System.StageName)-$(System.JobName)' diff --git a/eng/scripts/scan_wheel_binaries.py b/eng/scripts/scan_wheel_binaries.py new file mode 100644 index 000000000..e030174a9 --- /dev/null +++ b/eng/scripts/scan_wheel_binaries.py @@ -0,0 +1,215 @@ +#!/usr/bin/env python3 +"""Scan the native payload of built wheels, with per-file SARIF coverage checks.""" + +import argparse +import json +import re +import shutil +import struct +import subprocess +import sys +import zipfile +from pathlib import Path, PurePosixPath +from urllib.parse import unquote, urlsplit + +from assert_macho_arch import macho_arches + +NATIVE_NAME = re.compile(r"\.(pyd|dll|exe|rll|dylib|so(?:\..+)?)$", re.IGNORECASE) +MACHO_MAGICS = { + b"\xce\xfa\xed\xfe", + b"\xcf\xfa\xed\xfe", + b"\xfe\xed\xfa\xce", + b"\xfe\xed\xfa\xcf", + b"\xca\xfe\xba\xbe", + b"\xca\xfe\xba\xbf", +} + + +def binary_format(header): + if header.startswith(b"\x7fELF"): + return "ELF" + if header.startswith(b"MZ"): + return "PE" + if header[:4] in MACHO_MAGICS: + return "Mach-O" + return None + + +def extract_binaries(wheel, destination): + """Keep archive paths for adjacent PDB lookup; never extract outside the root.""" + binaries = {} + with zipfile.ZipFile(wheel) as archive: + for entry in archive.infolist(): + if entry.is_dir(): + continue + name = PurePosixPath(entry.filename) + if ( + name.is_absolute() + or ".." in name.parts + or "\\" in entry.filename + or ":" in entry.filename + ): + raise ValueError(f"Invalid wheel member: {entry.filename}") + with archive.open(entry) as source: + header = source.read(4) + kind = binary_format(header) + if not kind and not NATIVE_NAME.search(name.name) and name.suffix.lower() != ".pdb": + continue + if NATIVE_NAME.search(name.name) and not kind: + raise ValueError(f"Unrecognized native binary: {entry.filename}") + target = destination.joinpath(*name.parts) + target.parent.mkdir(parents=True, exist_ok=True) + with archive.open(entry) as source, target.open("xb") as output: + shutil.copyfileobj(source, output) + if kind: + binaries[target.resolve().as_uri()] = kind + if not binaries: + raise ValueError(f"No native binaries in {wheel.name}") + return binaries + + +def check_macho_stack(path): + """Cover every slice, including MH_BUNDLE, which BinSkim BA5002 skips.""" + data = path.read_bytes() + if not macho_arches(data): + raise ValueError(f"Malformed Mach-O: {path.name}") + magic = struct.unpack_from(">I", data)[0] + offsets = [0] + if magic in (0xCAFEBABE, 0xCAFEBABF): + count = struct.unpack_from(">I", data, 4)[0] + stride, offset_format = (20, ">I") if magic == 0xCAFEBABE else (32, ">Q") + offsets = [ + struct.unpack_from(offset_format, data, 8 + index * stride + 8)[0] + for index in range(count) + ] + for offset in offsets: + endian = ( + "<" if data[offset : offset + 4] in (b"\xce\xfa\xed\xfe", b"\xcf\xfa\xed\xfe") else ">" + ) + flags = struct.unpack_from(endian + "I", data, offset + 24)[0] + if flags & 0x20000: # MH_ALLOW_STACK_EXECUTION + raise ValueError(f"Executable Mach-O stack: {path.name}") + + +def canonical_uri(uri): + # BinSkim emits absolute file URIs, including percent-encoded wheel/member names. + return unquote(uri).casefold() if sys.platform == "win32" else unquote(uri) + + +def verify_report(report, binaries): + runs = json.loads(report.read_text(encoding="utf-8-sig")).get("runs", []) + if not runs: + raise ValueError("BinSkim produced no runs") + seen = set() + evaluated = {} + errors = [] + for run in runs: + invocations = run.get("invocations", []) + if not invocations or any(i.get("executionSuccessful") is not True for i in invocations): + errors.append("BinSkim did not complete successfully") + for invocation in invocations: + for key in ("toolExecutionNotifications", "toolConfigurationNotifications"): + if any(n.get("level") == "error" for n in invocation.get(key, [])): + errors.append("BinSkim reported an analysis error") + for result in run.get("results", []): + if result.get("kind", "fail") == "fail" and result.get("level", "warning") == "error": + errors.append(f"BinSkim {result.get('ruleId', 'unknown')} failed") + for location in result.get("locations", []): + artifact = location.get("physicalLocation", {}).get("artifactLocation", {}) + if "index" in artifact and "uri" not in artifact: + artifact = run["artifacts"][artifact["index"]]["location"] + uri = artifact.get("uri", "") + if uri: + seen.add(canonical_uri(uri)) + if result.get("kind", "fail") in ("pass", "fail"): + evaluated.setdefault(canonical_uri(uri), set()).add(result.get("ruleId")) + missing = {canonical_uri(uri) for uri in binaries} - seen + if missing: + errors.append(f"BinSkim omitted {len(missing)} native binary/binaries") + for uri, kind in binaries.items(): + # These ELF checks apply to shared libraries, not only executables. + if kind == "ELF" and not {"BA3006", "BA3010", "BA3011"} <= evaluated.get( + canonical_uri(uri), set() + ): + errors.append("BinSkim did not evaluate the required ELF checks") + if kind == "PE" and not evaluated.get(canonical_uri(uri)): + errors.append("BinSkim did not evaluate any PE checks") + if errors: + raise ValueError("; ".join(errors)) + + +def scan_wheels(wheel_dir, work_dir, binskim, expected_wheels, symbols=None): + wheels = sorted(wheel_dir.rglob("*.whl")) + if not wheels: + raise ValueError("No wheels to scan") + if len(wheels) != expected_wheels: + raise ValueError(f"Expected {expected_wheels} wheels, found {len(wheels)}") + work_dir.mkdir(parents=True, exist_ok=False) + reports = work_dir / "reports" + reports.mkdir() + symbol_dirs = ( + sorted({str(p.parent.resolve()) for p in symbols.rglob("*.pdb")}) if symbols else [] + ) + inventory = [] + failures = [] + for index, wheel in enumerate(wheels): + destination = work_dir / str(index) + binaries = extract_binaries(wheel, destination) + for uri, kind in binaries.items(): + if kind == "Mach-O": + # URI conversion is only for local files just extracted above. + local_path = unquote(urlsplit(uri).path) + if sys.platform == "win32": + local_path = local_path.lstrip("/") + check_macho_stack(Path(local_path)) + report = reports / f"{index}.sarif" + command = [ + str(binskim), + "analyze", + str(destination.resolve() / "*"), + "--recurse", + "true", + "--output", + str(report.resolve()), + "--kind", + "Fail;Pass;NotApplicable", + "--level", + "Error;Warning;Note", + "--trace", + "TargetsScanned", + "--quiet", + "true", + ] + if symbol_dirs: + command += ["--local-symbol-directories", ";".join(symbol_dirs)] + result = subprocess.run(command, check=False) + try: + verify_report(report, binaries) + if result.returncode: + raise ValueError(f"BinSkim exit code {result.returncode}") + except (ValueError, OSError) as error: + failures.append(f"{wheel.name}: {error}") + inventory.append({"wheel": wheel.name, "binaries": binaries, "report": report.name}) + print(f"{wheel.name}: {len(binaries)} native binaries") + (reports / "inventory.json").write_text(json.dumps(inventory, indent=2), encoding="utf-8") + if failures: + raise ValueError("\n".join(failures)) + print(f"Scanned {len(wheels)} wheels, {sum(len(i['binaries']) for i in inventory)} binaries") + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("wheel_dir", type=Path) + parser.add_argument("work_dir", type=Path) + parser.add_argument("binskim", type=Path) + parser.add_argument("--expected-wheels", type=int, required=True) + parser.add_argument("--symbols", type=Path) + args = parser.parse_args() + try: + scan_wheels(args.wheel_dir, args.work_dir, args.binskim, args.expected_wheels, args.symbols) + except (ValueError, OSError, zipfile.BadZipFile) as error: + parser.exit(1, f"{error}\n") + + +if __name__ == "__main__": + main() diff --git a/mssql_python/pybind/CMakeLists.txt b/mssql_python/pybind/CMakeLists.txt index c75eb23e4..c7e258ff4 100644 --- a/mssql_python/pybind/CMakeLists.txt +++ b/mssql_python/pybind/CMakeLists.txt @@ -1,6 +1,10 @@ cmake_minimum_required(VERSION 3.15) project(ddbc_bindings) +if(NOT CMAKE_CONFIGURATION_TYPES AND NOT CMAKE_BUILD_TYPE) + set(CMAKE_BUILD_TYPE Release CACHE STRING "Build configuration" FORCE) +endif() + # Set C++ standard set(CMAKE_CXX_STANDARD 17) set(CMAKE_CXX_STANDARD_REQUIRED ON) @@ -346,6 +350,9 @@ endif() target_link_libraries(ddbc_bindings PRIVATE simdutf::simdutf) +include(cmake/NativeBuildOptions.cmake) +ddbc_native_build_options(ddbc_bindings) + # Compiler definitions target_compile_definitions(ddbc_bindings PRIVATE HAVE_SNPRINTF diff --git a/mssql_python/pybind/build.sh b/mssql_python/pybind/build.sh index 2afec2069..fa0b47d44 100755 --- a/mssql_python/pybind/build.sh +++ b/mssql_python/pybind/build.sh @@ -101,21 +101,23 @@ echo "[DIAGNOSTIC] Changed to build directory: ${BUILD_DIR}" # Configure CMake (with Clang coverage instrumentation on Linux only - codecov is not supported for macOS) echo "[DIAGNOSTIC] Running CMake configure" +CONFIGURATION="${CMAKE_BUILD_TYPE:-Release}" if [[ "$COVERAGE_MODE" == "true" && "$OS" == "Linux" ]]; then echo "[ACTION] Configuring for Linux with Clang coverage instrumentation" cmake -DARCHITECTURE="$DETECTED_ARCH" \ + -DCMAKE_BUILD_TYPE="$CONFIGURATION" \ -DCMAKE_C_COMPILER=clang \ -DCMAKE_CXX_COMPILER=clang++ \ - -DCMAKE_CXX_FLAGS="-fprofile-instr-generate -fcoverage-mapping" \ - -DCMAKE_C_FLAGS="-fprofile-instr-generate -fcoverage-mapping" \ + -DCMAKE_CXX_FLAGS="${CXXFLAGS:-} -fprofile-instr-generate -fcoverage-mapping" \ + -DCMAKE_C_FLAGS="${CFLAGS:-} -fprofile-instr-generate -fcoverage-mapping" \ "${SOURCE_DIR}" else if [[ "$OS" == "macOS" ]]; then echo "[ACTION] Configuring for macOS (default build)" - cmake -DMACOS_STRING_FIX=ON "${SOURCE_DIR}" + cmake -DMACOS_STRING_FIX=ON -DCMAKE_BUILD_TYPE="$CONFIGURATION" "${SOURCE_DIR}" else echo "[ACTION] Configuring for Linux with architecture: $DETECTED_ARCH" - cmake -DARCHITECTURE="$DETECTED_ARCH" "${SOURCE_DIR}" + cmake -DARCHITECTURE="$DETECTED_ARCH" -DCMAKE_BUILD_TYPE="$CONFIGURATION" "${SOURCE_DIR}" fi fi @@ -126,8 +128,8 @@ if [ $? -ne 0 ]; then fi # Build the project -echo "[DIAGNOSTIC] Running CMake build with: cmake --build . --config Release" -cmake --build . --config Release +echo "[DIAGNOSTIC] Running CMake build with: cmake --build . --config $CONFIGURATION" +cmake --build . --config "$CONFIGURATION" # Check if build succeeded if [ $? -ne 0 ]; then diff --git a/mssql_python/pybind/cmake/NativeBuildOptions.cmake b/mssql_python/pybind/cmake/NativeBuildOptions.cmake new file mode 100644 index 000000000..c07ea9762 --- /dev/null +++ b/mssql_python/pybind/cmake/NativeBuildOptions.cmake @@ -0,0 +1,34 @@ +set(DDBC_NATIVE_BUILD_CONFIG "${CMAKE_CURRENT_LIST_DIR}/../native_build_config.h") + +function(ddbc_native_build_options target) + if(NOT CMAKE_CXX_COMPILER_ID MATCHES "^(GNU|Clang|AppleClang)$" OR MSVC) + return() + endif() + + # Do not downgrade an inherited stack-protector-all setting. + if(NOT CMAKE_CXX_FLAGS MATCHES "(^| )-fstack-protector-all( |$)") + set(stronger_configs "") + foreach(config DEBUG RELEASE RELWITHDEBINFO MINSIZEREL ${CMAKE_CONFIGURATION_TYPES} ${CMAKE_BUILD_TYPE}) + string(TOUPPER "${config}" config_upper) + if(CMAKE_CXX_FLAGS_${config_upper} MATCHES "(^| )-fstack-protector-all( |$)") + list(APPEND stronger_configs "$") + endif() + endforeach() + if(stronger_configs) + list(JOIN stronger_configs "," stronger_configs) + target_compile_options(${target} PRIVATE + "$<$>:-fstack-protector-strong>") + else() + target_compile_options(${target} PRIVATE -fstack-protector-strong) + endif() + endif() + + if(CMAKE_SYSTEM_NAME STREQUAL "Linux") + # Define the libc default before any system header, without replacing a + # caller's definition or enabling it for unoptimized debug builds. + target_compile_options(${target} PRIVATE + -include "${DDBC_NATIVE_BUILD_CONFIG}") + target_link_options(${target} PRIVATE + "LINKER:-z,relro" "LINKER:-z,now" "LINKER:-z,noexecstack") + endif() +endfunction() diff --git a/mssql_python/pybind/native_build_config.h b/mssql_python/pybind/native_build_config.h new file mode 100644 index 000000000..5fc1d9d98 --- /dev/null +++ b/mssql_python/pybind/native_build_config.h @@ -0,0 +1,5 @@ +#pragma once + +#if defined(__linux__) && defined(__OPTIMIZE__) && !defined(_FORTIFY_SOURCE) +#define _FORTIFY_SOURCE 2 +#endif diff --git a/tests/test_037_native_build_checks.py b/tests/test_037_native_build_checks.py new file mode 100644 index 000000000..f63d7e071 --- /dev/null +++ b/tests/test_037_native_build_checks.py @@ -0,0 +1,371 @@ +"""Behavior checks for native build options and built-wheel scan coverage.""" + +import ctypes +import importlib.util +import json +import os +from pathlib import Path +import shutil +import struct +import subprocess +import sys +import zipfile + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +SCRIPT = ROOT / "eng/scripts/scan_wheel_binaries.py" +sys.path.insert(0, str(SCRIPT.parent)) +try: + spec = importlib.util.spec_from_file_location("wheel_scan", SCRIPT) + scan = importlib.util.module_from_spec(spec) + spec.loader.exec_module(scan) +finally: + sys.path.pop(0) + + +def wheel_at(path, members): + with zipfile.ZipFile(path, "w") as archive: + for name, data in members.items(): + archive.writestr(name, data) + return path + + +def test_inventory_includes_versioned_libraries_resources_and_renamed_binaries(tmp_path): + members = { + "package/nested/lib.so.2.1": b"\x7fELF", + "package/locale/messages.rll": b"MZxx", + "package/core.pyd": b"MZxx", + "package/unusual.payload": b"\x7fELF", + "package/not_native.txt": b"text", + } + wheel = wheel_at(tmp_path / "data.whl", members) + binaries = scan.extract_binaries(wheel, tmp_path / "payload") + assert sorted(binaries.values()) == ["ELF", "ELF", "PE", "PE"] + assert len(binaries) == 4 + + +@pytest.mark.parametrize("name", ["../escape.so", "/escape.so", "pkg\\escape.so", "C:escape.so"]) +def test_invalid_wheel_paths_fail(tmp_path, name): + wheel = wheel_at(tmp_path / "data.whl", {name: b"\x7fELF"}) + with pytest.raises(ValueError, match="Invalid wheel member"): + scan.extract_binaries(wheel, tmp_path / "payload") + + +@pytest.mark.parametrize("members", [{"empty.py": b""}, {"bad.so.2": b"invalid"}]) +def test_missing_or_unrecognized_binaries_fail(tmp_path, members): + wheel = wheel_at(tmp_path / "data.whl", members) + with pytest.raises(ValueError, match="No native binaries|Unrecognized native binary"): + scan.extract_binaries(wheel, tmp_path / "payload") + + +def report_for(uri): + return { + "runs": [ + { + "invocations": [{"executionSuccessful": True}], + "results": [ + { + "ruleId": rule, + "kind": "pass", + "level": "none", + "locations": [{"physicalLocation": {"artifactLocation": {"uri": uri}}}], + } + for rule in ("BA3006", "BA3010", "BA3011") + ], + } + ] + } + + +def test_complete_report_passes(tmp_path): + uri = (tmp_path / "lib.so.2").as_uri() + report = tmp_path / "scan.sarif" + report.write_text(json.dumps(report_for(uri))) + scan.verify_report(report, {uri: "ELF"}) + + +def test_resource_pe_can_pass_without_executable_checks(tmp_path): + uri = (tmp_path / "messages.rll").as_uri() + document = report_for(uri) + document["runs"][0]["results"] = [document["runs"][0]["results"][0]] + document["runs"][0]["results"][0]["ruleId"] = "BA2009" + report = tmp_path / "scan.sarif" + report.write_text(json.dumps(document)) + scan.verify_report(report, {uri: "PE"}) + + +def test_pe_with_only_not_applicable_results_fails(tmp_path): + uri = (tmp_path / "bad.pyd").as_uri() + document = report_for(uri) + for result in document["runs"][0]["results"]: + result["kind"] = "notApplicable" + report = tmp_path / "scan.sarif" + report.write_text(json.dumps(document)) + with pytest.raises(ValueError, match="did not evaluate any PE"): + scan.verify_report(report, {uri: "PE"}) + + +@pytest.mark.parametrize( + "change", + [ + "no_runs", + "no_invocations", + "failed_invocation", + "notification", + "missing_file", + "no_results", + "missing_relro", + "missing_now", + "missing_stack", + "not_applicable", + "failed_rule", + ], +) +def test_incomplete_or_failed_reports_fail(tmp_path, change): + uri = (tmp_path / "lib.so.2").as_uri() + document = report_for(uri) + run = document["runs"][0] + if change == "no_runs": + document["runs"] = [] + elif change == "no_invocations": + run["invocations"] = [] + elif change == "failed_invocation": + run["invocations"][0]["executionSuccessful"] = False + elif change == "notification": + run["invocations"][0]["toolExecutionNotifications"] = [{"level": "error"}] + elif change == "missing_file": + uri = (tmp_path / "other.so.2").as_uri() + elif change == "no_results": + run["results"] = [] + elif change.startswith("missing_"): + rule = {"missing_relro": "BA3010", "missing_now": "BA3011", "missing_stack": "BA3006"}[ + change + ] + run["results"] = [r for r in run["results"] if r["ruleId"] != rule] + elif change == "not_applicable": + for result in run["results"]: + result["kind"] = "notApplicable" + elif change == "failed_rule": + run["results"][0].update(kind="fail", level="error") + report = tmp_path / "scan.sarif" + report.write_text(json.dumps(document)) + with pytest.raises(ValueError): + scan.verify_report(report, {uri: "ELF"}) + + +def thin_macho(cpu, flags=0): + # MH_BUNDLE with a valid LC_UUID command. + return ( + struct.pack("IIIII", cpu, 0, offset, len(data), 0) + body += data + offset += len(data) + return struct.pack(">II", 0xCAFEBABE, len(slices)) + table + body + + +def test_universal_bundle_without_pie_passes_stack_check(tmp_path): + path = tmp_path / "bundle.so" + path.write_bytes(fat_macho([(cpu, thin_macho(cpu)) for cpu in (0x01000007, 0x0100000C)])) + scan.check_macho_stack(path) + + +@pytest.mark.parametrize("bad_slice", [0, 1]) +def test_executable_stack_in_either_macho_slice_fails(tmp_path, bad_slice): + path = tmp_path / "bundle.so" + path.write_bytes( + fat_macho( + [ + (cpu, thin_macho(cpu, 0x20000 if index == bad_slice else 0)) + for index, cpu in enumerate((0x01000007, 0x0100000C)) + ] + ) + ) + with pytest.raises(ValueError, match="Executable Mach-O stack"): + scan.check_macho_stack(path) + + +def test_truncated_macho_fails(tmp_path): + path = tmp_path / "bundle.so" + path.write_bytes(thin_macho(0x01000007)[:32]) + with pytest.raises(ValueError, match="Malformed Mach-O"): + scan.check_macho_stack(path) + + +@pytest.mark.parametrize("count", [0, 1]) +def test_missing_wheels_fail_before_scanner_runs(tmp_path, count): + if count: + wheel_at(tmp_path / "data.whl", {"lib.so.2": b"\x7fELF"}) + with pytest.raises(ValueError, match="No wheels|Expected 2 wheels"): + scan.scan_wheels(tmp_path, tmp_path / "work", Path("not-a-scanner"), 2) + + +def test_scanner_failure_is_not_hidden_by_valid_report(tmp_path, monkeypatch): + wheel_at(tmp_path / "data.whl", {"lib.so.2": b"\x7fELF"}) + + def failing_scanner(command, **kwargs): + payload = tmp_path / "work/0/lib.so.2" + report = Path(command[command.index("--output") + 1]) + report.write_text(json.dumps(report_for(payload.as_uri()))) + return subprocess.CompletedProcess(command, 1) + + monkeypatch.setattr(scan.subprocess, "run", failing_scanner) + with pytest.raises(ValueError, match="exit code 1"): + scan.scan_wheels(tmp_path, tmp_path / "work", Path("scanner"), 1) + + +def test_successful_scan_records_inventory_and_supplies_symbols(tmp_path, monkeypatch): + wheel_at(tmp_path / "data.whl", {"nested/lib.so.2": b"\x7fELF"}) + symbols = tmp_path / "symbols" + symbols.mkdir() + (symbols / "binding.pdb").touch() + + def scanner(command, **kwargs): + assert command[2] == str(tmp_path / "work/0/*") + assert command[command.index("--recurse") + 1] == "true" + assert command[command.index("--local-symbol-directories") + 1] == str(symbols) + payload = tmp_path / "work/0/nested/lib.so.2" + report = Path(command[command.index("--output") + 1]) + report.write_text(json.dumps(report_for(payload.as_uri()))) + return subprocess.CompletedProcess(command, 0) + + monkeypatch.setattr(scan.subprocess, "run", scanner) + scan.scan_wheels(tmp_path, tmp_path / "work", Path("scanner"), 1, symbols) + inventory = json.loads((tmp_path / "work/reports/inventory.json").read_text()) + assert len(inventory) == 1 + assert list(inventory[0]["binaries"].values()) == ["ELF"] + + +def test_zero_exit_without_report_fails(tmp_path, monkeypatch): + wheel_at(tmp_path / "data.whl", {"lib.so.2": b"\x7fELF"}) + monkeypatch.setattr( + scan.subprocess, "run", lambda command, **kwargs: subprocess.CompletedProcess(command, 0) + ) + with pytest.raises(ValueError): + scan.scan_wheels(tmp_path, tmp_path / "work", Path("scanner"), 1) + + +@pytest.mark.parametrize( + "configuration,coverage", [(None, False), ("Debug", False), ("Debug", True)] +) +def test_build_entry_point_preserves_requested_configuration(tmp_path, configuration, coverage): + if sys.platform == "win32": + pytest.skip("Unix build entry point") + source = tmp_path / "mssql_python/pybind" + source.mkdir(parents=True) + (source / "probe.cpp").write_text("int probe() { return 0; }\n") + shutil.copy(ROOT / "mssql_python/pybind/build.sh", source / "build.sh") + tools = tmp_path / "tools" + tools.mkdir() + cmake = tools / "cmake" + cmake.write_text( + '#!/bin/sh\nprintf \'%s\\n\' "$@" >> "$BUILD_CALLS"\n' + 'if [ "$1" = "--build" ]; then touch probe.so; fi\n' + ) + uname = tools / "uname" + uname.write_text('#!/bin/sh\nif [ "$1" = "-s" ]; then echo Linux; else echo x86_64; fi\n') + cmake.chmod(0o755) + uname.chmod(0o755) + env = dict(os.environ, PATH=f"{tools}{os.pathsep}{os.environ['PATH']}") + env.pop("CMAKE_BUILD_TYPE", None) + if configuration: + env["CMAKE_BUILD_TYPE"] = configuration + env["CXXFLAGS"] = "-D_FORTIFY_SOURCE=3 -fstack-protector-all" + env["BUILD_CALLS"] = str(tmp_path / "calls") + subprocess.run( + ["bash", "build.sh"] + (["codecov"] if coverage else []), + cwd=source, + env=env, + check=True, + capture_output=True, + text=True, + ) + calls = (tmp_path / "calls").read_text().splitlines() + expected = configuration or "Release" + assert f"-DCMAKE_BUILD_TYPE={expected}" in calls + assert calls[calls.index("--config") + 1] == expected + if coverage: + flags = next(arg for arg in calls if arg.startswith("-DCMAKE_CXX_FLAGS=")) + assert "-D_FORTIFY_SOURCE=3 -fstack-protector-all" in flags + assert "-fprofile-instr-generate -fcoverage-mapping" in flags + + +@pytest.mark.parametrize( + "configuration,flags,fortify,optimized", + [ + ("Release", "", 2, 1), + ("Debug", "", 0, 0), + ("Release", "-D_FORTIFY_SOURCE=3 -fstack-protector-all", 3, 1), + ], +) +def test_native_options_build_real_shared_module( + tmp_path, configuration, flags, fortify, optimized +): + if sys.platform not in ("linux", "darwin") or not shutil.which("cmake"): + pytest.skip("Requires the native GCC/Clang CMake toolchain") + source = tmp_path / "source" + source.mkdir() + module = ROOT / "mssql_python/pybind/cmake/NativeBuildOptions.cmake" + (source / "CMakeLists.txt").write_text( + "cmake_minimum_required(VERSION 3.15)\nproject(probe CXX)\n" + "add_library(probe MODULE probe.cpp)\n" + f'include("{module.as_posix()}")\nddbc_native_build_options(probe)\n' + ) + (source / "probe.cpp").write_text( + "#include \n" + 'extern "C" int fortify_level() {\n' + "#ifdef _FORTIFY_SOURCE\nreturn _FORTIFY_SOURCE;\n#else\nreturn 0;\n#endif\n}\n" + 'extern "C" int optimized() {\n' + "#ifdef __OPTIMIZE__\nreturn 1;\n#else\nreturn 0;\n#endif\n}\n" + 'extern "C" int buffer(const char *s, unsigned n) {\n' + "char b[64]; memcpy(b, s, n); return b[n - 1]; }\n" + ) + build = tmp_path / "build" + subprocess.run( + [ + "cmake", + "-S", + str(source), + "-B", + str(build), + f"-DCMAKE_BUILD_TYPE={configuration}", + f"-DCMAKE_CXX_FLAGS={flags}", + "-DCMAKE_EXPORT_COMPILE_COMMANDS=ON", + ], + check=True, + capture_output=True, + text=True, + ) + subprocess.run(["cmake", "--build", str(build)], check=True, capture_output=True, text=True) + binary = build / "libprobe.so" + library = ctypes.CDLL(str(binary)) + assert library.optimized() == optimized + if sys.platform == "linux": + assert library.fortify_level() == fortify + headers = subprocess.check_output(["readelf", "-W", "-l", str(binary)], text=True) + stack = next(line for line in headers.splitlines() if "GNU_STACK" in line) + assert "RWE" not in stack + assert "GNU_RELRO" in headers + dynamic = subprocess.check_output(["readelf", "-d", str(binary)], text=True) + assert "BIND_NOW" in dynamic + symbols = subprocess.check_output(["readelf", "-Ws", str(binary)], text=True) + assert "__stack_chk_fail" in symbols + else: + scan.check_macho_stack(binary) + commands = json.loads((build / "compile_commands.json").read_text()) + command = commands[0]["command"] + if "-fstack-protector-all" in flags: + assert "-fstack-protector-all" in command + assert "-fstack-protector-strong" not in command + else: + assert "-fstack-protector-strong" in command From 47edc33a17b0b096c54219121d9befaa8a896ba7 Mon Sep 17 00:00:00 2001 From: Gaurav Sharma <223556219+Copilot@users.noreply.github.com> Date: Thu, 10 Sep 2026 16:25:46 +0530 Subject: [PATCH 2/2] CHORE: run native build checks before wheel test isolation Skip checkout-only tests when source helpers are absent from installed-wheel test layouts, and run the checks before isolation in both Linux build flows. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../stages/build-linux-single-stage.yml | 10 ++++++-- tests/test_037_native_build_checks.py | 23 +++++++++++++++++++ 2 files changed, 31 insertions(+), 2 deletions(-) diff --git a/OneBranchPipelines/stages/build-linux-single-stage.yml b/OneBranchPipelines/stages/build-linux-single-stage.yml index a307cbd15..c3937a8be 100644 --- a/OneBranchPipelines/stages/build-linux-single-stage.yml +++ b/OneBranchPipelines/stages/build-linux-single-stage.yml @@ -294,6 +294,10 @@ stages: cd /workspace; python setup.py bdist_wheel; + # Check build tooling from the checkout before wheel-only isolation. + python -m pip install -q pytest; + python -m pytest --noconftest tests/test_037_native_build_checks.py -v; + # Step 5: Install wheel in isolated directory for testing echo "Installing wheel in isolated test environment..."; TEST_DIR="/test_isolated_${PYBIN}"; @@ -316,7 +320,6 @@ stages: # Step 7: Setup test environment echo "Setting up test environment..."; - $PY -m pip install -q pytest; cp -r /workspace/tests $TEST_DIR/ || echo "WARNING: No tests directory"; # Some tests read repo-side helper scripts/workflows (e.g. .github/scripts/prepare_fork_coverage_comment.py). cp -r /workspace/.github $TEST_DIR/ || echo "WARNING: No .github directory"; @@ -364,6 +367,10 @@ stages: cd /workspace; python setup.py bdist_wheel; + # Check build tooling from the checkout before wheel-only isolation. + python -m pip install -q pytest; + python -m pytest --noconftest tests/test_037_native_build_checks.py -v; + # Step 5: Install wheel in isolated directory for testing echo "Installing wheel in isolated test environment..."; TEST_DIR="/test_isolated_${PYBIN}"; @@ -386,7 +393,6 @@ stages: # Step 7: Setup test environment echo "Setting up test environment..."; - $PY -m pip install -q pytest; cp -r /workspace/tests $TEST_DIR/ || echo "WARNING: No tests directory"; # Some tests read repo-side helper scripts/workflows (e.g. .github/scripts/prepare_fork_coverage_comment.py). cp -r /workspace/.github $TEST_DIR/ || echo "WARNING: No .github directory"; diff --git a/tests/test_037_native_build_checks.py b/tests/test_037_native_build_checks.py index f63d7e071..b89cfcfdc 100644 --- a/tests/test_037_native_build_checks.py +++ b/tests/test_037_native_build_checks.py @@ -15,6 +15,12 @@ ROOT = Path(__file__).resolve().parents[1] SCRIPT = ROOT / "eng/scripts/scan_wheel_binaries.py" +if not SCRIPT.is_file(): + pytest.skip( + f"native build checks require checkout sources ({SCRIPT}); skipping", + allow_module_level=True, + ) + sys.path.insert(0, str(SCRIPT.parent)) try: spec = importlib.util.spec_from_file_location("wheel_scan", SCRIPT) @@ -31,6 +37,23 @@ def wheel_at(path, members): return path +def test_source_less_wheel_layout_skips_checkout_checks(tmp_path): + tests = tmp_path / "tests" + tests.mkdir() + shutil.copy(__file__, tests / Path(__file__).name) + shutil.copy(ROOT / "pytest.ini", tmp_path / "pytest.ini") + result = subprocess.run( + [sys.executable, "-m", "pytest", "--noconftest", "-q", str(tests)], + cwd=tmp_path, + capture_output=True, + text=True, + ) + # No tests collected (5), not a collection error (2). The full wheel suite + # continues with its other modules; the checkout-only CI invocation must run tests. + assert result.returncode == 5, result.stdout + result.stderr + assert "1 skipped" in result.stdout + + def test_inventory_includes_versioned_libraries_resources_and_renamed_binaries(tmp_path): members = { "package/nested/lib.so.2.1": b"\x7fELF",