diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 1ca2926..bf6d1f1 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -812,49 +812,19 @@ jobs: "@ | Set-Content build-gates/windows-x86_64-msvc-owned-cuda.txt if ($status -ne 0) { exit $status } - # Hollow-green guard for the compiled backend, with no PE-inspection - # dependency. `--features cuda` is the only thing that makes the build - # link cudart/cublas, and on Windows those are LOAD-TIME imports: an - # exe that imports them cannot start when they are absent from PATH - # (STATUS_DLL_NOT_FOUND, 0xC0000135) even though its --version path - # never calls into them. So the same probe answers both halves: - # off-PATH must FAIL with 0xC0000135 -> the CUDA backend is baked in - # on-PATH must PASS -> the resolved DLL set is right - # A worker that silently compiled CPU-only prints --version both times, - # which this step refuses. It also records, in CI, the sidecar fact the - # release/installer PR must solve: the worker is not self-contained. - # Caveat: the discriminator depends on cudart/cublas staying load-time - # imports; if a future build moves them to delay-load, the off-PATH run - # exits 0 and this step reports "cuda feature is not compiled" — a false - # failure that points at the real cause rather than hiding it. - - name: Assert the packaged worker really carries the CUDA backend + # Derive sidecars from the same verified runtime components used to build. + # No independently maintained DLL manifest; nvcc and driver files are not + # redistribution inputs. The packager carries component licenses/hashes. + - name: Package and verify Windows CUDA sidecars working-directory: synapse shell: pwsh run: | - $ErrorActionPreference = 'Continue' - $exe = (Resolve-Path 'target/release/ck-synapse-worker-cuda.exe').Path - $isolated = Join-Path $env:RUNNER_TEMP 'cuda-isolated' - New-Item -ItemType Directory -Force $isolated | Out-Null - Copy-Item $exe $isolated - # PATH narrowed to the OS directories: no CUDA toolkit, no sidecars. - $env:PATH = "C:\Windows\system32;C:\Windows" - & (Join-Path $isolated 'ck-synapse-worker-cuda.exe') --version 2>&1 | Out-Null - $isolatedCode = $LASTEXITCODE - if ($isolatedCode -eq 0) { - throw "owned-CUDA worker ran without CUDA DLLs on PATH (exit 0): the cuda feature is not compiled into this binary — refusing to record a hollow-green gate" - } - if ($isolatedCode -ne -1073741515) { - throw "owned-CUDA worker failed off-PATH with exit $isolatedCode, expected -1073741515 (0xC0000135 STATUS_DLL_NOT_FOUND)" - } - # CUDA 13's redist layout keeps the runtime DLLs under bin\x64 (the - # nvcc drivers sit in bin); both directories are needed, as verified - # locally. With them restored the worker must start. - $env:PATH = "$env:CUDA_PATH\bin\x64;$env:CUDA_PATH\bin;C:\Windows\system32;C:\Windows" - & $exe --version - if ($LASTEXITCODE -ne 0) { - throw "owned-CUDA worker --version failed with CUDA on PATH (exit $LASTEXITCODE)" - } - "cuda_import_probe=ok off_path=$isolatedCode on_path=0" >> $env:GITHUB_STEP_SUMMARY + ./scripts/package-owned-cuda.ps1 ` + -Worker target/release/ck-synapse-worker-cuda.exe ` + -RuntimeComponents @("$env:RUNNER_TEMP/x-cuda_cudart", "$env:RUNNER_TEMP/x-libcublas") ` + -Output build-gates/owned-cuda-windows-x64.zip + ./scripts/test-owned-cuda-package.ps1 -Archive build-gates/owned-cuda-windows-x64.zip + 'cuda_package_probe=passed; GPU execution requires a GPU runner' >> $env:GITHUB_STEP_SUMMARY - name: Retain Windows owned-CUDA gate evidence if: always() diff --git a/README.md b/README.md index a6c81af..d6de79c 100644 --- a/README.md +++ b/README.md @@ -69,3 +69,58 @@ Example user-tier `~/.config/cortexkit/synapse.jsonc` (project configs must omit Tests can point at a file with `SYNAPSE_CONFIG_PATH`. Only one synapse module per machine (singleton lease); a second instance refuses to start. + +### Owned-CUDA hardware floor + +`ck-synapse-worker-cuda` implements `--probe-floor` (hidden, like the +`--test-abort*` surfaces). It prints one JSON object and exits 0: + +```json +{"driver_api": 13030, "compute_capability": {"major": 8, "minor": 9}} +``` + +The module probes the configured `worker_bin`, the engine's worker-binary +environment override, or the sibling `ck-synapse-worker-cuda`, in that order. +It caches one result per process unless both environment readings parse +successfully. The child wait is bounded to 10 seconds; stdout is capped at +4096 bytes and each pipe completion wait is bounded to another 100 ms. +A missing binary, non-zero exit, timeout, or invalid output produces +`HardwareUnavailable`. Refusal and model evidence carry diagnostic context, +including the last 4096 bytes of stderr when available, under `observed`. +Failed probes do not fabricate numeric hardware readings. + +The environment overrides the probe only as a complete, parseable pair. +Otherwise both readings come from the probe; partial overrides are not merged: + +- `SYNAPSE_CUDA_DRIVER_API` (alias `CUDA_DRIVER_API`) — the raw CUDA **driver + API** integer from `cuDriverGetVersion()`, not the marketing driver version. + For example, a measured driver API value is `13030`. `610.88` is not a valid + API integer; without a parseable alias, it causes fallback to the probe. +- `SYNAPSE_CUDA_COMPUTE_CAPABILITY` (alias `CUDA_COMPUTE_CAPABILITY`) — device + 0's compute capability as `major.minor`, for example `8.9`. +- `SYNAPSE_CUDA_PACKAGING_DRIVER` — optional; the driver string a packaging + build was tested against, carried into the refusal for diagnostics. + +### Windows owned-CUDA package + +The manual Windows CUDA gate packages the worker with runtime DLLs derived +from the same pinned `cuda_cudart` and `libcublas` redistribution archives +used for compilation. `scripts/package-owned-cuda.ps1` places the executable +and DLLs at the ZIP root, includes component licenses, and records source +components and SHA-256 hashes in `manifest.json`. The NVIDIA driver is not +bundled and must already be installed. + +Windows worker builds require CUDA 13. The build script rejects other toolkit +major versions before linking, because this package resolves CUDA 13 DLL names. + +`scripts/test-owned-cuda-package.ps1 -Archive -RequireGpu` extracts a +fresh copy, verifies hashes, and checks no-sidecar `--version`, actionable +missing-library refusal, and a real hardware-floor probe with adjacent DLLs +and CUDA removed from PATH. Without `-RequireGpu`, a runner without NVIDIA +hardware may report an explicit driver/device refusal; this is not a GPU +execution pass. Neither mode loads model weights or certifies embeddings. + +The worker delays its cuBLASLt import and checks runtime library loading +before CUDA calls. No global PATH changes or extra DLL search directories +are needed. Release-matrix publication remains separate from this manual +gate artifact. diff --git a/crates/synapse-engine-cuda/src/cuda.rs b/crates/synapse-engine-cuda/src/cuda.rs index 7ba1034..32a000d 100644 --- a/crates/synapse-engine-cuda/src/cuda.rs +++ b/crates/synapse-engine-cuda/src/cuda.rs @@ -60,8 +60,46 @@ mod enabled { context: NonNull, } + /// Keep successfully loaded libraries resident for all subsequent FFI calls. + /// Resolve explicitly before touching a delay import so failure is a Rust + /// error, not an unhandled Windows loader exception. + fn ensure_libraries_loaded() -> Result<()> { + #[cfg(target_env = "msvc")] + { + use std::sync::LazyLock; + static LOADED: LazyLock> = LazyLock::new(|| { + #[link(name = "kernel32")] + unsafe extern "system" { + fn LoadLibraryW(name: *const u16) -> *mut c_void; + } + for name in [ + "cublasLt64_13.dll", + "cublas64_13.dll", + "cudart64_13.dll", + "nvcuda.dll", + ] { + let wide: Vec = name.encode_utf16().chain(Some(0)).collect(); + // Windows searches the executable directory and installed + // system locations without changing PATH or global policy. + if unsafe { LoadLibraryW(wide.as_ptr()) }.is_null() { + return Err(format!( + "cannot load CUDA library {name}: {}", + std::io::Error::last_os_error() + )); + } + } + Ok(()) + }); + if let Err(message) = &*LOADED { + anyhow::bail!("{message}"); + } + } + Ok(()) + } + impl DeviceBinding { fn capture() -> Result { + ensure_libraries_loaded()?; cuda_driver_check(unsafe { cuInit(0) }, "cuInit")?; let mut runtime_device = 0; cuda_runtime_check( @@ -110,12 +148,59 @@ mod enabled { } pub fn ensure_available() -> Result<()> { + ensure_libraries_loaded()?; cuda_driver_check(unsafe { cuInit(0) }, "cuInit")?; let version = unsafe { synapse_cuda_cublaslt_version() }; ensure!(version > 0, "cuBLASLt did not report a version"); Ok(()) } + /// Read the driver API version and device 0's compute capability. + /// + /// This runs before an owned-CUDA load is admitted, so it deliberately + /// touches nothing else: no context is retained, no model is loaded, and + /// no weights are mapped. The reading is what the floor predicate is + /// applied to, which is why it reports the raw numbers rather than a + /// verdict. + pub fn probe_hardware_floor() -> Result { + ensure_libraries_loaded()?; + cuda_driver_check(unsafe { cuInit(0) }, "cuInit")?; + let mut driver_api = 0; + cuda_driver_check( + unsafe { cuDriverGetVersion(&mut driver_api) }, + "cuDriverGetVersion", + )?; + let mut device = 0; + cuda_driver_check(unsafe { cuDeviceGet(&mut device, 0) }, "cuDeviceGet")?; + let mut major = 0; + cuda_driver_check( + unsafe { + cuDeviceGetAttribute( + &mut major, + CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MAJOR, + device, + ) + }, + "cuDeviceGetAttribute(COMPUTE_CAPABILITY_MAJOR)", + )?; + let mut minor = 0; + cuda_driver_check( + unsafe { + cuDeviceGetAttribute( + &mut minor, + CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MINOR, + device, + ) + }, + "cuDeviceGetAttribute(COMPUTE_CAPABILITY_MINOR)", + )?; + Ok(crate::HardwareFloorProbe { + driver_api: driver_api as u32, + compute_major: major as u32, + compute_minor: minor as u32, + }) + } + pub struct MiniLmContext { binding: DeviceBinding, raw: NonNull, @@ -465,8 +550,16 @@ mod enabled { } } + /// `CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MAJOR` from `cuda.h`. + const CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MAJOR: i32 = 75; + /// `CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MINOR` from `cuda.h`. + const CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MINOR: i32 = 76; + unsafe extern "C" { fn cuInit(flags: u32) -> i32; + fn cuDriverGetVersion(version: *mut i32) -> i32; + fn cuDeviceGet(device: *mut i32, ordinal: i32) -> i32; + fn cuDeviceGetAttribute(value: *mut i32, attrib: i32, device: i32) -> i32; fn cuCtxGetDevice(device: *mut i32) -> i32; fn cuCtxSetCurrent(context: *mut c_void) -> i32; fn cuDevicePrimaryCtxRetain(context: *mut *mut c_void, device: i32) -> i32; @@ -551,6 +644,10 @@ mod enabled { bail!("owned CUDA requires a non-macOS build with cargo feature `cuda`") } + pub fn probe_hardware_floor() -> Result { + bail!("owned CUDA requires a non-macOS build with cargo feature `cuda`") + } + pub struct MiniLmContext; impl MiniLmContext { pub fn new(_graphs: bool) -> Result { @@ -629,4 +726,6 @@ mod enabled { } } -pub use enabled::{ensure_available, MiniLmContext, ModernBertContext, Qwen3Context}; +pub use enabled::{ + ensure_available, probe_hardware_floor, MiniLmContext, ModernBertContext, Qwen3Context, +}; diff --git a/crates/synapse-engine-cuda/src/lib.rs b/crates/synapse-engine-cuda/src/lib.rs index 6de832d..9af36f4 100644 --- a/crates/synapse-engine-cuda/src/lib.rs +++ b/crates/synapse-engine-cuda/src/lib.rs @@ -18,6 +18,8 @@ use synapse_core::{ mod cuda; mod model; +pub use cuda::probe_hardware_floor; + pub const ENGINE_VERSION: &str = "owned-cuda-v1"; /// The source revision from which the CUDA kernels were ported. pub const KERNEL_REVISION: &str = "4d0ded67c30286fe2be37cc7413359ad745dd751"; @@ -183,6 +185,17 @@ pub fn build_identity(family: ModelFamily, dtype: StorageDType) -> CudaBuildIden } } +/// A hardware-floor reading taken before any owned-CUDA worker is spawned. +/// +/// Carried separately from [`device_meets_floor`] so the caller can log or +/// refuse on the observed values rather than on a bare boolean. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct HardwareFloorProbe { + pub driver_api: u32, + pub compute_major: u32, + pub compute_minor: u32, +} + /// Hardware-floor predicate used by capability probes before worker creation. #[must_use] pub fn device_meets_floor(driver_api: u32, compute_major: u32, compute_minor: u32) -> bool { diff --git a/crates/synapse-module/src/lib.rs b/crates/synapse-module/src/lib.rs index 8043393..fb3641c 100644 --- a/crates/synapse-module/src/lib.rs +++ b/crates/synapse-module/src/lib.rs @@ -2499,7 +2499,7 @@ fn normalize_catalog_model( identity_override: None, }) } else if engine_name == CUDA_WORKER_ENGINE { - Some(owned_cuda_catalog_config( + let mut profile = owned_cuda_catalog_config( model.owned_family.as_deref(), model.owned_dtype.as_deref(), model.owned_execution.as_deref(), @@ -2526,7 +2526,10 @@ fn normalize_catalog_model( .get("minimum_cuda_driver_api") .and_then(|value| value.parse().ok()), }, - )?) + )?; + profile.config_locator = model.config_locator.clone(); + profile.extra_locators = model.extra_locators.clone(); + Some(profile) } else { None }; @@ -5280,33 +5283,71 @@ async fn load_catalog_model_task( fn stored_owned_profile( spec: &StoredModelConfig, ) -> Result, WireOperationError> { - if spec.engine != "owned-metal" { - return Ok(None); + if spec.engine == "owned-metal" { + let family = spec + .owned_family + .as_deref() + .ok_or_else(|| artifact_invalid_error("owned-metal catalog entry is missing family"))?; + let dtype = spec + .owned_dtype + .as_deref() + .ok_or_else(|| artifact_invalid_error("owned-metal catalog entry is missing dtype"))?; + return Ok(Some(OwnedCatalogConfig { + family: OwnedFamily::parse(family) + .map_err(|error| artifact_invalid_error(error.to_string()))?, + dtype: OwnedDType::parse(dtype) + .map_err(|error| artifact_invalid_error(error.to_string()))?, + execution: spec + .owned_execution + .clone() + .unwrap_or_else(|| "explicit".to_string()), + attention_units: spec + .owned_attention_units + .unwrap_or(OWNED_DEFAULT_ATTENTION_UNITS), + config_locator: spec.config_locator.clone(), + extra_locators: spec.extra_locators.clone(), + identity_override: None, + })); } - let family = spec - .owned_family - .as_deref() - .ok_or_else(|| artifact_invalid_error("owned-metal catalog entry is missing family"))?; - let dtype = spec - .owned_dtype - .as_deref() - .ok_or_else(|| artifact_invalid_error("owned-metal catalog entry is missing dtype"))?; - Ok(Some(OwnedCatalogConfig { - family: OwnedFamily::parse(family) - .map_err(|error| artifact_invalid_error(error.to_string()))?, - dtype: OwnedDType::parse(dtype) - .map_err(|error| artifact_invalid_error(error.to_string()))?, - execution: spec - .owned_execution - .clone() - .unwrap_or_else(|| "explicit".to_string()), - attention_units: spec - .owned_attention_units - .unwrap_or(OWNED_DEFAULT_ATTENTION_UNITS), - config_locator: spec.config_locator.clone(), - extra_locators: spec.extra_locators.clone(), - identity_override: None, - })) + if spec.engine == CUDA_WORKER_ENGINE { + // A persisted owned-cuda row carries the same owned_*/config/extra + // fields as metal; rebuild through the CUDA builder so the engine + // identity and floors are re-derived from the stored build flags + // instead of a second hand-rolled profile. + let mut profile = owned_cuda_catalog_config( + spec.owned_family.as_deref(), + spec.owned_dtype.as_deref(), + spec.owned_execution.as_deref(), + spec.owned_attention_units, + OwnedCudaDeclaredIdentity { + kernel_revision: spec + .engine_identity + .build_flags + .get("kernel_revision") + .map(String::as_str), + ptx_virtual_arch: spec + .engine_identity + .build_flags + .get("ptx_virtual_arch") + .map(String::as_str), + minimum_device_cc: spec + .engine_identity + .build_flags + .get("minimum_device_cc") + .and_then(|value| value.parse().ok()), + minimum_cuda_driver_api: spec + .engine_identity + .build_flags + .get("minimum_cuda_driver_api") + .and_then(|value| value.parse().ok()), + }, + ) + .map_err(|error| artifact_invalid_error(error.to_string()))?; + profile.config_locator = spec.config_locator.clone(); + profile.extra_locators = spec.extra_locators.clone(); + return Ok(Some(profile)); + } + Ok(None) } fn assemble_owned_model_package( @@ -5323,15 +5364,22 @@ fn assemble_owned_model_package( return Ok(model_path.to_path_buf()); } if !profile.extra_locators.is_empty() { - return Err(artifact_invalid_error( - "sharded owned-metal packages are reserved but not supported in wave 1", - )); + return Err(artifact_invalid_error(format!( + "sharded {} packages are reserved but not supported in wave 1", + spec.engine + ))); } let config_locator = profile.config_locator.as_ref().ok_or_else(|| { - artifact_invalid_error("owned-metal model package is missing files.config") + artifact_invalid_error(format!( + "{} model package is missing files.config", + spec.engine + )) })?; let config = locator_path(config_locator, model_cache)?; let package_key = spec.artifact_digest.trim_start_matches("sha256:"); + // Both owned backends resolve the same `config.json` + `model.safetensors` + // layout from this root, keyed by digest, so metal's existing populated + // packages are reused rather than re-copied for a cuda row. let packages_root = model_cache.root().join("owned-metal-models"); let package_root = packages_root.join(package_key); if package_root.join("config.json").is_file() @@ -5339,21 +5387,20 @@ fn assemble_owned_model_package( { return Ok(package_root); } - fs::create_dir_all(&packages_root).map_err(|error| { - io_to_load_error("create owned-metal package root", &packages_root, &error) - })?; + fs::create_dir_all(&packages_root) + .map_err(|error| io_to_load_error("create owned package root", &packages_root, &error))?; let temporary = packages_root.join(format!(".{package_key}.{}.tmp", std::process::id())); if temporary.exists() { fs::remove_dir_all(&temporary).map_err(|error| { - io_to_load_error("remove stale owned-metal package temp", &temporary, &error) + io_to_load_error("remove stale owned package temp", &temporary, &error) })?; } fs::create_dir_all(&temporary) - .map_err(|error| io_to_load_error("create owned-metal package temp", &temporary, &error))?; + .map_err(|error| io_to_load_error("create owned package temp", &temporary, &error))?; fs::copy(model_path, temporary.join("model.safetensors")) - .map_err(|error| io_to_load_error("copy owned-metal model", model_path, &error))?; + .map_err(|error| io_to_load_error("copy owned model", model_path, &error))?; fs::copy(&config.path, temporary.join("config.json")) - .map_err(|error| io_to_load_error("copy owned-metal config", &config.path, &error))?; + .map_err(|error| io_to_load_error("copy owned config", &config.path, &error))?; match fs::rename(&temporary, &package_root) { Ok(()) => {} Err(_) if package_root.is_dir() => { @@ -5361,7 +5408,7 @@ fn assemble_owned_model_package( } Err(error) => { return Err(io_to_load_error( - "publish owned-metal model package", + "publish owned model package", &package_root, &error, )) @@ -5388,7 +5435,7 @@ fn load_catalog_model_blocking( spec.model_id ))); } - ensure_owned_cuda_floor()?; + ensure_owned_cuda_floor(spec.worker_bin.as_deref())?; } let model_path = locator_path(&spec.model_locator, &model_cache)?; let tokenizer_path = locator_path(&spec.tokenizer_locator, &model_cache)?; @@ -5991,7 +6038,7 @@ fn locator_path( } } -fn owned_cuda_floor_decision() -> CudaFloorDecision { +fn owned_cuda_floor_decision(worker: Option<&Path>) -> CudaFloorDecision { let driver_api = ["SYNAPSE_CUDA_DRIVER_API", "CUDA_DRIVER_API"] .into_iter() .find_map(|name| { @@ -6008,14 +6055,152 @@ fn owned_cuda_floor_decision() -> CudaFloorDecision { }); let packaging_driver = env::var("SYNAPSE_CUDA_PACKAGING_DRIVER").ok(); let (Some(driver_api), Some((major, minor))) = (driver_api, compute) else { - return CudaFloorDecision::Unsupported { - reason: synapse_core::CudaUnsupportedReason::HardwareUnavailable, - observed: None, + // The environment is the override; when it is silent, ask the worker. + // The module deliberately does not link the CUDA driver, so the probe + // has to run in the worker process and report its numbers back. + return match owned_cuda_probe_floor(worker) { + Ok(reading) => evaluate_cuda_floor( + reading.driver_api, + reading.compute_major, + reading.compute_minor, + packaging_driver, + ), + Err(_) => CudaFloorDecision::Unsupported { + reason: synapse_core::CudaUnsupportedReason::HardwareUnavailable, + observed: None, + }, }; }; evaluate_cuda_floor(driver_api, major, minor, packaging_driver) } +/// A hardware reading reported by `ck-synapse-worker-cuda --probe-floor`. +#[derive(Clone, Copy, Debug)] +struct OwnedCudaFloorReading { + driver_api: u32, + compute_major: u32, + compute_minor: u32, +} + +static OWNED_CUDA_PROBE: std::sync::LazyLock< + Mutex>>, +> = std::sync::LazyLock::new(|| Mutex::new(HashMap::new())); + +/// Cache successes and failures per worker; a complete environment override skips it. +fn owned_cuda_probe_floor(worker: Option<&Path>) -> Result { + let worker = worker + .map(Path::to_path_buf) + .or_else(|| env::var_os(worker_binary_env_var(CUDA_WORKER_ENGINE)).map(PathBuf::from)) + .or_else(|| resolve_worker_binary_sibling(CUDA_WORKER_ENGINE)) + .ok_or_else(|| "CUDA floor probe worker binary not found".to_string())?; + let worker = fs::canonicalize(&worker).unwrap_or(worker); + let mut cache = OWNED_CUDA_PROBE + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + // Failed probes stay cached deliberately until module restart, for this worker only. + cache + .entry(worker) + .or_insert_with_key(|worker| { + let mut command = std::process::Command::new(worker); + command.arg("--probe-floor"); + run_owned_cuda_probe(&mut command, Duration::from_secs(10)) + }) + .clone() +} + +fn run_owned_cuda_probe( + command: &mut std::process::Command, + timeout: Duration, +) -> Result { + let deadline = std::time::Instant::now() + timeout; + command + .stdin(std::process::Stdio::null()) + .stdout(std::process::Stdio::piped()) + .stderr(std::process::Stdio::piped()); + let mut child = command + .spawn() + .map_err(|error| format!("spawn CUDA floor probe: {error}"))?; + let stdout = child.stdout.take().expect("piped stdout"); + let mut stderr = child.stderr.take().expect("piped stderr"); + let (stdout_tx, stdout_rx) = std::sync::mpsc::sync_channel(1); + let (stderr_tx, stderr_rx) = std::sync::mpsc::sync_channel(1); + std::thread::spawn(move || { + let mut bytes = Vec::new(); + let result = stdout.take(4097).read_to_end(&mut bytes).map(|_| bytes); + let _ = stdout_tx.send(result); + }); + std::thread::spawn(move || { + let mut tail = Vec::new(); + let mut chunk = [0_u8; 4096]; + while let Ok(count) = stderr.read(&mut chunk) { + if count == 0 { + break; + } + let discard = (tail.len() + count).saturating_sub(4096); + tail.drain(..discard); + tail.extend_from_slice(&chunk[..count]); + } + let _ = stderr_tx.send(String::from_utf8_lossy(&tail).into_owned()); + }); + let status = loop { + match child.try_wait() { + Ok(Some(status)) => break Ok(status), + Ok(None) if std::time::Instant::now() < deadline => { + std::thread::sleep( + Duration::from_millis(20) + .min(deadline.saturating_duration_since(std::time::Instant::now())), + ); + } + other => { + let _ = child.kill(); + let _ = child.wait(); + break Err(match other { + Err(error) => format!("wait for CUDA floor probe: {error}"), + _ => "CUDA floor probe timed out".to_string(), + }); + } + } + }; + // Bound pipe completion too: a descendant may still hold an inherited pipe. + let stderr = stderr_rx + .recv_timeout(deadline.saturating_duration_since(std::time::Instant::now())) + .unwrap_or_default(); + let fail = |reason: String| format!("{reason}; stderr: {stderr}"); + let status = status.map_err(&fail)?; + if !status.success() { + return Err(fail(format!("CUDA floor probe exited {status}"))); + } + let stdout = stdout_rx + .recv_timeout(deadline.saturating_duration_since(std::time::Instant::now())) + .map_err(|error| fail(format!("CUDA floor probe stdout: {error}")))? + .map_err(|error| fail(format!("read CUDA floor probe stdout: {error}")))?; + if stdout.len() > 4096 { + return Err(fail( + "CUDA floor probe stdout exceeds 4096 bytes".to_string(), + )); + } + let parsed: Value = serde_json::from_slice(&stdout) + .map_err(|error| fail(format!("invalid CUDA floor probe JSON: {error}")))?; + let reading = || { + Some(OwnedCudaFloorReading { + driver_api: parsed.get("driver_api")?.as_u64()?.try_into().ok()?, + compute_major: parsed + .get("compute_capability")? + .get("major")? + .as_u64()? + .try_into() + .ok()?, + compute_minor: parsed + .get("compute_capability")? + .get("minor")? + .as_u64()? + .try_into() + .ok()?, + }) + }; + reading().ok_or_else(|| fail("invalid CUDA floor probe hardware fields".to_string())) +} + fn parse_compute_capability(value: &str) -> Option<(u32, u32)> { let mut parts = value.trim().split('.'); let major = parts.next()?.parse().ok()?; @@ -6023,39 +6208,73 @@ fn parse_compute_capability(value: &str) -> Option<(u32, u32)> { parts.next().is_none().then_some((major, minor)) } -fn ensure_owned_cuda_floor() -> Result<(), WireOperationError> { - let decision = owned_cuda_floor_decision(); +fn owned_cuda_floor_observed(decision: &CudaFloorDecision, worker: Option<&Path>) -> Value { + let error = match decision { + CudaFloorDecision::Unsupported { observed: None, .. } => { + owned_cuda_probe_floor(worker).err() + } + _ => None, + }; + floor_observed_with_probe_error(decision, error.as_deref()) +} + +fn floor_observed_with_probe_error(decision: &CudaFloorDecision, error: Option<&str>) -> Value { + match decision { + CudaFloorDecision::Supported { observed } + | CudaFloorDecision::Unsupported { + observed: Some(observed), + .. + } => serde_json::to_value(observed).unwrap_or(Value::Null), + CudaFloorDecision::Unsupported { observed: None, .. } => error + .map(|stderr| json!({ "probe_stderr": stderr })) + .unwrap_or(Value::Null), + } +} + +fn ensure_owned_cuda_floor(worker: Option<&Path>) -> Result<(), WireOperationError> { + let decision = owned_cuda_floor_decision(worker); if decision.is_supported() { return Ok(()); } - let observed = match &decision { - CudaFloorDecision::Unsupported { observed, .. } => observed, - CudaFloorDecision::Supported { .. } => unreachable!(), - }; + let observed = owned_cuda_floor_observed(&decision, worker); Err(WireOperationError::from_stable( StableError::owned_cuda_unsupported(), format!( "owned-cuda floor refused before worker creation: decision={}, observed={}", decision.refusal_code().unwrap_or("owned_cuda_unsupported"), - serde_json::to_string(observed).unwrap_or_else(|_| "null".to_string()), + observed, ), )) } -fn owned_cuda_evidence(state: &ModuleState, model: &EmbeddingModel) -> Option { +async fn owned_cuda_evidence( + state: &ModuleState, + model: &EmbeddingModel, +) -> Result, WireOperationError> { if model.engine_identity.engine != CUDA_WORKER_ENGINE { - return None; + return Ok(None); } - let decision = owned_cuda_floor_decision(); - let observed = match &decision { - CudaFloorDecision::Supported { observed } - | CudaFloorDecision::Unsupported { - observed: Some(observed), - .. - } => serde_json::to_value(observed).ok(), - CudaFloorDecision::Unsupported { observed: None, .. } => None, - }; - Some(json!({ + let worker = state + .runtime + .catalog + .lock() + .ok() + .and_then(|catalog| { + catalog + .get(&model.model_id) + .map(|slot| slot.spec.worker_bin.clone()) + }) + .ok_or_else(|| { + artifact_invalid_error(format!("missing catalog entry for '{}'", model.model_id)) + })?; + let (decision, observed) = tokio::task::spawn_blocking(move || { + let decision = owned_cuda_floor_decision(worker.as_deref()); + let observed = owned_cuda_floor_observed(&decision, worker.as_deref()); + (decision, observed) + }) + .await + .map_err(|error| transient_model_load_error(format!("CUDA evidence task failed: {error}")))?; + Ok(Some(json!({ "engine": CUDA_WORKER_ENGINE, "backend": model.engine_identity.build_flags.get("backend"), "ptx_virtual_arch": model.engine_identity.build_flags.get("ptx_virtual_arch").cloned().unwrap_or_else(|| OWNED_CUDA_PTX_VIRTUAL_ARCH.to_string()), @@ -6071,7 +6290,7 @@ fn owned_cuda_evidence(state: &ModuleState, model: &EmbeddingModel) -> Option) -> WireOperationError { @@ -6142,6 +6361,61 @@ fn model_load_scratch_path(job_id: &str) -> PathBuf { )) } +fn model_load_owned_profile( + engine: &str, + root: &Path, + params: &ModelLoadParams, + config: Option<&ModelCacheMeta>, + extra: &[ModelCacheMeta], +) -> Result, WireOperationError> { + if engine != "owned-metal" && engine != CUDA_WORKER_ENGINE { + return Ok(None); + } + let config = config.ok_or_else(|| { + artifact_invalid_error(format!("{engine} model.load requires files.config")) + })?; + let locator = Some(ModelAssetLocator::CacheDigest { + digest: config.digest.clone(), + }); + let extras = extra + .iter() + .map(|meta| ModelAssetLocator::CacheDigest { + digest: meta.digest.clone(), + }) + .collect(); + let profile = if engine == "owned-metal" { + owned_catalog_config( + root, + params.family.as_deref(), + params.dtype.as_deref(), + params.execution.as_deref(), + params.attention_units, + locator, + extras, + ) + } else { + owned_cuda_catalog_config( + params.family.as_deref(), + params.dtype.as_deref(), + params.execution.as_deref(), + params.attention_units, + OwnedCudaDeclaredIdentity { + kernel_revision: None, + ptx_virtual_arch: None, + minimum_device_cc: None, + minimum_cuda_driver_api: None, + }, + ) + .map(|mut profile| { + profile.config_locator = locator; + profile.extra_locators = extras; + profile + }) + } + .map_err(|error| artifact_invalid_error(error.to_string()))?; + Ok(Some(profile)) +} + async fn execute_model_load_job(state: Arc, job_id: String, params: ModelLoadParams) { if !matches!( state @@ -6274,36 +6548,13 @@ async fn execute_model_load_job(state: Arc, job_id: String, params: config_meta.as_ref(), &extra_metas, ); - let owned = if engine_name == "owned-metal" { - if config_meta.is_none() { - return Err(artifact_invalid_error( - "owned-metal model.load requires files.config", - )); - } - Some( - owned_catalog_config( - temp_dir, - params.family.as_deref(), - params.dtype.as_deref(), - params.execution.as_deref(), - params.attention_units, - config_meta - .as_ref() - .map(|meta| ModelAssetLocator::CacheDigest { - digest: meta.digest.clone(), - }), - extra_metas - .iter() - .map(|meta| ModelAssetLocator::CacheDigest { - digest: meta.digest.clone(), - }) - .collect(), - ) - .map_err(|error| artifact_invalid_error(error.to_string()))?, - ) - } else { - None - }; + let owned = model_load_owned_profile( + &engine_name, + temp_dir, + ¶ms, + config_meta.as_ref(), + &extra_metas, + )?; let spec = build_loaded_catalog_model( ¶ms, &engine_name, @@ -11929,11 +12180,12 @@ async fn execute_embed_probe_for_model( true }; let passed = quality_passed && placement_passed; + let cuda_evidence = owned_cuda_evidence(state, &model).await?; let certification_evidence = json!({ "task": "embed", "metrics": evidence, "ane_placement_share": placement_share, - "cuda": owned_cuda_evidence(state, &model), + "cuda": cuda_evidence, }); let performance = if passed { let cold_load_ms = @@ -11968,7 +12220,7 @@ async fn execute_embed_probe_for_model( "worst_decile": state.runtime.probe.worst_decile_rank_overlap_threshold, "ane_placement_share": state.runtime.probe.ane_placement_threshold, }, - "cuda": owned_cuda_evidence(state, &model), + "cuda": cuda_evidence, "performance": performance, }), certified_vectors: passed.then_some(vectors), @@ -15003,6 +15255,132 @@ fn now_ms() -> u64 { #[cfg(test)] mod tests { use super::*; + #[test] + fn cuda_floor_probe_retains_child_failure_and_rejects_bad_json() { + let mut failed = std::process::Command::new(if cfg!(windows) { "cmd.exe" } else { "sh" }); + if cfg!(windows) { + failed.args(["/D", "/C", "echo driver unavailable 1>&2 & exit /b 7"]); + } else { + failed.args(["-c", "echo 'driver unavailable' >&2; exit 7"]); + } + let error = run_owned_cuda_probe(&mut failed, Duration::from_secs(2)).unwrap_err(); + assert!(error.contains("driver unavailable"), "{error}"); + assert!(error.contains("exited"), "{error}"); + let mut malformed = + std::process::Command::new(if cfg!(windows) { "cmd.exe" } else { "sh" }); + if cfg!(windows) { + malformed.args(["/D", "/C", "echo invalid-json"]); + } else { + malformed.args(["-c", "echo invalid-json"]); + } + let error = run_owned_cuda_probe(&mut malformed, Duration::from_secs(2)).unwrap_err(); + assert!(error.contains("invalid CUDA floor probe JSON"), "{error}"); + } + + #[test] + fn cuda_floor_probe_cache_is_keyed_per_worker_binary() { + let root = std::env::temp_dir().join(format!( + "synapse-probe-key-{}-{}", + std::process::id(), + TEST_STATE_COUNTER.fetch_add(1, Ordering::Relaxed) + )); + fs::create_dir_all(&root).unwrap(); + let good = root.join(if cfg!(windows) { "good.cmd" } else { "good.sh" }); + let bad = root.join(if cfg!(windows) { "bad.cmd" } else { "bad.sh" }); + let header = if cfg!(windows) { + "@echo off\r\n" + } else { + "#!/bin/sh\n" + }; + let json = r#"{"driver_api":13030,"compute_capability":{"major":8,"minor":9}}"#; + let success = if cfg!(windows) { + format!("{header}echo {json}\r\n") + } else { + format!("{header}echo '{json}'\n") + }; + fs::write(&good, &success).unwrap(); + fs::write( + &bad, + format!( + "{header}echo missing-library >&2\n{}\n", + if cfg!(windows) { "exit /b 9" } else { "exit 9" } + ), + ) + .unwrap(); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + for path in [&good, &bad] { + fs::set_permissions(path, fs::Permissions::from_mode(0o700)).unwrap(); + } + } + let failure = owned_cuda_probe_floor(Some(&bad)).unwrap_err(); + assert!(failure.contains("missing-library"), "{failure}"); + let reading = owned_cuda_probe_floor(Some(&good)).unwrap(); + assert_eq!( + ( + reading.driver_api, + reading.compute_major, + reading.compute_minor + ), + (13030, 8, 9) + ); + // Failure caching is deliberate, but must not contaminate another worker. + fs::write(&bad, success).unwrap(); + assert_eq!(owned_cuda_probe_floor(Some(&bad)).unwrap_err(), failure); + assert_eq!( + owned_cuda_probe_floor(Some(&good)).unwrap().driver_api, + 13030 + ); + fs::remove_dir_all(root).unwrap(); + } + + #[test] + #[ignore = "requires a staged CUDA worker and supported GPU; run explicitly"] + fn cuda_floor_probe_matches_real_worker_binary_output() { + let worker = env::var_os("SYNAPSE_TEST_CUDA_WORKER") + .map(PathBuf::from) + .unwrap_or_else(|| { + PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("../../target/release") + .join(if cfg!(windows) { + "ck-synapse-worker-cuda.exe" + } else { + "ck-synapse-worker-cuda" + }) + }); + assert!( + worker.is_file(), + "stage CUDA worker at {}", + worker.display() + ); + let reading = run_owned_cuda_probe( + std::process::Command::new(&worker).arg("--probe-floor"), + Duration::from_secs(10), + ) + .expect("real worker probe"); + assert!(reading.driver_api >= synapse_core::OWNED_CUDA_MINIMUM_DRIVER_API); + assert!( + reading.compute_major as f32 + reading.compute_minor as f32 / 10.0 + >= synapse_core::OWNED_CUDA_MINIMUM_DEVICE_CC + ); + } + + #[test] + fn cuda_floor_failure_evidence_preserves_stderr_without_fabricating_hardware() { + let unavailable = CudaFloorDecision::Unsupported { + reason: synapse_core::CudaUnsupportedReason::HardwareUnavailable, + observed: None, + }; + let observed = + floor_observed_with_probe_error(&unavailable, Some("CUDA driver unavailable")); + assert_eq!(observed["probe_stderr"], "CUDA driver unavailable"); + assert!(observed.get("driver_api").is_none()); + let below_floor = evaluate_cuda_floor(11000, 8, 9, None); + let observed = floor_observed_with_probe_error(&below_floor, Some("stale error")); + assert_eq!(observed["driver_api"], 11000); + assert!(observed.get("probe_stderr").is_none()); + } #[test] fn probe_report_separates_certification_from_serving_admission() { @@ -16967,6 +17345,79 @@ mod tests { .expect("load test tokenizer") } + #[tokio::test(flavor = "current_thread")] + async fn cuda_certification_evidence_uses_model_worker_without_blocking() { + let (storage_dir, descriptor) = test_storage_descriptor("cuda-evidence"); + let store = Arc::new(SynapseStore::open(&descriptor).expect("open test store")); + let profile = test_machine_profile("test-os"); + store + .activate_profile(&profile, 1, 1000) + .expect("activate profile"); + let state = test_module_state(store, profile); + let worker = storage_dir.join(if cfg!(windows) { + "probe.cmd" + } else { + "probe.sh" + }); + let output = r#"{"driver_api":13030,"compute_capability":{"major":8,"minor":9}}"#; + let script = if cfg!(windows) { + format!("@echo off\r\nping -n 3 127.0.0.1 >nul\r\necho {output}\r\n") + } else { + format!("#!/bin/sh\nsleep 2\nprintf '%s\\n' '{output}'\n") + }; + fs::write(&worker, script).expect("write probe"); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + fs::set_permissions(&worker, fs::Permissions::from_mode(0o700)).unwrap(); + } + let mut spec = stuck_model_spec(); + spec.engine_identity.engine = CUDA_WORKER_ENGINE.to_string(); + spec.worker_bin = Some(worker); + state + .runtime + .catalog + .lock() + .unwrap() + .get_mut(&spec.model_id) + .unwrap() + .spec = spec.clone(); + // This test exercises evidence collection only; no engine inference occurs. + let mut engine = OwnedMetalEmbedEngine::new(OwnedFamily::MiniLm, OwnedDType::F16); + let loaded_model = engine.insert_test_model("evidence-test".to_string(), 384, vec![128]); + let model = EmbeddingModel { + model_id: spec.model_id, + task: ModelTask::Embed, + loaded_model, + backend: EmbedBackend::Owned(Arc::new(Mutex::new(engine))), + tokenizer: make_test_tokenizer(&storage_dir, 128), + numeric_profile_id: spec.numeric_profile_id, + fingerprint: spec.fingerprint.clone(), + certification_fingerprint: spec.fingerprint, + engine_identity: spec.engine_identity, + owned_tokenizer_policy: None, + owned_decode_resolution_refusal: None, + }; + let evidence = owned_cuda_evidence(&state, &model); + tokio::pin!(evidence); + // A blocking probe on this single-thread executor would complete before + // the timer can run. The model-specific probe must instead remain pending. + tokio::select! { + biased; + result = &mut evidence => panic!("probe completed before executor progressed: {result:?}"), + _ = tokio::time::sleep(Duration::from_millis(100)) => {} + } + let evidence = evidence + .await + .expect("evidence task") + .expect("CUDA evidence"); + assert_eq!(evidence["floor_state"], "supported"); + assert_eq!(evidence["observed"]["driver_api"], 13030); + // Windows keeps the tokenizer file mapped while the process runs, so + // scratch removal is best effort here; assertions above are the contract. + let _ = fs::remove_dir_all(storage_dir); + } + #[test] fn models_list_rows_carry_all_contract_fields_matching_enforced_sources() { use crate::worker_host::{WorkerEngine, WorkerHostConfig}; @@ -17237,6 +17688,206 @@ mod tests { assert_eq!(unloaded_row["certified"], false); } + /// Regresses the native CUDA model.load path: a source=file owned-cuda load + /// with family=qwen3/dtype=f16/execution=supervised must persist a complete + /// owned profile, survive the restart normalization round-trip, and reach a + /// package directory whose config.json the CUDA engine resolves. + #[test] + fn owned_cuda_model_load_persists_owned_profile_and_assembles_package() { + let scratch = std::env::temp_dir().join(format!( + "synapse-owned-cuda-load-{}-{}", + std::process::id(), + TEST_STATE_COUNTER.fetch_add(1, Ordering::Relaxed) + )); + std::fs::create_dir_all(&scratch).expect("create scratch dir"); + + let model_src = scratch.join("model.safetensors"); + let tokenizer_src = scratch.join("tokenizer.json"); + let config_src = scratch.join("config.json"); + let header = br#"{"embedding.weight":{"dtype":"F16","shape":[2],"data_offsets":[0,4]}}"#; + let mut model_bytes = (header.len() as u64).to_le_bytes().to_vec(); + model_bytes.extend_from_slice(header); + model_bytes.extend_from_slice(&[0; 4]); + std::fs::write(&model_src, &model_bytes).expect("write model"); + std::fs::write(&tokenizer_src, b"{}").expect("write tokenizer"); + std::fs::write(&config_src, b"{}").expect("write config"); + // model.load construction: the same params execute_model_load_job + // parses, and the same owned profile it must now build for owned-cuda. + // source=file joins the top-level path with each file locator. + let params: ModelLoadParams = serde_json::from_value(json!({ + "source": "file", + "path": scratch.to_string_lossy(), + "engine": CUDA_WORKER_ENGINE, + "task": "embed", + "model_id": "native-qwen3-load", + "family": "qwen3", + "dtype": "f16", + "execution": "supervised", + "max_tokens": 512, + "files": { + "model": "model.safetensors", + "tokenizer": "tokenizer.json", + "config": "config.json" + } + })) + .expect("model.load params must parse"); + let engine_name = canonical_engine_name(¶ms.engine); + assert_eq!(engine_name, CUDA_WORKER_ENGINE); + + let sources = resolve_model_load_sources(¶ms).expect("sources must resolve"); + let model_digest = format!("sha256:{}", sha256_hex(&model_bytes)); + let tokenizer_digest = format!( + "sha256:{}", + sha256_hex(&std::fs::read(&tokenizer_src).expect("read tokenizer")) + ); + let config_digest = format!( + "sha256:{}", + sha256_hex(&std::fs::read(&config_src).expect("read config")) + ); + + let meta = |digest: String, + source_url: String, + format: String, + tokenizer_digest: Option| ModelCacheMeta { + digest, + source_url, + format, + sanitized_tokenizer_digest: tokenizer_digest, + validation_state: synapse_core::CacheValidationState::Valid, + pins: Vec::new(), + tombstone: None, + }; + let model_meta = meta( + model_digest.clone(), + local_file_url(&model_src), + default_artifact_format(&engine_name), + Some(tokenizer_digest.clone()), + ); + let tokenizer_meta = meta( + tokenizer_digest.clone(), + local_file_url(&tokenizer_src), + "tokenizer_json".to_string(), + None, + ); + let config_meta = meta( + config_digest.clone(), + local_file_url(&config_src), + "json".to_string(), + None, + ); + + let owned_profile = + model_load_owned_profile(&engine_name, &scratch, ¶ms, Some(&config_meta), &[]) + .expect("load profile must build") + .expect("CUDA load must retain its profile"); + + let package_digest = package_digest(&model_meta, &tokenizer_meta, Some(&config_meta), &[]); + let spec = build_loaded_catalog_model( + ¶ms, + &engine_name, + &sources, + &model_meta, + &tokenizer_meta, + package_digest, + Vec::new(), + Some(owned_profile), + &InlineConfig::default(), + &JobConfig::default(), + ) + .expect("catalog model must build"); + // The persisted row is the defect: pre-fix these were all None because + // no owned profile reached build_stored_model_config. + assert_eq!(spec.engine, CUDA_WORKER_ENGINE); + assert_eq!(spec.owned_family.as_deref(), Some("qwen3-0.6b")); + assert_eq!(spec.owned_dtype.as_deref(), Some("f16")); + assert_eq!(spec.owned_execution.as_deref(), Some("supervised")); + assert_eq!( + spec.config_locator, + Some(ModelAssetLocator::CacheDigest { + digest: config_digest.clone() + }) + ); + assert_eq!(spec.artifact_format, "safetensors-package"); + + // Restart round-trip: the stored row is re-read through + // normalize_catalog_model and must rehydrate the same owned profile. + let restored = normalize_catalog_model( + spec.clone(), + &InlineConfig::default(), + &JobConfig::default(), + ) + .expect("stored row must normalize"); + assert_eq!(restored.owned_family, spec.owned_family); + assert_eq!(restored.owned_dtype, spec.owned_dtype); + assert_eq!(restored.config_locator, spec.config_locator); + assert_eq!(restored.fingerprint, spec.fingerprint); + + let rehydrated = stored_owned_profile(&restored) + .expect("stored owned-cuda profile must rehydrate") + .expect("owned-cuda row must yield a profile"); + assert_eq!(rehydrated.family, OwnedFamily::Qwen3); + assert_eq!(rehydrated.dtype, OwnedDType::F16); + assert_eq!(rehydrated.execution, "supervised"); + assert_eq!( + rehydrated.config_locator, + Some(ModelAssetLocator::CacheDigest { + digest: config_digest.clone() + }) + ); + // Terminal token policy: qwen3 reserves one token, so a 512-token + // budget must become 511 through the owned profile. + assert_eq!( + owned_tokenizer_max_tokens(spec.max_tokens, Some(&rehydrated)), + 511 + ); + + // The package the CUDA worker loads: assemble from the cache so the + // engine sees a directory holding config.json + model.safetensors. + let cache_root = scratch.join("cache"); + std::fs::create_dir_all(cache_root.join("blobs")).expect("create cache blobs"); + let model_cache = ModelCache::new(&cache_root); + std::fs::write(model_cache.blob_path(&model_digest), &model_bytes) + .expect("stage model blob"); + std::fs::write( + model_cache.blob_path(&config_digest), + std::fs::read(&config_src).expect("read config"), + ) + .expect("stage config blob"); + let package = assemble_owned_model_package( + &restored, + model_cache.blob_path(&model_digest).as_path(), + &model_cache, + &rehydrated, + ) + .expect("owned-cuda package must assemble"); + assert!(package.is_dir(), "package must be a directory"); + assert!(package.join("config.json").is_file()); + assert!(package.join("model.safetensors").is_file()); + + // resolve_model_root is the contract the CUDA engine enforces on the + // effective path: a bare file resolves to its parent only when the + // parent holds config.json, which the assembled package guarantees. + let runtime_config = model_runtime_config( + &restored, + &package, + &[], + model_cache.root(), + DEFAULT_MICROLLM_MAX_TOKENS, + None, + ); + assert_eq!( + runtime_config.values["model_path"], + package.to_string_lossy() + ); + assert_eq!(runtime_config.values["backend"], "cuda-ptx"); + assert_eq!( + runtime_config.values["ptx_virtual_arch"], + OWNED_CUDA_PTX_VIRTUAL_ARCH + ); + + let _ = std::fs::remove_dir_all(&scratch); + } + #[test] fn models_list_every_row_carries_max_tokens_and_matches_enforced_source() { let (_storage_dir, descriptor) = test_storage_descriptor("models-list-source-match"); diff --git a/crates/synapse-worker-cuda/build.rs b/crates/synapse-worker-cuda/build.rs new file mode 100644 index 0000000..9749911 --- /dev/null +++ b/crates/synapse-worker-cuda/build.rs @@ -0,0 +1,31 @@ +fn main() { + if std::env::var_os("CARGO_FEATURE_CUDA").is_some() + && std::env::var("CARGO_CFG_TARGET_ENV").as_deref() == Ok("msvc") + { + let cuda_root = std::env::var_os("CUDA_HOME") + .or_else(|| std::env::var_os("CUDA_PATH")) + .map(std::path::PathBuf::from) + .expect("Windows owned-CUDA packaging requires CUDA_HOME or CUDA_PATH"); + let header = cuda_root.join("include/cuda.h"); + println!("cargo:rerun-if-env-changed=CUDA_HOME"); + println!("cargo:rerun-if-env-changed=CUDA_PATH"); + println!("cargo:rerun-if-changed={}", header.display()); + let contents = + std::fs::read_to_string(&header).expect("cannot read CUDA toolkit include/cuda.h"); + let version = contents.lines().find_map(|line| { + let mut fields = line.split_whitespace(); + (fields.next() == Some("#define") && fields.next() == Some("CUDA_VERSION")) + .then(|| fields.next()?.parse::().ok()) + .flatten() + }); + assert!( + version.is_some_and(|version| version / 1000 == 13), + "Windows owned-CUDA packaging requires CUDA 13; CUDA 12 DLL names are incompatible" + ); + // Link arguments on the engine rlib do not propagate to this executable. + println!("cargo:rustc-link-lib=delayimp"); + // The CUDA 13 import libraries link cudart/cublas statically; cuBLASLt + // is the remaining load-time DLL (verified with dumpbin /dependents). + println!("cargo:rustc-link-arg=/DELAYLOAD:cublasLt64_13.dll"); + } +} diff --git a/crates/synapse-worker-cuda/src/main.rs b/crates/synapse-worker-cuda/src/main.rs index ed8aef9..7d136b1 100644 --- a/crates/synapse-worker-cuda/src/main.rs +++ b/crates/synapse-worker-cuda/src/main.rs @@ -68,6 +68,27 @@ fn version_probe() -> bool { } } +/// Print the observed hardware floor as a single JSON object and exit 0. +/// +/// Only the CUDA-enabled build can answer; a build without the feature prints +/// the error to stderr and exits non-zero so the caller records the refusal +/// rather than mistaking silence for a pass. +fn probe_floor() -> Result<()> { + #[cfg(feature = "cuda")] + { + let probe = synapse_engine_cuda::probe_hardware_floor()?; + println!( + "{{\"driver_api\":{},\"compute_capability\":{{\"major\":{},\"minor\":{}}}}}", + probe.driver_api, probe.compute_major, probe.compute_minor + ); + Ok(()) + } + #[cfg(not(feature = "cuda"))] + { + anyhow::bail!("--probe-floor requires a build with cargo feature `cuda`") + } +} + /// Build the identity announced in the worker HELLO handshake. pub fn engine_identity() -> synapse_core::EngineIdentity { owned_cuda_engine_identity("worker", "f16", KERNEL_REVISION) @@ -77,6 +98,9 @@ fn main() -> Result<()> { if version_probe() { return Ok(()); } + if std::env::args().skip(1).any(|arg| arg == "--probe-floor") { + return probe_floor(); + } let args = Args::parse(); let hello = WorkerHello { v: WORKER_PROTOCOL_VERSION, diff --git a/scripts/package-owned-cuda.ps1 b/scripts/package-owned-cuda.ps1 new file mode 100644 index 0000000..f3b5e75 --- /dev/null +++ b/scripts/package-owned-cuda.ps1 @@ -0,0 +1,45 @@ +param( + [Parameter(Mandatory)][string]$Worker, + [Parameter(Mandatory)][string[]]$RuntimeComponents, + [Parameter(Mandatory)][string]$Output +) +$ErrorActionPreference = 'Stop' +$stage = Join-Path ([IO.Path]::GetTempPath()) ([guid]::NewGuid().ToString()) +New-Item -ItemType Directory $stage | Out-Null +try { + Copy-Item $Worker (Join-Path $stage 'ck-synapse-worker-cuda.exe') + $entries = @() + foreach ($component in $RuntimeComponents) { + $root = (Resolve-Path $component).Path + $dlls = @(Get-ChildItem $root -Recurse -File -Filter '*.dll') + if (!$dlls.Count) { throw "No runtime DLLs in $root" } + foreach ($file in $dlls) { + $destination = Join-Path $stage $file.Name + if (Test-Path $destination) { throw "Duplicate runtime filename: $($file.Name)" } + Copy-Item $file.FullName $destination + $entries += [ordered]@{ + file = $file.Name + sha256 = (Get-FileHash $destination -Algorithm SHA256).Hash.ToLowerInvariant() + component = Split-Path $root -Leaf + source = $file.FullName.Substring($root.Length + 1).Replace('\', '/') + } + } + $licenseDir = Join-Path $stage ('licenses/' + (Split-Path $root -Leaf)) + New-Item -ItemType Directory -Force $licenseDir | Out-Null + $licenses = @(Get-ChildItem $root -Recurse -File | Where-Object { $_.Name -match '^(LICENSE|EULA|COPYING)' }) + if (!$licenses.Count) { throw "Missing redistribution license in $root" } + foreach ($license in $licenses) { + $destination = Join-Path $licenseDir $license.FullName.Substring($root.Length + 1) + New-Item -ItemType Directory -Force (Split-Path $destination) | Out-Null + Copy-Item $license.FullName $destination + } + } + [ordered]@{ + schema = 1 + worker_sha256 = (Get-FileHash (Join-Path $stage 'ck-synapse-worker-cuda.exe') -Algorithm SHA256).Hash.ToLowerInvariant() + runtime_files = $entries + driver = 'System-installed NVIDIA driver; not bundled' + } | ConvertTo-Json -Depth 5 | Set-Content (Join-Path $stage 'manifest.json') -Encoding UTF8 + Compress-Archive -Path (Join-Path $stage '*') -DestinationPath $Output -Force + Get-FileHash $Output -Algorithm SHA256 +} finally { Remove-Item $stage -Recurse -Force } diff --git a/scripts/test-owned-cuda-package.ps1 b/scripts/test-owned-cuda-package.ps1 new file mode 100644 index 0000000..c9f37df --- /dev/null +++ b/scripts/test-owned-cuda-package.ps1 @@ -0,0 +1,62 @@ +param( + [Parameter(Mandatory)][string]$Archive, + [switch]$RequireGpu +) +$ErrorActionPreference = 'Stop' +$root = Join-Path ([IO.Path]::GetTempPath()) ([guid]::NewGuid().ToString()) +$savedPath = $env:PATH +New-Item -ItemType Directory $root | Out-Null +function Invoke-Worker([string]$Exe, [string]$Argument) { + $info = New-Object Diagnostics.ProcessStartInfo + $info.FileName = $Exe + $info.Arguments = $Argument + $info.WorkingDirectory = Split-Path $Exe + $info.UseShellExecute = $false + $info.RedirectStandardOutput = $true + $info.RedirectStandardError = $true + $process = [Diagnostics.Process]::Start($info) + $stdout = $process.StandardOutput.ReadToEndAsync() + $stderr = $process.StandardError.ReadToEndAsync() + try { + if (!$process.WaitForExit(15000)) { $process.Kill(); throw 'Worker probe exceeded 15 seconds' } + return @{ Code = $process.ExitCode; Out = $stdout.GetAwaiter().GetResult(); Err = $stderr.GetAwaiter().GetResult() } + } finally { $process.Dispose() } +} +try { + $package = Join-Path $root 'package' + Expand-Archive $Archive $package + $exe = Join-Path $package 'ck-synapse-worker-cuda.exe' + $manifest = Get-Content (Join-Path $package 'manifest.json') -Raw | ConvertFrom-Json + if ((Get-FileHash $exe -Algorithm SHA256).Hash.ToLowerInvariant() -ne $manifest.worker_sha256) { throw 'Worker hash mismatch' } + foreach ($entry in $manifest.runtime_files) { + if ((Get-FileHash (Join-Path $package $entry.file) -Algorithm SHA256).Hash.ToLowerInvariant() -ne $entry.sha256) { throw "Hash mismatch: $($entry.file)" } + } + $empty = Join-Path $root 'no-sidecars' + New-Item -ItemType Directory $empty | Out-Null + Copy-Item $exe $empty + $env:PATH = "$env:SystemRoot\System32;$env:SystemRoot" + $isolatedExe = Join-Path $empty 'ck-synapse-worker-cuda.exe' + $version = Invoke-Worker $isolatedExe '--version' + if ($version.Code -ne 0 -or $version.Out -notmatch '^ck-synapse-worker-cuda ') { throw "No-DLL version failed: $($version.Err)" } + $missing = Invoke-Worker $isolatedExe '--probe-floor' + if ($missing.Code -eq 0 -or $missing.Err -notmatch 'cannot load CUDA library') { throw "Missing-DLL refusal failed: $($missing.Code) $($missing.Err)" } + $present = Invoke-Worker $exe '--probe-floor' + if ($present.Code -eq 0) { + $floor = $present.Out | ConvertFrom-Json + if ($floor.driver_api -le 0 -or $floor.compute_capability.major -le 0) { throw 'Invalid floor JSON' } + if ($RequireGpu -and ($floor.driver_api -lt 12040 -or $floor.compute_capability.major -lt 7 -or ($floor.compute_capability.major -eq 7 -and $floor.compute_capability.minor -lt 5))) { + throw 'GPU below owned-CUDA floor: driver API >= 12040 and compute capability >= 7.5 required' + } + Write-Output "PASS packaged GPU probe: $($present.Out.Trim())" + } elseif ($RequireGpu) { + throw "Packaged GPU probe failed: $($present.Code) $($present.Err)" + } elseif ($present.Err -match 'cannot load CUDA library (cublas|cudart)' -or $present.Err -notmatch '(nvcuda.dll|cuInit|cuDeviceGet)') { + throw "Packaged runtime resolution failed: $($present.Code) $($present.Err)" + } else { + Write-Output "GPU execution not available on this runner: $($present.Err.Trim())" + } + Write-Output 'PASS archive hashes, no-DLL version, missing-DLL refusal, side-by-side runtime resolution' +} finally { + $env:PATH = $savedPath + Remove-Item $root -Recurse -Force +}