diff --git a/src/daemon/bootstrap.c b/src/daemon/bootstrap.c index 50f189036..0e4cbed61 100644 --- a/src/daemon/bootstrap.c +++ b/src/daemon/bootstrap.c @@ -6,13 +6,17 @@ #include "daemon/ipc.h" #include "daemon/service.h" #include "foundation/compat.h" +#include "foundation/compat_fs.h" +#include "foundation/log.h" #include "foundation/platform.h" +#include #include #include #include #include #include +#include #ifdef _WIN32 #ifndef WIN32_LEAN_AND_MEAN @@ -38,6 +42,14 @@ extern char **environ; enum { BOOTSTRAP_RETRY_NS = 1000000, + /* #1828: how often a waiting client re-reads the start-failure log + * (the wait loop itself ticks every millisecond). */ + BOOTSTRAP_START_FAILURE_CHECK_MS = 50, + /* A record this many seconds OLDER than the client's own spawn instant is + * still accepted: a sibling client's attempt that just failed is the same + * evidence, and second-granularity clocks need the slack anyway. */ + BOOTSTRAP_START_FAILURE_SKEW_S = 2, + BOOTSTRAP_START_FAILURE_LOG_CAP = 65536, BOOTSTRAP_COORDINATION_CLEANUP_MS = 500, BOOTSTRAP_PATH_CAP = 4096, }; @@ -292,6 +304,270 @@ static _Noreturn void bootstrap_cleanup_fail_stop(const char *component) { #endif } +static const char BOOTSTRAP_START_FAILURE_LOG_NAME[] = "cbm-daemon.start-failures.log"; + +bool cbm_daemon_bootstrap_log_directory(char *out, size_t capacity) { + if (!out || capacity == 0) { + return false; + } + out[0] = '\0'; + const char *cache = cbm_resolve_cache_dir(); + if (!cache || !cache[0]) { + return false; + } + int written = snprintf(out, capacity, "%s/logs", cache); + if (written <= 0 || (size_t)written >= capacity) { + out[0] = '\0'; + return false; + } + return true; +} + +/* One record field: the line format is tab-separated, so a tab or newline in + * a value (a path, in theory) becomes '?' rather than a parser ambiguity. */ +static void bootstrap_record_field(char *out, size_t capacity, const char *value) { + size_t used = 0; + for (const char *cursor = value ? value : ""; *cursor && used + 1 < capacity; cursor++) { + unsigned char ch = (unsigned char)*cursor; + out[used++] = ch == '\t' || ch == '\n' || ch == '\r' ? '?' : (char)ch; + } + out[used] = '\0'; +} + +static uint64_t bootstrap_current_pid(void) { +#ifdef _WIN32 + return (uint64_t)GetCurrentProcessId(); +#else + return (uint64_t)getpid(); +#endif +} + +bool cbm_daemon_bootstrap_start_failure_record(const char *log_directory, + const cbm_daemon_ipc_endpoint_t *endpoint, + const char *component) { + if (!log_directory || !log_directory[0] || !endpoint || !component || !component[0]) { + return false; + } + const char *runtime_dir = cbm_daemon_ipc_endpoint_runtime_dir(endpoint); + if (!runtime_dir || !runtime_dir[0]) { + return false; + } + cbm_daemon_ipc_listen_failure_t detail; + if (!cbm_daemon_ipc_listen_failure_detail(&detail)) { + memset(&detail, 0, sizeof(detail)); + } + char safe_runtime_dir[BOOTSTRAP_PATH_CAP]; + char safe_component[CBM_DAEMON_BOOTSTRAP_COMPONENT_CAP]; + char safe_stage[CBM_DAEMON_IPC_LISTEN_FAILURE_STAGE_CAP]; + char safe_path[CBM_DAEMON_IPC_LISTEN_FAILURE_PATH_CAP]; + bootstrap_record_field(safe_runtime_dir, sizeof(safe_runtime_dir), runtime_dir); + bootstrap_record_field(safe_component, sizeof(safe_component), component); + bootstrap_record_field(safe_stage, sizeof(safe_stage), detail.stage); + bootstrap_record_field(safe_path, sizeof(safe_path), detail.path); + FILE *stream = cbm_daemon_ipc_private_log_open(log_directory, BOOTSTRAP_START_FAILURE_LOG_NAME, + BOOTSTRAP_START_FAILURE_LOG_CAP); + if (!stream) { + return false; + } + int written = + fprintf(stream, "v1\t%llu\t%llu\t%s\t%s\t%s\t%d\t%s\n", (unsigned long long)time(NULL), + (unsigned long long)bootstrap_current_pid(), safe_runtime_dir, safe_component, + safe_stage, detail.errno_value, safe_path); + bool ok = written > 0 && fflush(stream) == 0; + if (fclose(stream) != 0) { + ok = false; + } + return ok; +} + +/* Parse one "v1\tts\tpid\truntime_dir\tcomponent\tstage\terrno\tpath" line + * (line is NUL-terminated, newline already stripped, and is modified). */ +static bool bootstrap_start_failure_parse(char *line, char **runtime_dir_out, + cbm_daemon_bootstrap_start_failure_t *out) { + char *fields[8]; + size_t count = 0; + char *cursor = line; + while (count < 8) { + fields[count++] = cursor; + char *tab = strchr(cursor, '\t'); + if (!tab) { + break; + } + *tab = '\0'; + cursor = tab + 1; + } + if (count != 8 || strcmp(fields[0], "v1") != 0) { + return false; + } + memset(out, 0, sizeof(*out)); + out->recorded_at_s = strtoull(fields[1], NULL, 10); + out->pid = strtoull(fields[2], NULL, 10); + *runtime_dir_out = fields[3]; + (void)snprintf(out->component, sizeof(out->component), "%s", fields[4]); + (void)snprintf(out->stage, sizeof(out->stage), "%s", fields[5]); + out->errno_value = (int)strtol(fields[6], NULL, 10); + (void)snprintf(out->path, sizeof(out->path), "%s", fields[7]); + return true; +} + +static FILE *bootstrap_start_failure_open(const char *log_directory, int *status_out) { + char path[BOOTSTRAP_PATH_CAP]; + int written = + snprintf(path, sizeof(path), "%s/%s", log_directory, BOOTSTRAP_START_FAILURE_LOG_NAME); + if (written <= 0 || written >= (int)sizeof(path)) { + *status_out = -1; + return NULL; + } +#ifndef _WIN32 + struct stat status; + if (lstat(path, &status) != 0) { + *status_out = errno == ENOENT ? 0 : -1; + return NULL; + } + if (!S_ISREG(status.st_mode) || status.st_uid != geteuid() || status.st_nlink != 1) { + *status_out = -1; + return NULL; + } +#endif + /* The directory is the trust boundary (owner-only, no symlinked + * ancestry); the same validation guards cbm-daemon.log itself. */ + if (!cbm_daemon_ipc_private_directory_secure(log_directory)) { + *status_out = -1; + return NULL; + } + FILE *file = cbm_fopen(path, "rb"); + if (!file) { + *status_out = errno == ENOENT ? 0 : -1; + return NULL; + } + *status_out = 1; + return file; +} + +int cbm_daemon_bootstrap_start_failure_read(const char *log_directory, + const cbm_daemon_ipc_endpoint_t *endpoint, + uint64_t not_before_s, + cbm_daemon_bootstrap_start_failure_t *out_failure) { + if (!out_failure) { + return -1; + } + memset(out_failure, 0, sizeof(*out_failure)); + if (!log_directory || !log_directory[0] || !endpoint) { + return -1; + } + const char *runtime_dir = cbm_daemon_ipc_endpoint_runtime_dir(endpoint); + if (!runtime_dir || !runtime_dir[0]) { + return -1; + } + int status = 0; + FILE *file = bootstrap_start_failure_open(log_directory, &status); + if (!file) { + return status; + } + char *buffer = malloc(BOOTSTRAP_START_FAILURE_LOG_CAP + 1U); + if (!buffer) { + (void)fclose(file); + return -1; + } + size_t used = fread(buffer, 1, BOOTSTRAP_START_FAILURE_LOG_CAP, file); + bool read_ok = !ferror(file); + (void)fclose(file); + buffer[used] = '\0'; + int found = 0; + if (read_ok) { + char *line = buffer; + while (line && *line) { + char *end = strchr(line, '\n'); + if (!end) { + break; /* a partial trailing line is a record still being written */ + } + *end = '\0'; + char *record_runtime_dir = NULL; + cbm_daemon_bootstrap_start_failure_t candidate; + if (bootstrap_start_failure_parse(line, &record_runtime_dir, &candidate) && + strcmp(record_runtime_dir, runtime_dir) == 0 && + candidate.recorded_at_s >= not_before_s) { + *out_failure = candidate; /* later lines are newer */ + found = 1; + } + line = end + 1; + } + } else { + found = -1; + } + free(buffer); + return found; +} + +void cbm_daemon_bootstrap_start_failure_format(const cbm_daemon_bootstrap_start_failure_t *failure, + const char *log_directory, char *out, + size_t capacity) { + if (!out || capacity == 0) { + return; + } + if (!failure) { + out[0] = '\0'; + return; + } + const char *what = failure->stage[0] ? failure->stage + : failure->component[0] ? failure->component + : "startup"; + char where[BOOTSTRAP_PATH_CAP]; + if (log_directory && log_directory[0]) { + (void)snprintf(where, sizeof(where), "see %s/cbm-daemon.log", log_directory); + } else { + (void)snprintf(where, sizeof(where), "see the daemon log"); + } + if (failure->errno_value != 0 && failure->path[0]) { + (void)snprintf(out, capacity, + "CBM daemon failed to start: %s failed with %s (%s) at %s; %s", what, + cbm_errno_name(failure->errno_value), strerror(failure->errno_value), + failure->path, where); + } else if (failure->errno_value != 0) { + (void)snprintf(out, capacity, "CBM daemon failed to start: %s failed with %s (%s); %s", + what, cbm_errno_name(failure->errno_value), strerror(failure->errno_value), + where); + } else if (failure->path[0]) { + (void)snprintf(out, capacity, "CBM daemon failed to start: %s failed at %s; %s", what, + failure->path, where); + } else { + (void)snprintf(out, capacity, "CBM daemon failed to start: %s failed; %s", what, where); + } +} + +/* After this attempt spawned a daemon: did that daemon (or a sibling attempt + * moments earlier) record a start failure? A found record ends the wait with + * its cause -- respawning a deterministically failing daemon until the + * deadline only produced "active or starting" (#1828). */ +static bool bootstrap_start_failure_detected(const cbm_daemon_bootstrap_config_t *config, + const cbm_daemon_bootstrap_ops_t *ops, + uint64_t spawn_wall_s, bool force, + uint64_t *next_check_ms, + cbm_daemon_bootstrap_result_t *result_out) { + if (spawn_wall_s == 0 || !ops->start_failure_probe) { + return false; + } + uint64_t now_ms = cbm_now_ms(); + if (!force && now_ms < *next_check_ms) { + return false; + } + *next_check_ms = now_ms + BOOTSTRAP_START_FAILURE_CHECK_MS; + uint64_t not_before_s = spawn_wall_s > BOOTSTRAP_START_FAILURE_SKEW_S + ? spawn_wall_s - BOOTSTRAP_START_FAILURE_SKEW_S + : 0; + cbm_daemon_bootstrap_start_failure_t failure; + if (ops->start_failure_probe(ops->context, config->endpoint, not_before_s, &failure) != 1) { + return false; + } + char logs[BOOTSTRAP_PATH_CAP]; + if (!cbm_daemon_bootstrap_log_directory(logs, sizeof(logs))) { + logs[0] = '\0'; + } + cbm_daemon_bootstrap_start_failure_format(&failure, logs, result_out->message, + sizeof(result_out->message)); + return true; +} + static void bootstrap_pause(uint64_t deadline) { uint64_t now = cbm_now_ms(); if (now >= deadline) { @@ -458,7 +734,15 @@ cbm_daemon_bootstrap_status_t cbm_daemon_bootstrap_execute_with_ops( bool lock_acquired = false; bool generation_observed = probe == CBM_DAEMON_BOOTSTRAP_PROBE_RESERVED || probe == CBM_DAEMON_BOOTSTRAP_PROBE_TERMINAL; + uint64_t spawn_wall_s = 0; + uint64_t next_failure_check_ms = 0; + bool start_failed = false; while (cbm_now_ms() < deadline) { + if (bootstrap_start_failure_detected(config, ops, spawn_wall_s, false, + &next_failure_check_ms, result_out)) { + start_failed = true; + break; + } if (!bootstrap_probe_is_waitable(probe)) { break; } @@ -508,11 +792,20 @@ cbm_daemon_bootstrap_status_t cbm_daemon_bootstrap_execute_with_ops( break; } + /* Never relaunch a daemon whose predecessor already recorded why it + * could not start; the record is written before that daemon lets go + * of its lifetime reservation, so this unconditional read sees it. */ + if (bootstrap_start_failure_detected(config, ops, spawn_wall_s, true, + &next_failure_check_ms, result_out)) { + start_failed = true; + break; + } cbm_daemon_bootstrap_launch_spec_t spec; bool spec_ready = config->spawn_permanent ? cbm_daemon_bootstrap_launch_spec_init_permanent(config->executable_path, &spec) : cbm_daemon_bootstrap_launch_spec_init(config->executable_path, &spec); + spawn_wall_s = (uint64_t)time(NULL); if (!spec_ready || !ops->startup_lock_prepare_handoff(ops->context, startup_lock) || !ops->spawn_daemon(ops->context, &spec)) { probe = CBM_DAEMON_BOOTSTRAP_PROBE_ERROR; @@ -537,7 +830,15 @@ cbm_daemon_bootstrap_status_t cbm_daemon_bootstrap_execute_with_ops( if (!bootstrap_probe_is_waitable(probe)) { break; } + if (bootstrap_start_failure_detected(config, ops, spawn_wall_s, false, + &next_failure_check_ms, result_out)) { + start_failed = true; + break; + } } while (cbm_now_ms() < deadline); + if (start_failed) { + break; + } if (!lock_acquired) { continue; } @@ -555,7 +856,9 @@ cbm_daemon_bootstrap_status_t cbm_daemon_bootstrap_execute_with_ops( } result_out->status = CBM_DAEMON_BOOTSTRAP_FAILED; - if (muted_holder_pid != 0) { + if (start_failed) { + /* result_out->message already names the recorded cause. */ + } else if (muted_holder_pid != 0) { /* The one diagnostic the 2026-08-29 zombie recovery had to assemble by * hand from process, pipe, and log correlation: name the pid that * holds the endpoint without answering, and say what to do with it. */ @@ -773,6 +1076,17 @@ static bool bootstrap_production_unlock(void *context, cbm_daemon_bootstrap_lock return released; } +static int bootstrap_production_start_failure_probe( + void *context, const cbm_daemon_ipc_endpoint_t *endpoint, uint64_t not_before_s, + cbm_daemon_bootstrap_start_failure_t *out_failure) { + (void)context; + char logs[BOOTSTRAP_PATH_CAP]; + if (!cbm_daemon_bootstrap_log_directory(logs, sizeof(logs))) { + return -1; + } + return cbm_daemon_bootstrap_start_failure_read(logs, endpoint, not_before_s, out_failure); +} + static bool bootstrap_production_handoff(void *context, cbm_daemon_bootstrap_lock_t lock) { (void)context; return cbm_daemon_ipc_startup_lock_prepare_handoff((cbm_daemon_ipc_startup_lock_t *)lock); @@ -1041,10 +1355,26 @@ static void bootstrap_production_diagnostic(void *context, const char *message) (void)fflush(stderr); } +#ifdef CBM_ENABLE_TEST_SEAMS +static cbm_daemon_bootstrap_spawn_fn g_bootstrap_spawn_override_for_test; +static void *g_bootstrap_spawn_override_context_for_test; + +void cbm_daemon_bootstrap_spawn_override_set_for_test(cbm_daemon_bootstrap_spawn_fn spawn, + void *context) { + g_bootstrap_spawn_override_context_for_test = context; + g_bootstrap_spawn_override_for_test = spawn; +} + +static bool bootstrap_seam_spawn(void *context, const cbm_daemon_bootstrap_launch_spec_t *spec) { + (void)context; + return g_bootstrap_spawn_override_for_test(g_bootstrap_spawn_override_context_for_test, spec); +} +#endif + cbm_daemon_bootstrap_status_t cbm_daemon_bootstrap_execute( const cbm_daemon_bootstrap_config_t *config, cbm_daemon_bootstrap_result_t *result_out) { bootstrap_production_context_t context = {0}; - const cbm_daemon_bootstrap_ops_t ops = { + cbm_daemon_bootstrap_ops_t ops = { .context = &context, .cohort_acquire = bootstrap_production_cohort_acquire, .cohort_release = bootstrap_production_cohort_release, @@ -1054,6 +1384,12 @@ cbm_daemon_bootstrap_status_t cbm_daemon_bootstrap_execute( .startup_lock_release = bootstrap_production_unlock, .spawn_daemon = bootstrap_production_spawn, .visible_diagnostic = bootstrap_production_diagnostic, + .start_failure_probe = bootstrap_production_start_failure_probe, }; +#ifdef CBM_ENABLE_TEST_SEAMS + if (g_bootstrap_spawn_override_for_test) { + ops.spawn_daemon = bootstrap_seam_spawn; + } +#endif return cbm_daemon_bootstrap_execute_with_ops(config, &ops, result_out); } diff --git a/src/daemon/bootstrap.h b/src/daemon/bootstrap.h index acb6c657b..8b1e797dd 100644 --- a/src/daemon/bootstrap.h +++ b/src/daemon/bootstrap.h @@ -119,6 +119,47 @@ cbm_daemon_bootstrap_probe_status_t cbm_daemon_bootstrap_classify_failed_connect typedef void *cbm_daemon_bootstrap_lock_t; typedef void *cbm_daemon_bootstrap_cohort_t; +/* Durable daemon start-failure record (#1828). A detached daemon has no + * stderr and no channel back to the client that launched it; when its + * listener publication fails (a full /tmp, for instance) the client used to + * respawn it for the whole startup deadline and then report "active or + * starting". The host now appends one record per failed start to + * /logs/cbm-daemon.start-failures.log -- the same owner-only log + * directory as cbm-daemon.log, deliberately NOT the runtime directory: that + * one is exactly what a full /tmp cannot write, and a new artifact there + * would entangle stale-generation cleanup. A waiting client reads the newest + * record for its endpoint that is not older than its own spawn and fails + * immediately with the cause. Diagnostic only: policy never reads it. */ +enum { CBM_DAEMON_BOOTSTRAP_COMPONENT_CAP = 32 }; +typedef struct { + uint64_t recorded_at_s; /* wall clock, seconds */ + uint64_t pid; + char component[CBM_DAEMON_BOOTSTRAP_COMPONENT_CAP]; /* host component that refused */ + char stage[CBM_DAEMON_IPC_LISTEN_FAILURE_STAGE_CAP]; /* IPC stage, "" when none */ + int errno_value; /* 0 when unknown */ + char path[CBM_DAEMON_IPC_LISTEN_FAILURE_PATH_CAP]; /* "" when none */ +} cbm_daemon_bootstrap_start_failure_t; + +/* /logs, the owner-only directory that holds every daemon log. */ +bool cbm_daemon_bootstrap_log_directory(char *out, size_t capacity); +/* Append one record for this process's most recent listener failure detail + * (cbm_daemon_ipc_listen_failure_detail) under `component`. */ +bool cbm_daemon_bootstrap_start_failure_record(const char *log_directory, + const cbm_daemon_ipc_endpoint_t *endpoint, + const char *component); +/* Newest record for `endpoint` recorded at or after not_before_s. Returns 1 + * and fills out_failure, 0 when none matches, -1 when the log directory or + * file cannot be validated. */ +int cbm_daemon_bootstrap_start_failure_read(const char *log_directory, + const cbm_daemon_ipc_endpoint_t *endpoint, + uint64_t not_before_s, + cbm_daemon_bootstrap_start_failure_t *out_failure); +/* The client-facing sentence: "CBM daemon failed to start: failed + * with ENOSPC (No space left on device) at ; see ". */ +void cbm_daemon_bootstrap_start_failure_format(const cbm_daemon_bootstrap_start_failure_t *failure, + const char *log_directory, char *out, + size_t capacity); + /* Injectable OS/runtime boundary used by the deterministic unit contract. * Production callers use cbm_daemon_bootstrap_execute(), whose built-in * operations delegate to daemon IPC/runtime and write visible diagnostics to @@ -149,6 +190,12 @@ typedef struct { bool (*startup_lock_release)(void *context, cbm_daemon_bootstrap_lock_t *lock_io); bool (*spawn_daemon)(void *context, const cbm_daemon_bootstrap_launch_spec_t *spec); void (*visible_diagnostic)(void *context, const char *message); + /* Optional. After this attempt spawned a daemon, report a start-failure + * record for the endpoint not older than not_before_s (1 found, 0 none, + * -1 unreadable). A found record ends the wait immediately (#1828). */ + int (*start_failure_probe)(void *context, const cbm_daemon_ipc_endpoint_t *endpoint, + uint64_t not_before_s, + cbm_daemon_bootstrap_start_failure_t *out_failure); } cbm_daemon_bootstrap_ops_t; cbm_daemon_bootstrap_status_t cbm_daemon_bootstrap_execute( @@ -156,6 +203,17 @@ cbm_daemon_bootstrap_status_t cbm_daemon_bootstrap_execute( /* Test seam for the same state machine. All callbacks are synchronous and * borrowed for the duration of the call. */ +#ifdef CBM_ENABLE_TEST_SEAMS +/* Replace ONLY the production spawn while every other production operation + * (cohort, probe, startup lock, handoff) stays real. A test forks a real + * daemon host in place of exec'ing the product binary, so the complete + * client/daemon rendezvous runs in one deterministic process tree (#1828). */ +typedef bool (*cbm_daemon_bootstrap_spawn_fn)(void *context, + const cbm_daemon_bootstrap_launch_spec_t *spec); +void cbm_daemon_bootstrap_spawn_override_set_for_test(cbm_daemon_bootstrap_spawn_fn spawn, + void *context); +#endif + cbm_daemon_bootstrap_status_t cbm_daemon_bootstrap_execute_with_ops( const cbm_daemon_bootstrap_config_t *config, const cbm_daemon_bootstrap_ops_t *ops, cbm_daemon_bootstrap_result_t *result_out); diff --git a/src/daemon/host.c b/src/daemon/host.c index f475658b8..dfab03e13 100644 --- a/src/daemon/host.c +++ b/src/daemon/host.c @@ -5,6 +5,7 @@ #include "daemon/host_internal.h" #include "daemon/application.h" +#include "daemon/bootstrap.h" #include "daemon/runtime.h" #include "daemon/project_lock.h" #include "daemon/version_cohort.h" @@ -146,6 +147,18 @@ static bool host_log_open(char conflict_log_out[HOST_PATH_CAP]) { return true; } +/* #1828: a detached daemon has no stderr; its start failure must reach the + * client that is waiting for it, or that client burns its whole deadline and + * reports the opposite of the truth. */ +static void host_start_failure_record(const cbm_daemon_ipc_endpoint_t *endpoint, + const char *component) { + char logs[HOST_PATH_CAP]; + if (!cbm_daemon_bootstrap_log_directory(logs, sizeof(logs)) || + !cbm_daemon_bootstrap_start_failure_record(logs, endpoint, component)) { + cbm_log_error("daemon.start_failure_record_failed", "component", component); + } +} + static void host_log_close(void) { cbm_log_set_sink(NULL); if (g_host_log_mutex_initialized) { @@ -1051,10 +1064,16 @@ int cbm_daemon_host_run(const cbm_daemon_host_config_t *config) { }; cbm_daemon_runtime_service_t *service = cbm_daemon_runtime_service_start_reserved(&runtime_config, &lifetime_reservation); + if (!service) { + cbm_log_error("daemon.start_failed", "component", "runtime"); + /* Recorded while the lifetime reservation is still held: a client + * that watches this generation vanish finds the cause already there + * and never launches a doomed replacement. */ + host_start_failure_record(config->endpoint, "runtime"); + } cbm_daemon_ipc_lifetime_reservation_release(lifetime_reservation); lifetime_reservation = NULL; if (!service) { - cbm_log_error("daemon.start_failed", "component", "runtime"); host_state_free(&host); host_log_close(); host_participant_guard_close(&participant_guard); diff --git a/src/daemon/ipc.c b/src/daemon/ipc.c index a706a7ca5..b44aff82a 100644 --- a/src/daemon/ipc.c +++ b/src/daemon/ipc.c @@ -9,6 +9,7 @@ #include "foundation/compat_thread.h" #include "foundation/log.h" #include "foundation/macos_acl.h" +#include "foundation/platform.h" #include "foundation/private_file_lock_internal.h" #include "foundation/sha256.h" #include "foundation/secure_random.h" @@ -53,6 +54,60 @@ const char *cbm_daemon_ipc_validation_detail(void) { return ipc_validation_detail_buffer; } +static cbm_daemon_ipc_listen_failure_t ipc_listen_failure; + +/* The failure recorder and its reset are only reached from the POSIX socket + * listener (all callers live in the `#ifndef _WIN32` block below); the Windows + * named-pipe listener does its own reporting. Guarding them keeps the always- + * compiled struct + accessor cross-platform while avoiding -Werror, + * -Wunused-function on the Windows build (#1828 CI). */ +#ifndef _WIN32 +static void ipc_listen_failure_reset(void) { + memset(&ipc_listen_failure, 0, sizeof(ipc_listen_failure)); +} + +/* Record and log one listener failure. `name` is an artifact inside the + * endpoint's runtime directory, "" for the directory itself, or NULL when + * the stage has no path. The log line names the errno symbolically and the + * full path, so `daemon.ipc.listen_failed stage=pending_publication` alone + * (the whole trace #1828's reporter had) can no longer happen. */ +static void ipc_listen_failed(const char *runtime_dir, const char *stage, int errno_value, + const char *name) { + ipc_listen_failure_reset(); + (void)snprintf(ipc_listen_failure.stage, sizeof(ipc_listen_failure.stage), "%s", + stage ? stage : ""); + ipc_listen_failure.errno_value = errno_value; + if (runtime_dir && name) { + int written = name[0] ? snprintf(ipc_listen_failure.path, sizeof(ipc_listen_failure.path), + "%s/%s", runtime_dir, name) + : snprintf(ipc_listen_failure.path, sizeof(ipc_listen_failure.path), + "%s", runtime_dir); + if (written <= 0 || written >= (int)sizeof(ipc_listen_failure.path)) { + ipc_listen_failure.path[0] = '\0'; + } + } + const char *errno_name = errno_value != 0 ? cbm_errno_name(errno_value) : NULL; + if (errno_name && ipc_listen_failure.path[0]) { + cbm_log_error("daemon.ipc.listen_failed", "stage", stage, "errno", errno_name, "path", + ipc_listen_failure.path); + } else if (errno_name) { + cbm_log_error("daemon.ipc.listen_failed", "stage", stage, "errno", errno_name); + } else if (ipc_listen_failure.path[0]) { + cbm_log_error("daemon.ipc.listen_failed", "stage", stage, "path", ipc_listen_failure.path); + } else { + cbm_log_error("daemon.ipc.listen_failed", "stage", stage); + } +} +#endif /* !_WIN32 */ + +bool cbm_daemon_ipc_listen_failure_detail(cbm_daemon_ipc_listen_failure_t *out) { + if (!out) { + return false; + } + *out = ipc_listen_failure; + return ipc_listen_failure.stage[0] != '\0'; +} + static bool instance_key_valid(const char *key) { if (!key) { return false; @@ -122,6 +177,19 @@ void cbm_daemon_ipc_windows_legacy_guard_release_failures_set_for_test(unsigned memory_order_release); } +#ifndef _WIN32 +static atomic_int g_posix_record_write_failure_errno_for_test; +#endif + +void cbm_daemon_ipc_posix_record_write_failure_set_for_test(int errno_value) { +#ifndef _WIN32 + atomic_store_explicit(&g_posix_record_write_failure_errno_for_test, errno_value, + memory_order_release); +#else + (void)errno_value; +#endif +} + #ifdef _WIN32 static cbm_daemon_ipc_startup_gate_fn g_startup_gate_for_test; static void *g_startup_gate_context_for_test; @@ -1815,6 +1883,29 @@ static bool posix_fd_write_all(int fd, const uint8_t *buffer, size_t length) { return true; } +/* The failing step of the most recent record publication in this process: + * its errno (0 when a validation predicate, not a syscall, refused) and the + * artifact name it was operating on. Read by the listener right after a + * publish returns false. */ +static int posix_publish_failure_errno; +static char posix_publish_failure_name[NAME_MAX + 1]; + +static void posix_publish_failure_note(int errno_value, const char *name) { + posix_publish_failure_errno = errno_value; + (void)snprintf(posix_publish_failure_name, sizeof(posix_publish_failure_name), "%s", + name ? name : ""); +} + +static bool posix_record_fd_write(int fd, const uint8_t *buffer, size_t length) { + int injected = + atomic_load_explicit(&g_posix_record_write_failure_errno_for_test, memory_order_acquire); + if (injected != 0) { + errno = injected; + return false; + } + return posix_fd_write_all(fd, buffer, length); +} + static bool posix_fd_pread_all(int fd, uint8_t *buffer, size_t length) { size_t offset = 0; while (offset < length) { @@ -2169,50 +2260,75 @@ static bool posix_socket_record_publish(const cbm_daemon_ipc_endpoint_t *endpoin return false; } struct stat existing; + errno = 0; if (fstatat(endpoint->dir_fd, record_name, &existing, AT_SYMLINK_NOFOLLOW) == 0 || errno != ENOENT) { + posix_publish_failure_note(errno == 0 ? EEXIST : errno, record_name); return false; } uint8_t record[POSIX_SOCKET_RECORD_SIZE]; if (!posix_socket_record_encode(magic, source, record)) { + posix_publish_failure_note(0, record_name); return false; } char temp_name[NAME_MAX + 1]; if (!posix_socket_record_temp_name(record_name, temp_name)) { + posix_publish_failure_note(ENAMETOOLONG, record_name); return false; } int fd = openat(endpoint->dir_fd, temp_name, O_WRONLY | O_CREAT | O_EXCL | O_CLOEXEC | O_NOFOLLOW | O_NONBLOCK, 0600); if (fd < 0) { + posix_publish_failure_note(errno, temp_name); return false; } struct stat created = {0}; bool temp_exists = true; bool stable_linked = false; + errno = 0; bool ok = fd_set_cloexec(fd) && fchmod(fd, 0600) == 0 && private_regular_file_snapshot(endpoint->dir_fd, temp_name, fd, 1, &created) && - posix_fd_write_all(fd, record, sizeof(record)) && fsync(fd) == 0 && + posix_record_fd_write(fd, record, sizeof(record)) && fsync(fd) == 0 && private_regular_file_snapshot(endpoint->dir_fd, temp_name, fd, 1, &created) && created.st_size == (off_t)POSIX_SOCKET_RECORD_SIZE && endpoint_runtime_still_valid(endpoint); + if (!ok) { + posix_publish_failure_note(errno, temp_name); + } if (ok) { posix_record_publication_stage_reached(magic, false); + errno = 0; ok = posix_linkat_no_follow(endpoint->dir_fd, temp_name, endpoint->dir_fd, record_name) == 0; stable_linked = ok; + if (!ok) { + posix_publish_failure_note(errno, record_name); + } } if (ok) { posix_record_publication_stage_reached(magic, true); + errno = 0; ok = posix_path_unlink_regular_if_matches(endpoint->dir_fd, temp_name, created.st_dev, created.st_ino, 2); temp_exists = !ok; + if (!ok) { + posix_publish_failure_note(errno, temp_name); + } } if (ok) { + errno = 0; ok = posix_directory_sync(endpoint->dir_fd); + if (!ok) { + posix_publish_failure_note(errno, ""); + } } + errno = 0; if (close(fd) != 0) { + if (ok) { + posix_publish_failure_note(errno, temp_name); + } ok = false; } @@ -2224,6 +2340,9 @@ static bool posix_socket_record_publish(const cbm_daemon_ipc_endpoint_t *endpoin posix_socket_identity_equal(&published_record.identity, &source->identity) && strcmp(published_record.anchor_name, source->anchor_name) == 0 && published_status.st_dev == created.st_dev && published_status.st_ino == created.st_ino; + if (!ok) { + posix_publish_failure_note(0, record_name); + } } if (!ok) { if (stable_linked) { @@ -2539,8 +2658,11 @@ cbm_daemon_ipc_listener_t *cbm_daemon_ipc_listen_reserved( cbm_daemon_ipc_lifetime_reservation_t **reservation_io) { cbm_daemon_ipc_lifetime_reservation_t *lifetime_reservation = reservation_io ? *reservation_io : NULL; + ipc_listen_failure_reset(); + posix_publish_failure_note(0, NULL); + const char *runtime_dir = endpoint ? endpoint->runtime_dir : NULL; if (!lifetime_reservation_matches_endpoint(endpoint, lifetime_reservation)) { - cbm_log_error("daemon.ipc.listen_failed", "stage", "reservation_validation"); + ipc_listen_failed(runtime_dir, "reservation_validation", 0, NULL); return NULL; } /* Stale removal happens only under the startup lock, before a daemon host @@ -2550,7 +2672,7 @@ cbm_daemon_ipc_listener_t *cbm_daemon_ipc_listen_reserved( char pending_temp_name[NAME_MAX + 1]; if (!posix_socket_record_temp_name(endpoint->socket_identity_name, identity_temp_name) || !posix_socket_record_temp_name(endpoint->socket_pending_name, pending_temp_name)) { - cbm_log_error("daemon.ipc.listen_failed", "stage", "temp_names"); + ipc_listen_failed(runtime_dir, "temp_names", ENAMETOOLONG, endpoint->socket_pending_name); return NULL; } struct stat existing; @@ -2559,27 +2681,32 @@ cbm_daemon_ipc_listener_t *cbm_daemon_ipc_listen_reserved( endpoint->socket_pending_name, identity_temp_name, pending_temp_name, }; bool namespace_absent = true; + const char *occupied_name = ""; + int occupied_errno = 0; for (size_t index = 0; index < sizeof(required_absent) / sizeof(required_absent[0]); index++) { + errno = 0; if (fstatat(endpoint->dir_fd, required_absent[index], &existing, AT_SYMLINK_NOFOLLOW) == 0 || errno != ENOENT) { namespace_absent = false; + occupied_name = required_absent[index]; + occupied_errno = errno == 0 ? EEXIST : errno; break; } } if (!endpoint_runtime_still_valid(endpoint) || !namespace_absent) { - cbm_log_error("daemon.ipc.listen_failed", "stage", "namespace_validation"); + ipc_listen_failed(runtime_dir, "namespace_validation", occupied_errno, occupied_name); return NULL; } int fd = local_socket_new(); if (fd < 0) { - cbm_log_error("daemon.ipc.listen_failed", "stage", "socket_creation"); + ipc_listen_failed(runtime_dir, "socket_creation", errno, NULL); return NULL; } cbm_daemon_ipc_listener_t *listener = calloc(1, sizeof(*listener)); if (!listener) { - cbm_log_error("daemon.ipc.listen_failed", "stage", "listener_allocation"); + ipc_listen_failed(runtime_dir, "listener_allocation", ENOMEM, NULL); (void)close(fd); return NULL; } @@ -2597,7 +2724,8 @@ cbm_daemon_ipc_listener_t *cbm_daemon_ipc_listen_reserved( if (listener->dir_fd < 0 || !fd_set_cloexec(listener->dir_fd) || !listener->runtime_dir || !listener->address || !listener->socket_name || !listener->socket_anchor_name || !listener->socket_identity_name || !listener->socket_pending_name) { - cbm_log_error("daemon.ipc.listen_failed", "stage", "listener_initialization"); + ipc_listen_failed(runtime_dir, "listener_initialization", + listener->dir_fd < 0 ? errno : ENOMEM, NULL); if (listener->dir_fd >= 0) { (void)close(listener->dir_fd); } @@ -2615,20 +2743,19 @@ cbm_daemon_ipc_listener_t *cbm_daemon_ipc_listen_reserved( struct sockaddr_un address; socklen_t address_length; if (!unix_address_set(&address, endpoint->socket_anchor_address, &address_length)) { - cbm_log_error("daemon.ipc.listen_failed", "stage", "socket_address"); + ipc_listen_failed(runtime_dir, "socket_address", ENAMETOOLONG, + endpoint->socket_anchor_name); cbm_daemon_ipc_listener_close(listener); return NULL; } if (bind(fd, (const struct sockaddr *)&address, address_length) != 0) { - int bind_error = errno; - char error_text[32]; - (void)snprintf(error_text, sizeof(error_text), "%d", bind_error); - cbm_log_error("daemon.ipc.listen_failed", "bind_errno", error_text); + ipc_listen_failed(runtime_dir, "socket_bind", errno, endpoint->socket_anchor_name); cbm_daemon_ipc_listener_close(listener); return NULL; } struct stat bound_status; + errno = 0; bool bound_path_ok = fstatat(endpoint->dir_fd, endpoint->socket_anchor_name, &bound_status, AT_SYMLINK_NOFOLLOW) == 0 && S_ISSOCK(bound_status.st_mode) && bound_status.st_uid == geteuid(); @@ -2648,11 +2775,13 @@ cbm_daemon_ipc_listener_t *cbm_daemon_ipc_listen_reserved( anchor_status.st_ino != bound_status.st_ino || !posix_socket_identity_from_stat(&anchor_status, &anchor_identity) || !posix_directory_sync(endpoint->dir_fd)) { + int security_errno = errno; if (bound_path_ok) { posix_bound_socket_unlink_if_matches(endpoint->dir_fd, endpoint->socket_anchor_name, bound_status.st_dev, bound_status.st_ino); } - cbm_log_error("daemon.ipc.listen_failed", "stage", "socket_security"); + ipc_listen_failed(runtime_dir, "socket_security", security_errno, + endpoint->socket_anchor_name); cbm_daemon_ipc_listener_close(listener); return NULL; } @@ -2669,16 +2798,19 @@ cbm_daemon_ipc_listener_t *cbm_daemon_ipc_listen_reserved( endpoint, endpoint->socket_pending_name, POSIX_SOCKET_PENDING_MAGIC, &pending, &pending_status.st_dev, &pending_status.st_ino); if (!pending_published) { - cbm_log_error("daemon.ipc.listen_failed", "stage", "pending_publication"); + ipc_listen_failed(runtime_dir, "pending_publication", posix_publish_failure_errno, + posix_publish_failure_name); posix_publication_abort(endpoint, &anchor_identity, false, NULL, false, NULL); cbm_daemon_ipc_listener_close(listener); return NULL; } posix_publication_stage_reached(CBM_DAEMON_IPC_POSIX_PUBLICATION_PENDING_DURABLE); + errno = 0; bool stable_linked = posix_linkat_no_follow(endpoint->dir_fd, endpoint->socket_anchor_name, endpoint->dir_fd, endpoint->socket_name) == 0 && posix_directory_sync(endpoint->dir_fd); + int stable_errno = stable_linked ? 0 : errno; posix_socket_identity_t stable_identity = {0}; posix_socket_identity_t committed_identity = {0}; struct stat stable_status = {0}; @@ -2693,7 +2825,7 @@ cbm_daemon_ipc_listener_t *cbm_daemon_ipc_listen_reserved( posix_socket_identity_equal(&stable_identity, &committed_identity) && posix_socket_inode_equal(&anchor_identity, &committed_identity); if (!stable_valid) { - cbm_log_error("daemon.ipc.listen_failed", "stage", "stable_publication"); + ipc_listen_failed(runtime_dir, "stable_publication", stable_errno, endpoint->socket_name); posix_publication_abort(endpoint, &anchor_identity, pending_published, &pending_status, false, NULL); cbm_daemon_ipc_listener_close(listener); @@ -2712,7 +2844,8 @@ cbm_daemon_ipc_listener_t *cbm_daemon_ipc_listen_reserved( endpoint, endpoint->socket_identity_name, POSIX_SOCKET_MARKER_MAGIC, &marker, &marker_status.st_dev, &marker_status.st_ino); if (!marker_published) { - cbm_log_error("daemon.ipc.listen_failed", "stage", "marker_publication"); + ipc_listen_failed(runtime_dir, "marker_publication", posix_publish_failure_errno, + posix_publish_failure_name); posix_publication_abort(endpoint, &committed_identity, pending_published, &pending_status, false, NULL); cbm_daemon_ipc_listener_close(listener); @@ -2722,10 +2855,11 @@ cbm_daemon_ipc_listener_t *cbm_daemon_ipc_listen_reserved( listener->identity_inode = marker_status.st_ino; posix_publication_stage_reached(CBM_DAEMON_IPC_POSIX_PUBLICATION_MARKER_DURABLE); + errno = 0; if (!posix_path_unlink_regular_if_matches(endpoint->dir_fd, endpoint->socket_pending_name, pending_status.st_dev, pending_status.st_ino, 1) || !posix_directory_sync(endpoint->dir_fd)) { - cbm_log_error("daemon.ipc.listen_failed", "stage", "pending_removal"); + ipc_listen_failed(runtime_dir, "pending_removal", errno, endpoint->socket_pending_name); posix_publication_abort(endpoint, &committed_identity, true, &pending_status, marker_published, &marker_status); cbm_daemon_ipc_listener_close(listener); diff --git a/src/daemon/ipc.h b/src/daemon/ipc.h index 2cdf792dd..daccf6900 100644 --- a/src/daemon/ipc.h +++ b/src/daemon/ipc.h @@ -91,6 +91,25 @@ bool cbm_daemon_ipc_private_directory_secure(const char *directory_path); * validation refusal (empty string when none). Diagnostic only — callers * append it to their error messages; policy decisions never read it. */ const char *cbm_daemon_ipc_validation_detail(void); + +/* This process's most recent listener-publication failure: the stage that + * refused, the errno the failing step reported (0 when the step reported + * none), and the artifact path it was operating on ("" when no path applies). + * Every `daemon.ipc.listen_failed` log line carries the same three fields. + * Reset at the start of each listen attempt. Returns false while no failure + * has been recorded. Diagnostic only: the daemon host copies it into its + * durable start-failure record so a waiting client can name the cause instead + * of burning its full startup deadline (#1828). */ +enum { + CBM_DAEMON_IPC_LISTEN_FAILURE_STAGE_CAP = 32, + CBM_DAEMON_IPC_LISTEN_FAILURE_PATH_CAP = 4096 +}; +typedef struct { + char stage[CBM_DAEMON_IPC_LISTEN_FAILURE_STAGE_CAP]; + int errno_value; + char path[CBM_DAEMON_IPC_LISTEN_FAILURE_PATH_CAP]; +} cbm_daemon_ipc_listen_failure_t; +bool cbm_daemon_ipc_listen_failure_detail(cbm_daemon_ipc_listen_failure_t *out); #ifdef CBM_ENABLE_TEST_SEAMS /* #1537: seed the detail so a test can prove the CLI refusal surfaces it. */ void cbm_daemon_ipc_set_validation_detail_for_testing(const char *detail); diff --git a/src/daemon/ipc_internal.h b/src/daemon/ipc_internal.h index b791a6b02..4b7cf1fd5 100644 --- a/src/daemon/ipc_internal.h +++ b/src/daemon/ipc_internal.h @@ -95,6 +95,11 @@ typedef void (*cbm_daemon_ipc_posix_publication_hook_fn)( void cbm_daemon_ipc_posix_publication_hook_set_for_test( cbm_daemon_ipc_posix_publication_hook_fn hook, void *context); void cbm_daemon_ipc_windows_legacy_guard_release_failures_set_for_test(unsigned int count); +/* #1828: while non-zero, every POSIX socket-record publication write fails + * with this errno. This is the deterministic stand-in for a full runtime + * filesystem (tmpfs ENOSPC): the socket binds, the record file is created, + * and only its data write is refused -- exactly the reporter's failure shape. */ +void cbm_daemon_ipc_posix_record_write_failure_set_for_test(int errno_value); /* Deterministic-interleaving seam: fires on the Windows startup path once the * startup lock is held, before the rendezvous handoff. A test parks here to diff --git a/src/foundation/platform.c b/src/foundation/platform.c index 14e36c0d8..cf9a0f469 100644 --- a/src/foundation/platform.c +++ b/src/foundation/platform.c @@ -543,6 +543,60 @@ const char *cbm_app_local_dir(void) { /* ── Cache directory ────────────────────────── */ +const char *cbm_errno_name(int error) { + static const struct { + int value; + const char *name; + } names[] = { + {EACCES, "EACCES"}, + {EAGAIN, "EAGAIN"}, + {EBUSY, "EBUSY"}, + {EEXIST, "EEXIST"}, + {EFBIG, "EFBIG"}, + {EINTR, "EINTR"}, + {EINVAL, "EINVAL"}, + {EIO, "EIO"}, + {EISDIR, "EISDIR"}, + {ELOOP, "ELOOP"}, + {EMFILE, "EMFILE"}, + {EMLINK, "EMLINK"}, + {ENAMETOOLONG, "ENAMETOOLONG"}, + {ENFILE, "ENFILE"}, + {ENOENT, "ENOENT"}, + {ENOMEM, "ENOMEM"}, + {ENOSPC, "ENOSPC"}, + {ENOTDIR, "ENOTDIR"}, + {ENOTEMPTY, "ENOTEMPTY"}, + {ENXIO, "ENXIO"}, + {EPERM, "EPERM"}, + {EROFS, "EROFS"}, + {ETXTBSY, "ETXTBSY"}, + {EXDEV, "EXDEV"}, + {ETIMEDOUT, "ETIMEDOUT"}, + {ECONNREFUSED, "ECONNREFUSED"}, + {EADDRINUSE, "EADDRINUSE"}, + {ENOTSOCK, "ENOTSOCK"}, + {EPIPE, "EPIPE"}, +#ifdef EDQUOT + {EDQUOT, "EDQUOT"}, +#endif +#ifdef EOVERFLOW + {EOVERFLOW, "EOVERFLOW"}, +#endif +#ifdef ENOTSUP + {ENOTSUP, "ENOTSUP"}, +#endif + }; + for (size_t i = 0; i < sizeof(names) / sizeof(names[0]); i++) { + if (names[i].value == error) { + return names[i].name; + } + } + static CBM_TLS char fallback[16]; + (void)snprintf(fallback, sizeof(fallback), "%d", error); + return fallback; +} + const char *cbm_resolve_cache_dir(void) { static CBM_TLS char buf[CBM_SZ_4K]; static const char missing[] = "\x1f" diff --git a/src/foundation/platform.h b/src/foundation/platform.h index 938cf5905..cc58e8429 100644 --- a/src/foundation/platform.h +++ b/src/foundation/platform.h @@ -94,6 +94,13 @@ uint64_t cbm_now_ns(void); /* Monotonic millisecond timestamp. */ uint64_t cbm_now_ms(void); +/* Symbolic name for an errno value ("ENOSPC"), or the decimal number when the + * value is not in the portable table. The fallback lives in thread-local + * storage; copy it before the next call on the same thread. Diagnostics only: + * a log line that says `errno=ENOSPC path=...` is a one-line diagnosis where + * `stage=pending_publication` alone cost a reporter hours (#1828). */ +const char *cbm_errno_name(int error); + /* ── System info ───────────────────────────────────────────────── */ /* Number of available CPU cores. */ diff --git a/tests/test_daemon_bootstrap.c b/tests/test_daemon_bootstrap.c index a9c8a368e..0114d6329 100644 --- a/tests/test_daemon_bootstrap.c +++ b/tests/test_daemon_bootstrap.c @@ -1,18 +1,30 @@ /* RED contract for early process-role classification. */ #include "test_framework.h" +#include "test_helpers.h" #include "daemon/bootstrap.h" +#include "daemon/host.h" #include "daemon/ipc.h" +#include "daemon/ipc_internal.h" +#include "daemon/runtime.h" #include "daemon/service.h" #include "foundation/compat.h" #include "foundation/compat_fs.h" #include "foundation/compat_thread.h" #include "foundation/platform.h" +#include #include #include #include +#include #include +#include +#ifndef _WIN32 +#include +#include +#include +#endif enum { BOOTSTRAP_TEST_PATH_CAP = 1024, @@ -874,6 +886,253 @@ TEST(daemon_bootstrap_darwin_launch_failure_is_synchronous) { } #endif +#ifndef _WIN32 +enum { BOOTSTRAP_ENOSPC_MAX_CHILDREN = 16, BOOTSTRAP_ENOSPC_LOG_CAP = 65536 }; + +typedef struct { + char parent[BOOTSTRAP_TEST_PATH_CAP]; + cbm_daemon_build_identity_t identity; + pid_t children[BOOTSTRAP_ENOSPC_MAX_CHILDREN]; + size_t child_count; + size_t spawn_calls; +} bootstrap_enospc_host_t; + +/* The production spawn exec's the product binary; this one forks a REAL daemon + * host (cbm_daemon_host_run, the same entry `--cbm-daemon-internal` reaches) + * whose record publication fails with ENOSPC through the inherited seam. */ +static bool bootstrap_enospc_host_spawn(void *opaque, + const cbm_daemon_bootstrap_launch_spec_t *spec) { + bootstrap_enospc_host_t *state = opaque; + if (!spec || !spec->detached) { + return false; + } + state->spawn_calls++; + if (state->child_count >= BOOTSTRAP_ENOSPC_MAX_CHILDREN) { + /* RED-run guard only: a client that keeps respawning a doomed daemon + * for its whole deadline must not fork without bound. */ + return true; + } + pid_t child = fork(); + if (child < 0) { + return false; + } + if (child == 0) { + cbm_daemon_ipc_endpoint_t *endpoint = cbm_daemon_bootstrap_endpoint_new(state->parent); + atomic_int stop_requested = ATOMIC_VAR_INIT(0); + cbm_daemon_host_config_t config = { + .endpoint = endpoint, + .identity = state->identity, + .executable_path = "/enospc-host-test", + .stop_requested = &stop_requested, + }; + int run_result = endpoint ? cbm_daemon_host_run(&config) : 0; + _exit(run_result == -1 ? 0 : 50); + } + state->children[state->child_count++] = child; + return true; +} + +static void bootstrap_enospc_reap(bootstrap_enospc_host_t *state, int *nonzero_exits) { + *nonzero_exits = 0; + for (size_t i = 0; i < state->child_count; i++) { + int status = 0; + pid_t waited; + do { + waited = waitpid(state->children[i], &status, WNOHANG); + } while (waited < 0 && errno == EINTR); + if (waited == 0) { + (void)kill(state->children[i], SIGKILL); + do { + waited = waitpid(state->children[i], &status, 0); + } while (waited < 0 && errno == EINTR); + } + if (!WIFEXITED(status) || WEXITSTATUS(status) != 0) { + (*nonzero_exits)++; + } + } +} + +static bool bootstrap_read_file(const char *path, char *out, size_t capacity) { + FILE *file = cbm_fopen(path, "rb"); + if (!file) { + return false; + } + size_t used = fread(out, 1, capacity - 1, file); + out[used] = '\0'; + (void)fclose(file); + return true; +} + +/* The record contract behind the fail-fast: a real listener failure in this + * process is recorded and read back with stage, errno, and path; a record + * older than the reader's spawn or for another endpoint is not evidence. */ +TEST(daemon_bootstrap_start_failure_record_round_trip) { + bootstrap_endpoint_fixture_t fixture; + bootstrap_endpoint_fixture_t other; + ASSERT_TRUE(bootstrap_endpoint_fixture_start(&fixture, "failure-record")); + char other_parent[BOOTSTRAP_TEST_PATH_CAP]; + int other_written = + snprintf(other_parent, sizeof(other_parent), "%s/other-XXXXXX", fixture.parent); + ASSERT(other_written > 0 && other_written < (int)sizeof(other_parent)); + ASSERT_TRUE(cbm_mkdtemp(other_parent) != NULL); + memset(&other, 0, sizeof(other)); + other.endpoint = cbm_daemon_ipc_endpoint_new("1828000000000002", other_parent); + ASSERT_TRUE(other.endpoint != NULL); + char logs[BOOTSTRAP_TEST_PATH_CAP]; + int logs_written = snprintf(logs, sizeof(logs), "%s/logs", fixture.parent); + ASSERT(logs_written > 0 && logs_written < (int)sizeof(logs)); + char expected_path[BOOTSTRAP_TEST_PATH_CAP]; + int path_written = snprintf(expected_path, sizeof(expected_path), "%s.pending.tmp", + cbm_daemon_ipc_endpoint_address(fixture.endpoint)); + ASSERT(path_written > 0 && path_written < (int)sizeof(expected_path)); + + cbm_daemon_ipc_posix_record_write_failure_set_for_test(ENOSPC); + cbm_daemon_ipc_listener_t *listener = cbm_daemon_ipc_listen(fixture.endpoint); + cbm_daemon_ipc_posix_record_write_failure_set_for_test(0); + ASSERT_TRUE(listener == NULL); + + uint64_t now_s = (uint64_t)time(NULL); + bool recorded = cbm_daemon_bootstrap_start_failure_record(logs, fixture.endpoint, "runtime"); + cbm_daemon_bootstrap_start_failure_t failure; + int found = + cbm_daemon_bootstrap_start_failure_read(logs, fixture.endpoint, now_s - 2, &failure); + cbm_daemon_bootstrap_start_failure_t stale; + int stale_found = + cbm_daemon_bootstrap_start_failure_read(logs, fixture.endpoint, now_s + 60, &stale); + cbm_daemon_bootstrap_start_failure_t foreign; + int foreign_found = + cbm_daemon_bootstrap_start_failure_read(logs, other.endpoint, now_s - 2, &foreign); + char message[CBM_DAEMON_CONFLICT_MESSAGE_SIZE]; + cbm_daemon_bootstrap_start_failure_format(&failure, logs, message, sizeof(message)); + + cbm_daemon_ipc_endpoint_free(other.endpoint); + (void)th_rmtree(fixture.parent); + bootstrap_endpoint_fixture_finish(&fixture); + + ASSERT_TRUE(recorded); + ASSERT_EQ(found, 1); + ASSERT_STR_EQ(failure.component, "runtime"); + ASSERT_STR_EQ(failure.stage, "pending_publication"); + ASSERT_EQ(failure.errno_value, ENOSPC); + ASSERT_STR_EQ(failure.path, expected_path); + ASSERT_TRUE(failure.pid == (uint64_t)getpid()); + ASSERT_EQ(stale_found, 0); + ASSERT_EQ(foreign_found, 0); + ASSERT_TRUE(strstr(message, "CBM daemon failed to start: pending_publication failed with " + "ENOSPC (") != NULL); + ASSERT_TRUE(strstr(message, expected_path) != NULL); + ASSERT_TRUE(strstr(message, "/cbm-daemon.log") != NULL); + PASS(); +} + +/* #1828: a daemon that dies at publication (full /tmp) left every client + * waiting the full 30 s and then reporting "active or starting" -- the + * opposite of the truth. A real host is spawned against a runtime directory + * whose record writes fail with ENOSPC; the client must report "failed to + * start" naming the errno and the path by ending the wait on the recorded + * cause, not by exhausting the deadline. The proof is the surfaced record and + * the single spawn (see the assertions), never a wall-clock measurement. */ +TEST(daemon_bootstrap_fails_fast_when_daemon_dies_at_publication) { + const char *old_cache = getenv("CBM_CACHE_DIR"); + char *saved_cache = old_cache ? cbm_strdup(old_cache) : NULL; + bool snapshot_ok = !old_cache || saved_cache; + + bootstrap_endpoint_fixture_t fixture; + bool fixture_ok = snapshot_ok && bootstrap_endpoint_fixture_start(&fixture, "enospc-host"); + char cache[BOOTSTRAP_TEST_PATH_CAP] = {0}; + char daemon_log[BOOTSTRAP_TEST_PATH_CAP] = {0}; + char expected_path[BOOTSTRAP_TEST_PATH_CAP] = {0}; + int cache_written = + fixture_ok ? snprintf(cache, sizeof(cache), "%s/cache", fixture.parent) : -1; + int log_written = + fixture_ok ? snprintf(daemon_log, sizeof(daemon_log), "%s/logs/cbm-daemon.log", cache) : -1; + int path_written = fixture_ok ? snprintf(expected_path, sizeof(expected_path), "%s.pending.tmp", + cbm_daemon_ipc_endpoint_address(fixture.endpoint)) + : -1; + bool environment_ready = cache_written > 0 && cache_written < (int)sizeof(cache) && + log_written > 0 && log_written < (int)sizeof(daemon_log) && + path_written > 0 && path_written < (int)sizeof(expected_path) && + cbm_mkdir_p(cache, 0700) && cbm_setenv("CBM_CACHE_DIR", cache, 1) == 0; + + char self_build[CBM_DAEMON_BUILD_FINGERPRINT_SIZE] = {0}; + bool identity_ready = environment_ready && cbm_daemon_runtime_process_build_fingerprint( + (uint64_t)getpid(), self_build); + static bootstrap_enospc_host_t host; + memset(&host, 0, sizeof(host)); + (void)snprintf(host.parent, sizeof(host.parent), "%s", fixture_ok ? fixture.parent : ""); + host.identity = bootstrap_identity("2.4.0", self_build); + + cbm_daemon_bootstrap_config_t config = { + .role = CBM_DAEMON_PROCESS_MCP_CLIENT, + .endpoint = fixture.endpoint, + .identity = &host.identity, + .executable_path = "/enospc-host-test", + .connect_timeout_ms = 200, + .startup_timeout_ms = 30000, + }; + cbm_daemon_bootstrap_result_t result; + memset(&result, 0, sizeof(result)); + cbm_daemon_bootstrap_status_t status = CBM_DAEMON_BOOTSTRAP_FAILED; + if (identity_ready) { + cbm_daemon_ipc_posix_record_write_failure_set_for_test(ENOSPC); + cbm_daemon_bootstrap_spawn_override_set_for_test(bootstrap_enospc_host_spawn, &host); + status = cbm_daemon_bootstrap_execute(&config, &result); + cbm_daemon_bootstrap_spawn_override_set_for_test(NULL, NULL); + cbm_daemon_ipc_posix_record_write_failure_set_for_test(0); + } + int nonzero_exits = 0; + bootstrap_enospc_reap(&host, &nonzero_exits); + + static char log[BOOTSTRAP_ENOSPC_LOG_CAP]; + log[0] = '\0'; + bool log_read = bootstrap_read_file(daemon_log, log, sizeof(log)); + const char *listen_failed = strstr(log, "msg=daemon.ipc.listen_failed"); + bool daemon_named_cause = listen_failed && strstr(listen_failed, "errno=ENOSPC") != NULL && + strstr(listen_failed, expected_path) != NULL; + bool message_names_failure = strstr(result.message, "failed to start") != NULL; + bool message_names_errno = strstr(result.message, "ENOSPC") != NULL; + bool message_names_path = strstr(result.message, expected_path) != NULL; + bool stale_wording = strstr(result.message, "active or starting") != NULL; + + if (saved_cache) { + (void)cbm_setenv("CBM_CACHE_DIR", saved_cache, 1); + } else { + (void)cbm_unsetenv("CBM_CACHE_DIR"); + } + free(saved_cache); + if (fixture_ok) { + (void)th_rmtree(fixture.parent); + bootstrap_endpoint_fixture_finish(&fixture); + } + + ASSERT_TRUE(snapshot_ok); + ASSERT_TRUE(fixture_ok); + ASSERT_TRUE(environment_ready); + ASSERT_TRUE(identity_ready); + ASSERT_EQ(status, CBM_DAEMON_BOOTSTRAP_FAILED); + ASSERT_TRUE(result.daemon_spawned); + ASSERT_TRUE(host.child_count >= 1); + ASSERT_TRUE(log_read); + ASSERT_TRUE(daemon_named_cause); + /* Fast-fail is proven by the MECHANISM, never by wall-clock (O9: a gate + * never asserts a transient timing window). The recorded ENOSPC cause is + * surfaced verbatim ("failed to start" + errno + path) and the slow + * "active or starting" timeout wording is absent -- that message is emitted + * ONLY on the fast-fail break (cbm_daemon_bootstrap_start_failure_format), + * never on the 30 s deadline path -- and the client stopped after exactly + * one spawn instead of respawning a doomed daemon until the deadline. Any + * regression to the pre-#1828 30 s hang trips these deterministically. */ + ASSERT_FALSE(stale_wording); + ASSERT_TRUE(message_names_failure); + ASSERT_TRUE(message_names_errno); + ASSERT_TRUE(message_names_path); + ASSERT_EQ(host.child_count, 1U); + ASSERT_EQ(nonzero_exits, 0); + bootstrap_endpoint_fixture_finish(&fixture); + PASS(); +} +#endif + SUITE(daemon_bootstrap) { RUN_TEST(daemon_bootstrap_classifies_default_and_ui_as_mcp_clients); RUN_TEST(daemon_bootstrap_classifies_stateless_commands_without_client); @@ -903,4 +1162,8 @@ SUITE(daemon_bootstrap) { #ifdef __APPLE__ RUN_TEST(daemon_bootstrap_darwin_launch_failure_is_synchronous); #endif +#ifndef _WIN32 + RUN_TEST(daemon_bootstrap_start_failure_record_round_trip); + RUN_TEST(daemon_bootstrap_fails_fast_when_daemon_dies_at_publication); +#endif } diff --git a/tests/test_daemon_ipc.c b/tests/test_daemon_ipc.c index 9bb550948..c57fa7bb6 100644 --- a/tests/test_daemon_ipc.c +++ b/tests/test_daemon_ipc.c @@ -13,6 +13,7 @@ #include "daemon/ipc_internal.h" #include "foundation/compat.h" #include "foundation/compat_thread.h" +#include "foundation/log.h" #include "foundation/platform.h" #include "foundation/private_file_lock_internal.h" #include "foundation/subprocess.h" @@ -4952,6 +4953,83 @@ TEST(daemon_ipc_posix_world_writable_ancestor_still_refused_issue1537) { } #endif /* !_WIN32 */ +#ifndef _WIN32 +enum { IPC_TEST_LOG_CAPTURE_CAP = 16384 }; +static char ipc_test_log_capture[IPC_TEST_LOG_CAPTURE_CAP]; +static size_t ipc_test_log_capture_used; + +static void ipc_test_log_capture_sink(const char *line) { + if (!line) { + return; + } + size_t length = strlen(line); + if (ipc_test_log_capture_used + length + 2 > sizeof(ipc_test_log_capture)) { + return; + } + memcpy(ipc_test_log_capture + ipc_test_log_capture_used, line, length); + ipc_test_log_capture_used += length; + ipc_test_log_capture[ipc_test_log_capture_used++] = '\n'; + ipc_test_log_capture[ipc_test_log_capture_used] = '\0'; +} + +/* #1828: a full /tmp made every daemon start die at pending publication, and + * the only durable trace was `daemon.ipc.listen_failed stage=pending_publication` + * -- no syscall, no errno, no path. The reporter needed hours (and a wrong + * `df` on the wrong mount) to find the cause. The failure line must name the + * errno and the exact artifact path that could not be written. */ +TEST(daemon_ipc_listen_failure_names_errno_and_path) { + static const char key[] = "1828000000000001"; + char parent[TEST_PATH_CAP] = {0}; + char runtime_dir[TEST_PATH_CAP] = {0}; + char socket_path[TEST_PATH_CAP] = {0}; + char pending_path[TEST_PATH_CAP] = {0}; + char expected_path[TEST_PATH_CAP + 16] = {0}; + cbm_daemon_ipc_endpoint_t *endpoint = NULL; + cbm_daemon_ipc_listener_t *listener = NULL; + + bool parent_ok = ipc_test_parent_new(parent, "enospc-diag"); + if (parent_ok) { + endpoint = cbm_daemon_ipc_endpoint_new(key, parent); + } + if (endpoint) { + ipc_test_copy_path(runtime_dir, cbm_daemon_ipc_endpoint_runtime_dir(endpoint)); + ipc_test_copy_path(socket_path, cbm_daemon_ipc_endpoint_address(endpoint)); + } + bool paths_ok = endpoint && ipc_test_socket_pending_path(pending_path, socket_path) && + snprintf(expected_path, sizeof(expected_path), "path=%s.tmp", pending_path) > 0; + if (paths_ok) { + ipc_test_log_capture_used = 0; + ipc_test_log_capture[0] = '\0'; + cbm_daemon_ipc_posix_record_write_failure_set_for_test(ENOSPC); + cbm_log_set_sink_ex(ipc_test_log_capture_sink, CBM_LOG_SINK_REPLACE); + listener = cbm_daemon_ipc_listen(endpoint); + cbm_log_set_sink(NULL); + cbm_daemon_ipc_posix_record_write_failure_set_for_test(0); + } + const char *failed = strstr(ipc_test_log_capture, "msg=daemon.ipc.listen_failed"); + bool stage_named = failed && strstr(failed, "stage=pending_publication") != NULL; + bool errno_named = failed && strstr(failed, "errno=ENOSPC") != NULL; + bool path_named = failed && strstr(failed, expected_path) != NULL; + struct stat leftover; + bool namespace_clean = paths_ok && lstat(socket_path, &leftover) != 0 && errno == ENOENT && + lstat(pending_path, &leftover) != 0 && errno == ENOENT; + + cbm_daemon_ipc_listener_close(listener); + cbm_daemon_ipc_endpoint_free(endpoint); + th_cleanup(parent_ok ? parent : NULL); + + ASSERT_TRUE(parent_ok); + ASSERT_TRUE(paths_ok); + ASSERT_TRUE(listener == NULL); + ASSERT_TRUE(failed != NULL); + ASSERT_TRUE(stage_named); + ASSERT_TRUE(errno_named); + ASSERT_TRUE(path_named); + ASSERT_TRUE(namespace_clean); + PASS(); +} +#endif + SUITE(daemon_ipc) { RUN_TEST(daemon_ipc_pending_timeout_race_returns_completed_io); RUN_TEST(daemon_ipc_pending_wait_failure_cancels_and_drains); @@ -5019,5 +5097,8 @@ SUITE(daemon_ipc) { RUN_TEST(daemon_ipc_posix_private_directory_rejects_world_writable_ancestor); RUN_TEST(daemon_ipc_posix_private_log_rejects_symlinks_and_is_owner_only); RUN_TEST(daemon_ipc_posix_rejects_non_socket_and_symlink_endpoints); +#ifndef _WIN32 + RUN_TEST(daemon_ipc_listen_failure_names_errno_and_path); +#endif #endif }