From 823b6fe0a4aa3cdb80a0f487b2666caaa9a334b8 Mon Sep 17 00:00:00 2001 From: robertjamesprior <83608739+robertjamesprior@users.noreply.github.com> Date: Wed, 2 Sep 2026 21:02:49 +0000 Subject: [PATCH 1/2] Clarify NVENC live view startup failures --- server/pkg/gst/gst.go | 21 +++++++++++++- server/pkg/gst/gst_test.go | 56 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 76 insertions(+), 1 deletion(-) create mode 100644 server/pkg/gst/gst_test.go diff --git a/server/pkg/gst/gst.go b/server/pkg/gst/gst.go index a9aa67092..530341822 100644 --- a/server/pkg/gst/gst.go +++ b/server/pkg/gst/gst.go @@ -8,6 +8,7 @@ package gst import "C" import ( "fmt" + "strings" "sync" "sync/atomic" "time" @@ -72,7 +73,8 @@ func CreatePipeline(pipelineStr string) (Pipeline, error) { if gstError != nil { defer C.g_error_free(gstError) - return nil, fmt.Errorf("(pipeline error) %s", C.GoString(gstError.message)) + msg := annotatePipelineError(pipelineStr, C.GoString(gstError.message)) + return nil, fmt.Errorf("(pipeline error) %s", msg) } p := &pipeline{ @@ -90,6 +92,23 @@ func CreatePipeline(pipelineStr string) (Pipeline, error) { return p, nil } +func annotatePipelineError(pipelineStr, msg string) string { + lowerMsg := strings.ToLower(msg) + if !strings.Contains(pipelineStr, "nvh264enc") { + return msg + } + + if !strings.Contains(lowerMsg, "nvh264enc") { + return msg + } + + if !strings.Contains(lowerMsg, "no element") && !strings.Contains(lowerMsg, "no such element or plugin") { + return msg + } + + return msg + " (live view could not initialize NVENC/CUDA; on GPU browsers this usually means GPU memory is exhausted by replay or browser GPU load. Reduce the browser resolution or stop replay, then try live view again.)" +} + func (p *pipeline) Src() string { return p.src } diff --git a/server/pkg/gst/gst_test.go b/server/pkg/gst/gst_test.go new file mode 100644 index 000000000..6e16799c6 --- /dev/null +++ b/server/pkg/gst/gst_test.go @@ -0,0 +1,56 @@ +package gst + +import ( + "strings" + "testing" +) + +func TestAnnotatePipelineError(t *testing.T) { + t.Parallel() + + const hint = "live view could not initialize NVENC/CUDA" + + tests := []struct { + name string + pipelineStr string + msg string + wantHint bool + }{ + { + name: "adds hint for missing nvh264enc in gpu pipeline", + pipelineStr: "ximagesrc ! cudaupload ! nvh264enc name=encoder ! appsink name=appsink", + msg: `no element "nvh264enc"`, + wantHint: true, + }, + { + name: "adds hint for plugin wording", + pipelineStr: "ximagesrc ! cudaupload ! nvh264enc name=encoder ! appsink name=appsink", + msg: "No such element or plugin 'nvh264enc'", + wantHint: true, + }, + { + name: "leaves unrelated encoder errors alone", + pipelineStr: "ximagesrc ! x264enc name=encoder ! appsink name=appsink", + msg: `no element "x264enc"`, + wantHint: false, + }, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + got := annotatePipelineError(tt.pipelineStr, tt.msg) + hasHint := got != tt.msg + + if hasHint != tt.wantHint { + t.Fatalf("annotatePipelineError(%q, %q) hint=%v want %v; got %q", tt.pipelineStr, tt.msg, hasHint, tt.wantHint, got) + } + + if tt.wantHint && !strings.Contains(got, hint) { + t.Fatalf("annotatePipelineError(%q, %q) = %q, want substring %q", tt.pipelineStr, tt.msg, got, hint) + } + }) + } +} From 7ae62fcbb8d3a6183c58d2e6ca3e2fec28a112f8 Mon Sep 17 00:00:00 2001 From: Steven Miller Date: Fri, 4 Sep 2026 11:41:23 -0400 Subject: [PATCH 2/2] Classify NVENC startup failures with CUDA (#20) ## summary - probe the CUDA driver immediately when GStreamer cannot create `nvh264enc` - distinguish GPU memory exhaustion, missing GPUs, unavailable driver symbols, and non-CUDA NVENC/plugin failures - load `libcuda.so.1` at runtime so non-GPU images retain no CUDA build dependency, while keeping initialized CUDA process state mapped safely - release the global pipeline lock before running the diagnostic so a slow CUDA driver cannot stall existing streams or unrelated pipelines ## testing - `go test ./...` - `go test -race ./pkg/gst -count=1` - `go build -o /tmp/neko ./cmd/neko` - validated dynamic probing against fake CUDA drivers returning success, `CUDA_ERROR_OUT_OF_MEMORY`, and `CUDA_ERROR_NO_DEVICE` - validated on a real GPU browser: healthy CUDA and NVENC pipeline creation passed; forced GPU-memory exhaustion produced GStreamer's missing-`nvh264enc` error and the detailed `CUDA_ERROR_OUT_OF_MEMORY (2)` diagnosis; the slow-probe lock regression passed --------- Co-authored-by: sjmiller609 <7516283+sjmiller609@users.noreply.github.com> --- server/pkg/gst/gst.c | 80 +++++++++++++++++++++++ server/pkg/gst/gst.go | 75 +++++++++++++++++++--- server/pkg/gst/gst.h | 1 + server/pkg/gst/gst_test.go | 127 ++++++++++++++++++++++++++++++++----- 4 files changed, 256 insertions(+), 27 deletions(-) diff --git a/server/pkg/gst/gst.c b/server/pkg/gst/gst.c index f1258a8bd..f1cc9a14a 100644 --- a/server/pkg/gst/gst.c +++ b/server/pkg/gst/gst.c @@ -1,5 +1,85 @@ #include "gst.h" +#include + +#define CUDA_ERROR_NO_DEVICE 100 + +typedef int CUdevice; +typedef void *CUcontext; +typedef int CUresult; + +typedef CUresult (*CuInit)(unsigned int flags); +typedef CUresult (*CuDeviceGetCount)(int *count); +typedef CUresult (*CuDeviceGet)(CUdevice *device, int ordinal); +typedef CUresult (*CuCtxCreate)(CUcontext *context, unsigned int flags, CUdevice device); +typedef CUresult (*CuCtxDestroy)(CUcontext context); +typedef CUresult (*CuGetErrorName)(CUresult error, const char **name); + +static int cuda_probe_result(void *library, CUresult code, const char *stage, + CuGetErrorName getErrorName, char **resultStage, char **errorName) { + const char *name = NULL; + if (getErrorName(code, &name) != 0 || name == NULL) { + name = "CUDA_ERROR_UNKNOWN"; + } + + *resultStage = g_strdup(stage); + *errorName = g_strdup(name); + dlclose(library); + return code; +} + +int gstreamer_cuda_context_probe(char **stage, char **errorName) { + void *library = dlopen("libcuda.so.1", RTLD_LAZY | RTLD_LOCAL | RTLD_NODELETE); + if (library == NULL) { + *stage = g_strdup("loading the CUDA driver"); + *errorName = g_strdup("CUDA_DRIVER_LIBRARY_UNAVAILABLE"); + return -1; + } + + CuInit cuInit = (CuInit)dlsym(library, "cuInit"); + CuDeviceGetCount cuDeviceGetCount = (CuDeviceGetCount)dlsym(library, "cuDeviceGetCount"); + CuDeviceGet cuDeviceGet = (CuDeviceGet)dlsym(library, "cuDeviceGet"); + CuCtxCreate cuCtxCreate = (CuCtxCreate)dlsym(library, "cuCtxCreate_v2"); + CuCtxDestroy cuCtxDestroy = (CuCtxDestroy)dlsym(library, "cuCtxDestroy_v2"); + CuGetErrorName cuGetErrorName = (CuGetErrorName)dlsym(library, "cuGetErrorName"); + if (cuInit == NULL || cuDeviceGetCount == NULL || cuDeviceGet == NULL || + cuCtxCreate == NULL || cuCtxDestroy == NULL || cuGetErrorName == NULL) { + *stage = g_strdup("resolving CUDA driver symbols"); + *errorName = g_strdup("CUDA_DRIVER_SYMBOL_UNAVAILABLE"); + dlclose(library); + return -2; + } + + CUresult result = cuInit(0); + if (result != 0) { + return cuda_probe_result(library, result, "initializing CUDA", cuGetErrorName, stage, errorName); + } + + int deviceCount = 0; + result = cuDeviceGetCount(&deviceCount); + if (result != 0) { + return cuda_probe_result(library, result, "querying CUDA devices", cuGetErrorName, stage, errorName); + } + if (deviceCount == 0) { + return cuda_probe_result(library, CUDA_ERROR_NO_DEVICE, "querying CUDA devices", cuGetErrorName, stage, errorName); + } + + CUdevice device; + result = cuDeviceGet(&device, 0); + if (result != 0) { + return cuda_probe_result(library, result, "selecting a CUDA device", cuGetErrorName, stage, errorName); + } + + CUcontext context; + result = cuCtxCreate(&context, 0, device); + if (result != 0) { + return cuda_probe_result(library, result, "creating a CUDA context", cuGetErrorName, stage, errorName); + } + + cuCtxDestroy(context); + return cuda_probe_result(library, 0, "creating a CUDA context", cuGetErrorName, stage, errorName); +} + static void gstreamer_pipeline_log(GstPipelineCtx *ctx, char* level, const char* format, ...) { va_list argptr; va_start(argptr, format); diff --git a/server/pkg/gst/gst.go b/server/pkg/gst/gst.go index 530341822..569dda641 100644 --- a/server/pkg/gst/gst.go +++ b/server/pkg/gst/gst.go @@ -2,6 +2,7 @@ package gst /* #cgo pkg-config: gstreamer-1.0 gstreamer-app-1.0 gstreamer-video-1.0 +#cgo LDFLAGS: -ldl #include "gst.h" */ @@ -60,20 +61,24 @@ type pipeline struct { } func CreatePipeline(pipelineStr string) (Pipeline, error) { + return createPipeline(pipelineStr, probeCUDAContext) +} + +func createPipeline(pipelineStr string, probe func() cudaProbeResult) (Pipeline, error) { id := atomic.AddInt32(&pSerial, 1) pipelineStrUnsafe := C.CString(pipelineStr) defer C.free(unsafe.Pointer(pipelineStrUnsafe)) pipelinesLock.Lock() - defer pipelinesLock.Unlock() var gstError *C.GError ctx := C.gstreamer_pipeline_create(pipelineStrUnsafe, C.int(id), &gstError) if gstError != nil { + pipelinesLock.Unlock() defer C.g_error_free(gstError) - msg := annotatePipelineError(pipelineStr, C.GoString(gstError.message)) + msg := annotatePipelineError(pipelineStr, C.GoString(gstError.message), probe) return nil, fmt.Errorf("(pipeline error) %s", msg) } @@ -89,24 +94,74 @@ func CreatePipeline(pipelineStr string) (Pipeline, error) { } pipelines[p.id] = p + pipelinesLock.Unlock() return p, nil } -func annotatePipelineError(pipelineStr, msg string) string { - lowerMsg := strings.ToLower(msg) - if !strings.Contains(pipelineStr, "nvh264enc") { +const ( + cudaDriverLibraryUnavailable = -1 + cudaDriverSymbolUnavailable = -2 + cudaSuccess = 0 + cudaErrorOutOfMemory = 2 + cudaErrorNoDevice = 100 +) + +type cudaProbeResult struct { + code int + stage string + name string +} + +func annotatePipelineError(pipelineStr, msg string, probe func() cudaProbeResult) string { + if !isMissingNVENCElementError(pipelineStr, msg) { return msg } - if !strings.Contains(lowerMsg, "nvh264enc") { - return msg + return fmt.Sprintf("%s (%s)", msg, nvencFailureDetail(probe())) +} + +func isMissingNVENCElementError(pipelineStr, msg string) bool { + lowerMsg := strings.ToLower(msg) + return strings.Contains(pipelineStr, "nvh264enc") && + strings.Contains(lowerMsg, "nvh264enc") && + (strings.Contains(lowerMsg, "no element") || strings.Contains(lowerMsg, "no such element or plugin")) +} + +func probeCUDAContext() cudaProbeResult { + var stage, name *C.char + code := int(C.gstreamer_cuda_context_probe(&stage, &name)) + + if stage != nil { + defer C.g_free(C.gpointer(stage)) + } + if name != nil { + defer C.g_free(C.gpointer(name)) } - if !strings.Contains(lowerMsg, "no element") && !strings.Contains(lowerMsg, "no such element or plugin") { - return msg + return cudaProbeResult{ + code: code, + stage: C.GoString(stage), + name: C.GoString(name), } +} - return msg + " (live view could not initialize NVENC/CUDA; on GPU browsers this usually means GPU memory is exhausted by replay or browser GPU load. Reduce the browser resolution or stop replay, then try live view again.)" +func nvencFailureDetail(probe cudaProbeResult) string { + const prefix = "live view could not initialize NVENC/CUDA" + + switch probe.code { + case cudaDriverLibraryUnavailable: + return prefix + ": the CUDA driver library is unavailable" + case cudaDriverSymbolUnavailable: + return prefix + ": required CUDA driver symbols are unavailable" + case cudaSuccess: + return prefix + ": the CUDA context probe succeeded; possible causes are failed GStreamer nvcodec registration, an unavailable NVIDIA encode library, a driver or capability mismatch, or exhausted NVENC sessions" + case cudaErrorOutOfMemory: + return fmt.Sprintf("%s: CUDA reported %s (%d) while %s. GPU memory is exhausted; reduce browser resolution or stop replay/browser GPU load, then restart Neko before retrying live view", prefix, probe.name, probe.code, probe.stage) + case cudaErrorNoDevice: + return fmt.Sprintf("%s: CUDA reported %s (%d) while %s; no CUDA-capable GPU is available to Neko", prefix, probe.name, probe.code, probe.stage) + default: + return fmt.Sprintf("%s: CUDA reported %s (%d) while %s", prefix, probe.name, probe.code, probe.stage) + } } func (p *pipeline) Src() string { diff --git a/server/pkg/gst/gst.h b/server/pkg/gst/gst.h index bdbd03472..4636166ab 100644 --- a/server/pkg/gst/gst.h +++ b/server/pkg/gst/gst.h @@ -38,3 +38,4 @@ gboolean gstreamer_pipeline_set_prop_int(GstPipelineCtx *ctx, char *binName, cha gboolean gstreamer_pipeline_set_caps_framerate(GstPipelineCtx *ctx, const gchar* binName, gint numerator, gint denominator); gboolean gstreamer_pipeline_set_caps_resolution(GstPipelineCtx *ctx, const gchar* binName, gint width, gint height); gboolean gstreamer_pipeline_emit_video_keyframe(GstPipelineCtx *ctx); +int gstreamer_cuda_context_probe(char **stage, char **errorName); diff --git a/server/pkg/gst/gst_test.go b/server/pkg/gst/gst_test.go index 6e16799c6..52ac0bf2b 100644 --- a/server/pkg/gst/gst_test.go +++ b/server/pkg/gst/gst_test.go @@ -3,36 +3,39 @@ package gst import ( "strings" "testing" + "time" ) -func TestAnnotatePipelineError(t *testing.T) { +func TestIsMissingNVENCElementError(t *testing.T) { t.Parallel() - const hint = "live view could not initialize NVENC/CUDA" - tests := []struct { name string pipelineStr string msg string - wantHint bool + want bool }{ { - name: "adds hint for missing nvh264enc in gpu pipeline", + name: "missing nvh264enc in gpu pipeline", pipelineStr: "ximagesrc ! cudaupload ! nvh264enc name=encoder ! appsink name=appsink", msg: `no element "nvh264enc"`, - wantHint: true, + want: true, }, { - name: "adds hint for plugin wording", + name: "alternate plugin wording", pipelineStr: "ximagesrc ! cudaupload ! nvh264enc name=encoder ! appsink name=appsink", msg: "No such element or plugin 'nvh264enc'", - wantHint: true, + want: true, }, { - name: "leaves unrelated encoder errors alone", + name: "unrelated encoder", pipelineStr: "ximagesrc ! x264enc name=encoder ! appsink name=appsink", msg: `no element "x264enc"`, - wantHint: false, + }, + { + name: "other nvh264enc error", + pipelineStr: "ximagesrc ! cudaupload ! nvh264enc name=encoder ! appsink name=appsink", + msg: "could not link cudaupload to nvh264enc", }, } @@ -41,15 +44,105 @@ func TestAnnotatePipelineError(t *testing.T) { t.Run(tt.name, func(t *testing.T) { t.Parallel() - got := annotatePipelineError(tt.pipelineStr, tt.msg) - hasHint := got != tt.msg - - if hasHint != tt.wantHint { - t.Fatalf("annotatePipelineError(%q, %q) hint=%v want %v; got %q", tt.pipelineStr, tt.msg, hasHint, tt.wantHint, got) + if got := isMissingNVENCElementError(tt.pipelineStr, tt.msg); got != tt.want { + t.Fatalf("isMissingNVENCElementError(%q, %q) = %v, want %v", tt.pipelineStr, tt.msg, got, tt.want) } + }) + } +} + +func TestCreatePipelineReleasesLockBeforeSlowCUDAProbe(t *testing.T) { + probeStarted := make(chan struct{}) + releaseProbe := make(chan struct{}) + pipelineDone := make(chan error, 1) + + go func() { + _, err := createPipeline("nvh264enc_missing", func() cudaProbeResult { + close(probeStarted) + <-releaseProbe + return cudaProbeResult{code: cudaErrorOutOfMemory, stage: "creating a CUDA context", name: "CUDA_ERROR_OUT_OF_MEMORY"} + }) + pipelineDone <- err + }() + + select { + case <-probeStarted: + case err := <-pipelineDone: + t.Fatalf("pipeline creation returned before running probe: %v", err) + case <-time.After(time.Second): + t.Fatal("timed out waiting for CUDA probe") + } + + lockAcquired := make(chan struct{}) + go func() { + pipelinesLock.Lock() + pipelinesLock.Unlock() + close(lockAcquired) + }() + + select { + case <-lockAcquired: + close(releaseProbe) + case <-time.After(time.Second): + close(releaseProbe) + <-pipelineDone + <-lockAcquired + t.Fatal("pipeline lock remained held during CUDA probe") + } + + if err := <-pipelineDone; err == nil { + t.Fatal("createPipeline() error = nil, want missing element error") + } +} + +func TestNVENCFailureDetail(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + probe cudaProbeResult + want string + }{ + { + name: "driver library unavailable", + probe: cudaProbeResult{code: cudaDriverLibraryUnavailable}, + want: "CUDA driver library is unavailable", + }, + { + name: "driver symbols unavailable", + probe: cudaProbeResult{code: cudaDriverSymbolUnavailable}, + want: "required CUDA driver symbols are unavailable", + }, + { + name: "cuda succeeds", + probe: cudaProbeResult{code: cudaSuccess}, + want: "CUDA context probe succeeded", + }, + { + name: "out of memory", + probe: cudaProbeResult{code: cudaErrorOutOfMemory, stage: "creating a CUDA context", name: "CUDA_ERROR_OUT_OF_MEMORY"}, + want: "CUDA_ERROR_OUT_OF_MEMORY (2) while creating a CUDA context. GPU memory is exhausted", + }, + { + name: "no device", + probe: cudaProbeResult{code: cudaErrorNoDevice, stage: "querying CUDA devices", name: "CUDA_ERROR_NO_DEVICE"}, + want: "CUDA_ERROR_NO_DEVICE (100) while querying CUDA devices; no CUDA-capable GPU is available", + }, + { + name: "other cuda failure", + probe: cudaProbeResult{code: 999, stage: "initializing CUDA", name: "CUDA_ERROR_UNKNOWN"}, + want: "CUDA_ERROR_UNKNOWN (999) while initializing CUDA", + }, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + t.Parallel() - if tt.wantHint && !strings.Contains(got, hint) { - t.Fatalf("annotatePipelineError(%q, %q) = %q, want substring %q", tt.pipelineStr, tt.msg, got, hint) + got := nvencFailureDetail(tt.probe) + if !strings.Contains(got, tt.want) { + t.Fatalf("nvencFailureDetail() = %q, want substring %q", got, tt.want) } }) }