diff --git a/server/pkg/gst/gst.c b/server/pkg/gst/gst.c index f1258a8bd..f1cc9a14a 100644 --- a/server/pkg/gst/gst.c +++ b/server/pkg/gst/gst.c @@ -1,5 +1,85 @@ #include "gst.h" +#include + +#define CUDA_ERROR_NO_DEVICE 100 + +typedef int CUdevice; +typedef void *CUcontext; +typedef int CUresult; + +typedef CUresult (*CuInit)(unsigned int flags); +typedef CUresult (*CuDeviceGetCount)(int *count); +typedef CUresult (*CuDeviceGet)(CUdevice *device, int ordinal); +typedef CUresult (*CuCtxCreate)(CUcontext *context, unsigned int flags, CUdevice device); +typedef CUresult (*CuCtxDestroy)(CUcontext context); +typedef CUresult (*CuGetErrorName)(CUresult error, const char **name); + +static int cuda_probe_result(void *library, CUresult code, const char *stage, + CuGetErrorName getErrorName, char **resultStage, char **errorName) { + const char *name = NULL; + if (getErrorName(code, &name) != 0 || name == NULL) { + name = "CUDA_ERROR_UNKNOWN"; + } + + *resultStage = g_strdup(stage); + *errorName = g_strdup(name); + dlclose(library); + return code; +} + +int gstreamer_cuda_context_probe(char **stage, char **errorName) { + void *library = dlopen("libcuda.so.1", RTLD_LAZY | RTLD_LOCAL | RTLD_NODELETE); + if (library == NULL) { + *stage = g_strdup("loading the CUDA driver"); + *errorName = g_strdup("CUDA_DRIVER_LIBRARY_UNAVAILABLE"); + return -1; + } + + CuInit cuInit = (CuInit)dlsym(library, "cuInit"); + CuDeviceGetCount cuDeviceGetCount = (CuDeviceGetCount)dlsym(library, "cuDeviceGetCount"); + CuDeviceGet cuDeviceGet = (CuDeviceGet)dlsym(library, "cuDeviceGet"); + CuCtxCreate cuCtxCreate = (CuCtxCreate)dlsym(library, "cuCtxCreate_v2"); + CuCtxDestroy cuCtxDestroy = (CuCtxDestroy)dlsym(library, "cuCtxDestroy_v2"); + CuGetErrorName cuGetErrorName = (CuGetErrorName)dlsym(library, "cuGetErrorName"); + if (cuInit == NULL || cuDeviceGetCount == NULL || cuDeviceGet == NULL || + cuCtxCreate == NULL || cuCtxDestroy == NULL || cuGetErrorName == NULL) { + *stage = g_strdup("resolving CUDA driver symbols"); + *errorName = g_strdup("CUDA_DRIVER_SYMBOL_UNAVAILABLE"); + dlclose(library); + return -2; + } + + CUresult result = cuInit(0); + if (result != 0) { + return cuda_probe_result(library, result, "initializing CUDA", cuGetErrorName, stage, errorName); + } + + int deviceCount = 0; + result = cuDeviceGetCount(&deviceCount); + if (result != 0) { + return cuda_probe_result(library, result, "querying CUDA devices", cuGetErrorName, stage, errorName); + } + if (deviceCount == 0) { + return cuda_probe_result(library, CUDA_ERROR_NO_DEVICE, "querying CUDA devices", cuGetErrorName, stage, errorName); + } + + CUdevice device; + result = cuDeviceGet(&device, 0); + if (result != 0) { + return cuda_probe_result(library, result, "selecting a CUDA device", cuGetErrorName, stage, errorName); + } + + CUcontext context; + result = cuCtxCreate(&context, 0, device); + if (result != 0) { + return cuda_probe_result(library, result, "creating a CUDA context", cuGetErrorName, stage, errorName); + } + + cuCtxDestroy(context); + return cuda_probe_result(library, 0, "creating a CUDA context", cuGetErrorName, stage, errorName); +} + static void gstreamer_pipeline_log(GstPipelineCtx *ctx, char* level, const char* format, ...) { va_list argptr; va_start(argptr, format); diff --git a/server/pkg/gst/gst.go b/server/pkg/gst/gst.go index a9aa67092..569dda641 100644 --- a/server/pkg/gst/gst.go +++ b/server/pkg/gst/gst.go @@ -2,12 +2,14 @@ package gst /* #cgo pkg-config: gstreamer-1.0 gstreamer-app-1.0 gstreamer-video-1.0 +#cgo LDFLAGS: -ldl #include "gst.h" */ import "C" import ( "fmt" + "strings" "sync" "sync/atomic" "time" @@ -59,20 +61,25 @@ type pipeline struct { } func CreatePipeline(pipelineStr string) (Pipeline, error) { + return createPipeline(pipelineStr, probeCUDAContext) +} + +func createPipeline(pipelineStr string, probe func() cudaProbeResult) (Pipeline, error) { id := atomic.AddInt32(&pSerial, 1) pipelineStrUnsafe := C.CString(pipelineStr) defer C.free(unsafe.Pointer(pipelineStrUnsafe)) pipelinesLock.Lock() - defer pipelinesLock.Unlock() var gstError *C.GError ctx := C.gstreamer_pipeline_create(pipelineStrUnsafe, C.int(id), &gstError) if gstError != nil { + pipelinesLock.Unlock() defer C.g_error_free(gstError) - return nil, fmt.Errorf("(pipeline error) %s", C.GoString(gstError.message)) + msg := annotatePipelineError(pipelineStr, C.GoString(gstError.message), probe) + return nil, fmt.Errorf("(pipeline error) %s", msg) } p := &pipeline{ @@ -87,9 +94,76 @@ func CreatePipeline(pipelineStr string) (Pipeline, error) { } pipelines[p.id] = p + pipelinesLock.Unlock() return p, nil } +const ( + cudaDriverLibraryUnavailable = -1 + cudaDriverSymbolUnavailable = -2 + cudaSuccess = 0 + cudaErrorOutOfMemory = 2 + cudaErrorNoDevice = 100 +) + +type cudaProbeResult struct { + code int + stage string + name string +} + +func annotatePipelineError(pipelineStr, msg string, probe func() cudaProbeResult) string { + if !isMissingNVENCElementError(pipelineStr, msg) { + return msg + } + + return fmt.Sprintf("%s (%s)", msg, nvencFailureDetail(probe())) +} + +func isMissingNVENCElementError(pipelineStr, msg string) bool { + lowerMsg := strings.ToLower(msg) + return strings.Contains(pipelineStr, "nvh264enc") && + strings.Contains(lowerMsg, "nvh264enc") && + (strings.Contains(lowerMsg, "no element") || strings.Contains(lowerMsg, "no such element or plugin")) +} + +func probeCUDAContext() cudaProbeResult { + var stage, name *C.char + code := int(C.gstreamer_cuda_context_probe(&stage, &name)) + + if stage != nil { + defer C.g_free(C.gpointer(stage)) + } + if name != nil { + defer C.g_free(C.gpointer(name)) + } + + return cudaProbeResult{ + code: code, + stage: C.GoString(stage), + name: C.GoString(name), + } +} + +func nvencFailureDetail(probe cudaProbeResult) string { + const prefix = "live view could not initialize NVENC/CUDA" + + switch probe.code { + case cudaDriverLibraryUnavailable: + return prefix + ": the CUDA driver library is unavailable" + case cudaDriverSymbolUnavailable: + return prefix + ": required CUDA driver symbols are unavailable" + case cudaSuccess: + return prefix + ": the CUDA context probe succeeded; possible causes are failed GStreamer nvcodec registration, an unavailable NVIDIA encode library, a driver or capability mismatch, or exhausted NVENC sessions" + case cudaErrorOutOfMemory: + return fmt.Sprintf("%s: CUDA reported %s (%d) while %s. GPU memory is exhausted; reduce browser resolution or stop replay/browser GPU load, then restart Neko before retrying live view", prefix, probe.name, probe.code, probe.stage) + case cudaErrorNoDevice: + return fmt.Sprintf("%s: CUDA reported %s (%d) while %s; no CUDA-capable GPU is available to Neko", prefix, probe.name, probe.code, probe.stage) + default: + return fmt.Sprintf("%s: CUDA reported %s (%d) while %s", prefix, probe.name, probe.code, probe.stage) + } +} + func (p *pipeline) Src() string { return p.src } diff --git a/server/pkg/gst/gst.h b/server/pkg/gst/gst.h index bdbd03472..4636166ab 100644 --- a/server/pkg/gst/gst.h +++ b/server/pkg/gst/gst.h @@ -38,3 +38,4 @@ gboolean gstreamer_pipeline_set_prop_int(GstPipelineCtx *ctx, char *binName, cha gboolean gstreamer_pipeline_set_caps_framerate(GstPipelineCtx *ctx, const gchar* binName, gint numerator, gint denominator); gboolean gstreamer_pipeline_set_caps_resolution(GstPipelineCtx *ctx, const gchar* binName, gint width, gint height); gboolean gstreamer_pipeline_emit_video_keyframe(GstPipelineCtx *ctx); +int gstreamer_cuda_context_probe(char **stage, char **errorName); diff --git a/server/pkg/gst/gst_test.go b/server/pkg/gst/gst_test.go new file mode 100644 index 000000000..52ac0bf2b --- /dev/null +++ b/server/pkg/gst/gst_test.go @@ -0,0 +1,149 @@ +package gst + +import ( + "strings" + "testing" + "time" +) + +func TestIsMissingNVENCElementError(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + pipelineStr string + msg string + want bool + }{ + { + name: "missing nvh264enc in gpu pipeline", + pipelineStr: "ximagesrc ! cudaupload ! nvh264enc name=encoder ! appsink name=appsink", + msg: `no element "nvh264enc"`, + want: true, + }, + { + name: "alternate plugin wording", + pipelineStr: "ximagesrc ! cudaupload ! nvh264enc name=encoder ! appsink name=appsink", + msg: "No such element or plugin 'nvh264enc'", + want: true, + }, + { + name: "unrelated encoder", + pipelineStr: "ximagesrc ! x264enc name=encoder ! appsink name=appsink", + msg: `no element "x264enc"`, + }, + { + name: "other nvh264enc error", + pipelineStr: "ximagesrc ! cudaupload ! nvh264enc name=encoder ! appsink name=appsink", + msg: "could not link cudaupload to nvh264enc", + }, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + if got := isMissingNVENCElementError(tt.pipelineStr, tt.msg); got != tt.want { + t.Fatalf("isMissingNVENCElementError(%q, %q) = %v, want %v", tt.pipelineStr, tt.msg, got, tt.want) + } + }) + } +} + +func TestCreatePipelineReleasesLockBeforeSlowCUDAProbe(t *testing.T) { + probeStarted := make(chan struct{}) + releaseProbe := make(chan struct{}) + pipelineDone := make(chan error, 1) + + go func() { + _, err := createPipeline("nvh264enc_missing", func() cudaProbeResult { + close(probeStarted) + <-releaseProbe + return cudaProbeResult{code: cudaErrorOutOfMemory, stage: "creating a CUDA context", name: "CUDA_ERROR_OUT_OF_MEMORY"} + }) + pipelineDone <- err + }() + + select { + case <-probeStarted: + case err := <-pipelineDone: + t.Fatalf("pipeline creation returned before running probe: %v", err) + case <-time.After(time.Second): + t.Fatal("timed out waiting for CUDA probe") + } + + lockAcquired := make(chan struct{}) + go func() { + pipelinesLock.Lock() + pipelinesLock.Unlock() + close(lockAcquired) + }() + + select { + case <-lockAcquired: + close(releaseProbe) + case <-time.After(time.Second): + close(releaseProbe) + <-pipelineDone + <-lockAcquired + t.Fatal("pipeline lock remained held during CUDA probe") + } + + if err := <-pipelineDone; err == nil { + t.Fatal("createPipeline() error = nil, want missing element error") + } +} + +func TestNVENCFailureDetail(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + probe cudaProbeResult + want string + }{ + { + name: "driver library unavailable", + probe: cudaProbeResult{code: cudaDriverLibraryUnavailable}, + want: "CUDA driver library is unavailable", + }, + { + name: "driver symbols unavailable", + probe: cudaProbeResult{code: cudaDriverSymbolUnavailable}, + want: "required CUDA driver symbols are unavailable", + }, + { + name: "cuda succeeds", + probe: cudaProbeResult{code: cudaSuccess}, + want: "CUDA context probe succeeded", + }, + { + name: "out of memory", + probe: cudaProbeResult{code: cudaErrorOutOfMemory, stage: "creating a CUDA context", name: "CUDA_ERROR_OUT_OF_MEMORY"}, + want: "CUDA_ERROR_OUT_OF_MEMORY (2) while creating a CUDA context. GPU memory is exhausted", + }, + { + name: "no device", + probe: cudaProbeResult{code: cudaErrorNoDevice, stage: "querying CUDA devices", name: "CUDA_ERROR_NO_DEVICE"}, + want: "CUDA_ERROR_NO_DEVICE (100) while querying CUDA devices; no CUDA-capable GPU is available", + }, + { + name: "other cuda failure", + probe: cudaProbeResult{code: 999, stage: "initializing CUDA", name: "CUDA_ERROR_UNKNOWN"}, + want: "CUDA_ERROR_UNKNOWN (999) while initializing CUDA", + }, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + got := nvencFailureDetail(tt.probe) + if !strings.Contains(got, tt.want) { + t.Fatalf("nvencFailureDetail() = %q, want substring %q", got, tt.want) + } + }) + } +}