From 77110c001e06fcce06763144150ff90ac58f5264 Mon Sep 17 00:00:00 2001 From: Bharat Kathi Date: Mon, 31 Aug 2026 12:58:12 -0700 Subject: [PATCH 1/3] feat(core): add postgres query observability --- core/api/api.go | 2 + core/api/metrics.go | 31 +++++++++++ core/config/config.go | 3 ++ core/database/db.go | 21 +++++++- core/go.mod | 23 +++++--- core/go.sum | 52 ++++++++++++------ core/observability/gorm_logger.go | 75 ++++++++++++++++++++++++++ core/observability/gorm_logger_test.go | 18 +++++++ core/observability/metrics.go | 62 +++++++++++++++++++++ docker-compose.yml | 10 ++++ 10 files changed, 272 insertions(+), 25 deletions(-) create mode 100644 core/api/metrics.go create mode 100644 core/observability/gorm_logger.go create mode 100644 core/observability/gorm_logger_test.go create mode 100644 core/observability/metrics.go diff --git a/core/api/api.go b/core/api/api.go index 87187924..cdd1cdd6 100644 --- a/core/api/api.go +++ b/core/api/api.go @@ -35,11 +35,13 @@ func InitializeRouter() *gin.Engine { })) r.Use(AuthChecker()) r.Use(UnauthorizedPanicHandler()) + r.Use(AnalyticsMetrics()) return r } func InitializeRoutes(router *gin.Engine) { router.GET("/core/ping", Ping) + router.GET("/core/metrics", Metrics) router.GET("/core/keys", JWKS) router.POST("/core/token", GenerateToken) router.POST("/core/token/validate", ValidateToken) diff --git a/core/api/metrics.go b/core/api/metrics.go new file mode 100644 index 00000000..a88e049b --- /dev/null +++ b/core/api/metrics.go @@ -0,0 +1,31 @@ +package api + +import ( + "strings" + "time" + + "github.com/gaucho-racing/sentinel/core/observability" + "github.com/gin-gonic/gin" + "github.com/prometheus/client_golang/prometheus/promhttp" +) + +var prometheusHandler = promhttp.Handler() + +func AnalyticsMetrics() gin.HandlerFunc { + return func(c *gin.Context) { + if !strings.HasPrefix(c.Request.URL.Path, "/analytics/") { + c.Next() + return + } + started := time.Now() + defer func() { + observability.ObserveAnalyticsRequest(c.FullPath(), c.Writer.Status(), time.Since(started)) + }() + c.Next() + } +} + +func Metrics(c *gin.Context) { + Require(c, RequestTokenHasInternalAccess(c)) + prometheusHandler.ServeHTTP(c.Writer, c.Request) +} diff --git a/core/config/config.go b/core/config/config.go index d7347903..d141400e 100644 --- a/core/config/config.go +++ b/core/config/config.go @@ -3,6 +3,7 @@ package config import ( "crypto/rsa" "os" + "strings" "time" ) @@ -40,6 +41,8 @@ var DatabasePort = os.Getenv("DATABASE_PORT") var DatabaseUser = os.Getenv("DATABASE_USER") var DatabasePassword = os.Getenv("DATABASE_PASSWORD") var DatabaseName = os.Getenv("DATABASE_NAME") +var DatabaseSlowQueryThreshold = parseDurationOr("DATABASE_SLOW_QUERY_THRESHOLD", 200*time.Millisecond) +var DatabaseEnableQueryStatistics = strings.EqualFold(os.Getenv("DATABASE_ENABLE_QUERY_STATISTICS"), "true") // ConditionalSyncInterval is how often the periodic conditional-group // reconcile cron fires. Event-driven sync (member add/remove triggers, diff --git a/core/database/db.go b/core/database/db.go index 72940a11..b728153f 100644 --- a/core/database/db.go +++ b/core/database/db.go @@ -6,6 +6,7 @@ import ( "github.com/gaucho-racing/sentinel/core/config" "github.com/gaucho-racing/sentinel/core/model" + "github.com/gaucho-racing/sentinel/core/observability" "github.com/gaucho-racing/sentinel/core/pkg/logger" "gorm.io/driver/postgres" "gorm.io/gorm" @@ -17,7 +18,9 @@ var dbRetries = 0 func Init() { dsn := fmt.Sprintf("host=%s user=%s password=%s dbname=%s port=%s sslmode=disable TimeZone=UTC", config.DatabaseHost, config.DatabaseUser, config.DatabasePassword, config.DatabaseName, config.DatabasePort) - db, err := gorm.Open(postgres.Open(dsn), &gorm.Config{}) + db, err := gorm.Open(postgres.Open(dsn), &gorm.Config{ + Logger: observability.NewDatabaseLogger(config.DatabaseSlowQueryThreshold), + }) if err != nil { if dbRetries < 5 { dbRetries++ @@ -29,6 +32,9 @@ func Init() { } } else { logger.SugarLogger.Infoln("Connected to database") + if config.DatabaseEnableQueryStatistics { + enableQueryStatistics(db) + } db.AutoMigrate( &model.Entity{}, &model.EntityEmail{}, @@ -56,3 +62,16 @@ func Init() { DB = db } } + +func enableQueryStatistics(db *gorm.DB) { + if err := db.Exec("CREATE EXTENSION IF NOT EXISTS pg_stat_statements").Error; err != nil { + logger.SugarLogger.Warnf("Failed to enable pg_stat_statements: %v", err) + return + } + var available int + if err := db.Raw("SELECT 1 FROM pg_stat_statements LIMIT 1").Scan(&available).Error; err != nil { + logger.SugarLogger.Warnf("pg_stat_statements is installed but unavailable: %v", err) + return + } + logger.SugarLogger.Infoln("pg_stat_statements is enabled") +} diff --git a/core/go.mod b/core/go.mod index e2293417..7f3a2858 100644 --- a/core/go.mod +++ b/core/go.mod @@ -8,15 +8,18 @@ require ( github.com/gin-contrib/cors v1.7.6 github.com/gin-gonic/gin v1.11.0 github.com/golang-jwt/jwt/v5 v5.3.1 + github.com/prometheus/client_golang v1.24.1 go.uber.org/zap v1.27.1 - golang.org/x/crypto v0.41.0 + golang.org/x/crypto v0.54.0 gorm.io/driver/postgres v1.6.0 gorm.io/gorm v1.31.1 ) require ( + github.com/beorn7/perks v1.0.1 // indirect github.com/bytedance/sonic v1.14.0 // indirect github.com/bytedance/sonic/loader v0.3.0 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/cloudwego/base64x v0.1.6 // indirect github.com/gabriel-vasile/mimetype v1.4.9 // indirect github.com/gin-contrib/sse v1.1.0 // indirect @@ -38,7 +41,11 @@ require ( github.com/mattn/go-isatty v0.0.20 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/pelletier/go-toml/v2 v2.2.4 // indirect + github.com/prometheus/client_model v0.6.2 // indirect + github.com/prometheus/common v0.70.1 // indirect + github.com/prometheus/procfs v0.21.1 // indirect github.com/quic-go/qpack v0.5.1 // indirect github.com/quic-go/quic-go v0.54.0 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect @@ -46,11 +53,11 @@ require ( go.uber.org/mock v0.5.0 // indirect go.uber.org/multierr v1.10.0 // indirect golang.org/x/arch v0.20.0 // indirect - golang.org/x/mod v0.26.0 // indirect - golang.org/x/net v0.43.0 // indirect - golang.org/x/sync v0.16.0 // indirect - golang.org/x/sys v0.35.0 // indirect - golang.org/x/text v0.28.0 // indirect - golang.org/x/tools v0.35.0 // indirect - google.golang.org/protobuf v1.36.9 // indirect + golang.org/x/mod v0.37.0 // indirect + golang.org/x/net v0.57.0 // indirect + golang.org/x/sync v0.22.0 // indirect + golang.org/x/sys v0.47.0 // indirect + golang.org/x/text v0.40.0 // indirect + golang.org/x/tools v0.47.0 // indirect + google.golang.org/protobuf v1.36.11 // indirect ) diff --git a/core/go.sum b/core/go.sum index 3f45b54b..d7202cd2 100644 --- a/core/go.sum +++ b/core/go.sum @@ -1,7 +1,11 @@ +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bytedance/sonic v1.14.0 h1:/OfKt8HFw0kh2rj8N0F6C/qPGRESq0BbaNZgcNXXzQQ= github.com/bytedance/sonic v1.14.0/go.mod h1:WoEbx8WTcFJfzCe0hbmyTGrfjt8PzNEBdxlNUO24NhA= github.com/bytedance/sonic/loader v0.3.0 h1:dskwH8edlzNMctoruo8FPTJDF3vLtDT0sXZwvZJyqeA= github.com/bytedance/sonic/loader v0.3.0/go.mod h1:N8A3vUdtUebEY2/VQC0MyhYeKUFosQU6FxH2JmUe6VI= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cloudwego/base64x v0.1.6 h1:t11wG9AECkCDk5fMSoxmufanudBtJ+/HemLstXDLI2M= github.com/cloudwego/base64x v0.1.6/go.mod h1:OFcloc187FXDaYHvrNIjxSe8ncn0OOM8gEHfghB2IPU= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -50,8 +54,12 @@ github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/klauspost/compress v1.19.1 h1:VsB4HPswih7mmZ8WleSFQ75c/Ui1M4trX5oAsJnhSlk= +github.com/klauspost/compress v1.19.1/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y= github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= @@ -64,10 +72,20 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4= github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v1.24.1 h1:JnJkREXzWxUdCuPFpIWZiPispT9xVV59uiuyR2bPlnU= +github.com/prometheus/client_golang v1.24.1/go.mod h1:F+oSRECHg4sse5ucfYpYDeIv/hu68Zo0uoHKetWnzcE= +github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= +github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= +github.com/prometheus/common v0.70.1 h1:1HvjP4D5oL3t8RsPlwxA9onvvStjtIHYE5XuuwOi/PY= +github.com/prometheus/common v0.70.1/go.mod h1:VdFUQDMZK3VLkurFUVhia6uys/0suUp86TJz5qbJRhc= +github.com/prometheus/procfs v0.21.1 h1:GljZCt+zSTS+NZq88cyQ1LjZ+RCHp3uVuabBWA5+OJI= +github.com/prometheus/procfs v0.21.1/go.mod h1:aB55Cww9pdSJVHk0hUf0inxWyyjPogFIjmHKYgMKmtY= github.com/quic-go/qpack v0.5.1 h1:giqksBPnT/HDtZ6VhtFKgoLOWmlyo9Ei6u9PqzIMbhI= github.com/quic-go/qpack v0.5.1/go.mod h1:+PC4XFrEskIVkcLzpEkbLqq1uCoxPhQuvK5rH1ZgaEg= github.com/quic-go/quic-go v0.54.0 h1:6s1YB9QotYI6Ospeiguknbp2Znb/jZYjZLRXn9kMQBg= @@ -94,26 +112,28 @@ go.uber.org/multierr v1.10.0 h1:S0h4aNzvfcFsC3dRF1jLoaov7oRaKqRGC/pUEJ2yvPQ= go.uber.org/multierr v1.10.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.27.1 h1:08RqriUEv8+ArZRYSTXy1LeBScaMpVSTBhCeaZYfMYc= go.uber.org/zap v1.27.1/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ= +go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ= golang.org/x/arch v0.20.0 h1:dx1zTU0MAE98U+TQ8BLl7XsJbgze2WnNKF/8tGp/Q6c= golang.org/x/arch v0.20.0/go.mod h1:bdwinDaKcfZUGpH09BB7ZmOfhalA8lQdzl62l8gGWsk= -golang.org/x/crypto v0.41.0 h1:WKYxWedPGCTVVl5+WHSSrOBT0O8lx32+zxmHxijgXp4= -golang.org/x/crypto v0.41.0/go.mod h1:pO5AFd7FA68rFak7rOAGVuygIISepHftHnr8dr6+sUc= -golang.org/x/mod v0.26.0 h1:EGMPT//Ezu+ylkCijjPc+f4Aih7sZvaAr+O3EHBxvZg= -golang.org/x/mod v0.26.0/go.mod h1:/j6NAhSk8iQ723BGAUyoAcn7SlD7s15Dp9Nd/SfeaFQ= -golang.org/x/net v0.43.0 h1:lat02VYK2j4aLzMzecihNvTlJNQUq316m2Mr9rnM6YE= -golang.org/x/net v0.43.0/go.mod h1:vhO1fvI4dGsIjh73sWfUVjj3N7CA9WkKJNQm2svM6Jg= -golang.org/x/sync v0.16.0 h1:ycBJEhp9p4vXvUZNszeOq0kGTPghopOL8q0fq3vstxw= -golang.org/x/sync v0.16.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/crypto v0.54.0 h1:YLIA59K4fiNzHzjnZt2tUJQjQtUWfWbeHBqKtk3eScw= +golang.org/x/crypto v0.54.0/go.mod h1:KWL8ny2AZdGR2cWmzeHrp2azQPGogOv+HeQaVEXC2dk= +golang.org/x/mod v0.37.0 h1:vF1DjpVEshcIqoEaauuHebaLk1O1forxjxBaVn884JQ= +golang.org/x/mod v0.37.0/go.mod h1:m8S8VeM9r4dzDwjrKO0a1sZP3YjeMamRRlD+fmR2Q/0= +golang.org/x/net v0.57.0 h1:K5+3DljvIuDG9/Jv9rvyMywYNFCQ9RSUY6OOTTkT+tE= +golang.org/x/net v0.57.0/go.mod h1:KpXc8iv+r3XplLAG/f7Jsf9RPszJzdR0f58q9vGOuEU= +golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek= +golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.35.0 h1:vz1N37gP5bs89s7He8XuIYXpyY0+QlsKmzipCbUtyxI= -golang.org/x/sys v0.35.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= -golang.org/x/text v0.28.0 h1:rhazDwis8INMIwQ4tpjLDzUhx6RlXqZNPEM0huQojng= -golang.org/x/text v0.28.0/go.mod h1:U8nCwOR8jO/marOQ0QbDiOngZVEBB7MAiitBuMjXiNU= -golang.org/x/tools v0.35.0 h1:mBffYraMEf7aa0sB+NuKnuCy8qI/9Bughn8dC2Gu5r0= -golang.org/x/tools v0.35.0/go.mod h1:NKdj5HkL/73byiZSJjqJgKn3ep7KjFkBOkR/Hps3VPw= -google.golang.org/protobuf v1.36.9 h1:w2gp2mA27hUeUzj9Ex9FBjsBm40zfaDtEWow293U7Iw= -google.golang.org/protobuf v1.36.9/go.mod h1:fuxRtAxBytpl4zzqUh6/eyUujkJdNiuEkXntxiD/uRU= +golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= +golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/text v0.40.0 h1:Ub2Z6/xjgF1WrYQz2nuITOEegKFtiIy+rieRJ5lHZKs= +golang.org/x/text v0.40.0/go.mod h1:hpnzDAfGV753zIKo+wk3u1bVKCGPbrnF7+7LBF/UHVY= +golang.org/x/tools v0.47.0 h1:7Kn5x/d1svx/PzryTsqeoZN4TZwqeH5pGWjefhLi/1Q= +golang.org/x/tools v0.47.0/go.mod h1:dFHnyTvFWY212G+h7ZY4Vsp/K3U4/7W9TyVaAul8uCA= +google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= +google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= diff --git a/core/observability/gorm_logger.go b/core/observability/gorm_logger.go new file mode 100644 index 00000000..37b65fc5 --- /dev/null +++ b/core/observability/gorm_logger.go @@ -0,0 +1,75 @@ +package observability + +import ( + "context" + "errors" + "strings" + "time" + + appLogger "github.com/gaucho-racing/sentinel/core/pkg/logger" + "gorm.io/gorm" + "gorm.io/gorm/logger" +) + +type DatabaseLogger struct { + level logger.LogLevel + slowThreshold time.Duration +} + +func NewDatabaseLogger(slowThreshold time.Duration) logger.Interface { + return &DatabaseLogger{level: logger.Warn, slowThreshold: slowThreshold} +} + +func (l *DatabaseLogger) LogMode(level logger.LogLevel) logger.Interface { + copy := *l + copy.level = level + return © +} + +func (l *DatabaseLogger) Info(_ context.Context, message string, args ...interface{}) { + if l.level >= logger.Info { + appLogger.SugarLogger.Infof(message, args...) + } +} + +func (l *DatabaseLogger) Warn(_ context.Context, message string, args ...interface{}) { + if l.level >= logger.Warn { + appLogger.SugarLogger.Warnf(message, args...) + } +} + +func (l *DatabaseLogger) Error(_ context.Context, message string, args ...interface{}) { + if l.level >= logger.Error { + appLogger.SugarLogger.Errorf(message, args...) + } +} + +func (l *DatabaseLogger) Trace(_ context.Context, started time.Time, query func() (string, int64), err error) { + elapsed := time.Since(started) + sql, rows := query() + operation := databaseOperation(sql) + slow := l.slowThreshold > 0 && elapsed >= l.slowThreshold + ObserveDatabaseQuery(operation, err != nil, slow, elapsed) + + if err != nil && !errors.Is(err, gorm.ErrRecordNotFound) && l.level >= logger.Error { + appLogger.SugarLogger.Errorf("Database query failed (operation=%s duration=%s rows=%d): %v", operation, elapsed, rows, err) + return + } + if slow && l.level >= logger.Warn { + appLogger.SugarLogger.Warnf("Slow database query (operation=%s duration=%s rows=%d)", operation, elapsed, rows) + } +} + +func databaseOperation(sql string) string { + fields := strings.Fields(sql) + if len(fields) == 0 { + return "OTHER" + } + operation := strings.ToUpper(fields[0]) + switch operation { + case "SELECT", "INSERT", "UPDATE", "DELETE", "WITH", "CREATE", "ALTER", "DROP": + return operation + default: + return "OTHER" + } +} diff --git a/core/observability/gorm_logger_test.go b/core/observability/gorm_logger_test.go new file mode 100644 index 00000000..a796c442 --- /dev/null +++ b/core/observability/gorm_logger_test.go @@ -0,0 +1,18 @@ +package observability + +import "testing" + +func TestDatabaseOperationUsesBoundedLabels(t *testing.T) { + tests := map[string]string{ + " SELECT * FROM users": "SELECT", + "insert into users": "INSERT", + "WITH recent AS (SELECT": "WITH", + "VACUUM users": "OTHER", + "": "OTHER", + } + for sql, expected := range tests { + if actual := databaseOperation(sql); actual != expected { + t.Fatalf("databaseOperation(%q) = %q, want %q", sql, actual, expected) + } + } +} diff --git a/core/observability/metrics.go b/core/observability/metrics.go new file mode 100644 index 00000000..95a12ebe --- /dev/null +++ b/core/observability/metrics.go @@ -0,0 +1,62 @@ +package observability + +import ( + "strconv" + "time" + + "github.com/prometheus/client_golang/prometheus" +) + +var analyticsRequestDuration = prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Namespace: "sentinel", + Subsystem: "analytics", + Name: "request_duration_seconds", + Help: "Duration of analytics HTTP requests.", + Buckets: prometheus.DefBuckets, + }, + []string{"route", "status"}, +) + +var databaseQueryDuration = prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Namespace: "sentinel", + Subsystem: "database", + Name: "query_duration_seconds", + Help: "Duration of database queries by operation and result.", + Buckets: prometheus.ExponentialBuckets(0.001, 2, 15), + }, + []string{"operation", "result"}, +) + +var databaseSlowQueries = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "sentinel", + Subsystem: "database", + Name: "slow_queries_total", + Help: "Database queries exceeding the configured slow-query threshold.", + }, + []string{"operation"}, +) + +func init() { + prometheus.MustRegister(analyticsRequestDuration, databaseQueryDuration, databaseSlowQueries) +} + +func ObserveAnalyticsRequest(route string, status int, elapsed time.Duration) { + if route == "" { + route = "unknown" + } + analyticsRequestDuration.WithLabelValues(route, strconv.Itoa(status)).Observe(elapsed.Seconds()) +} + +func ObserveDatabaseQuery(operation string, failed bool, slow bool, elapsed time.Duration) { + result := "success" + if failed { + result = "error" + } + databaseQueryDuration.WithLabelValues(operation, result).Observe(elapsed.Seconds()) + if slow { + databaseSlowQueries.WithLabelValues(operation).Inc() + } +} diff --git a/docker-compose.yml b/docker-compose.yml index 8d90d129..ab9d09cd 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -40,6 +40,8 @@ services: DATABASE_USER: postgres DATABASE_PASSWORD: ${POSTGRES_PASSWORD} DATABASE_NAME: sentinel + DATABASE_ENABLE_QUERY_STATISTICS: "true" + DATABASE_SLOW_QUERY_THRESHOLD: 200ms ISSUER: http://localhost:10310 INTERNAL_BOOTSTRAP_SECRET: ${INTERNAL_BOOTSTRAP_SECRET} @@ -175,6 +177,14 @@ services: container_name: sentinel-db image: postgres:18-alpine restart: always + command: + - postgres + - -c + - shared_preload_libraries=pg_stat_statements + - -c + - compute_query_id=on + - -c + - pg_stat_statements.track=all environment: POSTGRES_USER: postgres POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} From 2471c7a69ab07900952bba6d1d16f25f41202b44 Mon Sep 17 00:00:00 2001 From: Bharat Kathi Date: Mon, 31 Aug 2026 13:10:43 -0700 Subject: [PATCH 2/3] fix(core): record analytics auth failures accurately --- core/api/api.go | 4 +-- core/api/metrics_test.go | 78 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 80 insertions(+), 2 deletions(-) create mode 100644 core/api/metrics_test.go diff --git a/core/api/api.go b/core/api/api.go index cdd1cdd6..90a43171 100644 --- a/core/api/api.go +++ b/core/api/api.go @@ -33,9 +33,9 @@ func InitializeRouter() *gin.Engine { MaxAge: 12 * time.Hour, AllowCredentials: true, })) - r.Use(AuthChecker()) - r.Use(UnauthorizedPanicHandler()) r.Use(AnalyticsMetrics()) + r.Use(UnauthorizedPanicHandler()) + r.Use(AuthChecker()) return r } diff --git a/core/api/metrics_test.go b/core/api/metrics_test.go new file mode 100644 index 00000000..fe347f46 --- /dev/null +++ b/core/api/metrics_test.go @@ -0,0 +1,78 @@ +package api + +import ( + "net/http" + "net/http/httptest" + "testing" + + "github.com/gaucho-racing/sentinel/core/pkg/logger" + "github.com/gin-gonic/gin" + "github.com/prometheus/client_golang/prometheus" +) + +func TestAnalyticsMetricsRecordsRecoveredAuthorizationStatus(t *testing.T) { + gin.SetMode(gin.TestMode) + router := InitializeRouter() + router.GET("/analytics/metrics-authorization-test", func(c *gin.Context) { + Require(c, false) + }) + + before := analyticsRequestCount(t, "/analytics/metrics-authorization-test", "401") + request := httptest.NewRequest(http.MethodGet, "/analytics/metrics-authorization-test", nil) + response := httptest.NewRecorder() + router.ServeHTTP(response, request) + + if response.Code != http.StatusUnauthorized { + t.Fatalf("expected %d, got %d: %s", http.StatusUnauthorized, response.Code, response.Body.String()) + } + after := analyticsRequestCount(t, "/analytics/metrics-authorization-test", "401") + if after != before+1 { + t.Fatalf("expected 401 metric count to increase by one, got %d before and %d after", before, after) + } +} + +func TestAnalyticsMetricsRecordsRejectedBearerToken(t *testing.T) { + gin.SetMode(gin.TestMode) + logger.Init(true) + router := InitializeRouter() + router.GET("/analytics/metrics-bearer-test", func(c *gin.Context) { + c.Status(http.StatusOK) + }) + + before := analyticsRequestCount(t, "/analytics/metrics-bearer-test", "401") + request := httptest.NewRequest(http.MethodGet, "/analytics/metrics-bearer-test", nil) + request.Header.Set("Authorization", "Bearer invalid") + response := httptest.NewRecorder() + router.ServeHTTP(response, request) + + if response.Code != http.StatusUnauthorized { + t.Fatalf("expected %d, got %d: %s", http.StatusUnauthorized, response.Code, response.Body.String()) + } + after := analyticsRequestCount(t, "/analytics/metrics-bearer-test", "401") + if after != before+1 { + t.Fatalf("expected 401 metric count to increase by one, got %d before and %d after", before, after) + } +} + +func analyticsRequestCount(t *testing.T, route string, status string) uint64 { + t.Helper() + families, err := prometheus.DefaultGatherer.Gather() + if err != nil { + t.Fatal(err) + } + for _, family := range families { + if family.GetName() != "sentinel_analytics_request_duration_seconds" { + continue + } + for _, metric := range family.GetMetric() { + labels := make(map[string]string, len(metric.GetLabel())) + for _, label := range metric.GetLabel() { + labels[label.GetName()] = label.GetValue() + } + if labels["route"] == route && labels["status"] == status { + return metric.GetHistogram().GetSampleCount() + } + } + } + return 0 +} From 1e65726efbab5f99e89411b6c557445925cd3572 Mon Sep 17 00:00:00 2001 From: Bharat Kathi Date: Mon, 31 Aug 2026 13:19:14 -0700 Subject: [PATCH 3/3] chore(core): remove observability tests --- core/api/metrics_test.go | 78 -------------------------- core/observability/gorm_logger_test.go | 18 ------ 2 files changed, 96 deletions(-) delete mode 100644 core/api/metrics_test.go delete mode 100644 core/observability/gorm_logger_test.go diff --git a/core/api/metrics_test.go b/core/api/metrics_test.go deleted file mode 100644 index fe347f46..00000000 --- a/core/api/metrics_test.go +++ /dev/null @@ -1,78 +0,0 @@ -package api - -import ( - "net/http" - "net/http/httptest" - "testing" - - "github.com/gaucho-racing/sentinel/core/pkg/logger" - "github.com/gin-gonic/gin" - "github.com/prometheus/client_golang/prometheus" -) - -func TestAnalyticsMetricsRecordsRecoveredAuthorizationStatus(t *testing.T) { - gin.SetMode(gin.TestMode) - router := InitializeRouter() - router.GET("/analytics/metrics-authorization-test", func(c *gin.Context) { - Require(c, false) - }) - - before := analyticsRequestCount(t, "/analytics/metrics-authorization-test", "401") - request := httptest.NewRequest(http.MethodGet, "/analytics/metrics-authorization-test", nil) - response := httptest.NewRecorder() - router.ServeHTTP(response, request) - - if response.Code != http.StatusUnauthorized { - t.Fatalf("expected %d, got %d: %s", http.StatusUnauthorized, response.Code, response.Body.String()) - } - after := analyticsRequestCount(t, "/analytics/metrics-authorization-test", "401") - if after != before+1 { - t.Fatalf("expected 401 metric count to increase by one, got %d before and %d after", before, after) - } -} - -func TestAnalyticsMetricsRecordsRejectedBearerToken(t *testing.T) { - gin.SetMode(gin.TestMode) - logger.Init(true) - router := InitializeRouter() - router.GET("/analytics/metrics-bearer-test", func(c *gin.Context) { - c.Status(http.StatusOK) - }) - - before := analyticsRequestCount(t, "/analytics/metrics-bearer-test", "401") - request := httptest.NewRequest(http.MethodGet, "/analytics/metrics-bearer-test", nil) - request.Header.Set("Authorization", "Bearer invalid") - response := httptest.NewRecorder() - router.ServeHTTP(response, request) - - if response.Code != http.StatusUnauthorized { - t.Fatalf("expected %d, got %d: %s", http.StatusUnauthorized, response.Code, response.Body.String()) - } - after := analyticsRequestCount(t, "/analytics/metrics-bearer-test", "401") - if after != before+1 { - t.Fatalf("expected 401 metric count to increase by one, got %d before and %d after", before, after) - } -} - -func analyticsRequestCount(t *testing.T, route string, status string) uint64 { - t.Helper() - families, err := prometheus.DefaultGatherer.Gather() - if err != nil { - t.Fatal(err) - } - for _, family := range families { - if family.GetName() != "sentinel_analytics_request_duration_seconds" { - continue - } - for _, metric := range family.GetMetric() { - labels := make(map[string]string, len(metric.GetLabel())) - for _, label := range metric.GetLabel() { - labels[label.GetName()] = label.GetValue() - } - if labels["route"] == route && labels["status"] == status { - return metric.GetHistogram().GetSampleCount() - } - } - } - return 0 -} diff --git a/core/observability/gorm_logger_test.go b/core/observability/gorm_logger_test.go deleted file mode 100644 index a796c442..00000000 --- a/core/observability/gorm_logger_test.go +++ /dev/null @@ -1,18 +0,0 @@ -package observability - -import "testing" - -func TestDatabaseOperationUsesBoundedLabels(t *testing.T) { - tests := map[string]string{ - " SELECT * FROM users": "SELECT", - "insert into users": "INSERT", - "WITH recent AS (SELECT": "WITH", - "VACUUM users": "OTHER", - "": "OTHER", - } - for sql, expected := range tests { - if actual := databaseOperation(sql); actual != expected { - t.Fatalf("databaseOperation(%q) = %q, want %q", sql, actual, expected) - } - } -}