diff --git a/DockerFile b/DockerFile index 4dc1a52..d7f1c62 100644 --- a/DockerFile +++ b/DockerFile @@ -9,9 +9,7 @@ RUN go get github.com/akrylysov/algnhsa && \ FROM openeuler/openeuler:24.03 LABEL maintainer="Zhou Yi 1123678689@qq.com" -# 安装依赖工具 -RUN dnf install -y git wget tar gzip && \ - # 下载git-lfs +RUN dnf install -y git wget tar gzip tini && \ wget https://github.com/git-lfs/git-lfs/releases/download/v3.3.0/git-lfs-linux-amd64-v3.3.0.tar.gz && \ tar -xzf git-lfs-linux-amd64-v3.3.0.tar.gz && \ cd git-lfs-3.3.0 && \ @@ -26,4 +24,4 @@ COPY --chown=BigFiles:group --from=BUILDER /home/main /home/BigFiles/main COPY --chown=BigFiles:group --from=BUILDER /home/scripts/lfsNameQuery.py /home/BigFiles/lfsNameQuery.py EXPOSE 5000 -ENTRYPOINT ["/home/BigFiles/main"] \ No newline at end of file +ENTRYPOINT ["tini", "--", "/home/BigFiles/main"] \ No newline at end of file diff --git a/db/db.go b/db/db.go index 0799774..984187a 100644 --- a/db/db.go +++ b/db/db.go @@ -28,13 +28,12 @@ func Init(cfg config.DBConfig) error { }, ) if err != nil { - log.Fatal("Failed to connect to database", err) - return err + return fmt.Errorf("failed to connect to database: %w", err) } sqlDb, err := dbInstance.DB() if err != nil { - return err + return fmt.Errorf("failed to get underlying sql.DB: %w", err) } sqlDb.SetConnMaxLifetime(cfg.GetLifeDuration()) @@ -46,6 +45,11 @@ func Init(cfg config.DBConfig) error { return nil } +// RunMigration performs schema auto-migration once at startup. +func RunMigration() error { + return Db.AutoMigrate(&LfsObj{}) +} + // DB returns the current database instance. func DB() *gorm.DB { return Db @@ -67,11 +71,6 @@ type LfsObj struct { // InsertLFSObj 插入 LFS 元数据 func InsertLFSObj(obj LfsObj) error { - err := Db.AutoMigrate(&LfsObj{}) - if err != nil { - return err - } - var existingObj LfsObj if err := Db.Where("oid = ? AND repo = ? AND owner = ?", obj.Oid, obj.Repo, obj.Owner).First(&existingObj).Error; err == nil { diff --git a/main.go b/main.go index 308cedb..ae2a0b8 100644 --- a/main.go +++ b/main.go @@ -6,6 +6,8 @@ import ( "log" "net/http" "os" + "os/signal" + "syscall" "time" "github.com/huaweicloud/huaweicloud-sdk-go-obs/obs" @@ -60,33 +62,32 @@ func gatherOptions(fs *flag.FlagSet, args ...string) (options, error) { return o, err } -func initConfig(cfg *config.Config) { +func initConfig(cfg *config.Config) error { if err := server.Init(cfg); err != nil { - logrus.Errorf("load ValidateConfig, err:%s", err.Error()) - return + return fmt.Errorf("load ValidateConfig: %w", err) } if err := auth.Init(cfg); err != nil { - logrus.Errorf("load gitee config, err:%s", err.Error()) - return + return fmt.Errorf("load gitee config: %w", err) } if err := db.Init(cfg.DBConfig); err != nil { - logrus.Errorf("init database config, err:%s", err.Error()) - return + return fmt.Errorf("init database config: %w", err) } + + return nil } -func initObsClient(cfg *config.Config) { +func initObsClient(cfg *config.Config) error { var err error server.ObsClient, err = obs.New(cfg.ObsAccessKeyId, cfg.ObsSecretAccessKey, cfg.ObsRegion, obs.WithSignature(obs.SignatureObs)) server.Bucket = cfg.LfsBucket server.Prefit = cfg.Prefix if err != nil { - logrus.Errorf("failed to initialize OBS client: %v", err.Error()) - return + return fmt.Errorf("failed to initialize OBS client: %w", err) } + return nil } func main() { @@ -95,13 +96,11 @@ func main() { os.Args[1:]..., ) if err != nil { - logrus.Errorf("new options failed, err:%s", err.Error()) - return + logrus.Fatalf("new options failed, err:%s", err.Error()) } if err := o.Validate(); err != nil { - logrus.Errorf("Invalid options, err:%s", err.Error()) - return + logrus.Fatalf("Invalid options, err:%s", err.Error()) } if o.enableDebug { @@ -109,17 +108,42 @@ func main() { logrus.Debug("debug enable.") } + // Reap zombie child processes (e.g. git commands invoked by GetLFSMapping). + // Without this, zombie [git] processes accumulate and exhaust the node PID + // table, causing PIDPressure evictions. + sigChld := make(chan os.Signal, 1) + signal.Notify(sigChld, syscall.SIGCHLD) + go func() { + for range sigChld { + for { + // WNOHANG = 1: non-blocking wait — returns immediately if no child has exited. + pid, _ := syscall.Wait4(-1, nil, syscall.WNOHANG, nil) + if pid <= 0 { + break + } + } + } + }() + //cfg cfg := new(config.Config) if err := config.LoadConfig(o.service.ConfigFile, cfg, o.service.RemoveCfg); err != nil { - logrus.Errorf("load config, err:%s", err.Error()) - return + logrus.Fatalf("load config, err:%s", err.Error()) + } + + if err := initObsClient(cfg); err != nil { + logrus.Fatalf("init OBS client failed: %v", err) } - initObsClient(cfg) + if err := initConfig(cfg); err != nil { + logrus.Fatalf("init config failed: %v", err) + } - initConfig(cfg) + // Run database schema migration once at startup instead of on every insert. + if err := db.RunMigration(); err != nil { + logrus.Fatalf("run database migration failed: %v", err) + } s, err := server.New(server.Options{ Prefix: cfg.Prefix, @@ -132,6 +156,9 @@ func main() { IsGithubAuthorized: auth.GithubAuth(), SecretAccessKey: cfg.ObsSecretAccessKey, }) + if err != nil { + logrus.Fatalf("create server failed: %v", err) + } go server.StartScheduledTask() go server.ScheduledCheckOidAndFileName() @@ -144,12 +171,19 @@ func main() { IdleTimeout: 30 * time.Second, } - if err != nil { - log.Fatalln(err) - } + // Graceful shutdown: listen for SIGTERM/SIGINT and call srv.Shutdown() + quit := make(chan os.Signal, 1) + signal.Notify(quit, syscall.SIGTERM, syscall.SIGINT) + go func() { + <-quit + log.Println("shutting down server...") + if err := srv.Shutdown(nil); err != nil { + logrus.Errorf("server shutdown error: %v", err) + } + }() log.Println("serving on http://0.0.0.0:5000 ...") - if err := srv.ListenAndServe(); err != nil { - log.Fatalln(err) + if err := srv.ListenAndServe(); err != nil && err != http.ErrServerClosed { + logrus.Fatalf("server error: %v", err) } } diff --git a/server/server.go b/server/server.go index ff6d360..b06ea48 100644 --- a/server/server.go +++ b/server/server.go @@ -332,7 +332,14 @@ func (s *server) downloadObject(in *batch.RequestObject, out *batch.Object) { getObjectInput.Expires = int(s.ttl / time.Second) getObjectInput.Headers = map[string]string{contentType: obsHeader} // 生成下载对象的带授权信息的URL - v := s.generateDownloadUrl(getObjectInput) + v, err := s.generateDownloadUrl(getObjectInput) + if err != nil { + out.Error = &batch.ObjectError{ + Code: 500, + Message: err.Error(), + } + return + } out.Actions = &batch.Actions{ Download: &batch.Action{ @@ -366,7 +373,11 @@ func (s *server) uploadObject(in *batch.RequestObject, out *batch.Object) { putObjectInput.Headers = map[string]string{contentType: obsHeader} putObjectOutput, err := s.client.CreateSignedUrl(putObjectInput) if err != nil { - panic(err) + out.Error = &batch.ObjectError{ + Code: 500, + Message: fmt.Sprintf("failed to create signed upload URL: %v", err), + } + return } out.Actions = &batch.Actions{ @@ -387,39 +398,64 @@ func (s *server) getObjectMetadataInput(key string) (output *obs.GetObjectMetada } // 生成下载对象的带授权信息的URL -func (s *server) generateDownloadUrl(getObjectInput *obs.CreateSignedUrlInput) *url.URL { - // 生成下载对象的带授权信息的URL +func (s *server) generateDownloadUrl(getObjectInput *obs.CreateSignedUrlInput) (*url.URL, error) { getObjectOutput, err := s.client.CreateSignedUrl(getObjectInput) if err != nil { - panic(err) + return nil, fmt.Errorf("failed to create signed download URL: %w", err) } v, err := url.Parse(getObjectOutput.SignedUrl) - if err == nil { - v.Host = s.cdnDomain - v.Scheme = "https" - } else { - logrus.Infof("%s cannot be parsed", getObjectOutput.SignedUrl) - panic(err) + if err != nil { + return nil, fmt.Errorf("failed to parse signed URL: %w", err) } - return v + v.Host = s.cdnDomain + v.Scheme = "https" + return v, nil } func (s *server) healthCheck(w http.ResponseWriter, r *http.Request) { - response := batch.SuccessResponse{ - Message: "Success", - Data: "healthCheck success", + w.Header().Set(contentType, jsonHeader) + + dbOK := true + if db.Db != nil { + sqlDB, err := db.Db.DB() + if err != nil || sqlDB.Ping() != nil { + dbOK = false + } + } else { + dbOK = false + } + + obsOK := true + if ObsClient != nil { + _, err := ObsClient.ListBuckets(nil) + if err != nil { + obsOK = false + } + } else { + obsOK = false + } + + if !dbOK || !obsOK { + w.WriteHeader(http.StatusServiceUnavailable) + must(json.NewEncoder(w).Encode(batch.SuccessResponse{ + Message: "Unhealthy", + Data: fmt.Sprintf("db=%v obs=%v", dbOK, obsOK), + })) + return } - w.Header().Set(contentType, jsonHeader) w.WriteHeader(http.StatusOK) - must(json.NewEncoder(w).Encode(response)) + must(json.NewEncoder(w).Encode(batch.SuccessResponse{ + Message: "Success", + Data: "healthCheck success", + })) } // -- func must(err error) { if err != nil { - panic(err) + logrus.Errorf("encode response failed: %v", err) } } @@ -454,14 +490,20 @@ func (s *server) download(w http.ResponseWriter, r *http.Request) { Headers: map[string]string{contentType: obsHeader}, } - v := s.generateDownloadUrl(getObjectInput) + v, err := s.generateDownloadUrl(getObjectInput) + if err != nil { + w.Header().Set(contentType, jsonHeader) + w.WriteHeader(http.StatusInternalServerError) + json.NewEncoder(w).Encode(map[string]string{"error": err.Error()}) + return + } w.Header().Set(contentType, jsonHeader) response := map[string]string{"url": v.String()} w.WriteHeader(http.StatusOK) - err := json.NewEncoder(w).Encode(response) + err = json.NewEncoder(w).Encode(response) if err != nil { return } @@ -623,7 +665,17 @@ func checkOidFileName() { } +const maxRepoOidNameDepth = 10 + func checkRepoOidName(userInRepo auth.UserInRepo) (oidFileNameMap map[string]auth.FileInfo) { + return checkRepoOidNameWithDepth(userInRepo, 0) +} + +func checkRepoOidNameWithDepth(userInRepo auth.UserInRepo, depth int) (oidFileNameMap map[string]auth.FileInfo) { + if depth >= maxRepoOidNameDepth { + logrus.Errorf("checkRepoOidName exceeded max depth %d for owner:%v repo:%v", maxRepoOidNameDepth, userInRepo.Owner, userInRepo.Repo) + return nil + } oidFileNameMap, err := auth.GetLFSMapping(userInRepo) if err != nil { logrus.Errorf("get lfs mapping failed: %v", err) @@ -638,7 +690,7 @@ func checkRepoOidName(userInRepo auth.UserInRepo) (oidFileNameMap map[string]aut if repo.Parent.Fullname != "" { userInRepo.Owner = strings.Split(repo.Parent.Fullname, "/")[0] userInRepo.Repo = strings.Split(repo.Parent.Fullname, "/")[1] - return checkRepoOidName(userInRepo) + return checkRepoOidNameWithDepth(userInRepo, depth+1) } } return oidFileNameMap diff --git a/server/server_test.go b/server/server_test.go index 1fbef50..5b431cb 100644 --- a/server/server_test.go +++ b/server/server_test.go @@ -181,22 +181,19 @@ func Test_must(t *testing.T) { args args wantErr bool }{ - // 测试传入nil，期望不会触发panic，也就是正常执行 { name: "no error", args: args{err: nil}, wantErr: false, }, - // 测试传入一个具体错误，期望触发panic { - name: "panic error", - args: args{err: errors.New("panic error test")}, + name: "log error", + args: args{err: errors.New("log error test")}, wantErr: true, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - defer panicCheck(t, tt.wantErr) must(tt.args.err) }) } @@ -398,8 +395,8 @@ func Test_server_downloadObject(t *testing.T) { downloadUrl, _ := url.Parse("test.url") generateDownloadUrlPtr := reflect.ValueOf((*server).generateDownloadUrl) monkey.Patch(generateDownloadUrlPtr.Interface(), - func(s *server, getObjectInput *obs.CreateSignedUrlInput) *url.URL { - return downloadUrl + func(s *server, getObjectInput *obs.CreateSignedUrlInput) (*url.URL, error) { + return downloadUrl, nil }) defer monkey.Unpatch(generateDownloadUrlPtr.Interface()) s := &server{ @@ -456,7 +453,8 @@ func Test_server_generateDownloadUrl(t *testing.T) { isAuthorized: tt.fields.isAuthorized, } defer panicCheck(t, tt.wantErr) - if got := s.generateDownloadUrl(tt.args.getObjectInput); got != nil { + got, err := s.generateDownloadUrl(tt.args.getObjectInput) + if err == nil && got != nil { t.Errorf("generateDownloadUrl() = %v", got) } }) @@ -1145,9 +1143,9 @@ func TestDownload(t *testing.T) { defer monkey.UnpatchAll() // 模拟 generateDownloadUrl 的行为 - monkey.Patch((*server).generateDownloadUrl, func(s *server, input *obs.CreateSignedUrlInput) *url.URL { + monkey.Patch((*server).generateDownloadUrl, func(s *server, input *obs.CreateSignedUrlInput) (*url.URL, error) { u, _ := url.Parse(tt.mockOutput) - return u + return u, nil }) defer monkey.UnpatchAll()