refactor(firewall): queue rule operations and simplify synchronization

agent/app/api/v2/firewall.go

@@ -460,6 +460,8 @@ func handleFirewallRuleError(c *gin.Context, err error) {
 		helper.ErrorWithBusinessCode(c, http.StatusConflict, "FW_RULE_STALE", "ErrInvalidParams", err)
 	case errors.Is(err, repo.ErrFirewallRuleRevisionConflict):
 		helper.ErrorWithBusinessCode(c, http.StatusConflict, "FW_RULE_REVISION_CONFLICT", "ErrInvalidParams", err)
+	case errors.Is(err, filter.ErrManagedScopeChange):
+		helper.ErrorWithBusinessCode(c, http.StatusBadRequest, "FW_SCOPE_UNSUPPORTED", "ErrFirewallRuleScopeChange", err)
 	case errors.Is(err, filter.ErrUnsupportedScope), errors.Is(err, filter.ErrInvalidScope),
 		errors.Is(err, filter.ErrProviderUnavailable), errors.Is(err, filter.ErrAdapterUnavailable):
 		helper.ErrorWithBusinessCode(c, http.StatusBadRequest, "FW_SCOPE_UNSUPPORTED", "ErrInvalidParams", err)
@@ -488,6 +490,29 @@ func (b *BaseApi) LoadFirewallSettings(c *gin.Context) {
 	helper.SuccessWithData(c, data)
 }
 
+// @Tags Firewall
+// @Summary Queue firewall port whitelist update
+// @Description Returns a taskID; configuration save and per-rule results are recorded in the task log.
+// @Accept json
+// @Param request body dto.FirewallPortWhitelistUpdate true "request"
+// @Success 200 {object} dto.FilterChainOperationResponse
+// @Security ApiKeyAuth
+// @Security Timestamp
+// @Router /hosts/firewall/settings/whitelist [post]
+// @x-panel-log {"bodyKeys":["value"],"paramKeys":[],"BeforeFunctions":[],"formatZH":"更新防火墙端口白名单 [value]","formatEN":"update firewall port whitelist [value]"}
+func (b *BaseApi) UpdateFirewallPortWhitelist(c *gin.Context) {
+	var request dto.FirewallPortWhitelistUpdate
+	if err := helper.CheckBindAndValidate(&request, c); err != nil {
+		return
+	}
+	result, err := firewallSettingService.QueuePortWhitelist(request.Value)
+	if err != nil {
+		helper.InternalServer(c, err)
+		return
+	}
+	helper.SuccessWithData(c, result)
+}
+
 // @Tags Firewall
 // @Summary Operate firewall backend
 // @Accept json

agent/app/dto/firewall.go

@@ -62,6 +62,10 @@ type FirewallSettings struct {
 	PortWhitelist string               `json:"portWhiteList"`
 }
 
+type FirewallPortWhitelistUpdate struct {
+	Value string `json:"value" validate:"required"`
+}
+
 type FirewallBackendOperation struct {
 	Subsystem string `json:"subsystem" validate:"required,oneof=system forwarding docker"`
 	Backend   string `json:"backend" validate:"required,oneof=firewalld ufw iptables nftables"`

agent/app/dto/setting.go

@@ -35,7 +35,7 @@ type SettingUpdate struct {
 }
 
 type AgentSettingUpdate struct {
-	Key   string `json:"key" validate:"required,oneof=SystemIP DockerSockPath FileRecycleBin FirewallPortWhiteList"`
+	Key   string `json:"key" validate:"required,oneof=SystemIP DockerSockPath FileRecycleBin"`
 	Value string `json:"value"`
 }
 

agent/app/service/firewall.go

@@ -1825,7 +1825,7 @@ func (s *FirewallService) prepareManagedUpdate(
 		return preparedManagedUpdate{}, err
 	}
 	if after.Scope.Key() != before.Rule.Scope.Key() {
-		return preparedManagedUpdate{}, fmt.Errorf("%w: managed rule scope cannot be changed", filter.ErrUnsupportedScope)
+		return preparedManagedUpdate{}, filter.ErrManagedScopeChange
 	}
 	if !supportsManagedNativeKindTransition(before.Rule, after) {
 		return preparedManagedUpdate{}, fmt.Errorf("%w: native rule conversion requires an explicit workflow", filter.ErrUnsupportedScope)
@@ -2471,6 +2471,47 @@ func isFirewallPolicyIncompatible(err error) bool {
 		errors.Is(err, filter.ErrInvalidScope) || errors.Is(err, filter.ErrCompositeRule)
 }
 
+func (s *FirewallService) compileRestorableFirewallRules(
+	ctx context.Context,
+	stored model.FirewallRule,
+	provider filter.Provider,
+) (restorable, preserved []filter.DesiredRule, err error) {
+	compiled, err := s.compileStoredFirewallRules(ctx, stored, provider)
+	if err != nil {
+		return nil, nil, err
+	}
+	if !supportsManagedFilterChains(string(provider)) || !isProtectedSystemFirewallRule(stored) {
+		return compiled, nil, nil
+	}
+	loadRequired := s.requiredPorts
+	if loadRequired == nil {
+		loadRequired = LoadRequiredFirewallPortWhiteList
+	}
+	required, err := loadRequired()
+	if err != nil {
+		return nil, nil, err
+	}
+	requiredPorts := systemPorts(required)
+	for _, desired := range compiled {
+		covered := false
+		for _, port := range requiredPorts {
+			covered, err = filter.SameRuleContent(desired.Rule, systemPortRule(provider, port))
+			if err != nil {
+				return nil, nil, err
+			}
+			if covered {
+				break
+			}
+		}
+		if covered {
+			preserved = append(preserved, desired)
+		} else {
+			restorable = append(restorable, desired)
+		}
+	}
+	return restorable, preserved, nil
+}
+
 func (s *FirewallService) desiredFirewallRulesByScope(
 	ctx context.Context,
 	stored []model.FirewallRule,
@@ -2480,7 +2521,7 @@ func (s *FirewallService) desiredFirewallRulesByScope(
 	desired := make(map[string][]filter.DesiredRule)
 	var failures []filter.InventoryItem
 	for _, record := range stored {
-		compiled, err := s.compileStoredFirewallRules(ctx, record, provider)
+		compiled, _, err := s.compileRestorableFirewallRules(ctx, record, provider)
 		if err != nil {
 			rule := filter.FirewallRule{
 				UUID:     record.UUID,
@@ -2648,47 +2689,6 @@ func loadRequiredFirewallPorts(panelPort string) ([]firewall.PortWhitelist, erro
 	return firewall.NormalizeRequiredPorts(ports)
 }
 
-func (s *FirewallService) updatePortWhitelist(ctx context.Context, value string) error {
-	firewallRuleMutationMu.Lock()
-	defer firewallRuleMutationMu.Unlock()
-
-	ports, err := firewall.ParsePortWhitelist(value)
-	if err != nil {
-		return err
-	}
-	oldValue, err := settingRepo.GetValueByKey(constant.FirewallPortWhiteList)
-	if errors.Is(err, gorm.ErrRecordNotFound) {
-		oldValue = constant.FirewallPortWhiteListValue
-	} else if err != nil {
-		return err
-	}
-	oldPorts, err := firewall.ParsePortWhitelist(oldValue)
-	if err != nil {
-		return err
-	}
-	required, err := s.requiredPorts()
-	if err != nil {
-		return err
-	}
-	ctx = context.WithValue(ctx, panelPortWhitelistKey{}, firewall.NormalizePortWhitelist(append(oldPorts, required...)))
-	added := excludeFirewallPorts(excludeFirewallPorts(ports, oldPorts), required)
-	for _, port := range systemPorts(added) {
-		if err := s.ensureSystemPortLocked(ctx, port); err != nil {
-			return err
-		}
-	}
-	removed := excludeFirewallPorts(excludeFirewallPorts(oldPorts, ports), required)
-	return global.DB.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
-		txCtx := context.WithValue(ctx, constant.DB, tx)
-		if err := s.releaseSystemPorts(txCtx, systemPorts(removed)); err != nil {
-			return err
-		}
-		return tx.Where("key = ?", constant.FirewallPortWhiteList).
-			Assign(map[string]interface{}{"value": value}).
-			FirstOrCreate(&model.Setting{Key: constant.FirewallPortWhiteList}).Error
-	})
-}
-
 func (s *FirewallService) releaseSystemPorts(ctx context.Context, ports []dto.FirewallSystemPort) error {
 	portSet, err := normalizeSystemPorts(ports)
 	if err != nil || len(portSet) == 0 {

agent/app/service/firewall_setting.go

@@ -2,11 +2,18 @@ package service
 
 import (
 	"context"
+	"encoding/json"
 	"errors"
 	"fmt"
+	"sync"
 
 	"github.com/1Panel-dev/1Panel/agent/app/dto"
+	"github.com/1Panel-dev/1Panel/agent/app/model"
+	"github.com/1Panel-dev/1Panel/agent/app/repo"
+	"github.com/1Panel-dev/1Panel/agent/app/task"
 	"github.com/1Panel-dev/1Panel/agent/constant"
+	"github.com/1Panel-dev/1Panel/agent/global"
+	"github.com/1Panel-dev/1Panel/agent/i18n"
 	"github.com/1Panel-dev/1Panel/agent/utils/cmd"
 	"github.com/1Panel-dev/1Panel/agent/utils/firewall"
 	"github.com/1Panel-dev/1Panel/agent/utils/firewall/docker_guard"
@@ -15,15 +22,19 @@ import (
 	"github.com/1Panel-dev/1Panel/agent/utils/firewall/lifecycle"
 	"github.com/1Panel-dev/1Panel/agent/utils/firewall/nftables_helper"
 	"github.com/1Panel-dev/1Panel/agent/utils/firewall/ping"
+	"gorm.io/gorm"
 )
 
 type IFirewallSettingService interface {
+	QueuePortWhitelist(value string) (dto.FilterChainOperationResponse, error)
 	Load(context.Context) (dto.FirewallSettings, error)
 	Operate(context.Context, dto.FirewallBackendOperation) error
 }
 
 type FirewallSettingService struct{}
 
+var firewallWhitelistTaskMu sync.Mutex
+
 var ErrFirewallBackendCleanupRequired = errors.New("firewall backend cleanup required")
 
 func firewallBackendCleanupRequired(current, target string) error {
@@ -39,6 +50,180 @@ func NewIFirewallSettingService() IFirewallSettingService {
 	return &FirewallSettingService{}
 }
 
+func (s *FirewallSettingService) QueuePortWhitelist(value string) (dto.FilterChainOperationResponse, error) {
+	return s.queuePortWhitelist(value, newFirewallService())
+}
+
+func (s *FirewallSettingService) queuePortWhitelist(value string, firewallService *FirewallService) (dto.FilterChainOperationResponse, error) {
+	firewallWhitelistTaskMu.Lock()
+	defer firewallWhitelistTaskMu.Unlock()
+	if err := task.CheckScopeTaskIsExecuting(task.TaskScopeFirewall, 0); err != nil {
+		return dto.FilterChainOperationResponse{}, err
+	}
+	taskItem, err := task.NewTask(i18n.GetMsgByKey("FirewallWhitelistTask"), task.TaskUpdate, task.TaskScopeFirewall, "", 0)
+	if err != nil {
+		return dto.FilterChainOperationResponse{}, err
+	}
+	taskItem.AddSubTaskWithOps(taskItem.Name, func(t *task.Task) error {
+		succeeded, failed := 0, 0
+		err := s.applyPortWhitelist(t.TaskCtx, value, firewallService, func(status, label string, err error) {
+			switch status {
+			case "applied":
+				succeeded++
+				t.LogSuccess(label)
+			case "failed":
+				failed++
+				t.LogFailedWithErr(label, err)
+			default:
+				t.Log(i18n.GetWithName(status, label))
+			}
+		})
+		t.Log(i18n.GetMsgWithMap("FirewallRuleOperationResult", map[string]interface{}{
+			"succeeded": succeeded, "failed": failed,
+		}))
+		return err
+	}, nil, 0, 0)
+	if err := repo.NewITaskRepo().Save(context.Background(), taskItem.Task); err != nil {
+		closeUnstartedFirewallTask(taskItem)
+		return dto.FilterChainOperationResponse{}, fmt.Errorf("save firewall whitelist task: %w", err)
+	}
+	go func() { _ = taskItem.Execute() }()
+	return dto.FilterChainOperationResponse{TaskID: taskItem.TaskID, Queued: true}, nil
+}
+
+type whitelistReporter func(status, label string, err error)
+
+func (s *FirewallSettingService) applyPortWhitelist(ctx context.Context, value string, firewallService *FirewallService, report whitelistReporter) error {
+	firewallRuleMutationMu.Lock()
+	defer firewallRuleMutationMu.Unlock()
+	if err := ctx.Err(); err != nil {
+		return err
+	}
+	ports, err := firewall.ParsePortWhitelist(value)
+	if err != nil {
+		return err
+	}
+	required, err := firewallService.requiredPorts()
+	if err != nil {
+		return err
+	}
+	removed, err := s.savePortWhitelist(ctx, ports, required, firewallService)
+	if err != nil {
+		return err
+	}
+	report("FirewallWhitelistSaved", "", nil)
+	for _, port := range systemPorts(removed) {
+		report("FirewallWhitelistReleased", whitelistPortLabel(port), nil)
+	}
+
+	ctx = context.WithValue(ctx, panelPortWhitelistKey{}, required)
+	provider, providerErr := firewallService.selectedProvider(ctx)
+	ready := s.portWhitelistReadiness(provider, providerErr, firewallService)
+	return syncPortWhitelist(ctx, ports, required, ready, firewallService.ensureSystemPortLocked, report)
+}
+
+func (s *FirewallSettingService) savePortWhitelist(ctx context.Context, ports, required []firewall.PortWhitelist, firewallService *FirewallService) ([]firewall.PortWhitelist, error) {
+	var removed []firewall.PortWhitelist
+	err := global.DB.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
+		var setting model.Setting
+		err := tx.Where("key = ?", constant.FirewallPortWhiteList).First(&setting).Error
+		if errors.Is(err, gorm.ErrRecordNotFound) {
+			setting.Value = constant.FirewallPortWhiteListValue
+		} else if err != nil {
+			return err
+		}
+		previous, err := firewall.ParsePortWhitelist(setting.Value)
+		if err != nil {
+			return err
+		}
+		removed = excludeFirewallPorts(excludeFirewallPorts(previous, ports), required)
+		txCtx := context.WithValue(ctx, constant.DB, tx)
+		if err := firewallService.releaseSystemPorts(txCtx, systemPorts(removed)); err != nil {
+			return err
+		}
+		value, err := json.Marshal(ports)
+		if err != nil {
+			return err
+		}
+		return tx.Where("key = ?", constant.FirewallPortWhiteList).
+			Assign(map[string]interface{}{"value": string(value)}).
+			FirstOrCreate(&model.Setting{Key: constant.FirewallPortWhiteList}).Error
+	})
+	return removed, err
+}
+
+func (s *FirewallSettingService) portWhitelistReadiness(provider filter.Provider, providerErr error, firewallService *FirewallService) func(dto.FirewallSystemPort) (bool, error) {
+	type state struct {
+		ready bool
+		err   error
+	}
+	states := make(map[string]state)
+	return func(port dto.FirewallSystemPort) (bool, error) {
+		if providerErr != nil {
+			return false, providerErr
+		}
+		key := "service"
+		if isDirectFirewallProvider(provider) {
+			key = port.Family
+		}
+		if cached, ok := states[key]; ok {
+			return cached.ready, cached.err
+		}
+		var result state
+		if isDirectFirewallProvider(provider) {
+			initialized, bound, err := loadSystemFirewallFamilyStatus(string(provider), port.Family)
+			result = state{ready: initialized && bound, err: err}
+		} else {
+			client, err := firewallService.baseClient()
+			result.err = err
+			if err == nil {
+				result.ready, result.err = client.Status()
+			}
+		}
+		states[key] = result
+		return result.ready, result.err
+	}
+}
+
+func syncPortWhitelist(
+	ctx context.Context,
+	ports, required []firewall.PortWhitelist,
+	ready func(dto.FirewallSystemPort) (bool, error),
+	ensure func(context.Context, dto.FirewallSystemPort) error,
+	report whitelistReporter,
+) error {
+	var failures []error
+	for _, port := range systemPorts(ports) {
+		if err := ctx.Err(); err != nil {
+			return errors.Join(append(failures, err)...)
+		}
+		label := whitelistPortLabel(port)
+		if containsFirewallPort(required, firewall.PortWhitelist{Family: port.Family, Port: port.Port, Protocol: port.Protocol}) {
+			report("FirewallWhitelistRequired", label, nil)
+			continue
+		}
+		active, err := ready(port)
+		if err == nil && !active {
+			report("FirewallWhitelistDeferred", label, nil)
+			continue
+		}
+		if err == nil {
+			err = ensure(ctx, port)
+		}
+		if err != nil {
+			report("failed", label, err)
+			failures = append(failures, fmt.Errorf("%s: %w", label, err))
+			continue
+		}
+		report("applied", label, nil)
+	}
+	return errors.Join(failures...)
+}
+
+func whitelistPortLabel(port dto.FirewallSystemPort) string {
+	return fmt.Sprintf("%s %s/%s", port.Family, port.Port, port.Protocol)
+}
+
 func (s *FirewallSettingService) Load(ctx context.Context) (dto.FirewallSettings, error) {
 	result := dto.FirewallSettings{PingStatus: ping.LoadStatus()}
 	if ports, err := settingRepo.GetValueByKey(constant.FirewallPortWhiteList); err == nil {