Increase gateway replica count limit 3 → 9

mkdocs/docs/concepts/gateways.md

@@ -245,12 +245,7 @@ $ dstack gateway list
 
 </div>
 
-!!! warning "Experimental"
-    Replicated gateways are an experimental feature and currently have limitations:
-
-    - HTTPS is only supported for AWS gateways with the `acm` [certificate type](#certificate) and GCP gateways with the `gcp-cm` [certificate type](#certificate). For other gateways, use an external load balancer for TLS termination.
-    - All replicas are bound to the same backend and region.
-    - At most 3 replicas are allowed per gateway.
+Replicated gateways do not support automatic certificate issuance via Let's Encrypt (`certificate: { type: lets-encrypt }`). For TLS termination, use an external load balancer or one of the other [certificate types](#certificate).
 
 !!! info "Reference"
     For all gateway configuration options, refer to the [reference](../reference/dstack.yml/gateway.md).

mkdocs/docs/reference/env.md

@@ -148,6 +148,7 @@ For more details on the options below, refer to the [server deployment](../guide
 - `DSTACK_SERVER_METRICS_RUNNING_TTL_SECONDS`{ #DSTACK_SERVER_METRICS_RUNNING_TTL_SECONDS } – Maximum age of metrics samples for running jobs.
 - `DSTACK_SERVER_METRICS_FINISHED_TTL_SECONDS`{ #DSTACK_SERVER_METRICS_FINISHED_TTL_SECONDS } – Maximum age of metrics samples for finished jobs.
 - `DSTACK_SERVER_INSTANCE_HEALTH_TTL_SECONDS`{ #DSTACK_SERVER_INSTANCE_HEALTH_TTL_SECONDS } – Maximum age of instance health checks.
+- `DSTACK_SERVER_GATEWAY_MAX_REPLICAS`{ #DSTACK_SERVER_GATEWAY_MAX_REPLICAS } - Maximum number of replicas allowed for a gateway. Defaults to `9`.
 - `DSTACK_SERVER_INSTANCE_HEALTH_MIN_COLLECT_INTERVAL_SECONDS`{ #DSTACK_SERVER_INSTANCE_HEALTH_MIN_COLLECT_INTERVAL_SECONDS } – Minimum time interval between consecutive health checks of the same instance.
 - `DSTACK_SERVER_EVENTS_TTL_SECONDS`{ #DSTACK_SERVER_EVENTS_TTL_SECONDS } - Maximum age of event records. Set to `0` to disable event storage. Defaults to 30 days.
 - `DSTACK_SERVER_DEFAULT_DOCKER_REGISTRY`{ #DSTACK_SERVER_DEFAULT_DOCKER_REGISTRY } – A default Docker registry to use for job images that do not specify an explicit registry. E.g., if set to `registry.example`, then `image: ubuntu` becomes equivalent to `image: registry.example/ubuntu`. **Note**: This setting should only be used for configuring registries that act as a pull-through cache for Docker Hub. The default `dstack` images are also pulled from the configured registry.

src/dstack/_internal/server/services/gateways/__init__.py

@@ -159,10 +159,6 @@ def emit_gateway_replica_status_change_event(
 GATEWAY_CONNECT_DELAY = 10
 GATEWAY_CONFIGURE_ATTEMPTS = 50
 GATEWAY_CONFIGURE_DELAY = 3
-# Artificial limit to avoid doing too many per-replica operations (gateway replica provisioning,
-# service registration, etc) in a single pipeline tick. Can be lifted once the implementation is
-# more mature.
-GATEWAY_MAX_REPLICAS = 3  # documented in gateways.md, keep in sync
 
 
 async def list_project_gateways(
@@ -1237,9 +1233,10 @@ def _validate_gateway_configuration(configuration: GatewayConfiguration):
         configuration.replicas if configuration.replicas is not None else GATEWAY_REPLICAS_DEFAULT
     )
 
-    if replicas > GATEWAY_MAX_REPLICAS:
+    if replicas > settings.GATEWAY_MAX_REPLICAS:
         raise ServerClientError(
-            f"Cannot provision {replicas} gateway replicas. This server allows at most {GATEWAY_MAX_REPLICAS}"
+            f"Cannot provision {replicas} gateway replicas. This server allows at most"
+            f" {settings.GATEWAY_MAX_REPLICAS}"
         )
 
     if configuration.load_balancer is not None:

src/dstack/_internal/server/settings.py

@@ -83,6 +83,8 @@ def get_database_url() -> str:
 MAX_PROBES_PER_JOB = int(os.getenv("DSTACK_SERVER_MAX_PROBES_PER_JOB", 10))
 MAX_PROBE_TIMEOUT = int(os.getenv("DSTACK_SERVER_MAX_PROBE_TIMEOUT", 60 * 5))
 
+GATEWAY_MAX_REPLICAS = int(os.getenv("DSTACK_SERVER_GATEWAY_MAX_REPLICAS", 9))
+
 SERVER_CONFIG_DISABLED = os.getenv("DSTACK_SERVER_CONFIG_DISABLED") is not None
 SERVER_CONFIG_ENABLED = not SERVER_CONFIG_DISABLED
 

src/tests/_internal/server/routers/test_gateways.py

@@ -753,9 +753,9 @@ async def test_create_gateway_with_invalid_domain_interpolation(
                     "backend": "aws",
                     "region": "us",
                     "certificate": None,
-                    "replicas": 4,
+                    "replicas": 10,
                 },
-                "Cannot provision 4 gateway replicas. This server allows at most 3",
+                "Cannot provision 10 gateway replicas. This server allows at most 9",
                 id="replicas-exceed-max",
             ),
             pytest.param(