Introduce operations rate limits

multiapps-controller-core/src/main/java/org/cloudfoundry/multiapps/controller/core/Messages.java

@@ -198,6 +198,13 @@ public final class Messages {
     public static final String THREADS_FOR_FILE_STORAGE_UPLOAD_0 = "Threads for file storage upload: {0}";
     public static final String DELETED_ORPHANED_MTA_DESCRIPTORS_COUNT = "Deleted orphaned mta descriptors count: {0}";
     public static final String IS_HEALTH_CHECK_ENABLED = "Is health check enabled: {0}";
+    public static final String OPERATION_RATE_LIMITING_ENABLED = "Operation rate limiting enabled: {0}";
+    public static final String MAX_ACTIVE_OPERATIONS_PER_SPACE = "Max active operations per space: {0}";
+    public static final String MAX_ACTIVE_OPERATIONS_PER_USER = "Max active operations per user: {0}";
+    public static final String OPERATION_RATE_LIMIT_PER_SPACE_CAPACITY = "Operation rate limit per space capacity: {0}";
+    public static final String OPERATION_RATE_LIMIT_PER_SPACE_REFILL_PER_HOUR = "Operation rate limit per space refill per hour: {0}";
+    public static final String OPERATION_RATE_LIMIT_PER_USER_CAPACITY = "Operation rate limit per user capacity: {0}";
+    public static final String OPERATION_RATE_LIMIT_PER_USER_REFILL_PER_HOUR = "Operation rate limit per user refill per hour: {0}";
 
     // Debug messages
     public static final String DEPLOYMENT_DESCRIPTOR = "Deployment descriptor: {0}";

multiapps-controller-core/src/main/java/org/cloudfoundry/multiapps/controller/core/util/ApplicationConfiguration.java

@@ -100,6 +100,13 @@ public class ApplicationConfiguration {
     static final String CFG_THREADS_FOR_FILE_UPLOAD_TO_CONTROLLER = "THREADS_FOR_FILE_UPLOAD_TO_CONTROLLER";
     static final String CFG_THREADS_FOR_FILE_STORAGE_UPLOAD = "THREADS_FOR_FILE_STORAGE_UPLOAD";
     static final String CFG_IS_HEALTH_CHECK_ENABLED = "IS_HEALTH_CHECK_ENABLED";
+    static final String CFG_OPERATION_RATE_LIMITING_ENABLED = "OPERATION_RATE_LIMITING_ENABLED";
+    static final String CFG_MAX_ACTIVE_OPERATIONS_PER_SPACE = "MAX_ACTIVE_OPERATIONS_PER_SPACE";
+    static final String CFG_MAX_ACTIVE_OPERATIONS_PER_USER = "MAX_ACTIVE_OPERATIONS_PER_USER";
+    static final String CFG_OPERATION_RATE_LIMIT_PER_SPACE_CAPACITY = "OP_RATE_LIMIT_PER_SPACE_CAPACITY";
+    static final String CFG_OPERATION_RATE_LIMIT_PER_SPACE_REFILL_PER_HOUR = "OP_RATE_LIMIT_PER_SPACE_REFILL_PER_HOUR";
+    static final String CFG_OPERATION_RATE_LIMIT_PER_USER_CAPACITY = "OP_RATE_LIMIT_PER_USER_CAPACITY";
+    static final String CFG_OPERATION_RATE_LIMIT_PER_USER_REFILL_PER_HOUR = "OP_RATE_LIMIT_PER_USER_REFILL_PER_HOUR";
 
     private static final List<String> VCAP_APPLICATION_URIS_KEYS = List.of("full_application_uris", "application_uris", "uris");
 
@@ -158,6 +165,13 @@ public class ApplicationConfiguration {
     public static final int DEFAULT_THREADS_FOR_FILE_UPLOAD_TO_CONTROLLER = 6;
     public static final int DEFAULT_THREADS_FOR_FILE_STORAGE_UPLOAD = 7;
     public static final boolean DEFAULT_IS_HEALTH_CHECK_ENABLED = false;
+    public static final boolean DEFAULT_OPERATION_RATE_LIMITING_ENABLED = false;
+    public static final int DEFAULT_MAX_ACTIVE_OPERATIONS_PER_SPACE = 500;
+    public static final int DEFAULT_MAX_ACTIVE_OPERATIONS_PER_USER = 200;
+    public static final int DEFAULT_OPERATION_RATE_LIMIT_PER_SPACE_CAPACITY = 300;
+    public static final int DEFAULT_OPERATION_RATE_LIMIT_PER_SPACE_REFILL_PER_HOUR = 800;
+    public static final int DEFAULT_OPERATION_RATE_LIMIT_PER_USER_CAPACITY = 150;
+    public static final int DEFAULT_OPERATION_RATE_LIMIT_PER_USER_REFILL_PER_HOUR = 300;
 
     protected final Environment environment;
 
@@ -217,6 +231,13 @@ public class ApplicationConfiguration {
     private Integer threadsForFileStorageUpload;
     private Boolean isHealthCheckEnabled;
     private Set<String> objectStoreRegions;
+    private Boolean operationRateLimitingEnabled;
+    private Integer maxActiveOperationsPerSpace;
+    private Integer maxActiveOperationsPerUser;
+    private Integer operationRateLimitPerSpaceCapacity;
+    private Integer operationRateLimitPerSpaceRefillPerHour;
+    private Integer operationRateLimitPerUserCapacity;
+    private Integer operationRateLimitPerUserRefillPerHour;
 
     public ApplicationConfiguration() {
         this(new Environment());
@@ -286,7 +307,10 @@ private Set<String> getNotSensitiveConfigVariables() {
                       CFG_FLOWABLE_JOB_EXECUTOR_QUEUE_CAPACITY,
                       CFG_CONTROLLER_CLIENT_CONNECTION_POOL_SIZE,
                       CFG_CONTROLLER_CLIENT_THREAD_POOL_SIZE, CFG_CONTROLLER_CLIENT_RESPONSE_TIMEOUT, CFG_DB_TRANSACTION_TIMEOUT_IN_SECONDS,
-                      CFG_SNAKEYAML_MAX_ALIASES_FOR_COLLECTIONS, CFG_SERVICE_HANDLING_MAX_PARALLEL_THREADS);
+                      CFG_SNAKEYAML_MAX_ALIASES_FOR_COLLECTIONS, CFG_SERVICE_HANDLING_MAX_PARALLEL_THREADS,
+                      CFG_OPERATION_RATE_LIMITING_ENABLED, CFG_MAX_ACTIVE_OPERATIONS_PER_SPACE, CFG_MAX_ACTIVE_OPERATIONS_PER_USER,
+                      CFG_OPERATION_RATE_LIMIT_PER_SPACE_CAPACITY, CFG_OPERATION_RATE_LIMIT_PER_SPACE_REFILL_PER_HOUR,
+                      CFG_OPERATION_RATE_LIMIT_PER_USER_CAPACITY, CFG_OPERATION_RATE_LIMIT_PER_USER_REFILL_PER_HOUR);
     }
 
     public URL getControllerUrl() {
@@ -668,6 +692,55 @@ public boolean isHealthCheckEnabled() {
         return isHealthCheckEnabled;
     }
 
+    public boolean isOperationRateLimitingEnabled() {
+        if (operationRateLimitingEnabled == null) {
+            operationRateLimitingEnabled = isOperationRateLimitingEnabledThroughEnvironment();
+        }
+        return operationRateLimitingEnabled;
+    }
+
+    public Integer getMaxActiveOperationsPerSpace() {
+        if (maxActiveOperationsPerSpace == null) {
+            maxActiveOperationsPerSpace = getMaxActiveOperationsPerSpaceFromEnvironment();
+        }
+        return maxActiveOperationsPerSpace;
+    }
+
+    public Integer getMaxActiveOperationsPerUser() {
+        if (maxActiveOperationsPerUser == null) {
+            maxActiveOperationsPerUser = getMaxActiveOperationsPerUserFromEnvironment();
+        }
+        return maxActiveOperationsPerUser;
+    }
+
+    public Integer getOperationRateLimitPerSpaceCapacity() {
+        if (operationRateLimitPerSpaceCapacity == null) {
+            operationRateLimitPerSpaceCapacity = getOperationRateLimitPerSpaceCapacityFromEnvironment();
+        }
+        return operationRateLimitPerSpaceCapacity;
+    }
+
+    public Integer getOperationRateLimitPerSpaceRefillPerHour() {
+        if (operationRateLimitPerSpaceRefillPerHour == null) {
+            operationRateLimitPerSpaceRefillPerHour = getOperationRateLimitPerSpaceRefillPerHourFromEnvironment();
+        }
+        return operationRateLimitPerSpaceRefillPerHour;
+    }
+
+    public Integer getOperationRateLimitPerUserCapacity() {
+        if (operationRateLimitPerUserCapacity == null) {
+            operationRateLimitPerUserCapacity = getOperationRateLimitPerUserCapacityFromEnvironment();
+        }
+        return operationRateLimitPerUserCapacity;
+    }
+
+    public Integer getOperationRateLimitPerUserRefillPerHour() {
+        if (operationRateLimitPerUserRefillPerHour == null) {
+            operationRateLimitPerUserRefillPerHour = getOperationRateLimitPerUserRefillPerHourFromEnvironment();
+        }
+        return operationRateLimitPerUserRefillPerHour;
+    }
+
     private URL getControllerUrlFromEnvironment() {
         String controllerUrlString = environment.getString("CF_API");
         if (StringUtils.isEmpty(controllerUrlString)) {
@@ -1098,6 +1171,54 @@ public boolean isHealthCheckEnabledFromEnvironment() {
         return value;
     }
 
+    private Boolean isOperationRateLimitingEnabledThroughEnvironment() {
+        Boolean value = environment.getBoolean(CFG_OPERATION_RATE_LIMITING_ENABLED, DEFAULT_OPERATION_RATE_LIMITING_ENABLED);
+        logEnvironmentVariable(CFG_OPERATION_RATE_LIMITING_ENABLED, Messages.OPERATION_RATE_LIMITING_ENABLED, value);
+        return value;
+    }
+
+    private Integer getMaxActiveOperationsPerSpaceFromEnvironment() {
+        Integer value = environment.getPositiveInteger(CFG_MAX_ACTIVE_OPERATIONS_PER_SPACE, DEFAULT_MAX_ACTIVE_OPERATIONS_PER_SPACE);
+        logEnvironmentVariable(CFG_MAX_ACTIVE_OPERATIONS_PER_SPACE, Messages.MAX_ACTIVE_OPERATIONS_PER_SPACE, value);
+        return value;
+    }
+
+    private Integer getMaxActiveOperationsPerUserFromEnvironment() {
+        Integer value = environment.getPositiveInteger(CFG_MAX_ACTIVE_OPERATIONS_PER_USER, DEFAULT_MAX_ACTIVE_OPERATIONS_PER_USER);
+        logEnvironmentVariable(CFG_MAX_ACTIVE_OPERATIONS_PER_USER, Messages.MAX_ACTIVE_OPERATIONS_PER_USER, value);
+        return value;
+    }
+
+    private Integer getOperationRateLimitPerSpaceCapacityFromEnvironment() {
+        Integer value = environment.getPositiveInteger(CFG_OPERATION_RATE_LIMIT_PER_SPACE_CAPACITY,
+                                                       DEFAULT_OPERATION_RATE_LIMIT_PER_SPACE_CAPACITY);
+        logEnvironmentVariable(CFG_OPERATION_RATE_LIMIT_PER_SPACE_CAPACITY, Messages.OPERATION_RATE_LIMIT_PER_SPACE_CAPACITY, value);
+        return value;
+    }
+
+    private Integer getOperationRateLimitPerSpaceRefillPerHourFromEnvironment() {
+        Integer value = environment.getPositiveInteger(CFG_OPERATION_RATE_LIMIT_PER_SPACE_REFILL_PER_HOUR,
+                                                       DEFAULT_OPERATION_RATE_LIMIT_PER_SPACE_REFILL_PER_HOUR);
+        logEnvironmentVariable(CFG_OPERATION_RATE_LIMIT_PER_SPACE_REFILL_PER_HOUR,
+                               Messages.OPERATION_RATE_LIMIT_PER_SPACE_REFILL_PER_HOUR, value);
+        return value;
+    }
+
+    private Integer getOperationRateLimitPerUserCapacityFromEnvironment() {
+        Integer value = environment.getPositiveInteger(CFG_OPERATION_RATE_LIMIT_PER_USER_CAPACITY,
+                                                       DEFAULT_OPERATION_RATE_LIMIT_PER_USER_CAPACITY);
+        logEnvironmentVariable(CFG_OPERATION_RATE_LIMIT_PER_USER_CAPACITY, Messages.OPERATION_RATE_LIMIT_PER_USER_CAPACITY, value);
+        return value;
+    }
+
+    private Integer getOperationRateLimitPerUserRefillPerHourFromEnvironment() {
+        Integer value = environment.getPositiveInteger(CFG_OPERATION_RATE_LIMIT_PER_USER_REFILL_PER_HOUR,
+                                                       DEFAULT_OPERATION_RATE_LIMIT_PER_USER_REFILL_PER_HOUR);
+        logEnvironmentVariable(CFG_OPERATION_RATE_LIMIT_PER_USER_REFILL_PER_HOUR,
+                               Messages.OPERATION_RATE_LIMIT_PER_USER_REFILL_PER_HOUR, value);
+        return value;
+    }
+
     public Boolean isInternalEnvironment() {
         return environment.getBoolean(SAP_INTERNAL_DELIVERY, DEFAULT_SAP_INTERNAL_DELIVERY);
     }

multiapps-controller-core/src/test/java/org/cloudfoundry/multiapps/controller/core/util/ApplicationConfigurationTest.java

@@ -464,6 +464,131 @@
         Assertions.assertEquals(executorThreads, configuration.getSpringSchedulerTaskExecutorThreads());
     }
 
+    @Test
+    void testIsOperationRateLimitingEnabled() {
+        Mockito.when(environment.getBoolean(ApplicationConfiguration.CFG_OPERATION_RATE_LIMITING_ENABLED,
+                                            ApplicationConfiguration.DEFAULT_OPERATION_RATE_LIMITING_ENABLED))
+               .thenReturn(ApplicationConfiguration.DEFAULT_OPERATION_RATE_LIMITING_ENABLED);
+        assertEquals(ApplicationConfiguration.DEFAULT_OPERATION_RATE_LIMITING_ENABLED,
+                     configuration.isOperationRateLimitingEnabled());
+    }
+
+    @Test
+    void testIsOperationRateLimitingEnabledWithCustomValue() {
+        Mockito.when(environment.getBoolean(ApplicationConfiguration.CFG_OPERATION_RATE_LIMITING_ENABLED,
+                                            ApplicationConfiguration.DEFAULT_OPERATION_RATE_LIMITING_ENABLED))
+               .thenReturn(true);
+        assertTrue(configuration.isOperationRateLimitingEnabled());
+    }
+
+    @Test
+    void testGetMaxActiveOperationsPerSpace() {
+        Mockito.when(environment.getPositiveInteger(ApplicationConfiguration.CFG_MAX_ACTIVE_OPERATIONS_PER_SPACE,
+                                                    ApplicationConfiguration.DEFAULT_MAX_ACTIVE_OPERATIONS_PER_SPACE))
+               .thenReturn(ApplicationConfiguration.DEFAULT_MAX_ACTIVE_OPERATIONS_PER_SPACE);
+        assertEquals(ApplicationConfiguration.DEFAULT_MAX_ACTIVE_OPERATIONS_PER_SPACE,
+                     configuration.getMaxActiveOperationsPerSpace());
+    }
+
+    @Test
+    void testGetMaxActiveOperationsPerSpaceWithCustomValue() {
+        int customValue = 750;
+        Mockito.when(environment.getPositiveInteger(ApplicationConfiguration.CFG_MAX_ACTIVE_OPERATIONS_PER_SPACE,
+                                                    ApplicationConfiguration.DEFAULT_MAX_ACTIVE_OPERATIONS_PER_SPACE))
+               .thenReturn(customValue);
+        assertEquals(customValue, configuration.getMaxActiveOperationsPerSpace());
+    }
+
+    @Test
+    void testGetMaxActiveOperationsPerUser() {
+        Mockito.when(environment.getPositiveInteger(ApplicationConfiguration.CFG_MAX_ACTIVE_OPERATIONS_PER_USER,
+                                                    ApplicationConfiguration.DEFAULT_MAX_ACTIVE_OPERATIONS_PER_USER))
+               .thenReturn(ApplicationConfiguration.DEFAULT_MAX_ACTIVE_OPERATIONS_PER_USER);
+        assertEquals(ApplicationConfiguration.DEFAULT_MAX_ACTIVE_OPERATIONS_PER_USER,
+                     configuration.getMaxActiveOperationsPerUser());
+    }
+
+    @Test
+    void testGetMaxActiveOperationsPerUserWithCustomValue() {
+        int customValue = 250;
+        Mockito.when(environment.getPositiveInteger(ApplicationConfiguration.CFG_MAX_ACTIVE_OPERATIONS_PER_USER,
+                                                    ApplicationConfiguration.DEFAULT_MAX_ACTIVE_OPERATIONS_PER_USER))
+               .thenReturn(customValue);
+        assertEquals(customValue, configuration.getMaxActiveOperationsPerUser());
+    }
+
+    @Test
+    void testGetOperationRateLimitPerSpaceCapacity() {
+        Mockito.when(environment.getPositiveInteger(ApplicationConfiguration.CFG_OPERATION_RATE_LIMIT_PER_SPACE_CAPACITY,
+                                                    ApplicationConfiguration.DEFAULT_OPERATION_RATE_LIMIT_PER_SPACE_CAPACITY))
+               .thenReturn(ApplicationConfiguration.DEFAULT_OPERATION_RATE_LIMIT_PER_SPACE_CAPACITY);
+        assertEquals(ApplicationConfiguration.DEFAULT_OPERATION_RATE_LIMIT_PER_SPACE_CAPACITY,
+                     configuration.getOperationRateLimitPerSpaceCapacity());
+    }
+
+    @Test
+    void testGetOperationRateLimitPerSpaceCapacityWithCustomValue() {
+        int customValue = 400;
+        Mockito.when(environment.getPositiveInteger(ApplicationConfiguration.CFG_OPERATION_RATE_LIMIT_PER_SPACE_CAPACITY,
+                                                    ApplicationConfiguration.DEFAULT_OPERATION_RATE_LIMIT_PER_SPACE_CAPACITY))
+               .thenReturn(customValue);
+        assertEquals(customValue, configuration.getOperationRateLimitPerSpaceCapacity());
+    }
+
+    @Test
+    void testGetOperationRateLimitPerSpaceRefillPerHour() {
+        Mockito.when(environment.getPositiveInteger(ApplicationConfiguration.CFG_OPERATION_RATE_LIMIT_PER_SPACE_REFILL_PER_HOUR,
+                                                    ApplicationConfiguration.DEFAULT_OPERATION_RATE_LIMIT_PER_SPACE_REFILL_PER_HOUR))
+               .thenReturn(ApplicationConfiguration.DEFAULT_OPERATION_RATE_LIMIT_PER_SPACE_REFILL_PER_HOUR);
+        assertEquals(ApplicationConfiguration.DEFAULT_OPERATION_RATE_LIMIT_PER_SPACE_REFILL_PER_HOUR,
+                     configuration.getOperationRateLimitPerSpaceRefillPerHour());
+    }
+
+    @Test
+    void testGetOperationRateLimitPerSpaceRefillPerHourWithCustomValue() {
+        int customValue = 1000;
+        Mockito.when(environment.getPositiveInteger(ApplicationConfiguration.CFG_OPERATION_RATE_LIMIT_PER_SPACE_REFILL_PER_HOUR,
+                                                    ApplicationConfiguration.DEFAULT_OPERATION_RATE_LIMIT_PER_SPACE_REFILL_PER_HOUR))
+               .thenReturn(customValue);
+        assertEquals(customValue, configuration.getOperationRateLimitPerSpaceRefillPerHour());
+    }
+
+    @Test
+    void testGetOperationRateLimitPerUserCapacity() {
+        Mockito.when(environment.getPositiveInteger(ApplicationConfiguration.CFG_OPERATION_RATE_LIMIT_PER_USER_CAPACITY,
+                                                    ApplicationConfiguration.DEFAULT_OPERATION_RATE_LIMIT_PER_USER_CAPACITY))
+               .thenReturn(ApplicationConfiguration.DEFAULT_OPERATION_RATE_LIMIT_PER_USER_CAPACITY);
+        assertEquals(ApplicationConfiguration.DEFAULT_OPERATION_RATE_LIMIT_PER_USER_CAPACITY,
+                     configuration.getOperationRateLimitPerUserCapacity());
+    }
+
+    @Test
+    void testGetOperationRateLimitPerUserCapacityWithCustomValue() {
+        int customValue = 200;
+        Mockito.when(environment.getPositiveInteger(ApplicationConfiguration.CFG_OPERATION_RATE_LIMIT_PER_USER_CAPACITY,
+                                                    ApplicationConfiguration.DEFAULT_OPERATION_RATE_LIMIT_PER_USER_CAPACITY))
+               .thenReturn(customValue);
+        assertEquals(customValue, configuration.getOperationRateLimitPerUserCapacity());
+    }
+
+    @Test
+    void testGetOperationRateLimitPerUserRefillPerHour() {
+        Mockito.when(environment.getPositiveInteger(ApplicationConfiguration.CFG_OPERATION_RATE_LIMIT_PER_USER_REFILL_PER_HOUR,
+                                                    ApplicationConfiguration.DEFAULT_OPERATION_RATE_LIMIT_PER_USER_REFILL_PER_HOUR))
+               .thenReturn(ApplicationConfiguration.DEFAULT_OPERATION_RATE_LIMIT_PER_USER_REFILL_PER_HOUR);
+        assertEquals(ApplicationConfiguration.DEFAULT_OPERATION_RATE_LIMIT_PER_USER_REFILL_PER_HOUR,
+                     configuration.getOperationRateLimitPerUserRefillPerHour());
+    }
+
+    @Test
+    void testGetOperationRateLimitPerUserRefillPerHourWithCustomValue() {
+        int customValue = 500;
+        Mockito.when(environment.getPositiveInteger(ApplicationConfiguration.CFG_OPERATION_RATE_LIMIT_PER_USER_REFILL_PER_HOUR,
+                                                    ApplicationConfiguration.DEFAULT_OPERATION_RATE_LIMIT_PER_USER_REFILL_PER_HOUR))
+               .thenReturn(customValue);
+        assertEquals(customValue, configuration.getOperationRateLimitPerUserRefillPerHour());
+    }
+
     @Test
     void testGetFilteredEnv() {
         Map<String, String> filteredEnvironment = new HashMap<>();

multiapps-controller-persistence/src/main/resources/org/cloudfoundry/multiapps/controller/persistence/db/changelog/db-changelog-2.52.0-persistence.xml

@@ -0,0 +1,25 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<databaseChangeLog xmlns="http://www.liquibase.org/xml/ns/dbchangelog"
+                   xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+                   xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog
+         http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.6.xsd">
+
+    <changeSet author="sap.com" id="add_table_operation_rate_limit_bucket">
+        <preConditions onFail="MARK_RAN">
+            <not>
+                <tableExists tableName="operation_rate_limit_bucket"/>
+            </not>
+        </preConditions>
+
+        <createTable tableName="operation_rate_limit_bucket">
+            <column name="id" type="BIGINT">
+                <constraints primaryKey="true" primaryKeyName="operation_rate_limit_bucket_pk"/>
+            </column>
+
+            <column name="state" type="${small-blob.type}"/>
+
+            <column name="expires_at" type="BIGINT"/>
+        </createTable>
+    </changeSet>
+
+</databaseChangeLog>

multiapps-controller-persistence/src/main/resources/org/cloudfoundry/multiapps/controller/persistence/db/changelog/db-changelog.xml

@@ -47,5 +47,7 @@
 
     <include
             file="/org/cloudfoundry/multiapps/controller/persistence/db/changelog/db-changelog-2.43.0-persistence.xml"/>
-
+
+    <include
+            file="/org/cloudfoundry/multiapps/controller/persistence/db/changelog/db-changelog-2.52.0-persistence.xml"/>
 </databaseChangeLog>