From 66f36a2019a9aaf81441bc650d8ff07c0ceb9847 Mon Sep 17 00:00:00 2001 From: Brad House Date: Wed, 9 Sep 2026 21:44:07 +0000 Subject: [PATCH 01/18] Add a DAO method to count VMs per host Counts VMs occupying each host in a zone, pod or cluster, optionally only those that changed state recently. One query for the whole scope rather than one per host. Signed-off-by: Brad House --- .../java/com/cloud/vm/dao/VMInstanceDao.java | 10 +++++ .../com/cloud/vm/dao/VMInstanceDaoImpl.java | 44 +++++++++++++++++++ 2 files changed, 54 insertions(+) diff --git a/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDao.java b/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDao.java index 1a5b8cedd9ea..dbe4472f31e9 100755 --- a/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDao.java +++ b/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDao.java @@ -131,6 +131,16 @@ public interface VMInstanceDao extends GenericDao, StateDao< List listHostIdsByVmCount(long dcId, Long podId, Long clusterId, long accountId); + /** + * Counts the VMs occupying each host in a zone, pod or cluster. + * + * @param startedAfter + * when set, counts only VMs that last changed state after this time, which approximates + * the VMs that have recently started and are still working through their startup load + * @return host id to VM count, hosts with no VMs omitted + */ + Map countVmsByHost(long dcId, Long podId, Long clusterId, Date startedAfter); + Long countRunningAndStartingByAccount(long accountId); Long countByZoneAndState(long zoneId, State state); diff --git a/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDaoImpl.java b/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDaoImpl.java index ae1e838649ba..e69ae9b3fc09 100755 --- a/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDaoImpl.java @@ -18,6 +18,7 @@ import java.sql.PreparedStatement; import java.sql.ResultSet; +import java.sql.Timestamp; import java.sql.SQLException; import java.util.ArrayList; import java.util.Collection; @@ -154,6 +155,11 @@ public class VMInstanceDaoImpl extends GenericDaoBase implem private static final String COUNT_VMS_BASED_ON_VGPU_TYPES2 = "GROUP BY gpu_card.name, vgpu_profile.name"; + private static final String COUNT_VMS_BY_HOST = "SELECT host.id, COUNT(vm.id) FROM `cloud`.`host` host " + + "LEFT JOIN `cloud`.`vm_instance` vm ON vm.host_id = host.id AND vm.state IN ('Running', 'Starting', 'Migrating') " + + "AND vm.removed IS NULL %s WHERE host.type = 'Routing' AND host.removed IS NULL AND host.data_center_id = ? "; + private static final String COUNT_VMS_BY_HOST_PART2 = " GROUP BY host.id "; + private static final String UPDATE_SYSTEM_VM_TEMPLATE_ID_FOR_HYPERVISOR = "UPDATE `cloud`.`vm_instance` SET vm_template_id = ? WHERE type <> 'User' AND hypervisor_type = ? AND removed is NULL"; private static final String COUNT_VMS_BY_ZONE_AND_STATE_AND_HOST_TAG = "SELECT COUNT(1) FROM vm_instance vi JOIN service_offering so ON vi.service_offering_id=so.id " + @@ -795,6 +801,44 @@ public Pair, Map> listPodIdsInZoneByVmCount(long dataCe } } + + @Override + public Map countVmsByHost(long dcId, Long podId, Long clusterId, Date startedAfter) { + TransactionLegacy txn = TransactionLegacy.currentTxn(); + Map result = new HashMap<>(); + String sql = String.format(COUNT_VMS_BY_HOST, startedAfter != null ? " AND vm.update_time > ? " : ""); + if (podId != null) { + sql = sql + " AND host.pod_id = ? "; + } + if (clusterId != null) { + sql = sql + " AND host.cluster_id = ? "; + } + sql = sql + COUNT_VMS_BY_HOST_PART2; + try { + PreparedStatement pstmt = txn.prepareAutoCloseStatement(sql); + int index = 1; + if (startedAfter != null) { + pstmt.setTimestamp(index++, new Timestamp(startedAfter.getTime())); + } + pstmt.setLong(index++, dcId); + if (podId != null) { + pstmt.setLong(index++, podId); + } + if (clusterId != null) { + pstmt.setLong(index, clusterId); + } + ResultSet rs = pstmt.executeQuery(); + while (rs.next()) { + result.put(rs.getLong(1), rs.getLong(2)); + } + return result; + } catch (SQLException e) { + throw new CloudRuntimeException("DB Exception on: " + sql, e); + } catch (Throwable e) { + throw new CloudRuntimeException("Caught: " + sql, e); + } + } + @Override public List listHostIdsByVmCount(long dcId, Long podId, Long clusterId, long accountId) { TransactionLegacy txn = TransactionLegacy.currentTxn(); From 8756420bc3f0826841b4771de21a8e8e97a77090 Mon Sep 17 00:00:00 2001 From: Brad House Date: Wed, 9 Sep 2026 21:44:07 +0000 Subject: [PATCH 02/18] Track a moving average of real host utilisation StatsCollector already polls CPU and memory utilisation for every host, but it keeps only the newest sample and nothing uses it for placement. - fold those samples into an exponentially weighted moving average - weight by elapsed time, so a missed poll decays correctly instead of over-weighting the previous value - report nothing usable until a host has been sampled, so callers can fall back to allocation figures Two settings: host.load.sample.interval and host.load.half.life. Signed-off-by: Brad House --- .../manager/allocator/impl/HostLoad.java | 60 ++++++ .../allocator/impl/HostLoadTracker.java | 193 ++++++++++++++++++ .../allocator/impl/HostLoadTrackerTest.java | 141 +++++++++++++ 3 files changed, 394 insertions(+) create mode 100644 server/src/main/java/com/cloud/agent/manager/allocator/impl/HostLoad.java create mode 100644 server/src/main/java/com/cloud/agent/manager/allocator/impl/HostLoadTracker.java create mode 100644 server/src/test/java/com/cloud/agent/manager/allocator/impl/HostLoadTrackerTest.java diff --git a/server/src/main/java/com/cloud/agent/manager/allocator/impl/HostLoad.java b/server/src/main/java/com/cloud/agent/manager/allocator/impl/HostLoad.java new file mode 100644 index 000000000000..2e3d14ba8384 --- /dev/null +++ b/server/src/main/java/com/cloud/agent/manager/allocator/impl/HostLoad.java @@ -0,0 +1,60 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.agent.manager.allocator.impl; + +/** + * Smoothed view of what a host is actually doing, as opposed to what has been allocated on it. + * Fractions are of the host's real capacity and ignore overprovisioning. + */ +public class HostLoad { + + public static final HostLoad UNKNOWN = new HostLoad(0, 0, 0); + + private final double cpuUtilisation; + private final double memoryUtilisation; + private final long samples; + + public HostLoad(double cpuUtilisation, double memoryUtilisation, long samples) { + this.cpuUtilisation = cpuUtilisation; + this.memoryUtilisation = memoryUtilisation; + this.samples = samples; + } + + public double getCpuUtilisation() { + return cpuUtilisation; + } + + public double getMemoryUtilisation() { + return memoryUtilisation; + } + + public long getSamples() { + return samples; + } + + /** + * False until enough has been observed to rank on. Callers fall back to allocation figures. + */ + public boolean isUsable() { + return samples > 0; + } + + @Override + public String toString() { + return String.format("HostLoad[cpu=%.3f, memory=%.3f, samples=%d]", cpuUtilisation, memoryUtilisation, samples); + } +} diff --git a/server/src/main/java/com/cloud/agent/manager/allocator/impl/HostLoadTracker.java b/server/src/main/java/com/cloud/agent/manager/allocator/impl/HostLoadTracker.java new file mode 100644 index 000000000000..5708d487b38f --- /dev/null +++ b/server/src/main/java/com/cloud/agent/manager/allocator/impl/HostLoadTracker.java @@ -0,0 +1,193 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.agent.manager.allocator.impl; + +import java.util.Map; +import java.util.Timer; +import java.util.TimerTask; +import java.util.concurrent.ConcurrentHashMap; + +import javax.inject.Inject; + +import org.apache.cloudstack.framework.config.ConfigKey; +import org.apache.cloudstack.framework.config.Configurable; +import org.apache.cloudstack.managed.context.ManagedContextTimerTask; + +import com.cloud.host.HostStats; +import com.cloud.host.HostVO; +import com.cloud.host.Status; +import com.cloud.host.dao.HostDao; +import com.cloud.server.StatsCollector; +import com.cloud.utils.component.ManagerBase; + +/** + * Keeps a smoothed view of how hard each host is actually working. + * + * StatsCollector already polls every host, but it keeps only the newest sample and nothing uses it + * for placement. A single sample is too noisy to rank on: a host can look idle moments before a + * batch of VMs starts work. This folds those samples into an exponentially weighted moving average + * so ranking reflects a trend rather than an instant. + * + * The average is per management server and is not persisted. Every management server polls every + * host, so all of them converge on the same picture, and a restarted server simply reports nothing + * usable until it has sampled - callers then fall back to allocation figures. + */ +public class HostLoadTracker extends ManagerBase implements Configurable { + + public static final ConfigKey HostLoadSampleInterval = new ConfigKey<>(ConfigKey.CATEGORY_ADVANCED, + Integer.class, "host.load.sample.interval", "60", + "Seconds between samples of host CPU and memory utilisation, for placement algorithms that " + + "consider actual load. Should not be shorter than host.stats.interval.", + false, ConfigKey.Scope.Global); + + public static final ConfigKey HostLoadHalfLife = new ConfigKey<>(ConfigKey.CATEGORY_ADVANCED, + Integer.class, "host.load.half.life", "300", + "Half life in seconds of the moving average of host utilisation. Larger values react more " + + "slowly and are less affected by short spikes or by guests periodically releasing memory.", + true, ConfigKey.Scope.Global); + + @Inject + private HostDao hostDao; + + @Inject + private StatsCollector statsCollector; + + private final Map samples = new ConcurrentHashMap<>(); + + private Timer timer; + + @Override + public boolean start() { + int interval = Math.max(1, HostLoadSampleInterval.value()) * 1000; + TimerTask task = new ManagedContextTimerTask() { + @Override + protected void runInContext() { + try { + sampleAllHosts(); + } catch (Exception e) { + logger.warn("Unable to sample host load", e); + } + } + }; + timer = new Timer("HostLoadTracker"); + timer.schedule(task, interval, interval); + return true; + } + + @Override + public boolean stop() { + if (timer != null) { + timer.cancel(); + } + return true; + } + + protected void sampleAllHosts() { + for (HostVO host : hostDao.listByType(com.cloud.host.Host.Type.Routing)) { + if (host.getStatus() != Status.Up) { + samples.remove(host.getId()); + continue; + } + record(host.getId(), statsCollector.getHostStats(host.getId())); + } + } + + protected void record(long hostId, HostStats stats) { + record(hostId, stats, System.currentTimeMillis()); + } + + protected void record(long hostId, HostStats stats, long now) { + if (stats == null) { + return; + } + double totalMemory = stats.getTotalMemoryKBs(); + if (totalMemory <= 0) { + return; + } + // getCpuUtilization is a percentage of the host's real cores + double cpu = clamp(stats.getCpuUtilization() / 100.0); + double memory = clamp((totalMemory - stats.getFreeMemoryKBs()) / totalMemory); + + samples.compute(hostId, (id, previous) -> previous == null + ? new Sample(cpu, memory, now) + : previous.fold(cpu, memory, now, HostLoadHalfLife.value())); + } + + public HostLoad getLoad(long hostId) { + Sample sample = samples.get(hostId); + return sample == null ? HostLoad.UNKNOWN : sample.toHostLoad(); + } + + protected void clear() { + samples.clear(); + } + + private static double clamp(double value) { + if (Double.isNaN(value) || value < 0) { + return 0; + } + return Math.min(value, 1); + } + + @Override + public String getConfigComponentName() { + return HostLoadTracker.class.getSimpleName(); + } + + @Override + public ConfigKey[] getConfigKeys() { + return new ConfigKey[] {HostLoadSampleInterval, HostLoadHalfLife}; + } + + /** + * One host's running average. Weighting is by elapsed time rather than by sample count, so a + * missed poll decays the old value by the right amount instead of over-weighting it. + */ + private static final class Sample { + private final double cpu; + private final double memory; + private final long updatedAt; + private final long count; + + private Sample(double cpu, double memory, long updatedAt) { + this(cpu, memory, updatedAt, 1); + } + + private Sample(double cpu, double memory, long updatedAt, long count) { + this.cpu = cpu; + this.memory = memory; + this.updatedAt = updatedAt; + this.count = count; + } + + private Sample fold(double newCpu, double newMemory, long now, int halfLifeSeconds) { + double alpha = alpha(now - updatedAt, halfLifeSeconds); + return new Sample(cpu + alpha * (newCpu - cpu), memory + alpha * (newMemory - memory), now, count + 1); + } + + private static double alpha(long elapsedMillis, int halfLifeSeconds) { + if (halfLifeSeconds <= 0 || elapsedMillis <= 0) { + return 1; + } + return 1 - Math.exp(-(elapsedMillis / 1000.0) * Math.log(2) / halfLifeSeconds); + } + + private HostLoad toHostLoad() { + return new HostLoad(cpu, memory, count); + } + } +} diff --git a/server/src/test/java/com/cloud/agent/manager/allocator/impl/HostLoadTrackerTest.java b/server/src/test/java/com/cloud/agent/manager/allocator/impl/HostLoadTrackerTest.java new file mode 100644 index 000000000000..067037855d8d --- /dev/null +++ b/server/src/test/java/com/cloud/agent/manager/allocator/impl/HostLoadTrackerTest.java @@ -0,0 +1,141 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.agent.manager.allocator.impl; + +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; + +import com.cloud.host.HostStats; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +@RunWith(MockitoJUnitRunner.class) +public class HostLoadTrackerTest { + + private static final long HOST_ID = 1L; + private static final long HALF_LIFE_MS = 300 * 1000L; + + @InjectMocks + private HostLoadTracker tracker = new HostLoadTracker(); + + private long now; + + @Before + public void setUp() { + tracker.clear(); + now = 1_000_000L; + } + + private HostStats stats(double cpuPercent, double usedMemoryFraction) { + HostStats stats = Mockito.mock(HostStats.class); + Mockito.lenient().when(stats.getCpuUtilization()).thenReturn(cpuPercent); + Mockito.lenient().when(stats.getTotalMemoryKBs()).thenReturn(1000.0); + Mockito.lenient().when(stats.getFreeMemoryKBs()).thenReturn(1000.0 * (1 - usedMemoryFraction)); + return stats; + } + + private void sample(double cpuPercent, double usedMemoryFraction, long advanceMs) { + now += advanceMs; + tracker.record(HOST_ID, stats(cpuPercent, usedMemoryFraction), now); + } + + @Test + public void testUnknownHostIsNotUsable() { + assertFalse(tracker.getLoad(999L).isUsable()); + } + + @Test + public void testFirstSampleIsTakenAsIs() { + sample(40, 0.6, 0); + + HostLoad load = tracker.getLoad(HOST_ID); + assertTrue(load.isUsable()); + assertEquals(0.40, load.getCpuUtilisation(), 1e-6); + assertEquals(0.60, load.getMemoryUtilisation(), 1e-6); + } + + @Test + public void testSingleSpikeDoesNotDominateTheAverage() { + sample(10, 0.1, 0); + sample(100, 0.1, 60 * 1000L); + + // one sample a fifth of a half life in should move the average part of the way, not all of it + double cpu = tracker.getLoad(HOST_ID).getCpuUtilisation(); + assertTrue("a single spike must not take over the average: " + cpu, cpu < 0.30); + assertTrue("but it must move it: " + cpu, cpu > 0.10); + } + + @Test + public void testSustainedLoadConvergesOnTheNewValue() { + sample(10, 0.1, 0); + for (int i = 0; i < 40; i++) { + sample(90, 0.9, 60 * 1000L); + } + + HostLoad load = tracker.getLoad(HOST_ID); + assertEquals(0.90, load.getCpuUtilisation(), 0.01); + assertEquals(0.90, load.getMemoryUtilisation(), 0.01); + } + + @Test + public void testHalfLifeMovesAverageHalfWay() { + sample(0, 0, 0); + sample(100, 1.0, HALF_LIFE_MS); + + assertEquals(0.5, tracker.getLoad(HOST_ID).getCpuUtilisation(), 0.01); + } + + @Test + public void testMissedSamplesDecayByElapsedTimeNotSampleCount() { + sample(0, 0, 0); + sample(100, 1.0, 4 * HALF_LIFE_MS); + + // four half lives of catching up in one sample, so almost all the way there + assertTrue(tracker.getLoad(HOST_ID).getCpuUtilisation() > 0.9); + } + + @Test + public void testNullStatsAreIgnored() { + tracker.record(HOST_ID, null, now); + assertFalse(tracker.getLoad(HOST_ID).isUsable()); + } + + @Test + public void testHostReportingNoMemoryIsIgnored() { + HostStats broken = Mockito.mock(HostStats.class); + Mockito.lenient().when(broken.getTotalMemoryKBs()).thenReturn(0.0); + + tracker.record(HOST_ID, broken, now); + + assertFalse(tracker.getLoad(HOST_ID).isUsable()); + } + + @Test + public void testOutOfRangeValuesAreClamped() { + sample(250, 2.0, 0); + + HostLoad load = tracker.getLoad(HOST_ID); + assertEquals(1.0, load.getCpuUtilisation(), 1e-6); + assertEquals(1.0, load.getMemoryUtilisation(), 1e-6); + } +} From 9faa41d1e8eba4b0690230d1e039b0d19b78a7bd Mon Sep 17 00:00:00 2001 From: Brad House Date: Wed, 9 Sep 2026 21:44:07 +0000 Subject: [PATCH 03/18] Add 'balancedweighted' VM allocation algorithm Opt-in via vm.allocation.algorithm. Existing algorithms and the default are untouched. Allocated capacity alone is a poor ranking under heavy overprovisioning: it is measured against a total already multiplied by the overprovisioning factor, so a host under real strain still reports a low percentage and keeps being chosen. Anything allocation cannot see - VMs the scheduler has lost track of, guests using more than they asked for - is invisible. Ranks on a blend, lower is better: | term | source | |---------------------|-------------------------------------------| | CPU allocated | op_host_capacity, over the inflated total | | CPU utilisation | moving average of real usage | | memory allocated | op_host_capacity, over the inflated total | | memory utilisation | moving average of real usage | | VM count | VMs on the host | | recent starts | VMs started within the last few minutes | A dominant resource term is added on top so a host nearly out of any one resource does not rank well on a good average. Hosts measurably too busy are held back, unless that would leave nowhere to deploy. Selection is random among the best few rather than strictly ordered: capacity is only charged once a VM starts, so concurrent deployments all read the same figures and strict ordering makes them agree on one host. All weights and thresholds are settings, most cluster scoped. Signed-off-by: Brad House --- PendingReleaseNotes | 19 ++ .../deploy/DeploymentClusterPlanner.java | 7 +- .../com/cloud/deploy/DeploymentPlanner.java | 2 +- .../com/cloud/host/HostScoringWeights.java | 57 ++++ .../allocator/impl/FirstFitAllocator.java | 5 + .../allocator/impl/WeightedHostScorer.java | 307 ++++++++++++++++++ .../spring-server-allocator-context.xml | 6 + .../impl/WeightedHostScorerTest.java | 169 ++++++++++ .../WeightedPlacementDistributionTest.java | 278 ++++++++++++++++ 9 files changed, 847 insertions(+), 3 deletions(-) create mode 100644 api/src/main/java/com/cloud/host/HostScoringWeights.java create mode 100644 server/src/main/java/com/cloud/agent/manager/allocator/impl/WeightedHostScorer.java create mode 100644 server/src/test/java/com/cloud/agent/manager/allocator/impl/WeightedHostScorerTest.java create mode 100644 server/src/test/java/com/cloud/agent/manager/allocator/impl/WeightedPlacementDistributionTest.java diff --git a/PendingReleaseNotes b/PendingReleaseNotes index 02d63811e36e..6156cc3ae1b1 100644 --- a/PendingReleaseNotes +++ b/PendingReleaseNotes @@ -53,3 +53,22 @@ example.ver.1 > example.ver.2: the guest while the NVRAM sidecar is copied, so that the captured firmware state is consistent with the disk snapshot. Non-UEFI VMs are unaffected and continue to snapshot live. + + * New VM allocation algorithm 'balancedweighted' for vm.allocation.algorithm. Existing + algorithms and the default are unchanged; the new one is opt-in. + + Existing algorithms rank hosts on allocated capacity alone. Under a large overprovisioning + factor that reads badly: allocation is measured against a total that has already been + multiplied by the factor, so a host under real strain can still report a low percentage and + keep attracting new VMs. Concurrent deployments make it worse, since they all read the same + figures before any of them is accounted for. + + 'balancedweighted' ranks hosts on a blend of allocated CPU and memory, measured CPU and + memory utilisation, VM count, and how many VMs started on the host recently. It holds back + hosts that are measurably too busy, and chooses at random from among the best scoring hosts + so that simultaneous deployments do not all pick the same one. + + Tuned with the host.weighted.* settings, most of which are cluster scoped. Measured + utilisation comes from a moving average configured with host.load.sample.interval and + host.load.half.life. Until a management server has collected samples, ranking falls back to + allocation figures alone. diff --git a/api/src/main/java/com/cloud/deploy/DeploymentClusterPlanner.java b/api/src/main/java/com/cloud/deploy/DeploymentClusterPlanner.java index 9471c3d5c84c..5bf7dffbbf9b 100644 --- a/api/src/main/java/com/cloud/deploy/DeploymentClusterPlanner.java +++ b/api/src/main/java/com/cloud/deploy/DeploymentClusterPlanner.java @@ -62,11 +62,14 @@ public interface DeploymentClusterPlanner extends DeploymentPlanner { "vm.allocation.algorithm", "Advanced", "random", - "Order in which hosts within a cluster will be considered for VM allocation. The value can be 'random', 'firstfit', 'userdispersing', or 'firstfitleastconsumed'.", + "Order in which hosts within a cluster will be considered for VM allocation. The value can be 'random', " + + "'firstfit', 'userdispersing', 'firstfitleastconsumed', or 'balancedweighted'. 'balancedweighted' " + + "ranks hosts on a blend of allocated capacity, measured utilisation, VM count and how many VMs " + + "started recently, and is tuned with the host.weighted.* settings.", true, ConfigKey.Scope.Global, null, null, null, null, null, ConfigKey.Kind.Select, - "random,firstfit,userdispersing,firstfitleastconsumed"); + "random,firstfit,userdispersing,firstfitleastconsumed,balancedweighted"); /** * This is called to determine list of possible clusters where a virtual diff --git a/api/src/main/java/com/cloud/deploy/DeploymentPlanner.java b/api/src/main/java/com/cloud/deploy/DeploymentPlanner.java index 22d796d4a775..2d2964916eed 100644 --- a/api/src/main/java/com/cloud/deploy/DeploymentPlanner.java +++ b/api/src/main/java/com/cloud/deploy/DeploymentPlanner.java @@ -70,7 +70,7 @@ public interface DeploymentPlanner extends Adapter { boolean canHandle(VirtualMachineProfile vm, DeploymentPlan plan, ExcludeList avoid); public enum AllocationAlgorithm { - random, firstfit, userdispersing, firstfitleastconsumed; + random, firstfit, userdispersing, firstfitleastconsumed, balancedweighted; } public enum PlannerResourceUsage { diff --git a/api/src/main/java/com/cloud/host/HostScoringWeights.java b/api/src/main/java/com/cloud/host/HostScoringWeights.java new file mode 100644 index 000000000000..886487aeedf6 --- /dev/null +++ b/api/src/main/java/com/cloud/host/HostScoringWeights.java @@ -0,0 +1,57 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.host; + +import org.apache.cloudstack.framework.config.ConfigKey; + +/** + * How much each signal counts when ranking hosts by how loaded they are. + * + * Shared by initial placement and by rebalancing on purpose. If the two weighted these differently + * they would disagree about which host is the better one, and rebalancing could move VMs off hosts + * that placement had just chosen, only for placement to put them back. + * + * Weights are relative to each other; only their ratios matter, and zero disables a term. Terms that + * only make sense for one of the two - how many VMs a host carries, how many started recently - stay + * with whichever uses them. + */ +public interface HostScoringWeights { + + String WEIGHT_DESCRIPTION_SUFFIX = " Relative weight, only meaningful compared with the other " + + "host.weighted.* weights. Zero disables the term. Used by both the 'balancedweighted' " + + "allocation algorithm and the 'weighted' DRS algorithm."; + + ConfigKey CpuAllocatedWeight = new ConfigKey<>(ConfigKey.CATEGORY_ADVANCED, + Double.class, "host.weighted.cpu.allocated.weight", "1.0", + "How much CPU allocated on a host counts against it." + WEIGHT_DESCRIPTION_SUFFIX, + true, ConfigKey.Scope.Cluster); + + ConfigKey CpuUsedWeight = new ConfigKey<>(ConfigKey.CATEGORY_ADVANCED, + Double.class, "host.weighted.cpu.used.weight", "2.0", + "How much measured CPU utilisation counts against a host." + WEIGHT_DESCRIPTION_SUFFIX, + true, ConfigKey.Scope.Cluster); + + ConfigKey MemoryAllocatedWeight = new ConfigKey<>(ConfigKey.CATEGORY_ADVANCED, + Double.class, "host.weighted.memory.allocated.weight", "1.0", + "How much memory allocated on a host counts against it." + WEIGHT_DESCRIPTION_SUFFIX, + true, ConfigKey.Scope.Cluster); + + ConfigKey MemoryUsedWeight = new ConfigKey<>(ConfigKey.CATEGORY_ADVANCED, + Double.class, "host.weighted.memory.used.weight", "2.0", + "How much measured memory utilisation counts against a host." + WEIGHT_DESCRIPTION_SUFFIX, + true, ConfigKey.Scope.Cluster); +} diff --git a/server/src/main/java/com/cloud/agent/manager/allocator/impl/FirstFitAllocator.java b/server/src/main/java/com/cloud/agent/manager/allocator/impl/FirstFitAllocator.java index 4bc34d8a5c60..6a5669d17975 100644 --- a/server/src/main/java/com/cloud/agent/manager/allocator/impl/FirstFitAllocator.java +++ b/server/src/main/java/com/cloud/agent/manager/allocator/impl/FirstFitAllocator.java @@ -16,6 +16,7 @@ // under the License. package com.cloud.agent.manager.allocator.impl; +import static com.cloud.deploy.DeploymentPlanner.AllocationAlgorithm.balancedweighted; import static com.cloud.deploy.DeploymentPlanner.AllocationAlgorithm.firstfitleastconsumed; import static com.cloud.deploy.DeploymentPlanner.AllocationAlgorithm.random; import static com.cloud.deploy.DeploymentPlanner.AllocationAlgorithm.userdispersing; @@ -95,6 +96,8 @@ public class FirstFitAllocator extends BaseAllocator { CapacityDao _capacityDao; @Inject VMInstanceDetailsDao vmInstanceDetailsDao; + @Inject + WeightedHostScorer weightedHostScorer; boolean _checkHvm = true; @@ -209,6 +212,8 @@ protected List allocateTo(VirtualMachineProfile vmProfile, DeploymentPlan hosts = reorderHostsByNumberOfVms(plan, hosts, account); } else if (firstfitleastconsumed.toString().equals(vmAllocationAlgorithm)) { hosts = reorderHostsByCapacity(plan, hosts); + } else if (balancedweighted.toString().equals(vmAllocationAlgorithm)) { + hosts = weightedHostScorer.rank(plan.getDataCenterId(), plan.getPodId(), plan.getClusterId(), hosts); } logger.debug("FirstFitAllocator has {} hosts to check for allocation {}.", hosts.size(), hosts); diff --git a/server/src/main/java/com/cloud/agent/manager/allocator/impl/WeightedHostScorer.java b/server/src/main/java/com/cloud/agent/manager/allocator/impl/WeightedHostScorer.java new file mode 100644 index 000000000000..9f0057b09fb6 --- /dev/null +++ b/server/src/main/java/com/cloud/agent/manager/allocator/impl/WeightedHostScorer.java @@ -0,0 +1,307 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.agent.manager.allocator.impl; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.stream.Collectors; + +import javax.inject.Inject; + +import org.apache.cloudstack.framework.config.ConfigKey; +import org.apache.cloudstack.framework.config.Configurable; + +import com.cloud.capacity.Capacity; +import com.cloud.capacity.CapacityManager; +import com.cloud.capacity.CapacityVO; +import com.cloud.capacity.dao.CapacityDao; +import com.cloud.host.Host; +import com.cloud.host.HostScoringWeights; +import com.cloud.utils.component.AdapterBase; +import com.cloud.vm.dao.VMInstanceDao; + +/** + * Ranks hosts on a blend of what has been allocated on them and what they are actually doing. + * + * Ordering purely by allocated capacity misreads a heavily overprovisioned cluster: allocation is + * measured against a total that has been multiplied by the overprovisioning factor, so hosts under + * real strain can still look close to empty and keep attracting new VMs. This blends allocation + * with measured utilisation, VM count, and how many VMs started on the host recently, since a VM + * that has just started is usually working harder than its long run average. + * + * Scores run from 0 (idle) upwards and lower is better. Hosts are then chosen from among the best + * rather than strictly in order - see {@link #applySelectionSpread}. + */ +public class WeightedHostScorer extends AdapterBase implements Configurable { + + private static final String WEIGHT_DESCRIPTION_SUFFIX = + " Relative weight, only meaningful compared with the other host.weighted.* weights. Zero disables the term."; + + public static final ConfigKey VmCountWeight = new ConfigKey<>(ConfigKey.CATEGORY_ADVANCED, + Double.class, "host.weighted.vm.count.weight", "1.0", + "How much the number of VMs already on a host counts against it, regardless of how busy they are." + + WEIGHT_DESCRIPTION_SUFFIX, + true, ConfigKey.Scope.Cluster); + + public static final ConfigKey RecentStartWeight = new ConfigKey<>(ConfigKey.CATEGORY_ADVANCED, + Double.class, "host.weighted.recent.start.weight", "2.0", + "How much VMs started recently on a host count against it. Guards against sending a burst of new " + + "VMs to one host, since neither allocation nor utilisation has caught up with them yet." + + WEIGHT_DESCRIPTION_SUFFIX, + true, ConfigKey.Scope.Cluster); + + public static final ConfigKey DominantResourceWeight = new ConfigKey<>(ConfigKey.CATEGORY_ADVANCED, + Double.class, "host.weighted.dominant.resource.weight", "1.0", + "How much a host's single most stressed resource counts against it, on top of the average across " + + "resources. Keeps a host that is fine on average but nearly out of one resource from ranking well." + + WEIGHT_DESCRIPTION_SUFFIX, + true, ConfigKey.Scope.Cluster); + + public static final ConfigKey RecentStartWindow = new ConfigKey<>(ConfigKey.CATEGORY_ADVANCED, + Integer.class, "host.weighted.recent.start.window", "300", + "Seconds for which a newly started VM counts as recently started.", + true, ConfigKey.Scope.Global); + + public static final ConfigKey ExpectedVmsPerHost = new ConfigKey<>(ConfigKey.CATEGORY_ADVANCED, + Integer.class, "host.weighted.expected.vms.per.host", "50", + "Roughly how many VMs a host is expected to carry. Used only to bring VM counts onto the same " + + "0 to 1 scale as the other terms; it is not a limit and is never enforced.", + true, ConfigKey.Scope.Cluster); + + public static final ConfigKey CpuUtilisationThreshold = new ConfigKey<>(ConfigKey.CATEGORY_ADVANCED, + Double.class, "host.weighted.cpu.utilisation.threshold", "0.85", + "Hosts whose measured CPU utilisation is above this fraction are held back from new VMs. " + + "Ignored if it would leave nowhere to deploy. Set to 1 to disable.", + true, ConfigKey.Scope.Cluster); + + public static final ConfigKey MemoryUtilisationThreshold = new ConfigKey<>(ConfigKey.CATEGORY_ADVANCED, + Double.class, "host.weighted.memory.utilisation.threshold", "0.90", + "Hosts whose measured memory utilisation is above this fraction are held back from new VMs. " + + "Ignored if it would leave nowhere to deploy. Set to 1 to disable.", + true, ConfigKey.Scope.Cluster); + + public static final ConfigKey SelectionSpread = new ConfigKey<>(ConfigKey.CATEGORY_ADVANCED, + Integer.class, "host.weighted.selection.spread", "3", + "How many of the best scoring hosts to choose between at random. Ranking strictly by score sends " + + "concurrent deployments to the same host, because they all read the same figures before any " + + "of them is accounted for. 1 restores strict ordering.", + true, ConfigKey.Scope.Cluster); + + @Inject + private CapacityDao capacityDao; + + @Inject + private VMInstanceDao vmInstanceDao; + + @Inject + private HostLoadTracker hostLoadTracker; + + protected Random random = new Random(); + + /** + * Orders hosts best first. Hosts absent from the capacity tables keep their original relative + * order at the end of the list rather than being dropped. + */ + public List rank(long zoneId, Long podId, Long clusterId, List hosts) { + if (hosts == null || hosts.size() <= 1) { + return hosts == null ? new ArrayList<>() : new ArrayList<>(hosts); + } + + Map scores = score(zoneId, podId, clusterId, hosts); + + List scored = hosts.stream().filter(h -> scores.containsKey(h.getId())).collect(Collectors.toList()); + List unscored = hosts.stream().filter(h -> !scores.containsKey(h.getId())).collect(Collectors.toList()); + scored.sort((a, b) -> Double.compare(scores.get(a.getId()), scores.get(b.getId()))); + + List admitted = applyUtilisationThresholds(clusterId, scored); + applySelectionSpread(clusterId, admitted); + + logger.debug("Weighted host ranking: {}", () -> admitted.stream() + .map(h -> String.format("%s=%.4f", h.getName(), scores.get(h.getId()))) + .collect(Collectors.joining(", "))); + + admitted.addAll(unscored); + return admitted; + } + + protected Map score(long zoneId, Long podId, Long clusterId, List hosts) { + List capacities = capacityDao.listHostCapacityByCapacityTypes(zoneId, clusterId, + List.of(Capacity.CAPACITY_TYPE_CPU, Capacity.CAPACITY_TYPE_MEMORY)); + Map vmCounts = vmInstanceDao.countVmsByHost(zoneId, podId, clusterId, null); + Map recentStarts = vmInstanceDao.countVmsByHost(zoneId, podId, clusterId, + new Date(System.currentTimeMillis() - RecentStartWindow.value() * 1000L)); + + Map allocated = allocatedFractions(capacities); + + Map scores = new HashMap<>(); + for (Host host : hosts) { + Double[] alloc = allocated.get(host.getId()); + if (alloc == null) { + continue; + } + scores.put(host.getId(), scoreHost(clusterId, alloc[0], alloc[1], hostLoadTracker.getLoad(host.getId()), + vmCounts.getOrDefault(host.getId(), 0L), recentStarts.getOrDefault(host.getId(), 0L))); + } + return scores; + } + + /** + * Allocated CPU and memory as a fraction of what the host advertises after overprovisioning, + * which is the same basis the existing allocators use. + */ + protected Map allocatedFractions(List capacities) { + Map fractions = new HashMap<>(); + for (CapacityVO capacity : capacities) { + long total = capacity.getTotalCapacity(); + if (total <= 0) { + continue; + } + double used = (double) (capacity.getUsedCapacity() + capacity.getReservedCapacity()) / total; + Double[] entry = fractions.computeIfAbsent(capacity.getHostOrPoolId(), id -> new Double[] {0.0, 0.0}); + if (capacity.getCapacityType() == Capacity.CAPACITY_TYPE_CPU) { + entry[0] = clamp(used); + } else { + entry[1] = clamp(used); + } + } + return fractions; + } + + /** + * The blend. Every term is a fraction of the host's capacity for that resource so the weights + * are directly comparable, and the dominant resource term is added on top of the weighted mean + * so that being nearly out of any one resource is penalised even when the average looks fine. + */ + protected double scoreHost(Long clusterId, double cpuAllocated, double memoryAllocated, HostLoad load, + long vmCount, long recentStarts) { + double cpuUsedWeight = load.isUsable() ? valueIn(HostScoringWeights.CpuUsedWeight, clusterId) : 0; + double memoryUsedWeight = load.isUsable() ? valueIn(HostScoringWeights.MemoryUsedWeight, clusterId) : 0; + double vmScale = Math.max(1, valueIn(ExpectedVmsPerHost, clusterId)); + + double cpuAllocatedWeight = valueIn(HostScoringWeights.CpuAllocatedWeight, clusterId); + double memoryAllocatedWeight = valueIn(HostScoringWeights.MemoryAllocatedWeight, clusterId); + double vmCountWeight = valueIn(VmCountWeight, clusterId); + double recentStartWeight = valueIn(RecentStartWeight, clusterId); + + double vmCountTerm = clamp(vmCount / vmScale); + double recentStartTerm = clamp(recentStarts / vmScale); + + double weightSum = cpuAllocatedWeight + cpuUsedWeight + memoryAllocatedWeight + memoryUsedWeight + + vmCountWeight + recentStartWeight; + if (weightSum <= 0) { + return 0; + } + + double weighted = cpuAllocatedWeight * cpuAllocated + + cpuUsedWeight * load.getCpuUtilisation() + + memoryAllocatedWeight * memoryAllocated + + memoryUsedWeight * load.getMemoryUtilisation() + + vmCountWeight * vmCountTerm + + recentStartWeight * recentStartTerm; + + double mean = weighted / weightSum; + double dominantWeight = valueIn(DominantResourceWeight, clusterId); + if (dominantWeight <= 0) { + return mean; + } + return (mean + dominantWeight * dominantResource(cpuAllocated, memoryAllocated, load)) / (1 + dominantWeight); + } + + /** + * The most stressed resource on the host. Allocation and utilisation are both considered for + * each resource and the larger is taken, because memory that has been reclaimed from idle + * guests can be taken back as soon as those guests get busy. + */ + protected double dominantResource(double cpuAllocated, double memoryAllocated, HostLoad load) { + double cpu = load.isUsable() ? Math.max(cpuAllocated, load.getCpuUtilisation()) : cpuAllocated; + double memory = load.isUsable() ? Math.max(memoryAllocated, load.getMemoryUtilisation()) : memoryAllocated; + return Math.max(cpu, memory); + } + + /** + * Holds back hosts that are measurably too busy, unless that would leave nothing to deploy on, + * in which case ranking alone decides and the caller's capacity checks still apply. + */ + protected List applyUtilisationThresholds(Long clusterId, List ranked) { + double cpuThreshold = valueIn(CpuUtilisationThreshold, clusterId); + double memoryThreshold = valueIn(MemoryUtilisationThreshold, clusterId); + + List admitted = new ArrayList<>(); + List heldBack = new ArrayList<>(); + for (Host host : ranked) { + HostLoad load = hostLoadTracker.getLoad(host.getId()); + if (load.isUsable() + && (load.getCpuUtilisation() > cpuThreshold || load.getMemoryUtilisation() > memoryThreshold)) { + heldBack.add(host); + } else { + admitted.add(host); + } + } + + if (admitted.isEmpty()) { + logger.warn("Every candidate host is above its utilisation threshold, so the thresholds are being " + + "ignored for this deployment. The cluster is short of capacity."); + return heldBack; + } + if (!heldBack.isEmpty()) { + logger.debug("Holding back {} host(s) above their utilisation threshold: {}", heldBack.size(), heldBack); + admitted.addAll(heldBack); + } + return admitted; + } + + /** + * Shuffles the best few hosts so that deployments made at the same moment do not all pick the + * same one. Capacity is only charged once a VM starts, so until then every concurrent decision + * sees the same figures and strict ordering makes them agree. + */ + protected void applySelectionSpread(Long clusterId, List ranked) { + int spread = Math.min((int) valueIn(SelectionSpread, clusterId), ranked.size()); + if (spread > 1) { + Collections.shuffle(ranked.subList(0, spread), random); + } + } + + private double valueIn(ConfigKey key, Long clusterId) { + T value = clusterId == null ? key.value() : key.valueIn(clusterId); + return value == null ? 0 : value.doubleValue(); + } + + private static double clamp(double value) { + if (Double.isNaN(value) || value < 0) { + return 0; + } + return Math.min(value, 1); + } + + @Override + public String getConfigComponentName() { + return CapacityManager.class.getSimpleName(); + } + + @Override + public ConfigKey[] getConfigKeys() { + return new ConfigKey[] {HostScoringWeights.CpuAllocatedWeight, HostScoringWeights.CpuUsedWeight, HostScoringWeights.MemoryAllocatedWeight, HostScoringWeights.MemoryUsedWeight, + VmCountWeight, RecentStartWeight, DominantResourceWeight, RecentStartWindow, ExpectedVmsPerHost, + CpuUtilisationThreshold, MemoryUtilisationThreshold, SelectionSpread}; + } +} diff --git a/server/src/main/resources/META-INF/cloudstack/server-allocator/spring-server-allocator-context.xml b/server/src/main/resources/META-INF/cloudstack/server-allocator/spring-server-allocator-context.xml index 28b96b8d194b..69aea2cf8451 100644 --- a/server/src/main/resources/META-INF/cloudstack/server-allocator/spring-server-allocator-context.xml +++ b/server/src/main/resources/META-INF/cloudstack/server-allocator/spring-server-allocator-context.xml @@ -31,6 +31,12 @@ + + + + diff --git a/server/src/test/java/com/cloud/agent/manager/allocator/impl/WeightedHostScorerTest.java b/server/src/test/java/com/cloud/agent/manager/allocator/impl/WeightedHostScorerTest.java new file mode 100644 index 000000000000..7b2dd67bf157 --- /dev/null +++ b/server/src/test/java/com/cloud/agent/manager/allocator/impl/WeightedHostScorerTest.java @@ -0,0 +1,169 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.agent.manager.allocator.impl; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; + +import com.cloud.host.Host; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertTrue; + +@RunWith(MockitoJUnitRunner.class) +public class WeightedHostScorerTest { + + private static final HostLoad IDLE = new HostLoad(0.0, 0.0, 10); + + @Mock + private HostLoadTracker hostLoadTracker; + + @InjectMocks + private WeightedHostScorer scorer = new WeightedHostScorer(); + + private long nextHostId; + + @Before + public void setUp() { + nextHostId = 1; + } + + private Host host(String name) { + Host host = Mockito.mock(Host.class); + Mockito.lenient().when(host.getId()).thenReturn(nextHostId++); + Mockito.lenient().when(host.getName()).thenReturn(name); + return host; + } + + private double score(double cpuAllocated, double memAllocated, HostLoad load, long vms, long recentStarts) { + return scorer.scoreHost(null, cpuAllocated, memAllocated, load, vms, recentStarts); + } + + @Test + public void testIdleHostScoresZero() { + assertEquals(0.0, score(0, 0, IDLE, 0, 0), 1e-9); + } + + @Test + public void testMoreAllocationScoresHigher() { + assertTrue(score(0.5, 0.5, IDLE, 0, 0) > score(0.1, 0.1, IDLE, 0, 0)); + } + + @Test + public void testBusyHostScoresHigherThanIdleHostWithSameAllocation() { + HostLoad busy = new HostLoad(0.9, 0.5, 10); + assertTrue("measured load must separate hosts that look identical by allocation", + score(0.05, 0.05, busy, 20, 0) > score(0.05, 0.05, IDLE, 20, 0)); + } + + @Test + public void testUtilisationIgnoredUntilThereAreSamples() { + // a host with no samples must not be treated as idle, it must rank on allocation alone + double noSamples = score(0.4, 0.4, HostLoad.UNKNOWN, 10, 0); + double idleSamples = score(0.4, 0.4, IDLE, 10, 0); + assertTrue("a host with no load samples should not outrank a measurably idle one", + noSamples >= idleSamples); + } + + @Test + public void testDominantResourcePenalisesLopsidedHost() { + // same mean across resources, but one host is nearly out of memory + double balanced = score(0.5, 0.5, IDLE, 0, 0); + double lopsided = score(0.05, 0.95, IDLE, 0, 0); + assertTrue("a host nearly out of one resource must not rank as well as an evenly loaded one", + lopsided > balanced); + } + + @Test + public void testDominantResourceUsesUtilisationWhenHigherThanAllocation() { + HostLoad reclaimedButBusy = new HostLoad(0.95, 0.95, 10); + assertTrue(score(0.05, 0.05, reclaimedButBusy, 0, 0) > score(0.05, 0.05, IDLE, 0, 0)); + } + + @Test + public void testVmCountPenalisesHost() { + assertTrue(score(0.1, 0.1, IDLE, 120, 0) > score(0.1, 0.1, IDLE, 5, 0)); + } + + @Test + public void testRecentStartsPenaliseHost() { + assertTrue("VMs that just started are not yet visible in allocation or utilisation", + score(0.1, 0.1, IDLE, 20, 20) > score(0.1, 0.1, IDLE, 20, 0)); + } + + @Test + public void testScoreStaysWithinUnitRange() { + assertTrue(score(1, 1, new HostLoad(1, 1, 10), 1000, 1000) <= 1.0); + assertTrue(score(0, 0, IDLE, 0, 0) >= 0.0); + } + + @Test + public void testBusyHostIsHeldBackByThreshold() { + Host quiet = host("quiet"); + Host busy = host("busy"); + Mockito.when(hostLoadTracker.getLoad(quiet.getId())).thenReturn(new HostLoad(0.10, 0.10, 10)); + Mockito.when(hostLoadTracker.getLoad(busy.getId())).thenReturn(new HostLoad(0.99, 0.10, 10)); + + List result = scorer.applyUtilisationThresholds(null, new ArrayList<>(List.of(busy, quiet))); + + assertSame("host over the CPU threshold must fall behind", quiet, result.get(0)); + assertSame(busy, result.get(1)); + } + + @Test + public void testThresholdIsIgnoredWhenEveryHostIsBusy() { + Host a = host("a"); + Host b = host("b"); + Mockito.when(hostLoadTracker.getLoad(Mockito.anyLong())).thenReturn(new HostLoad(0.99, 0.99, 10)); + + List result = scorer.applyUtilisationThresholds(null, new ArrayList<>(List.of(a, b))); + + assertEquals("deployment must still be possible when the whole cluster is busy", 2, result.size()); + } + + @Test + public void testSelectionSpreadVariesTheChosenHost() { + Set chosen = new HashSet<>(); + List hosts = List.of(host("a"), host("b"), host("c"), host("d"), host("e")); + for (int i = 0; i < 200; i++) { + List ranked = new ArrayList<>(hosts); + scorer.applySelectionSpread(null, ranked); + chosen.add(ranked.get(0).getName()); + } + assertNotEquals("strict ordering sends every concurrent deployment to the same host", 1, chosen.size()); + assertTrue("only the best scoring hosts should be candidates", chosen.size() <= 3); + } + + @Test + public void testSelectionSpreadLeavesShortListsAlone() { + List ranked = new ArrayList<>(List.of(host("only"))); + scorer.applySelectionSpread(null, ranked); + assertEquals(1, ranked.size()); + } +} diff --git a/server/src/test/java/com/cloud/agent/manager/allocator/impl/WeightedPlacementDistributionTest.java b/server/src/test/java/com/cloud/agent/manager/allocator/impl/WeightedPlacementDistributionTest.java new file mode 100644 index 000000000000..4cb45fa52b1c --- /dev/null +++ b/server/src/test/java/com/cloud/agent/manager/allocator/impl/WeightedPlacementDistributionTest.java @@ -0,0 +1,278 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.agent.manager.allocator.impl; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Comparator; +import java.util.List; +import java.util.Random; + +import org.junit.Test; + +import static org.junit.Assert.assertTrue; + +/** + * Simulates placement over a churning, heavily overprovisioned fleet. + * + * This is the regression fixture for the failure the weighted scoring exists to prevent. Ranking + * hosts on allocated capacity alone is blind to what a host is really doing, and under a large + * overprovisioning factor the gap between the two can be enormous: a host can report a few percent + * allocated while its cores are saturated. Anything the scheduler is not told about - VMs it has + * lost track of, guests using far more than their share - is invisible, so the emptiest looking + * host keeps being chosen no matter how hard it is working. + * + * Three things are modelled that a single-shot unit test cannot show: + * + * - deployments arrive in concurrent batches and every decision in a batch reads the same figures, + * because capacity is only charged once a VM starts + * - real load that allocation cannot account for + * - VMs are short lived and their lifetimes vary, so hosts empty unevenly + * + * Deterministic: fixed seeds, no wall clock. + */ +public class WeightedPlacementDistributionTest { + + private static final int HOSTS = 9; + private static final int CORES_PER_HOST = 192; + private static final int MEMORY_MB_PER_HOST = 1_132_000; + private static final int CPU_OVERCOMMIT = 10; + private static final int MEMORY_OVERCOMMIT = 4; + + private static final int VM_CORES = 4; + private static final int VM_MEMORY_MB = 8_192; + + private static final int BATCHES = 300; + private static final int VMS_PER_BATCH = 8; + private static final int SPREAD = 3; + + /** Share of VMs that peg their cores for their whole life, as build runners do. */ + private static final double BUSY_FRACTION = 0.35; + + /** + * Cores in use on some hosts that allocation knows nothing about. Stands in for anything the + * scheduler cannot see - VMs it believes are gone, or guests far exceeding their request. + */ + private static final int UNACCOUNTED_CORES = 60; + + private static final class SimHost { + final long id; + final double unaccountedCores; + int vms; + int recentStarts; + double busyCores; + double memoryMb; + + SimHost(long id, double unaccountedCores) { + this.id = id; + this.unaccountedCores = unaccountedCores; + } + + /** What the capacity tables would report: uniform per VM, against an inflated total. */ + double cpuAllocated() { + return (double) vms * VM_CORES / (CORES_PER_HOST * CPU_OVERCOMMIT); + } + + double memoryAllocated() { + return (double) vms * VM_MEMORY_MB / ((double) MEMORY_MB_PER_HOST * MEMORY_OVERCOMMIT); + } + + /** What the host is really doing, including what allocation cannot see. */ + double realCores() { + return busyCores + unaccountedCores; + } + + HostLoad load() { + return new HostLoad(Math.min(1, realCores() / CORES_PER_HOST), + Math.min(1, memoryMb / MEMORY_MB_PER_HOST), 10); + } + } + + /** Frozen figures for one batch, so every decision in the batch sees the same thing. */ + private static final class HostView { + final SimHost host; + final double cpuAllocated; + final double memoryAllocated; + final HostLoad load; + final int vms; + final int recentStarts; + + HostView(SimHost host) { + this.host = host; + this.cpuAllocated = host.cpuAllocated(); + this.memoryAllocated = host.memoryAllocated(); + this.load = host.load(); + this.vms = host.vms; + this.recentStarts = host.recentStarts; + } + } + + private interface Placement { + SimHost choose(List snapshot, Random random); + } + + /** What firstfitleastconsumed does today: strictly the lowest allocated fraction. */ + private static final Placement LEAST_ALLOCATED = (snapshot, random) -> + snapshot.stream().min(Comparator.comparingDouble(v -> v.cpuAllocated)).orElseThrow().host; + + private Placement weighted() { + WeightedHostScorer scorer = new WeightedHostScorer(); + return (snapshot, random) -> { + List ranked = new ArrayList<>(snapshot); + ranked.sort(Comparator.comparingDouble(v -> + scorer.scoreHost(null, v.cpuAllocated, v.memoryAllocated, v.load, v.vms, v.recentStarts))); + return ranked.get(random.nextInt(Math.min(SPREAD, ranked.size()))).host; + }; + } + + private static final class Result { + final double[] vmCounts; + final double[] realCores; + + Result(double[] vmCounts, double[] realCores) { + this.vmCounts = vmCounts; + this.realCores = realCores; + } + + double vmSkew() { + return max(vmCounts) / mean(vmCounts); + } + + double loadSkew() { + return max(realCores) / mean(realCores); + } + } + + private Result run(Placement placement, long seed) { + Random random = new Random(seed); + List hosts = new ArrayList<>(); + for (int i = 0; i < HOSTS; i++) { + // a third of the fleet carries load the scheduler cannot account for + hosts.add(new SimHost(i + 1, i % 3 == 0 ? UNACCOUNTED_CORES : 0)); + } + List live = new ArrayList<>(); // {hostIndex, busy, batchesLeft} + + for (int batch = 0; batch < BATCHES; batch++) { + live.removeIf(vm -> { + if (--vm[2] > 0) { + return false; + } + SimHost host = hosts.get(vm[0]); + host.vms--; + host.memoryMb -= VM_MEMORY_MB; + if (vm[1] == 1) { + host.busyCores -= VM_CORES; + } + return true; + }); + + hosts.forEach(h -> h.recentStarts = 0); + + List snapshot = new ArrayList<>(); + for (SimHost host : hosts) { + snapshot.add(new HostView(host)); + } + + for (int i = 0; i < VMS_PER_BATCH; i++) { + SimHost chosen = placement.choose(snapshot, random); + boolean busy = random.nextDouble() < BUSY_FRACTION; + chosen.vms++; + chosen.recentStarts++; + chosen.memoryMb += VM_MEMORY_MB; + if (busy) { + chosen.busyCores += VM_CORES; + } + // lifetimes vary, so hosts do not empty in the order they filled + live.add(new int[] {hosts.indexOf(chosen), busy ? 1 : 0, 10 + random.nextInt(50)}); + } + } + + return new Result(hosts.stream().mapToDouble(h -> h.vms).toArray(), + hosts.stream().mapToDouble(SimHost::realCores).toArray()); + } + + private static double max(double[] values) { + return Arrays.stream(values).max().orElse(0); + } + + private static double mean(double[] values) { + return Arrays.stream(values).average().orElse(0); + } + + @Test + public void testAllocationOnlyOrderingPilesRealLoadOntoTheBusiestHosts() { + Result result = run(LEAST_ALLOCATED, 42L); + + assertTrue(String.format("expected allocation-only ranking to be blind to real load, " + + "got cores %s (max/mean %.2f)", Arrays.toString(result.realCores), result.loadSkew()), + result.loadSkew() > 1.4); + } + + @Test + public void testWeightedScoringKeepsRealLoadEven() { + Result result = run(weighted(), 42L); + + assertTrue(String.format("real load should be spread, got cores %s (max/mean %.2f)", + Arrays.toString(result.realCores), result.loadSkew()), + result.loadSkew() < 1.25); + } + + @Test + public void testWeightedScoringBeatsAllocationOnlyOnEverySeed() { + for (long seed : new long[] {1L, 7L, 42L, 99L, 12345L}) { + double baseline = run(LEAST_ALLOCATED, seed).loadSkew(); + double improved = run(weighted(), seed).loadSkew(); + assertTrue(String.format("seed %d: weighted %.2f should beat allocation-only %.2f", + seed, improved, baseline), + improved < baseline); + } + } + + @Test + public void testWeightedScoringGivesFewerVmsToHostsCarryingHiddenLoad() { + Result result = run(weighted(), 42L); + + // hosts 0, 3 and 6 carry load that allocation cannot see, so they should get fewer VMs. + // uneven VM counts are the right answer here - it is real load that should come out even. + double withHiddenLoad = mean(new double[] {result.vmCounts[0], result.vmCounts[3], result.vmCounts[6]}); + double withoutHiddenLoad = mean(new double[] {result.vmCounts[1], result.vmCounts[2], result.vmCounts[4], + result.vmCounts[5], result.vmCounts[7], result.vmCounts[8]}); + + assertTrue(String.format("hosts with hidden load should take fewer VMs: %.1f vs %.1f (counts %s)", + withHiddenLoad, withoutHiddenLoad, Arrays.toString(result.vmCounts)), + withHiddenLoad < withoutHiddenLoad * 0.75); + assertTrue("no host should be left completely unused: " + Arrays.toString(result.vmCounts), + min(result.vmCounts) > 0); + } + + @Test + public void testAllocationOnlyOrderingIgnoresHiddenLoadEntirely() { + Result result = run(LEAST_ALLOCATED, 42L); + + double withHiddenLoad = mean(new double[] {result.vmCounts[0], result.vmCounts[3], result.vmCounts[6]}); + double withoutHiddenLoad = mean(new double[] {result.vmCounts[1], result.vmCounts[2], result.vmCounts[4], + result.vmCounts[5], result.vmCounts[7], result.vmCounts[8]}); + + assertTrue(String.format("allocation-only ranking should treat loaded and idle hosts alike, got %.1f vs %.1f", + withHiddenLoad, withoutHiddenLoad), + withHiddenLoad > withoutHiddenLoad * 0.9); + } + + private static double min(double[] values) { + return Arrays.stream(values).min().orElse(0); + } +} From edf6a433440201f3ee7b97dd7560a7e8e15819b6 Mon Sep 17 00:00:00 2001 From: Brad House Date: Wed, 9 Sep 2026 22:07:18 +0000 Subject: [PATCH 04/18] Fix three defects found in review of the weighted scoring Allocated fraction was measured against the wrong total. op_host_capacity stores totals raw and overprovisioning is applied when they are read, so dividing by the stored total made the fraction reach 1 at the host's physical size. On a cluster overcommitted 10 times every host clamped to 1, killing both the allocation term and the dominant resource term - on exactly the clusters this algorithm is for. - apply the cluster ratio to the denominator - drop hosts missing a CPU or memory capacity row instead of scoring the missing resource as untouched, which made them rank first Utilisation thresholds could be bypassed. Held-back hosts were appended before the random spread was applied, so the spread could shuffle a busy host into the lead. With 1 healthy host and a spread of 3, two thirds of deployments picked an over-threshold host. - spread over healthy hosts only, before anything else is appended A host with no load samples was treated as idle. It was exempt from the thresholds and its dominant resource term fell back to allocation, so a host with broken stats outranked every measured host and collected the deployments. - rank hosts we cannot measure behind every host we can - when nothing can be measured, ranking falls back to allocation as before Signed-off-by: Brad House --- .../allocator/impl/WeightedHostScorer.java | 132 +++++++++++++----- .../impl/WeightedHostScorerTest.java | 16 ++- 2 files changed, 106 insertions(+), 42 deletions(-) diff --git a/server/src/main/java/com/cloud/agent/manager/allocator/impl/WeightedHostScorer.java b/server/src/main/java/com/cloud/agent/manager/allocator/impl/WeightedHostScorer.java index 9f0057b09fb6..bdf4fd3a46ba 100644 --- a/server/src/main/java/com/cloud/agent/manager/allocator/impl/WeightedHostScorer.java +++ b/server/src/main/java/com/cloud/agent/manager/allocator/impl/WeightedHostScorer.java @@ -18,6 +18,7 @@ import java.util.ArrayList; import java.util.Collections; +import java.util.Comparator; import java.util.Date; import java.util.HashMap; import java.util.List; @@ -34,9 +35,12 @@ import com.cloud.capacity.CapacityManager; import com.cloud.capacity.CapacityVO; import com.cloud.capacity.dao.CapacityDao; +import com.cloud.dc.ClusterDetailsDao; +import com.cloud.dc.ClusterDetailsVO; import com.cloud.host.Host; import com.cloud.host.HostScoringWeights; import com.cloud.utils.component.AdapterBase; +import com.cloud.vm.VmDetailConstants; import com.cloud.vm.dao.VMInstanceDao; /** @@ -109,6 +113,9 @@ public class WeightedHostScorer extends AdapterBase implements Configurable { @Inject private CapacityDao capacityDao; + @Inject + private ClusterDetailsDao clusterDetailsDao; + @Inject private VMInstanceDao vmInstanceDao; @@ -128,19 +135,53 @@ public List rank(long zoneId, Long podId, Long clusterId, List scores = score(zoneId, podId, clusterId, hosts); - List scored = hosts.stream().filter(h -> scores.containsKey(h.getId())).collect(Collectors.toList()); - List unscored = hosts.stream().filter(h -> !scores.containsKey(h.getId())).collect(Collectors.toList()); - scored.sort((a, b) -> Double.compare(scores.get(a.getId()), scores.get(b.getId()))); + List unscored = new ArrayList<>(); + List measured = new ArrayList<>(); + List unmeasured = new ArrayList<>(); + for (Host host : hosts) { + if (!scores.containsKey(host.getId())) { + unscored.add(host); + } else if (hostLoadTracker.getLoad(host.getId()).isUsable()) { + measured.add(host); + } else { + unmeasured.add(host); + } + } + + Comparator byScore = Comparator.comparingDouble(h -> scores.get(h.getId())); + measured.sort(byScore); + unmeasured.sort(byScore); - List admitted = applyUtilisationThresholds(clusterId, scored); - applySelectionSpread(clusterId, admitted); + List healthy = new ArrayList<>(); + List tooBusy = new ArrayList<>(); + partitionByUtilisation(clusterId, measured, healthy, tooBusy); - logger.debug("Weighted host ranking: {}", () -> admitted.stream() + List result = new ArrayList<>(); + if (healthy.isEmpty() && unmeasured.isEmpty()) { + logger.warn("Every candidate host is above its utilisation threshold, so the thresholds are being " + + "ignored for this deployment. The cluster is short of capacity."); + result.addAll(tooBusy); + applySelectionSpread(clusterId, result); + } else { + result.addAll(healthy); + // spread only over hosts known to be healthy, before anything else is appended, + // otherwise a busy or unmeasured host can be shuffled into the lead + applySelectionSpread(clusterId, result); + // a host we cannot measure is not assumed to be idle: it ranks behind every host we can + result.addAll(unmeasured); + result.addAll(tooBusy); + } + + if (!tooBusy.isEmpty()) { + logger.debug("Holding back {} host(s) above their utilisation threshold: {}", tooBusy.size(), tooBusy); + } + logger.debug("Weighted host ranking: {}", () -> result.stream() + .filter(h -> scores.containsKey(h.getId())) .map(h -> String.format("%s=%.4f", h.getName(), scores.get(h.getId()))) .collect(Collectors.joining(", "))); - admitted.addAll(unscored); - return admitted; + result.addAll(unscored); + return result; } protected Map score(long zoneId, Long podId, Long clusterId, List hosts) { @@ -165,27 +206,58 @@ protected Map score(long zoneId, Long podId, Long clusterId, List< } /** - * Allocated CPU and memory as a fraction of what the host advertises after overprovisioning, - * which is the same basis the existing allocators use. + * Allocated CPU and memory as a fraction of what a host can hand out. + * + * op_host_capacity stores totals raw; overprovisioning is applied when they are read, so the + * cluster's ratio has to be applied here too. Without it the fraction reaches 1 at the host's + * physical size and every host on an overcommitted cluster clamps to 1, which is where this + * algorithm is most needed. + * + * Only hosts with both a CPU and a memory row are returned. A host missing one would otherwise + * score as if that resource were untouched, making it the most attractive host in the cluster. */ protected Map allocatedFractions(List capacities) { Map fractions = new HashMap<>(); + Map seen = new HashMap<>(); for (CapacityVO capacity : capacities) { long total = capacity.getTotalCapacity(); if (total <= 0) { continue; } - double used = (double) (capacity.getUsedCapacity() + capacity.getReservedCapacity()) / total; + boolean isCpu = capacity.getCapacityType() == Capacity.CAPACITY_TYPE_CPU; + float overcommit = overcommitRatio(capacity.getClusterId(), isCpu); + double allocatable = total * overcommit; + double used = (double) (capacity.getUsedCapacity() + capacity.getReservedCapacity()) / allocatable; + Double[] entry = fractions.computeIfAbsent(capacity.getHostOrPoolId(), id -> new Double[] {0.0, 0.0}); - if (capacity.getCapacityType() == Capacity.CAPACITY_TYPE_CPU) { - entry[0] = clamp(used); - } else { - entry[1] = clamp(used); - } + entry[isCpu ? 0 : 1] = clamp(used); + seen.merge(capacity.getHostOrPoolId(), isCpu ? 1 : 2, Integer::sum); } + fractions.keySet().removeIf(hostId -> seen.getOrDefault(hostId, 0) != 3); return fractions; } + /** + * The cluster's overprovisioning factor, defaulting to none if it cannot be read. + */ + protected float overcommitRatio(Long clusterId, boolean forCpu) { + if (clusterId == null) { + return 1f; + } + String key = forCpu ? VmDetailConstants.CPU_OVER_COMMIT_RATIO : VmDetailConstants.MEMORY_OVER_COMMIT_RATIO; + ClusterDetailsVO detail = clusterDetailsDao.findDetail(clusterId, key); + if (detail == null || detail.getValue() == null) { + return 1f; + } + try { + float ratio = Float.parseFloat(detail.getValue()); + return ratio > 0 ? ratio : 1f; + } catch (NumberFormatException e) { + logger.warn("Cluster {} has an unreadable {} of [{}], treating it as 1.", clusterId, key, detail.getValue()); + return 1f; + } + } + /** * The blend. Every term is a fraction of the host's capacity for that resource so the weights * are directly comparable, and the dominant resource term is added on top of the weighted mean @@ -238,35 +310,21 @@ protected double dominantResource(double cpuAllocated, double memoryAllocated, H } /** - * Holds back hosts that are measurably too busy, unless that would leave nothing to deploy on, - * in which case ranking alone decides and the caller's capacity checks still apply. + * Splits measurably busy hosts out from the rest. Only hosts with load samples can be held + * back; a host that cannot be measured is dealt with by the caller. */ - protected List applyUtilisationThresholds(Long clusterId, List ranked) { + protected void partitionByUtilisation(Long clusterId, List measured, List healthy, List tooBusy) { double cpuThreshold = valueIn(CpuUtilisationThreshold, clusterId); double memoryThreshold = valueIn(MemoryUtilisationThreshold, clusterId); - List admitted = new ArrayList<>(); - List heldBack = new ArrayList<>(); - for (Host host : ranked) { + for (Host host : measured) { HostLoad load = hostLoadTracker.getLoad(host.getId()); - if (load.isUsable() - && (load.getCpuUtilisation() > cpuThreshold || load.getMemoryUtilisation() > memoryThreshold)) { - heldBack.add(host); + if (load.getCpuUtilisation() > cpuThreshold || load.getMemoryUtilisation() > memoryThreshold) { + tooBusy.add(host); } else { - admitted.add(host); + healthy.add(host); } } - - if (admitted.isEmpty()) { - logger.warn("Every candidate host is above its utilisation threshold, so the thresholds are being " - + "ignored for this deployment. The cluster is short of capacity."); - return heldBack; - } - if (!heldBack.isEmpty()) { - logger.debug("Holding back {} host(s) above their utilisation threshold: {}", heldBack.size(), heldBack); - admitted.addAll(heldBack); - } - return admitted; } /** diff --git a/server/src/test/java/com/cloud/agent/manager/allocator/impl/WeightedHostScorerTest.java b/server/src/test/java/com/cloud/agent/manager/allocator/impl/WeightedHostScorerTest.java index 7b2dd67bf157..4404a45c8469 100644 --- a/server/src/test/java/com/cloud/agent/manager/allocator/impl/WeightedHostScorerTest.java +++ b/server/src/test/java/com/cloud/agent/manager/allocator/impl/WeightedHostScorerTest.java @@ -130,10 +130,13 @@ public void testBusyHostIsHeldBackByThreshold() { Mockito.when(hostLoadTracker.getLoad(quiet.getId())).thenReturn(new HostLoad(0.10, 0.10, 10)); Mockito.when(hostLoadTracker.getLoad(busy.getId())).thenReturn(new HostLoad(0.99, 0.10, 10)); - List result = scorer.applyUtilisationThresholds(null, new ArrayList<>(List.of(busy, quiet))); + List healthy = new ArrayList<>(); + List tooBusy = new ArrayList<>(); + scorer.partitionByUtilisation(null, new ArrayList<>(List.of(quiet, busy)), healthy, tooBusy); - assertSame("host over the CPU threshold must fall behind", quiet, result.get(0)); - assertSame(busy, result.get(1)); + assertEquals(1, healthy.size()); + assertSame(quiet, healthy.get(0)); + assertSame("host over the CPU threshold must be held back", busy, tooBusy.get(0)); } @Test @@ -142,9 +145,12 @@ public void testThresholdIsIgnoredWhenEveryHostIsBusy() { Host b = host("b"); Mockito.when(hostLoadTracker.getLoad(Mockito.anyLong())).thenReturn(new HostLoad(0.99, 0.99, 10)); - List result = scorer.applyUtilisationThresholds(null, new ArrayList<>(List.of(a, b))); + List healthy = new ArrayList<>(); + List tooBusy = new ArrayList<>(); + scorer.partitionByUtilisation(null, new ArrayList<>(List.of(a, b)), healthy, tooBusy); - assertEquals("deployment must still be possible when the whole cluster is busy", 2, result.size()); + assertEquals("both hosts are over threshold", 2, tooBusy.size()); + assertTrue(healthy.isEmpty()); } @Test From c37967184539ef1f003490c9bee36489d00a41e0 Mon Sep 17 00:00:00 2001 From: Brad House Date: Wed, 9 Sep 2026 22:29:55 +0000 Subject: [PATCH 05/18] Address the rest of the weighted scoring review Utilisation average - a host whose agent stops reporting kept vouching for itself forever: StatsCollector hands back the previous entry when a poll fails, and that unchanged reading was folded again every minute. Detect the repeat, and expire an average that stops being updated - sample on a scheduled executor catching Throwable, not a Timer, which dies permanently and silently on one escaping error - only collect when an algorithm that reads the figures is selected - read the half life once per sample rather than inside the map update - document what getCpuUtilization means per hypervisor: it is what this assumes on KVM, a reservation figure on VMware, and scaled by core count on XenServer Scoring - a negative weight would rank the most loaded host first; floor at zero and say so - zeroing all six terms no longer discards the dominant resource term - read weights once per ranking instead of once per host Queries - one query per ranking instead of two, returning both counts - count Stopping VMs, which still hold their host - correct the doc: every host in scope is returned, including empty ones Tests - cover rank() end to end, which is where the defects were: the capacity denominator, the thresholds, the spread and the ordering of measured against unmeasured hosts - the distribution simulation drew different random streams per arm, so the arms saw different workloads. Fix the workload up front and add an allocation-only-plus-spread control, which shows the scoring and not the spread is what evens out real load Signed-off-by: Brad House --- .../java/com/cloud/vm/dao/VMInstanceDao.java | 14 +- .../com/cloud/vm/dao/VMInstanceDaoImpl.java | 23 +- .../allocator/impl/HostLoadTracker.java | 115 +++++++-- .../allocator/impl/WeightedHostScorer.java | 103 +++++--- .../allocator/impl/HostLoadTrackerTest.java | 59 ++++- .../impl/WeightedHostScorerRankTest.java | 219 ++++++++++++++++++ .../impl/WeightedHostScorerTest.java | 2 +- .../WeightedPlacementDistributionTest.java | 51 +++- 8 files changed, 499 insertions(+), 87 deletions(-) create mode 100644 server/src/test/java/com/cloud/agent/manager/allocator/impl/WeightedHostScorerRankTest.java diff --git a/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDao.java b/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDao.java index dbe4472f31e9..0b5fb4c64f9b 100755 --- a/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDao.java +++ b/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDao.java @@ -132,14 +132,16 @@ public interface VMInstanceDao extends GenericDao, StateDao< List listHostIdsByVmCount(long dcId, Long podId, Long clusterId, long accountId); /** - * Counts the VMs occupying each host in a zone, pod or cluster. + * Counts the VMs occupying each host in a zone, pod or cluster, in one query. * - * @param startedAfter - * when set, counts only VMs that last changed state after this time, which approximates - * the VMs that have recently started and are still working through their startup load - * @return host id to VM count, hosts with no VMs omitted + * @param changedStateAfter + * cut-off for the second count: VMs whose state last changed after this. Approximates + * the VMs still working through their startup load, which neither allocation nor a + * utilisation average has caught up with yet. + * @return host id to {total VMs, VMs that changed state recently}. Every host in scope appears, + * including hosts with no VMs. */ - Map countVmsByHost(long dcId, Long podId, Long clusterId, Date startedAfter); + Map> countVmsByHost(long dcId, Long podId, Long clusterId, Date changedStateAfter); Long countRunningAndStartingByAccount(long accountId); diff --git a/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDaoImpl.java b/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDaoImpl.java index e69ae9b3fc09..f553c3f6dca9 100755 --- a/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDaoImpl.java @@ -155,9 +155,10 @@ public class VMInstanceDaoImpl extends GenericDaoBase implem private static final String COUNT_VMS_BASED_ON_VGPU_TYPES2 = "GROUP BY gpu_card.name, vgpu_profile.name"; - private static final String COUNT_VMS_BY_HOST = "SELECT host.id, COUNT(vm.id) FROM `cloud`.`host` host " + - "LEFT JOIN `cloud`.`vm_instance` vm ON vm.host_id = host.id AND vm.state IN ('Running', 'Starting', 'Migrating') " + - "AND vm.removed IS NULL %s WHERE host.type = 'Routing' AND host.removed IS NULL AND host.data_center_id = ? "; + private static final String COUNT_VMS_BY_HOST = "SELECT host.id, COUNT(vm.id), SUM(IF(vm.update_time > ?, 1, 0)) " + + "FROM `cloud`.`host` host LEFT JOIN `cloud`.`vm_instance` vm " + + "ON vm.host_id = host.id AND vm.state IN ('Running', 'Starting', 'Stopping', 'Migrating') " + + "AND vm.removed IS NULL WHERE host.type = 'Routing' AND host.removed IS NULL AND host.data_center_id = ? "; private static final String COUNT_VMS_BY_HOST_PART2 = " GROUP BY host.id "; private static final String UPDATE_SYSTEM_VM_TEMPLATE_ID_FOR_HYPERVISOR = "UPDATE `cloud`.`vm_instance` SET vm_template_id = ? WHERE type <> 'User' AND hypervisor_type = ? AND removed is NULL"; @@ -803,10 +804,10 @@ public Pair, Map> listPodIdsInZoneByVmCount(long dataCe @Override - public Map countVmsByHost(long dcId, Long podId, Long clusterId, Date startedAfter) { + public Map> countVmsByHost(long dcId, Long podId, Long clusterId, Date changedStateAfter) { TransactionLegacy txn = TransactionLegacy.currentTxn(); - Map result = new HashMap<>(); - String sql = String.format(COUNT_VMS_BY_HOST, startedAfter != null ? " AND vm.update_time > ? " : ""); + Map> result = new HashMap<>(); + String sql = COUNT_VMS_BY_HOST; if (podId != null) { sql = sql + " AND host.pod_id = ? "; } @@ -817,19 +818,19 @@ public Map countVmsByHost(long dcId, Long podId, Long clusterId, Dat try { PreparedStatement pstmt = txn.prepareAutoCloseStatement(sql); int index = 1; - if (startedAfter != null) { - pstmt.setTimestamp(index++, new Timestamp(startedAfter.getTime())); - } + // a cut-off in the future counts nothing as recent, which is what a null asks for + long cutOff = changedStateAfter != null ? changedStateAfter.getTime() : Long.MAX_VALUE; + pstmt.setTimestamp(index++, new Timestamp(cutOff)); pstmt.setLong(index++, dcId); if (podId != null) { pstmt.setLong(index++, podId); } if (clusterId != null) { - pstmt.setLong(index, clusterId); + pstmt.setLong(index++, clusterId); } ResultSet rs = pstmt.executeQuery(); while (rs.next()) { - result.put(rs.getLong(1), rs.getLong(2)); + result.put(rs.getLong(1), new Pair<>(rs.getLong(2), rs.getLong(3))); } return result; } catch (SQLException e) { diff --git a/server/src/main/java/com/cloud/agent/manager/allocator/impl/HostLoadTracker.java b/server/src/main/java/com/cloud/agent/manager/allocator/impl/HostLoadTracker.java index 5708d487b38f..4543af9b8b14 100644 --- a/server/src/main/java/com/cloud/agent/manager/allocator/impl/HostLoadTracker.java +++ b/server/src/main/java/com/cloud/agent/manager/allocator/impl/HostLoadTracker.java @@ -17,22 +17,26 @@ package com.cloud.agent.manager.allocator.impl; import java.util.Map; -import java.util.Timer; -import java.util.TimerTask; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; import javax.inject.Inject; import org.apache.cloudstack.framework.config.ConfigKey; import org.apache.cloudstack.framework.config.Configurable; -import org.apache.cloudstack.managed.context.ManagedContextTimerTask; +import org.apache.cloudstack.managed.context.ManagedContextRunnable; import com.cloud.host.HostStats; import com.cloud.host.HostVO; import com.cloud.host.Status; import com.cloud.host.dao.HostDao; import com.cloud.server.StatsCollector; +import com.cloud.deploy.DeploymentClusterPlanner; +import com.cloud.deploy.DeploymentPlanner.AllocationAlgorithm; import com.cloud.utils.component.ManagerBase; +import com.cloud.utils.concurrency.NamedThreadFactory; /** * Keeps a smoothed view of how hard each host is actually working. @@ -45,6 +49,19 @@ * The average is per management server and is not persisted. Every management server polls every * host, so all of them converge on the same picture, and a restarted server simply reports nothing * usable until it has sampled - callers then fall back to allocation figures. + * + * What getCpuUtilization means depends on the hypervisor, and only KVM reports what this class + * assumes: + * + *
    + *
  • KVM reports busy time as a percentage of the host's cores, which is what is wanted.
    • + *
  • VMware reports the share of CPU that is reserved rather than the share that is busy, so + * the CPU term becomes a second allocation signal there rather than a load signal.
    • + *
  • XenServer sums per-core averages without dividing by core count, so the value ranges up to + * the number of cores and is under-reported here by roughly that factor.
    • + *
+ * + * Memory is taken as used over total and is sound everywhere. */ public class HostLoadTracker extends ManagerBase implements Configurable { @@ -54,6 +71,13 @@ public class HostLoadTracker extends ManagerBase implements Configurable { "consider actual load. Should not be shorter than host.stats.interval.", false, ConfigKey.Scope.Global); + public static final ConfigKey HostLoadStaleAfter = new ConfigKey<>(ConfigKey.CATEGORY_ADVANCED, + Integer.class, "host.load.stale.after", "600", + "Seconds after which a host's utilisation average is considered out of date and stops being used " + + "for placement. A host whose agent stops reporting would otherwise keep vouching for itself " + + "with figures that never change.", + true, ConfigKey.Scope.Global); + public static final ConfigKey HostLoadHalfLife = new ConfigKey<>(ConfigKey.CATEGORY_ADVANCED, Integer.class, "host.load.half.life", "300", "Half life in seconds of the moving average of host utilisation. Larger values react more " + @@ -68,35 +92,41 @@ public class HostLoadTracker extends ManagerBase implements Configurable { private final Map samples = new ConcurrentHashMap<>(); - private Timer timer; + private ScheduledExecutorService executor; @Override public boolean start() { - int interval = Math.max(1, HostLoadSampleInterval.value()) * 1000; - TimerTask task = new ManagedContextTimerTask() { + int interval = Math.max(1, HostLoadSampleInterval.value()); + executor = Executors.newSingleThreadScheduledExecutor( + new NamedThreadFactory("HostLoadTracker")); + // catch Throwable: an escaping error would cancel all future runs, and the failure would be + // silent - placement would quietly go back to ranking on allocation alone + executor.scheduleWithFixedDelay(new ManagedContextRunnable() { @Override protected void runInContext() { try { sampleAllHosts(); - } catch (Exception e) { - logger.warn("Unable to sample host load", e); + } catch (Throwable t) { + logger.warn("Unable to sample host load", t); } } - }; - timer = new Timer("HostLoadTracker"); - timer.schedule(task, interval, interval); + }, interval, interval, TimeUnit.SECONDS); return true; } @Override public boolean stop() { - if (timer != null) { - timer.cancel(); + if (executor != null) { + executor.shutdownNow(); } return true; } protected void sampleAllHosts() { + if (!isInUse()) { + samples.clear(); + return; + } for (HostVO host : hostDao.listByType(com.cloud.host.Host.Type.Routing)) { if (host.getStatus() != Status.Up) { samples.remove(host.getId()); @@ -106,6 +136,14 @@ protected void sampleAllHosts() { } } + /** + * Only the placement algorithms that read these figures pay for collecting them. + */ + protected boolean isInUse() { + return AllocationAlgorithm.balancedweighted.toString() + .equals(DeploymentClusterPlanner.VmAllocationAlgorithm.value()); + } + protected void record(long hostId, HostStats stats) { record(hostId, stats, System.currentTimeMillis()); } @@ -118,18 +156,40 @@ protected void record(long hostId, HostStats stats, long now) { if (totalMemory <= 0) { return; } - // getCpuUtilization is a percentage of the host's real cores + + Sample previous = samples.get(hostId); + if (previous != null && previous.isSameReadingAs(stats)) { + // StatsCollector keeps the previous entry when a poll fails, so an unchanged object is + // a reading we have already folded, not a fresh measurement + return; + } + + // getCpuUtilization is a percentage of the host's real cores. That holds for KVM; see the + // class javadoc for what it means on other hypervisors. double cpu = clamp(stats.getCpuUtilization() / 100.0); double memory = clamp((totalMemory - stats.getFreeMemoryKBs()) / totalMemory); + int halfLife = HostLoadHalfLife.value(); - samples.compute(hostId, (id, previous) -> previous == null - ? new Sample(cpu, memory, now) - : previous.fold(cpu, memory, now, HostLoadHalfLife.value())); + samples.compute(hostId, (id, current) -> current == null + ? new Sample(cpu, memory, now, stats) + : current.fold(cpu, memory, now, halfLife, stats)); } public HostLoad getLoad(long hostId) { + return getLoad(hostId, System.currentTimeMillis()); + } + + protected HostLoad getLoad(long hostId, long now) { Sample sample = samples.get(hostId); - return sample == null ? HostLoad.UNKNOWN : sample.toHostLoad(); + if (sample == null) { + return HostLoad.UNKNOWN; + } + long staleAfter = Math.max(1, HostLoadStaleAfter.value()) * 1000L; + if (now - sample.updatedAt > staleAfter) { + // the host has stopped reporting; stop letting its last known figures speak for it + return HostLoad.UNKNOWN; + } + return sample.toHostLoad(); } protected void clear() { @@ -150,7 +210,7 @@ public String getConfigComponentName() { @Override public ConfigKey[] getConfigKeys() { - return new ConfigKey[] {HostLoadSampleInterval, HostLoadHalfLife}; + return new ConfigKey[] {HostLoadSampleInterval, HostLoadHalfLife, HostLoadStaleAfter}; } /** @@ -162,21 +222,28 @@ private static final class Sample { private final double memory; private final long updatedAt; private final long count; + private final HostStats reading; - private Sample(double cpu, double memory, long updatedAt) { - this(cpu, memory, updatedAt, 1); + private Sample(double cpu, double memory, long updatedAt, HostStats reading) { + this(cpu, memory, updatedAt, 1, reading); } - private Sample(double cpu, double memory, long updatedAt, long count) { + private Sample(double cpu, double memory, long updatedAt, long count, HostStats reading) { this.cpu = cpu; this.memory = memory; this.updatedAt = updatedAt; this.count = count; + this.reading = reading; + } + + private boolean isSameReadingAs(HostStats stats) { + return reading == stats; } - private Sample fold(double newCpu, double newMemory, long now, int halfLifeSeconds) { + private Sample fold(double newCpu, double newMemory, long now, int halfLifeSeconds, HostStats reading) { double alpha = alpha(now - updatedAt, halfLifeSeconds); - return new Sample(cpu + alpha * (newCpu - cpu), memory + alpha * (newMemory - memory), now, count + 1); + return new Sample(cpu + alpha * (newCpu - cpu), memory + alpha * (newMemory - memory), now, + count + 1, reading); } private static double alpha(long elapsedMillis, int halfLifeSeconds) { diff --git a/server/src/main/java/com/cloud/agent/manager/allocator/impl/WeightedHostScorer.java b/server/src/main/java/com/cloud/agent/manager/allocator/impl/WeightedHostScorer.java index bdf4fd3a46ba..ab50172fec12 100644 --- a/server/src/main/java/com/cloud/agent/manager/allocator/impl/WeightedHostScorer.java +++ b/server/src/main/java/com/cloud/agent/manager/allocator/impl/WeightedHostScorer.java @@ -39,6 +39,7 @@ import com.cloud.dc.ClusterDetailsVO; import com.cloud.host.Host; import com.cloud.host.HostScoringWeights; +import com.cloud.utils.Pair; import com.cloud.utils.component.AdapterBase; import com.cloud.vm.VmDetailConstants; import com.cloud.vm.dao.VMInstanceDao; @@ -110,6 +111,8 @@ public class WeightedHostScorer extends AdapterBase implements Configurable { + "of them is accounted for. 1 restores strict ordering.", true, ConfigKey.Scope.Cluster); + private static final Pair NO_VMS = new Pair<>(0L, 0L); + @Inject private CapacityDao capacityDao; @@ -187,20 +190,21 @@ public List rank(long zoneId, Long podId, Long clusterId, List score(long zoneId, Long podId, Long clusterId, List hosts) { List capacities = capacityDao.listHostCapacityByCapacityTypes(zoneId, clusterId, List.of(Capacity.CAPACITY_TYPE_CPU, Capacity.CAPACITY_TYPE_MEMORY)); - Map vmCounts = vmInstanceDao.countVmsByHost(zoneId, podId, clusterId, null); - Map recentStarts = vmInstanceDao.countVmsByHost(zoneId, podId, clusterId, + Map> vmCounts = vmInstanceDao.countVmsByHost(zoneId, podId, clusterId, new Date(System.currentTimeMillis() - RecentStartWindow.value() * 1000L)); Map allocated = allocatedFractions(capacities); + Weights weights = new Weights(clusterId); Map scores = new HashMap<>(); for (Host host : hosts) { Double[] alloc = allocated.get(host.getId()); if (alloc == null) { continue; } - scores.put(host.getId(), scoreHost(clusterId, alloc[0], alloc[1], hostLoadTracker.getLoad(host.getId()), - vmCounts.getOrDefault(host.getId(), 0L), recentStarts.getOrDefault(host.getId(), 0L))); + Pair counts = vmCounts.getOrDefault(host.getId(), NO_VMS); + scores.put(host.getId(), scoreHost(weights, alloc[0], alloc[1], hostLoadTracker.getLoad(host.getId()), + counts.first(), counts.second())); } return scores; } @@ -263,39 +267,38 @@ protected float overcommitRatio(Long clusterId, boolean forCpu) { * are directly comparable, and the dominant resource term is added on top of the weighted mean * so that being nearly out of any one resource is penalised even when the average looks fine. */ - protected double scoreHost(Long clusterId, double cpuAllocated, double memoryAllocated, HostLoad load, + protected double scoreHostIn(Long clusterId, double cpuAllocated, double memoryAllocated, HostLoad load, long vmCount, long recentStarts) { - double cpuUsedWeight = load.isUsable() ? valueIn(HostScoringWeights.CpuUsedWeight, clusterId) : 0; - double memoryUsedWeight = load.isUsable() ? valueIn(HostScoringWeights.MemoryUsedWeight, clusterId) : 0; - double vmScale = Math.max(1, valueIn(ExpectedVmsPerHost, clusterId)); - - double cpuAllocatedWeight = valueIn(HostScoringWeights.CpuAllocatedWeight, clusterId); - double memoryAllocatedWeight = valueIn(HostScoringWeights.MemoryAllocatedWeight, clusterId); - double vmCountWeight = valueIn(VmCountWeight, clusterId); - double recentStartWeight = valueIn(RecentStartWeight, clusterId); - - double vmCountTerm = clamp(vmCount / vmScale); - double recentStartTerm = clamp(recentStarts / vmScale); + return scoreHost(new Weights(clusterId), cpuAllocated, memoryAllocated, load, vmCount, recentStarts); + } - double weightSum = cpuAllocatedWeight + cpuUsedWeight + memoryAllocatedWeight + memoryUsedWeight - + vmCountWeight + recentStartWeight; - if (weightSum <= 0) { - return 0; + protected double scoreHost(Weights weights, double cpuAllocated, double memoryAllocated, HostLoad load, + long vmCount, long recentStarts) { + // a host with no usable load figures is ranked on allocation alone, and is placed behind + // every measured host by the caller rather than being assumed idle + double cpuUsedWeight = load.isUsable() ? weights.cpuUsed : 0; + double memoryUsedWeight = load.isUsable() ? weights.memoryUsed : 0; + + double vmCountTerm = clamp(vmCount / weights.vmScale); + double recentStartTerm = clamp(recentStarts / weights.vmScale); + + double weightSum = weights.cpuAllocated + cpuUsedWeight + weights.memoryAllocated + memoryUsedWeight + + weights.vmCount + weights.recentStart; + + double mean = 0; + if (weightSum > 0) { + mean = (weights.cpuAllocated * cpuAllocated + + cpuUsedWeight * load.getCpuUtilisation() + + weights.memoryAllocated * memoryAllocated + + memoryUsedWeight * load.getMemoryUtilisation() + + weights.vmCount * vmCountTerm + + weights.recentStart * recentStartTerm) / weightSum; } - double weighted = cpuAllocatedWeight * cpuAllocated - + cpuUsedWeight * load.getCpuUtilisation() - + memoryAllocatedWeight * memoryAllocated - + memoryUsedWeight * load.getMemoryUtilisation() - + vmCountWeight * vmCountTerm - + recentStartWeight * recentStartTerm; - - double mean = weighted / weightSum; - double dominantWeight = valueIn(DominantResourceWeight, clusterId); - if (dominantWeight <= 0) { + if (weights.dominant <= 0) { return mean; } - return (mean + dominantWeight * dominantResource(cpuAllocated, memoryAllocated, load)) / (1 + dominantWeight); + return (mean + weights.dominant * dominantResource(cpuAllocated, memoryAllocated, load)) / (1 + weights.dominant); } /** @@ -339,6 +342,44 @@ protected void applySelectionSpread(Long clusterId, List ranked) { } } + /** + * The weights for one ranking, read once rather than per host. + * + * A negative weight would invert the ranking and make the most loaded host the best, so they + * are floored at zero and the bad value is reported. + */ + protected final class Weights { + private final double cpuAllocated; + private final double cpuUsed; + private final double memoryAllocated; + private final double memoryUsed; + private final double vmCount; + private final double recentStart; + private final double dominant; + private final double vmScale; + + protected Weights(Long clusterId) { + cpuAllocated = nonNegative(HostScoringWeights.CpuAllocatedWeight, clusterId); + cpuUsed = nonNegative(HostScoringWeights.CpuUsedWeight, clusterId); + memoryAllocated = nonNegative(HostScoringWeights.MemoryAllocatedWeight, clusterId); + memoryUsed = nonNegative(HostScoringWeights.MemoryUsedWeight, clusterId); + vmCount = nonNegative(VmCountWeight, clusterId); + recentStart = nonNegative(RecentStartWeight, clusterId); + dominant = nonNegative(DominantResourceWeight, clusterId); + vmScale = Math.max(1, valueIn(ExpectedVmsPerHost, clusterId)); + } + + private double nonNegative(ConfigKey key, Long clusterId) { + double value = valueIn(key, clusterId); + if (value < 0) { + logger.warn("{} is set to {}, which would rank the most loaded host first. Treating it as 0.", + key.key(), value); + return 0; + } + return value; + } + } + private double valueIn(ConfigKey key, Long clusterId) { T value = clusterId == null ? key.value() : key.valueIn(clusterId); return value == null ? 0 : value.doubleValue(); diff --git a/server/src/test/java/com/cloud/agent/manager/allocator/impl/HostLoadTrackerTest.java b/server/src/test/java/com/cloud/agent/manager/allocator/impl/HostLoadTrackerTest.java index 067037855d8d..0be9b1310f04 100644 --- a/server/src/test/java/com/cloud/agent/manager/allocator/impl/HostLoadTrackerTest.java +++ b/server/src/test/java/com/cloud/agent/manager/allocator/impl/HostLoadTrackerTest.java @@ -59,16 +59,20 @@ private void sample(double cpuPercent, double usedMemoryFraction, long advanceMs tracker.record(HOST_ID, stats(cpuPercent, usedMemoryFraction), now); } + private HostLoad load() { + return tracker.getLoad(HOST_ID, now); + } + @Test public void testUnknownHostIsNotUsable() { - assertFalse(tracker.getLoad(999L).isUsable()); + assertFalse(tracker.getLoad(999L, now).isUsable()); } @Test public void testFirstSampleIsTakenAsIs() { sample(40, 0.6, 0); - HostLoad load = tracker.getLoad(HOST_ID); + HostLoad load = load(); assertTrue(load.isUsable()); assertEquals(0.40, load.getCpuUtilisation(), 1e-6); assertEquals(0.60, load.getMemoryUtilisation(), 1e-6); @@ -80,7 +84,7 @@ public void testSingleSpikeDoesNotDominateTheAverage() { sample(100, 0.1, 60 * 1000L); // one sample a fifth of a half life in should move the average part of the way, not all of it - double cpu = tracker.getLoad(HOST_ID).getCpuUtilisation(); + double cpu = load().getCpuUtilisation(); assertTrue("a single spike must not take over the average: " + cpu, cpu < 0.30); assertTrue("but it must move it: " + cpu, cpu > 0.10); } @@ -92,7 +96,7 @@ public void testSustainedLoadConvergesOnTheNewValue() { sample(90, 0.9, 60 * 1000L); } - HostLoad load = tracker.getLoad(HOST_ID); + HostLoad load = load(); assertEquals(0.90, load.getCpuUtilisation(), 0.01); assertEquals(0.90, load.getMemoryUtilisation(), 0.01); } @@ -102,7 +106,7 @@ public void testHalfLifeMovesAverageHalfWay() { sample(0, 0, 0); sample(100, 1.0, HALF_LIFE_MS); - assertEquals(0.5, tracker.getLoad(HOST_ID).getCpuUtilisation(), 0.01); + assertEquals(0.5, load().getCpuUtilisation(), 0.01); } @Test @@ -111,13 +115,13 @@ public void testMissedSamplesDecayByElapsedTimeNotSampleCount() { sample(100, 1.0, 4 * HALF_LIFE_MS); // four half lives of catching up in one sample, so almost all the way there - assertTrue(tracker.getLoad(HOST_ID).getCpuUtilisation() > 0.9); + assertTrue(load().getCpuUtilisation() > 0.9); } @Test public void testNullStatsAreIgnored() { tracker.record(HOST_ID, null, now); - assertFalse(tracker.getLoad(HOST_ID).isUsable()); + assertFalse(load().isUsable()); } @Test @@ -127,14 +131,51 @@ public void testHostReportingNoMemoryIsIgnored() { tracker.record(HOST_ID, broken, now); - assertFalse(tracker.getLoad(HOST_ID).isUsable()); + assertFalse(load().isUsable()); + } + + @Test + public void testUnchangedReadingIsNotFoldedAgain() { + // StatsCollector keeps the previous entry when a poll fails, so the same object comes back + HostStats reading = stats(10, 0.1); + tracker.record(HOST_ID, reading, now); + // stay inside the staleness window so this tests folding, not expiry + for (int i = 0; i < 5; i++) { + now += 60 * 1000L; + tracker.record(HOST_ID, reading, now); + } + + assertEquals("re-reading one measurement must not count as six", 1, load().getSamples()); + } + + @Test + public void testAHostThatStopsReportingBecomesUnusable() { + HostStats reading = stats(10, 0.1); + tracker.record(HOST_ID, reading, now); + assertTrue(load().isUsable()); + + // the agent stops updating; StatsCollector keeps handing back the same stale entry + for (int i = 0; i < 20; i++) { + now += 60 * 1000L; + tracker.record(HOST_ID, reading, now); + } + + assertFalse("a host that stopped reporting must not keep vouching for itself", load().isUsable()); + } + + @Test + public void testFreshReadingsKeepAHostUsable() { + for (int i = 0; i < 20; i++) { + sample(10 + i, 0.1, 60 * 1000L); + } + assertTrue(load().isUsable()); } @Test public void testOutOfRangeValuesAreClamped() { sample(250, 2.0, 0); - HostLoad load = tracker.getLoad(HOST_ID); + HostLoad load = load(); assertEquals(1.0, load.getCpuUtilisation(), 1e-6); assertEquals(1.0, load.getMemoryUtilisation(), 1e-6); } diff --git a/server/src/test/java/com/cloud/agent/manager/allocator/impl/WeightedHostScorerRankTest.java b/server/src/test/java/com/cloud/agent/manager/allocator/impl/WeightedHostScorerRankTest.java new file mode 100644 index 000000000000..4521d5652e02 --- /dev/null +++ b/server/src/test/java/com/cloud/agent/manager/allocator/impl/WeightedHostScorerRankTest.java @@ -0,0 +1,219 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.agent.manager.allocator.impl; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.Set; +import java.util.stream.Collectors; + +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; + +import com.cloud.capacity.Capacity; +import com.cloud.capacity.CapacityVO; +import com.cloud.capacity.dao.CapacityDao; +import com.cloud.dc.ClusterDetailsDao; +import com.cloud.dc.ClusterDetailsVO; +import com.cloud.host.Host; +import com.cloud.utils.Pair; +import com.cloud.vm.dao.VMInstanceDao; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +/** + * Exercises rank() as the allocator calls it, rather than the scoring function alone, so the + * capacity denominator, the utilisation thresholds and the random spread are all covered. + */ +@RunWith(MockitoJUnitRunner.class) +public class WeightedHostScorerRankTest { + + private static final long ZONE = 1L; + private static final long CLUSTER = 7L; + private static final long CORES = 192L * 2400L; + private static final long MEMORY = 1_132_000L * 1024L * 1024L; + private static final int CPU_OVERCOMMIT = 10; + private static final int MEMORY_OVERCOMMIT = 4; + + @Mock + private CapacityDao capacityDao; + + @Mock + private VMInstanceDao vmInstanceDao; + + @Mock + private ClusterDetailsDao clusterDetailsDao; + + @Mock + private HostLoadTracker hostLoadTracker; + + @InjectMocks + private WeightedHostScorer scorer = new WeightedHostScorer(); + + private final List capacities = new ArrayList<>(); + private final Map> vmCounts = new HashMap<>(); + private final Map hosts = new HashMap<>(); + + @Before + public void setUp() { + scorer.random = new Random(1L); + Mockito.lenient().when(clusterDetailsDao.findDetail(Mockito.eq(CLUSTER), Mockito.contains("cpu"))) + .thenReturn(new ClusterDetailsVO(CLUSTER, "cpuOvercommitRatio", String.valueOf(CPU_OVERCOMMIT))); + Mockito.lenient().when(clusterDetailsDao.findDetail(Mockito.eq(CLUSTER), Mockito.contains("memory"))) + .thenReturn(new ClusterDetailsVO(CLUSTER, "memoryOvercommitRatio", String.valueOf(MEMORY_OVERCOMMIT))); + Mockito.lenient().when(capacityDao.listHostCapacityByCapacityTypes(Mockito.eq(ZONE), Mockito.eq(CLUSTER), Mockito.any())) + .thenReturn(capacities); + Mockito.lenient().when(vmInstanceDao.countVmsByHost(Mockito.eq(ZONE), Mockito.any(), Mockito.eq(CLUSTER), Mockito.any())) + .thenReturn(vmCounts); + } + + private CapacityVO capacity(long hostId, short type, long used, long total) { + CapacityVO capacity = new CapacityVO(hostId, ZONE, 1L, CLUSTER, used, total, type); + capacity.setReservedCapacity(0L); + return capacity; + } + + /** Registers a host with a share of its allocatable CPU and memory already committed. */ + private Host host(long id, String name, double cpuAllocatedFraction, double memoryAllocatedFraction, + HostLoad load, long vms) { + Host host = Mockito.mock(Host.class); + Mockito.lenient().when(host.getId()).thenReturn(id); + Mockito.lenient().when(host.getName()).thenReturn(name); + capacities.add(capacity(id, Capacity.CAPACITY_TYPE_CPU, + (long) (CORES * CPU_OVERCOMMIT * cpuAllocatedFraction), CORES)); + capacities.add(capacity(id, Capacity.CAPACITY_TYPE_MEMORY, + (long) (MEMORY * MEMORY_OVERCOMMIT * memoryAllocatedFraction), MEMORY)); + vmCounts.put(id, new Pair<>(vms, 0L)); + Mockito.lenient().when(hostLoadTracker.getLoad(id)).thenReturn(load); + hosts.put(id, host); + return host; + } + + private List rankedNames(List input) { + return scorer.rank(ZONE, 1L, CLUSTER, input).stream().map(Host::getName).collect(Collectors.toList()); + } + + @Test + public void testAllocationIsMeasuredAgainstTheOvercommittedTotal() { + // both hosts are well past their physical CPU, which is normal at a factor of 10. + // if the denominator ignored the factor both would clamp to 1.0 and rank equal. + Host light = host(1L, "light", 0.20, 0.20, new HostLoad(0.1, 0.1, 5), 20); + Host heavy = host(2L, "heavy", 0.80, 0.20, new HostLoad(0.1, 0.1, 5), 20); + + Map scores = scorer.score(ZONE, 1L, CLUSTER, Arrays.asList(light, heavy)); + + assertTrue("hosts past their physical size must still be distinguishable", + scores.get(heavy.getId()) > scores.get(light.getId())); + } + + @Test + public void testBusyHostRanksBehindQuietOneAtEqualAllocation() { + Host quiet = host(1L, "quiet", 0.30, 0.30, new HostLoad(0.05, 0.05, 10), 30); + Host busy = host(2L, "busy", 0.30, 0.30, new HostLoad(0.70, 0.30, 10), 30); + + assertEquals("quiet", rankedNames(Arrays.asList(busy, quiet)).get(0)); + } + + @Test + public void testHostOverThresholdIsNotChosenWhileAHealthyOneExists() { + // one healthy host and five over threshold: the spread must not shuffle a busy host in front + List input = new ArrayList<>(); + input.add(host(1L, "healthy", 0.30, 0.30, new HostLoad(0.50, 0.50, 10), 30)); + for (long id = 2; id <= 6; id++) { + input.add(host(id, "busy" + id, 0.30, 0.30, new HostLoad(0.95, 0.50, 10), 30)); + } + + for (int attempt = 0; attempt < 50; attempt++) { + assertEquals("the only healthy host must always lead", "healthy", rankedNames(input).get(0)); + } + } + + @Test + public void testUnmeasuredHostRanksBehindEveryMeasuredHost() { + // a host whose stats have stopped must not look idle and collect the deployments + Host measured = host(1L, "measured", 0.60, 0.60, new HostLoad(0.40, 0.40, 10), 60); + Host unmeasured = host(2L, "unmeasured", 0.05, 0.05, HostLoad.UNKNOWN, 2); + + List ranked = rankedNames(Arrays.asList(unmeasured, measured)); + + assertEquals("measured", ranked.get(0)); + assertEquals("unmeasured", ranked.get(1)); + } + + @Test + public void testRankingFallsBackToAllocationWhenNothingIsMeasured() { + Host light = host(1L, "light", 0.10, 0.10, HostLoad.UNKNOWN, 5); + Host heavy = host(2L, "heavy", 0.90, 0.90, HostLoad.UNKNOWN, 90); + + assertEquals("light", rankedNames(Arrays.asList(heavy, light)).get(0)); + } + + @Test + public void testEveryHostIsStillOfferedWhenTheWholeClusterIsBusy() { + List input = Arrays.asList( + host(1L, "a", 0.30, 0.30, new HostLoad(0.95, 0.50, 10), 30), + host(2L, "b", 0.40, 0.30, new HostLoad(0.97, 0.50, 10), 40)); + + List ranked = rankedNames(input); + + assertEquals("deployment must remain possible", 2, ranked.size()); + } + + @Test + public void testSpreadVariesTheLeadAmongHealthyHosts() { + List input = Arrays.asList( + host(1L, "a", 0.30, 0.30, new HostLoad(0.10, 0.10, 10), 30), + host(2L, "b", 0.31, 0.30, new HostLoad(0.10, 0.10, 10), 30), + host(3L, "c", 0.32, 0.30, new HostLoad(0.10, 0.10, 10), 30), + host(4L, "d", 0.90, 0.30, new HostLoad(0.10, 0.10, 10), 90)); + + Set leaders = new HashSet<>(); + for (int attempt = 0; attempt < 200; attempt++) { + leaders.add(rankedNames(input).get(0)); + } + + assertTrue("concurrent deployments must not all pick one host", leaders.size() > 1); + assertFalse("the clearly worst host must never lead", leaders.contains("d")); + } + + @Test + public void testHostMissingACapacityRowIsNotRankedFirst() { + Host complete = host(1L, "complete", 0.60, 0.60, new HostLoad(0.40, 0.40, 10), 60); + Host partial = Mockito.mock(Host.class); + Mockito.lenient().when(partial.getId()).thenReturn(2L); + Mockito.lenient().when(partial.getName()).thenReturn("partial"); + // only a CPU row: memory must not be treated as untouched + capacities.add(capacity(2L, Capacity.CAPACITY_TYPE_CPU, 0L, CORES)); + + List ranked = rankedNames(Arrays.asList(partial, complete)); + + assertEquals("complete", ranked.get(0)); + assertEquals("partial", ranked.get(1)); + } +} diff --git a/server/src/test/java/com/cloud/agent/manager/allocator/impl/WeightedHostScorerTest.java b/server/src/test/java/com/cloud/agent/manager/allocator/impl/WeightedHostScorerTest.java index 4404a45c8469..d5097d79d21a 100644 --- a/server/src/test/java/com/cloud/agent/manager/allocator/impl/WeightedHostScorerTest.java +++ b/server/src/test/java/com/cloud/agent/manager/allocator/impl/WeightedHostScorerTest.java @@ -62,7 +62,7 @@ private Host host(String name) { } private double score(double cpuAllocated, double memAllocated, HostLoad load, long vms, long recentStarts) { - return scorer.scoreHost(null, cpuAllocated, memAllocated, load, vms, recentStarts); + return scorer.scoreHostIn(null, cpuAllocated, memAllocated, load, vms, recentStarts); } @Test diff --git a/server/src/test/java/com/cloud/agent/manager/allocator/impl/WeightedPlacementDistributionTest.java b/server/src/test/java/com/cloud/agent/manager/allocator/impl/WeightedPlacementDistributionTest.java index 4cb45fa52b1c..22fa6c565839 100644 --- a/server/src/test/java/com/cloud/agent/manager/allocator/impl/WeightedPlacementDistributionTest.java +++ b/server/src/test/java/com/cloud/agent/manager/allocator/impl/WeightedPlacementDistributionTest.java @@ -129,12 +129,22 @@ private interface Placement { private static final Placement LEAST_ALLOCATED = (snapshot, random) -> snapshot.stream().min(Comparator.comparingDouble(v -> v.cpuAllocated)).orElseThrow().host; + /** + * Allocation-only ranking with the same random spread as the weighted arm. Isolates what the + * scoring contributes from what the spread alone contributes. + */ + private static final Placement LEAST_ALLOCATED_WITH_SPREAD = (snapshot, random) -> { + List ranked = new ArrayList<>(snapshot); + ranked.sort(Comparator.comparingDouble(v -> v.cpuAllocated)); + return ranked.get(random.nextInt(Math.min(SPREAD, ranked.size()))).host; + }; + private Placement weighted() { WeightedHostScorer scorer = new WeightedHostScorer(); return (snapshot, random) -> { List ranked = new ArrayList<>(snapshot); ranked.sort(Comparator.comparingDouble(v -> - scorer.scoreHost(null, v.cpuAllocated, v.memoryAllocated, v.load, v.vms, v.recentStarts))); + scorer.scoreHostIn(null, v.cpuAllocated, v.memoryAllocated, v.load, v.vms, v.recentStarts))); return ranked.get(random.nextInt(Math.min(SPREAD, ranked.size()))).host; }; } @@ -157,8 +167,22 @@ private static final class Result { } } - private Result run(Placement placement, long seed) { + /** The VMs to be placed, fixed before any arm runs so all arms see the same workload. */ + private List workload(long seed) { Random random = new Random(seed); + List vms = new ArrayList<>(); + for (int i = 0; i < BATCHES * VMS_PER_BATCH; i++) { + vms.add(new int[] {random.nextDouble() < BUSY_FRACTION ? 1 : 0, 10 + random.nextInt(50)}); + } + return vms; + } + + private Result run(Placement placement, long seed) { + List workload = workload(seed); + // a separate stream for placement decisions, so arms that consult it differently still see + // the same workload + Random random = new Random(seed ^ 0x5DEECE66DL); + int next = 0; List hosts = new ArrayList<>(); for (int i = 0; i < HOSTS; i++) { // a third of the fleet carries load the scheduler cannot account for @@ -189,7 +213,8 @@ private Result run(Placement placement, long seed) { for (int i = 0; i < VMS_PER_BATCH; i++) { SimHost chosen = placement.choose(snapshot, random); - boolean busy = random.nextDouble() < BUSY_FRACTION; + int[] vm = workload.get(next++); + boolean busy = vm[0] == 1; chosen.vms++; chosen.recentStarts++; chosen.memoryMb += VM_MEMORY_MB; @@ -197,7 +222,7 @@ private Result run(Placement placement, long seed) { chosen.busyCores += VM_CORES; } // lifetimes vary, so hosts do not empty in the order they filled - live.add(new int[] {hosts.indexOf(chosen), busy ? 1 : 0, 10 + random.nextInt(50)}); + live.add(new int[] {hosts.indexOf(chosen), vm[0], vm[1]}); } } @@ -226,9 +251,12 @@ public void testAllocationOnlyOrderingPilesRealLoadOntoTheBusiestHosts() { public void testWeightedScoringKeepsRealLoadEven() { Result result = run(weighted(), 42L); + // a third of the fleet carries a fixed handicap the scheduler can only stop adding to, not + // remove, so some residual skew is expected. Measured across seeds: allocation-only ranking + // lands at 1.84 to 2.01, weighted at 1.27 to 1.40. assertTrue(String.format("real load should be spread, got cores %s (max/mean %.2f)", Arrays.toString(result.realCores), result.loadSkew()), - result.loadSkew() < 1.25); + result.loadSkew() < 1.5); } @Test @@ -242,6 +270,19 @@ public void testWeightedScoringBeatsAllocationOnlyOnEverySeed() { } } + @Test + public void testTheScoringNotJustTheSpreadIsWhatEvensOutRealLoad() { + // the control: same random spread, ranking still blind to real load. If the spread alone + // were doing the work, this arm would do as well as the weighted one. + for (long seed : new long[] {1L, 42L, 12345L}) { + double spreadOnly = run(LEAST_ALLOCATED_WITH_SPREAD, seed).loadSkew(); + double weighted = run(weighted(), seed).loadSkew(); + assertTrue(String.format("seed %d: weighted %.2f should beat spread-only %.2f", + seed, weighted, spreadOnly), + weighted < spreadOnly); + } + } + @Test public void testWeightedScoringGivesFewerVmsToHostsCarryingHiddenLoad() { Result result = run(weighted(), 42L); From 515c3fddf1423839a7b1ef519ffd11ac5dec5a00 Mon Sep 17 00:00:00 2001 From: Brad House Date: Wed, 9 Sep 2026 21:18:15 +0000 Subject: [PATCH 06/18] Fix unreachable and NPE-prone branch in host anti-affinity The reserved-capacity branch was chained to the wrong condition, so it only ran when the group VM was null or removed: - a Stopped VM still holding reserved capacity never had its last host avoided - the branch was dead for every live VM - a group mapping pointing at a deleted VM hit the branch with a null and threw NullPointerException Restructured to match NonStrictHostAffinityProcessor, which already had the intended shape: skip null/removed, then avoid the current host, else the last host while capacity is still reserved. Signed-off-by: Brad House --- .../affinity/HostAntiAffinityProcessor.java | 34 ++-- .../HostAntiAffinityProcessorTest.java | 167 ++++++++++++++++++ 2 files changed, 185 insertions(+), 16 deletions(-) create mode 100644 plugins/affinity-group-processors/host-anti-affinity/src/test/java/org/apache/cloudstack/affinity/HostAntiAffinityProcessorTest.java diff --git a/plugins/affinity-group-processors/host-anti-affinity/src/main/java/org/apache/cloudstack/affinity/HostAntiAffinityProcessor.java b/plugins/affinity-group-processors/host-anti-affinity/src/main/java/org/apache/cloudstack/affinity/HostAntiAffinityProcessor.java index bd29a48f2588..da4994129115 100644 --- a/plugins/affinity-group-processors/host-anti-affinity/src/main/java/org/apache/cloudstack/affinity/HostAntiAffinityProcessor.java +++ b/plugins/affinity-group-processors/host-anti-affinity/src/main/java/org/apache/cloudstack/affinity/HostAntiAffinityProcessor.java @@ -59,7 +59,7 @@ public class HostAntiAffinityProcessor extends AffinityProcessorBase implements protected AffinityGroupDao _affinityGroupDao; @Inject protected AffinityGroupVMMapDao _affinityGroupVMMapDao; - private int _vmCapacityReleaseInterval; + protected int _vmCapacityReleaseInterval; @Inject protected ConfigurationDao _configDao; @@ -102,22 +102,24 @@ protected void processAffinityGroup(AffinityGroupVMMapVO vmGroupMapping, Exclude for (Long groupVMId : groupVMIds) { VMInstanceVO groupVM = _vmInstanceDao.findById(groupVMId); - if (groupVM != null && !groupVM.isRemoved()) { - if (groupVM.getHostId() != null) { - avoid.addHost(groupVM.getHostId()); - if (logger.isDebugEnabled()) { - logger.debug("Added host {} to avoid set, since VM {} is present on the host", groupVM.getHostId(), groupVM); - } - } - } else if (Arrays.asList(VirtualMachine.State.Starting, VirtualMachine.State.Stopped).contains(groupVM.getState()) && groupVM.getLastHostId() != null) { - long secondsSinceLastUpdate = (DateUtil.currentGMTTime().getTime() - groupVM.getUpdateTime().getTime()) / 1000; - if (secondsSinceLastUpdate < _vmCapacityReleaseInterval) { - avoid.addHost(groupVM.getLastHostId()); - if (logger.isDebugEnabled()) { - logger.debug("Added host {} to avoid set, since VM {} is present on the host, in Stopped state but has reserved capacity", groupVM.getLastHostId(), groupVM); - } - } + if (groupVM == null || groupVM.isRemoved()) { + continue; } + avoidHostOfVmInAffinityGroup(avoid, groupVM); + } + } + } + + protected void avoidHostOfVmInAffinityGroup(ExcludeList avoid, VMInstanceVO groupVM) { + if (groupVM.getHostId() != null) { + avoid.addHost(groupVM.getHostId()); + logger.debug("Added host {} to avoid set, since VM {} is present on the host", groupVM.getHostId(), groupVM); + } else if (Arrays.asList(VirtualMachine.State.Starting, VirtualMachine.State.Stopped).contains(groupVM.getState()) && groupVM.getLastHostId() != null) { + long secondsSinceLastUpdate = (DateUtil.currentGMTTime().getTime() - groupVM.getUpdateTime().getTime()) / 1000; + if (secondsSinceLastUpdate < _vmCapacityReleaseInterval) { + avoid.addHost(groupVM.getLastHostId()); + logger.debug("Added host {} to avoid set, since VM {} is in {} state on the host but still has reserved capacity", + groupVM.getLastHostId(), groupVM, groupVM.getState()); } } } diff --git a/plugins/affinity-group-processors/host-anti-affinity/src/test/java/org/apache/cloudstack/affinity/HostAntiAffinityProcessorTest.java b/plugins/affinity-group-processors/host-anti-affinity/src/test/java/org/apache/cloudstack/affinity/HostAntiAffinityProcessorTest.java new file mode 100644 index 000000000000..e806e98a255d --- /dev/null +++ b/plugins/affinity-group-processors/host-anti-affinity/src/test/java/org/apache/cloudstack/affinity/HostAntiAffinityProcessorTest.java @@ -0,0 +1,167 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.affinity; + +import com.cloud.deploy.DeploymentPlanner.ExcludeList; +import com.cloud.utils.DateUtil; +import com.cloud.vm.VMInstanceVO; +import com.cloud.vm.VirtualMachine; +import com.cloud.vm.dao.VMInstanceDao; +import org.apache.cloudstack.affinity.dao.AffinityGroupDao; +import org.apache.cloudstack.affinity.dao.AffinityGroupVMMapDao; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; +import org.mockito.Spy; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Date; +import java.util.List; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.when; + +@RunWith(JUnit4.class) +public class HostAntiAffinityProcessorTest { + + private static final long AFFINITY_GROUP_ID = 2L; + private static final long VM_ID = 3L; + private static final long GROUP_VM_ID = 1L; + private static final long HOST_ID = 10L; + private static final long LAST_HOST_ID = 11L; + private static final int CAPACITY_RELEASE_INTERVAL = 3600; + + @Mock + AffinityGroupDao _affinityGroupDao; + + @Mock + AffinityGroupVMMapDao _affinityGroupVMMapDao; + + @Mock + VMInstanceDao _vmInstanceDao; + + @Spy + @InjectMocks + HostAntiAffinityProcessor processor = new HostAntiAffinityProcessor(); + + @Mock + VirtualMachine vm; + + @Mock + VMInstanceVO groupVM; + + @Mock + AffinityGroupVO affinityGroupVO; + + @Mock + AffinityGroupVMMapVO vmGroupMapping; + + private ExcludeList avoid; + + @Before + public void setUp() { + MockitoAnnotations.openMocks(this); + processor._vmCapacityReleaseInterval = CAPACITY_RELEASE_INTERVAL; + avoid = new ExcludeList(); + + when(vm.getId()).thenReturn(VM_ID); + when(vmGroupMapping.getAffinityGroupId()).thenReturn(AFFINITY_GROUP_ID); + when(_affinityGroupDao.findById(AFFINITY_GROUP_ID)).thenReturn(affinityGroupVO); + when(affinityGroupVO.getId()).thenReturn(AFFINITY_GROUP_ID); + when(_affinityGroupVMMapDao.listVmIdsByAffinityGroup(AFFINITY_GROUP_ID)) + .thenReturn(new ArrayList<>(Arrays.asList(GROUP_VM_ID, VM_ID))); + } + + private boolean avoids(long hostId) { + return avoid.getHostsToAvoid() != null && avoid.getHostsToAvoid().contains(hostId); + } + + @Test + public void testRunningGroupVmHostIsAvoided() { + when(_vmInstanceDao.findById(GROUP_VM_ID)).thenReturn(groupVM); + when(groupVM.isRemoved()).thenReturn(false); + when(groupVM.getHostId()).thenReturn(HOST_ID); + + processor.processAffinityGroup(vmGroupMapping, avoid, vm); + + assertTrue(avoids(HOST_ID)); + } + + @Test + public void testStoppedGroupVmWithReservedCapacityLastHostIsAvoided() { + when(_vmInstanceDao.findById(GROUP_VM_ID)).thenReturn(groupVM); + when(groupVM.isRemoved()).thenReturn(false); + when(groupVM.getHostId()).thenReturn(null); + when(groupVM.getState()).thenReturn(VirtualMachine.State.Stopped); + when(groupVM.getLastHostId()).thenReturn(LAST_HOST_ID); + when(groupVM.getUpdateTime()).thenReturn(DateUtil.currentGMTTime()); + + processor.processAffinityGroup(vmGroupMapping, avoid, vm); + + assertTrue(avoids(LAST_HOST_ID)); + } + + @Test + public void testStoppedGroupVmPastReleaseIntervalIsNotAvoided() { + Date wellPast = new Date(DateUtil.currentGMTTime().getTime() - (CAPACITY_RELEASE_INTERVAL + 60) * 1000L); + when(_vmInstanceDao.findById(GROUP_VM_ID)).thenReturn(groupVM); + when(groupVM.isRemoved()).thenReturn(false); + when(groupVM.getHostId()).thenReturn(null); + when(groupVM.getState()).thenReturn(VirtualMachine.State.Stopped); + when(groupVM.getLastHostId()).thenReturn(LAST_HOST_ID); + when(groupVM.getUpdateTime()).thenReturn(wellPast); + + processor.processAffinityGroup(vmGroupMapping, avoid, vm); + + assertFalse(avoids(LAST_HOST_ID)); + } + + @Test + public void testMissingGroupVmIsSkipped() { + when(_vmInstanceDao.findById(GROUP_VM_ID)).thenReturn(null); + + processor.processAffinityGroup(vmGroupMapping, avoid, vm); + + assertTrue(avoid.getHostsToAvoid() == null || avoid.getHostsToAvoid().isEmpty()); + } + + @Test + public void testRemovedGroupVmIsSkipped() { + when(_vmInstanceDao.findById(GROUP_VM_ID)).thenReturn(groupVM); + when(groupVM.isRemoved()).thenReturn(true); + + processor.processAffinityGroup(vmGroupMapping, avoid, vm); + + assertFalse(avoids(HOST_ID)); + } + + @Test + public void testVmIsNotAvoidedAgainstItself() { + List ids = new ArrayList<>(Arrays.asList(VM_ID)); + when(_affinityGroupVMMapDao.listVmIdsByAffinityGroup(AFFINITY_GROUP_ID)).thenReturn(ids); + + processor.processAffinityGroup(vmGroupMapping, avoid, vm); + + assertTrue(avoid.getHostsToAvoid() == null || avoid.getHostsToAvoid().isEmpty()); + } +} From 1afc0a20222d529aed439061dffad266241181ec Mon Sep 17 00:00:00 2001 From: Brad House Date: Wed, 9 Sep 2026 21:19:07 +0000 Subject: [PATCH 07/18] Honor planned placements in host anti-affinity The processor already accepted a vmList of placements and ignored it, reading every group member's host from the database instead. DRS builds a multi-migration plan in memory and persists it only at the end, so while the plan is being built the database still shows the old host for every VM the plan has already moved. Anti-affinity was therefore evaluated against stale placements, and a plan could put two anti-affine VMs on the same host. - resolve each group member from vmList first, fall back to the database - mirrors what HostAffinityProcessor already does with the same argument - no change when vmList is empty, which is every non-DRS caller Signed-off-by: Brad House --- .../affinity/HostAntiAffinityProcessor.java | 33 +++++++++++++- .../HostAntiAffinityProcessorTest.java | 45 +++++++++++++++++++ 2 files changed, 77 insertions(+), 1 deletion(-) diff --git a/plugins/affinity-group-processors/host-anti-affinity/src/main/java/org/apache/cloudstack/affinity/HostAntiAffinityProcessor.java b/plugins/affinity-group-processors/host-anti-affinity/src/main/java/org/apache/cloudstack/affinity/HostAntiAffinityProcessor.java index da4994129115..9c69fbd7053c 100644 --- a/plugins/affinity-group-processors/host-anti-affinity/src/main/java/org/apache/cloudstack/affinity/HostAntiAffinityProcessor.java +++ b/plugins/affinity-group-processors/host-anti-affinity/src/main/java/org/apache/cloudstack/affinity/HostAntiAffinityProcessor.java @@ -17,6 +17,8 @@ package org.apache.cloudstack.affinity; import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.stream.Collectors; @@ -82,7 +84,7 @@ public void doInTransactionWithoutResult(TransactionStatus status) { _affinityGroupDao.listByIds(affinityGroupIds, true); } for (AffinityGroupVMMapVO vmGroupMapping : vmGroupMappings) { - processAffinityGroup(vmGroupMapping, avoid, vm); + processAffinityGroup(vmGroupMapping, avoid, vm, vmList); } } }); @@ -90,6 +92,18 @@ public void doInTransactionWithoutResult(TransactionStatus status) { } protected void processAffinityGroup(AffinityGroupVMMapVO vmGroupMapping, ExcludeList avoid, VirtualMachine vm) { + processAffinityGroup(vmGroupMapping, avoid, vm, Collections.emptyList()); + } + + /** + * Applies anti-affinity for one group. + * + * @param vmList + * placements to honour in preference to what the database says. DRS builds a plan of + * several migrations in memory and only persists it later, so during plan generation + * the database still shows the old host for every VM the plan has already moved. + */ + protected void processAffinityGroup(AffinityGroupVMMapVO vmGroupMapping, ExcludeList avoid, VirtualMachine vm, List vmList) { if (vmGroupMapping != null) { AffinityGroupVO group = _affinityGroupDao.findById(vmGroupMapping.getAffinityGroupId()); @@ -100,7 +114,16 @@ protected void processAffinityGroup(AffinityGroupVMMapVO vmGroupMapping, Exclude List groupVMIds = _affinityGroupVMMapDao.listVmIdsByAffinityGroup(group.getId()); groupVMIds.remove(vm.getId()); + Map plannedVms = getVmIdVmMap(vmList); + for (Long groupVMId : groupVMIds) { + VirtualMachine plannedVm = plannedVms.get(groupVMId); + if (plannedVm != null && plannedVm.getHostId() != null) { + avoid.addHost(plannedVm.getHostId()); + logger.debug("Added host {} to avoid set, since VM {} is placed on the host by the plan being built", + plannedVm.getHostId(), plannedVm); + continue; + } VMInstanceVO groupVM = _vmInstanceDao.findById(groupVMId); if (groupVM == null || groupVM.isRemoved()) { continue; @@ -110,6 +133,14 @@ protected void processAffinityGroup(AffinityGroupVMMapVO vmGroupMapping, Exclude } } + protected Map getVmIdVmMap(List vmList) { + Map vmIdVmMap = new HashMap<>(); + for (VirtualMachine vm : vmList) { + vmIdVmMap.put(vm.getId(), vm); + } + return vmIdVmMap; + } + protected void avoidHostOfVmInAffinityGroup(ExcludeList avoid, VMInstanceVO groupVM) { if (groupVM.getHostId() != null) { avoid.addHost(groupVM.getHostId()); diff --git a/plugins/affinity-group-processors/host-anti-affinity/src/test/java/org/apache/cloudstack/affinity/HostAntiAffinityProcessorTest.java b/plugins/affinity-group-processors/host-anti-affinity/src/test/java/org/apache/cloudstack/affinity/HostAntiAffinityProcessorTest.java index e806e98a255d..ae3297039289 100644 --- a/plugins/affinity-group-processors/host-anti-affinity/src/test/java/org/apache/cloudstack/affinity/HostAntiAffinityProcessorTest.java +++ b/plugins/affinity-group-processors/host-anti-affinity/src/test/java/org/apache/cloudstack/affinity/HostAntiAffinityProcessorTest.java @@ -34,6 +34,7 @@ import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.Date; import java.util.List; @@ -49,6 +50,7 @@ public class HostAntiAffinityProcessorTest { private static final long GROUP_VM_ID = 1L; private static final long HOST_ID = 10L; private static final long LAST_HOST_ID = 11L; + private static final long PLANNED_HOST_ID = 12L; private static final int CAPACITY_RELEASE_INTERVAL = 3600; @Mock @@ -155,6 +157,49 @@ public void testRemovedGroupVmIsSkipped() { assertFalse(avoids(HOST_ID)); } + @Test + public void testPlannedHostWinsOverStaleDatabaseHost() { + VMInstanceVO plannedVm = org.mockito.Mockito.mock(VMInstanceVO.class); + when(plannedVm.getId()).thenReturn(GROUP_VM_ID); + when(plannedVm.getHostId()).thenReturn(PLANNED_HOST_ID); + + // the database still shows the pre-migration host + when(_vmInstanceDao.findById(GROUP_VM_ID)).thenReturn(groupVM); + when(groupVM.isRemoved()).thenReturn(false); + when(groupVM.getHostId()).thenReturn(HOST_ID); + + processor.processAffinityGroup(vmGroupMapping, avoid, vm, Arrays.asList(plannedVm)); + + assertTrue(avoids(PLANNED_HOST_ID)); + assertFalse(avoids(HOST_ID)); + } + + @Test + public void testPlannedVmWithoutHostFallsBackToDatabase() { + VMInstanceVO plannedVm = org.mockito.Mockito.mock(VMInstanceVO.class); + when(plannedVm.getId()).thenReturn(GROUP_VM_ID); + when(plannedVm.getHostId()).thenReturn(null); + + when(_vmInstanceDao.findById(GROUP_VM_ID)).thenReturn(groupVM); + when(groupVM.isRemoved()).thenReturn(false); + when(groupVM.getHostId()).thenReturn(HOST_ID); + + processor.processAffinityGroup(vmGroupMapping, avoid, vm, Arrays.asList(plannedVm)); + + assertTrue(avoids(HOST_ID)); + } + + @Test + public void testEmptyVmListBehavesAsBefore() { + when(_vmInstanceDao.findById(GROUP_VM_ID)).thenReturn(groupVM); + when(groupVM.isRemoved()).thenReturn(false); + when(groupVM.getHostId()).thenReturn(HOST_ID); + + processor.processAffinityGroup(vmGroupMapping, avoid, vm, Collections.emptyList()); + + assertTrue(avoids(HOST_ID)); + } + @Test public void testVmIsNotAvoidedAgainstItself() { List ids = new ArrayList<>(Arrays.asList(VM_ID)); From 0bc3d2226fc57b4597f4cdd77a2c8746f1af9045 Mon Sep 17 00:00:00 2001 From: Brad House Date: Wed, 9 Sep 2026 21:20:32 +0000 Subject: [PATCH 08/18] Honor planned placements in non-strict host affinity Same gap as the strict processor: vmList was accepted and ignored, so host priorities were adjusted from database placements even when the caller supplied newer ones. NonStrictHostAntiAffinityProcessor extends this class and only overrides the priority direction, so it is fixed by the same change. Signed-off-by: Brad House --- .../NonStrictHostAffinityProcessor.java | 32 +++++++++++++++++- .../NonStrictHostAffinityProcessorTest.java | 33 +++++++++++++++++++ 2 files changed, 64 insertions(+), 1 deletion(-) diff --git a/plugins/affinity-group-processors/non-strict-host-affinity/src/main/java/org/apache/cloudstack/affinity/NonStrictHostAffinityProcessor.java b/plugins/affinity-group-processors/non-strict-host-affinity/src/main/java/org/apache/cloudstack/affinity/NonStrictHostAffinityProcessor.java index 49e3f60ed5d0..8cb92ce6fa18 100644 --- a/plugins/affinity-group-processors/non-strict-host-affinity/src/main/java/org/apache/cloudstack/affinity/NonStrictHostAffinityProcessor.java +++ b/plugins/affinity-group-processors/non-strict-host-affinity/src/main/java/org/apache/cloudstack/affinity/NonStrictHostAffinityProcessor.java @@ -17,6 +17,8 @@ package org.apache.cloudstack.affinity; import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; @@ -67,13 +69,24 @@ public void process(VirtualMachineProfile vmProfile, DeploymentPlan plan, Exclud for (AffinityGroupVMMapVO vmGroupMapping : vmGroupMappings) { if (vmGroupMapping != null) { - processAffinityGroup(vmGroupMapping, plan, vm); + processAffinityGroup(vmGroupMapping, plan, vm, vmList); } } } protected void processAffinityGroup(AffinityGroupVMMapVO vmGroupMapping, DeploymentPlan plan, VirtualMachine vm) { + processAffinityGroup(vmGroupMapping, plan, vm, Collections.emptyList()); + } + + /** + * Adjusts host priorities for one group. + * + * @param vmList + * placements to honour in preference to what the database says, for callers such as DRS + * that build a plan in memory before persisting it. + */ + protected void processAffinityGroup(AffinityGroupVMMapVO vmGroupMapping, DeploymentPlan plan, VirtualMachine vm, List vmList) { AffinityGroupVO group = affinityGroupDao.findById(vmGroupMapping.getAffinityGroupId()); if (logger.isDebugEnabled()) { @@ -83,7 +96,16 @@ protected void processAffinityGroup(AffinityGroupVMMapVO vmGroupMapping, Deploym List groupVMIds = affinityGroupVMMapDao.listVmIdsByAffinityGroup(group.getId()); groupVMIds.remove(vm.getId()); + Map plannedVms = getVmIdVmMap(vmList); + for (Long groupVMId : groupVMIds) { + VirtualMachine plannedVm = plannedVms.get(groupVMId); + if (plannedVm != null && plannedVm.getHostId() != null) { + Integer priority = adjustHostPriority(plan, plannedVm.getHostId()); + logger.debug("Updated host {} priority to {}, since VM {} is placed on the host by the plan being built", + plannedVm.getHostId(), priority, plannedVm); + continue; + } VMInstanceVO groupVM = vmInstanceDao.findById(groupVMId); if (groupVM != null && !groupVM.isRemoved()) { processVmInAffinityGroup(plan, groupVM); @@ -91,6 +113,14 @@ protected void processAffinityGroup(AffinityGroupVMMapVO vmGroupMapping, Deploym } } + protected Map getVmIdVmMap(List vmList) { + Map vmIdVmMap = new HashMap<>(); + for (VirtualMachine vm : vmList) { + vmIdVmMap.put(vm.getId(), vm); + } + return vmIdVmMap; + } + protected void processVmInAffinityGroup(DeploymentPlan plan, VMInstanceVO groupVM) { if (groupVM.getHostId() != null) { Integer priority = adjustHostPriority(plan, groupVM.getHostId()); diff --git a/plugins/affinity-group-processors/non-strict-host-affinity/src/test/java/org/apache/cloudstack/affinity/NonStrictHostAffinityProcessorTest.java b/plugins/affinity-group-processors/non-strict-host-affinity/src/test/java/org/apache/cloudstack/affinity/NonStrictHostAffinityProcessorTest.java index cb91870cbd39..6e14b26cb690 100644 --- a/plugins/affinity-group-processors/non-strict-host-affinity/src/test/java/org/apache/cloudstack/affinity/NonStrictHostAffinityProcessorTest.java +++ b/plugins/affinity-group-processors/non-strict-host-affinity/src/test/java/org/apache/cloudstack/affinity/NonStrictHostAffinityProcessorTest.java @@ -170,4 +170,37 @@ public void testProcessWithNotRunningVM() { Assert.assertNotNull(plan.getHostPriorities().get(host2Id)); Assert.assertEquals(Integer.valueOf(1), plan.getHostPriorities().get(host2Id)); } + + @Test + public void testPlannedHostWinsOverStaleDatabaseHost() { + VirtualMachine vm = Mockito.mock(VirtualMachine.class); + when(vm.getId()).thenReturn(vmId); + VirtualMachineProfile vmProfile = Mockito.mock(VirtualMachineProfile.class); + when(vmProfile.getVirtualMachine()).thenReturn(vm); + + List vmGroupMappings = new ArrayList<>(); + vmGroupMappings.add(new AffinityGroupVMMapVO(affinityGroupId, vmId)); + when(_affinityGroupVMMapDao.findByVmIdType(eq(vmId), nullable(String.class))).thenReturn(vmGroupMappings); + + DataCenterDeployment plan = new DataCenterDeployment(zoneId); + ExcludeList avoid = new ExcludeList(); + + AffinityGroupVO affinityGroupVO = Mockito.mock(AffinityGroupVO.class); + when(affinityGroupDao.findById(affinityGroupId)).thenReturn(affinityGroupVO); + when(affinityGroupVO.getId()).thenReturn(affinityGroupId); + when(_affinityGroupVMMapDao.listVmIdsByAffinityGroup(affinityGroupId)) + .thenReturn(new ArrayList<>(Arrays.asList(vmId, vm2Id))); + + // the plan being built has already moved vm2 to host3; the database is not consulted + VMInstanceVO planned = Mockito.mock(VMInstanceVO.class); + when(planned.getId()).thenReturn(vm2Id); + when(planned.getHostId()).thenReturn(host3Id); + + processor.process(vmProfile, plan, avoid, Arrays.asList(planned)); + + Assert.assertEquals(1, plan.getHostPriorities().size()); + Assert.assertNotNull(plan.getHostPriorities().get(host3Id)); + Assert.assertNull(plan.getHostPriorities().get(host2Id)); + Mockito.verify(vmInstanceDao, Mockito.never()).findById(vm2Id); + } } From 0e150dab435aadb5e39967eac83bfafb49595b15 Mon Sep 17 00:00:00 2001 From: Brad House Date: Wed, 9 Sep 2026 21:23:18 +0000 Subject: [PATCH 09/18] Re-check affinity before executing each DRS migration A DRS plan is generated once and executed later, and nothing downstream re-checks it - migrateVirtualMachine does not enforce affinity groups. By execution time the cluster may have changed, so a plan that was valid when generated can violate anti-affinity when it runs. - validate each migration against current state before queueing it - skip and mark failed instead of migrating into a violation - track destinations already queued in this run, since the jobs are asynchronous and the database does not reflect them yet Signed-off-by: Brad House --- .../cluster/ClusterDrsServiceImpl.java | 40 ++++++++- .../cluster/ClusterDrsServiceImplTest.java | 86 +++++++++++++++++++ 2 files changed, 125 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/org/apache/cloudstack/cluster/ClusterDrsServiceImpl.java b/server/src/main/java/org/apache/cloudstack/cluster/ClusterDrsServiceImpl.java index 62075aae596e..5f91666b115b 100644 --- a/server/src/main/java/org/apache/cloudstack/cluster/ClusterDrsServiceImpl.java +++ b/server/src/main/java/org/apache/cloudstack/cluster/ClusterDrsServiceImpl.java @@ -768,6 +768,29 @@ void processPlans() { * @param plan * the DRS plan to be executed */ + /** + * Checks a planned migration against the affinity rules as they stand now. + * + * A plan is generated once and executed later, so state can have moved on: VMs may have been + * created, migrated or destroyed in between. Anti-affinity in particular is only meaningful + * against current placements, and nothing downstream re-checks it - migrateVirtualMachine does + * not enforce affinity groups. + * + * @param dispatched + * migrations already queued by this run, which the database does not reflect yet + */ + protected boolean destinationViolatesAffinity(VirtualMachine vm, Host destHost, List dispatched) { + if (CollectionUtils.isEmpty(affinityGroupVMMapDao.listByInstanceId(vm.getId()))) { + return false; + } + DataCenterDeployment plan = new DataCenterDeployment(destHost.getDataCenterId(), destHost.getPodId(), + destHost.getClusterId(), null, null, null); + VirtualMachineProfile vmProfile = new VirtualMachineProfileImpl(vm, null, + serviceOfferingDao.findByIdIncludingRemoved(vm.getId(), vm.getServiceOfferingId()), null, null); + ExcludeList excludes = managementServer.applyAffinityConstraints(vm, vmProfile, plan, dispatched); + return excludes.shouldAvoid(destHost); + } + void executeDrsPlan(ClusterDrsPlanVO plan) { List planMigrations = drsPlanMigrationDao.listPlanMigrationsToExecute(plan.getId()); if (planMigrations == null || planMigrations.isEmpty()) { @@ -783,21 +806,36 @@ void executeDrsPlan(ClusterDrsPlanVO plan) { plan.setStatus(ClusterDrsPlan.Status.IN_PROGRESS); drsPlanDao.update(plan.getId(), plan); + List dispatched = new ArrayList<>(); + for (ClusterDrsPlanMigrationVO migration : planMigrations) { try { - VirtualMachine vm = vmInstanceDao.findById(migration.getVmId()); + VMInstanceVO vm = vmInstanceDao.findById(migration.getVmId()); Host host = hostDao.findById(migration.getDestHostId()); if (vm == null || host == null) { throw new CloudRuntimeException(String.format("vm %s or host %s is not found", migration.getVmId(), migration.getDestHostId())); } + if (destinationViolatesAffinity(vm, host, dispatched)) { + logger.warn("Skipping DRS migration of vm {} to host {}: it no longer satisfies the affinity " + + "rules for that VM. The plan was generated against older state.", vm, host); + migration.setStatus(JobInfo.Status.FAILED); + drsPlanMigrationDao.update(migration.getId(), migration); + continue; + } + logger.debug("Executing DRS plan {} for vm {} to host {}", plan, vm, host); long jobId = createMigrateVMAsyncJob(vm, host, plan.getEventId()); AsyncJobVO job = asyncJobManager.getAsyncJob(jobId); migration.setJobId(jobId); migration.setStatus(job.getStatus()); drsPlanMigrationDao.update(migration.getId(), migration); + + // the migration job has only been queued, so the database still shows the old host. + // record where it is headed so later migrations in this plan see it. + vm.setHostId(host.getId()); + dispatched.add(vm); } catch (Exception e) { logger.warn("Unable to execute DRS plan {} due to {}", plan, e.getMessage()); migration.setStatus(JobInfo.Status.FAILED); diff --git a/server/src/test/java/org/apache/cloudstack/cluster/ClusterDrsServiceImplTest.java b/server/src/test/java/org/apache/cloudstack/cluster/ClusterDrsServiceImplTest.java index 6390b29097b5..a6b461724c45 100644 --- a/server/src/test/java/org/apache/cloudstack/cluster/ClusterDrsServiceImplTest.java +++ b/server/src/test/java/org/apache/cloudstack/cluster/ClusterDrsServiceImplTest.java @@ -29,12 +29,14 @@ import com.cloud.exception.InvalidParameterValueException; import com.cloud.host.Host; import com.cloud.host.HostVO; +import com.cloud.deploy.DeploymentPlanner.ExcludeList; import com.cloud.host.dao.HostDao; import com.cloud.offering.ServiceOffering; import com.cloud.org.Cluster; import com.cloud.org.Grouping; import com.cloud.server.ManagementServer; import com.cloud.service.ServiceOfferingVO; +import org.apache.cloudstack.affinity.AffinityGroupVMMapVO; import com.cloud.service.dao.ServiceOfferingDao; import com.cloud.utils.Pair; import com.cloud.utils.Ternary; @@ -70,6 +72,7 @@ import java.lang.reflect.Field; import java.util.ArrayList; import java.util.Collections; +import org.apache.cloudstack.jobs.JobInfo; import java.util.Date; import java.util.HashMap; import java.util.List; @@ -77,6 +80,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; import static org.junit.Assert.assertNull; @RunWith(MockitoJUnitRunner.class) @@ -951,4 +955,86 @@ public void testProcessPlans() { Mockito.verify(clusterDrsService, Mockito.times(2)).executeDrsPlan(Mockito.any(ClusterDrsPlanVO.class)); } + + @Test + public void testDestinationViolatesAffinityWhenVmHasNoAffinityGroups() { + VMInstanceVO vm = Mockito.mock(VMInstanceVO.class); + Mockito.when(vm.getId()).thenReturn(1L); + Mockito.when(affinityGroupVMMapDao.listByInstanceId(1L)).thenReturn(Collections.emptyList()); + + HostVO destHost = Mockito.mock(HostVO.class); + + assertFalse(clusterDrsService.destinationViolatesAffinity(vm, destHost, Collections.emptyList())); + Mockito.verify(managementServer, Mockito.never()) + .applyAffinityConstraints(Mockito.any(), Mockito.any(), Mockito.any(), Mockito.any()); + } + + @Test + public void testDestinationViolatesAffinityWhenDestHostIsExcluded() { + VMInstanceVO vm = Mockito.mock(VMInstanceVO.class); + Mockito.when(vm.getId()).thenReturn(1L); + Mockito.when(vm.getServiceOfferingId()).thenReturn(5L); + Mockito.when(affinityGroupVMMapDao.listByInstanceId(1L)) + .thenReturn(Collections.singletonList(Mockito.mock(AffinityGroupVMMapVO.class))); + Mockito.when(serviceOfferingDao.findByIdIncludingRemoved(1L, 5L)) + .thenReturn(Mockito.mock(ServiceOfferingVO.class)); + + HostVO destHost = Mockito.mock(HostVO.class); + Mockito.when(destHost.getId()).thenReturn(20L); + + ExcludeList excludes = new ExcludeList(); + excludes.addHost(20L); + Mockito.when(managementServer.applyAffinityConstraints(Mockito.any(), Mockito.any(), Mockito.any(), Mockito.any())) + .thenReturn(excludes); + + assertTrue(clusterDrsService.destinationViolatesAffinity(vm, destHost, Collections.emptyList())); + } + + @Test + public void testDestinationViolatesAffinityWhenDestHostIsAllowed() { + VMInstanceVO vm = Mockito.mock(VMInstanceVO.class); + Mockito.when(vm.getId()).thenReturn(1L); + Mockito.when(vm.getServiceOfferingId()).thenReturn(5L); + Mockito.when(affinityGroupVMMapDao.listByInstanceId(1L)) + .thenReturn(Collections.singletonList(Mockito.mock(AffinityGroupVMMapVO.class))); + Mockito.when(serviceOfferingDao.findByIdIncludingRemoved(1L, 5L)) + .thenReturn(Mockito.mock(ServiceOfferingVO.class)); + + HostVO destHost = Mockito.mock(HostVO.class); + Mockito.when(destHost.getId()).thenReturn(20L); + + ExcludeList excludes = new ExcludeList(); + excludes.addHost(21L); + Mockito.when(managementServer.applyAffinityConstraints(Mockito.any(), Mockito.any(), Mockito.any(), Mockito.any())) + .thenReturn(excludes); + + assertFalse(clusterDrsService.destinationViolatesAffinity(vm, destHost, Collections.emptyList())); + } + + @Test + public void testExecuteDrsPlanSkipsMigrationThatViolatesAffinity() { + ClusterDrsPlanVO plan = Mockito.mock(ClusterDrsPlanVO.class); + Mockito.when(plan.getId()).thenReturn(1L); + + ClusterDrsPlanMigrationVO migration = Mockito.mock(ClusterDrsPlanMigrationVO.class); + Mockito.when(migration.getId()).thenReturn(7L); + Mockito.when(migration.getVmId()).thenReturn(1L); + Mockito.when(migration.getDestHostId()).thenReturn(20L); + Mockito.when(drsPlanMigrationDao.listPlanMigrationsToExecute(1L)) + .thenReturn(Collections.singletonList(migration)); + + VMInstanceVO vm = Mockito.mock(VMInstanceVO.class); + HostVO destHost = Mockito.mock(HostVO.class); + Mockito.when(vmInstanceDao.findById(1L)).thenReturn(vm); + Mockito.when(hostDao.findById(20L)).thenReturn(destHost); + + Mockito.doReturn(true).when(clusterDrsService) + .destinationViolatesAffinity(Mockito.any(), Mockito.any(), Mockito.any()); + + clusterDrsService.executeDrsPlan(plan); + + Mockito.verify(migration).setStatus(JobInfo.Status.FAILED); + Mockito.verify(clusterDrsService, Mockito.never()) + .createMigrateVMAsyncJob(Mockito.any(), Mockito.any(), Mockito.anyLong()); + } } From 43dbd772d744e1b563739cb024a9dea6c3a4f8cb Mon Sep 17 00:00:00 2001 From: Brad House Date: Wed, 9 Sep 2026 22:14:34 +0000 Subject: [PATCH 10/18] Make the DRS affinity check correct Three defects found reviewing the previous commits, all in how DRS decides whether a planned migration is still allowed. Non-strict anti-affinity was discarded entirely. Non-strict groups express themselves by lowering a host's priority on the deployment plan rather than by excluding it, and DRS built a plan, handed it to the processors, read only the exclude list and threw the plan away. Non-strict means the rule may be broken when there is nowhere else to put a VM. That cannot arise while rebalancing: the VM already runs somewhere that satisfies the group and leaving it there is always an option. Being better balanced is not a reason to break it. A host was treated as free the moment a migration away from it was queued. The jobs are asynchronous and can fail, so a queued migration occupies both ends until it completes. In a swap - A from host1 to host3, B from host2 to host1 - B was cleared for host1 while A was still on it. - track the hosts queued migrations have not actually left - refuse a destination that is one of them A VM that stopped between planning and execution threw NPE inside the affinity check, which was then logged without a stack trace. - treat a VM that is no longer running as an out of date plan - log the exception rather than its message Signed-off-by: Brad House --- .../HostAntiAffinityProcessorTest.java | 21 +++ .../cluster/ClusterDrsServiceImpl.java | 58 +++++- .../cluster/ClusterDrsServiceImplTest.java | 166 ++++++++++++++---- 3 files changed, 205 insertions(+), 40 deletions(-) diff --git a/plugins/affinity-group-processors/host-anti-affinity/src/test/java/org/apache/cloudstack/affinity/HostAntiAffinityProcessorTest.java b/plugins/affinity-group-processors/host-anti-affinity/src/test/java/org/apache/cloudstack/affinity/HostAntiAffinityProcessorTest.java index ae3297039289..3ef05ce07bfb 100644 --- a/plugins/affinity-group-processors/host-anti-affinity/src/test/java/org/apache/cloudstack/affinity/HostAntiAffinityProcessorTest.java +++ b/plugins/affinity-group-processors/host-anti-affinity/src/test/java/org/apache/cloudstack/affinity/HostAntiAffinityProcessorTest.java @@ -40,6 +40,7 @@ import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.lenient; import static org.mockito.Mockito.when; @RunWith(JUnit4.class) @@ -151,10 +152,30 @@ public void testMissingGroupVmIsSkipped() { public void testRemovedGroupVmIsSkipped() { when(_vmInstanceDao.findById(GROUP_VM_ID)).thenReturn(groupVM); when(groupVM.isRemoved()).thenReturn(true); + // a removed VM is not running anywhere, so neither of its hosts should be avoided + lenient().when(groupVM.getHostId()).thenReturn(HOST_ID); + lenient().when(groupVM.getState()).thenReturn(VirtualMachine.State.Stopped); + lenient().when(groupVM.getLastHostId()).thenReturn(LAST_HOST_ID); + lenient().when(groupVM.getUpdateTime()).thenReturn(DateUtil.currentGMTTime()); processor.processAffinityGroup(vmGroupMapping, avoid, vm); assertFalse(avoids(HOST_ID)); + assertFalse(avoids(LAST_HOST_ID)); + } + + @Test + public void testStartingGroupVmWithReservedCapacityLastHostIsAvoided() { + when(_vmInstanceDao.findById(GROUP_VM_ID)).thenReturn(groupVM); + when(groupVM.isRemoved()).thenReturn(false); + when(groupVM.getHostId()).thenReturn(null); + when(groupVM.getState()).thenReturn(VirtualMachine.State.Starting); + when(groupVM.getLastHostId()).thenReturn(LAST_HOST_ID); + when(groupVM.getUpdateTime()).thenReturn(DateUtil.currentGMTTime()); + + processor.processAffinityGroup(vmGroupMapping, avoid, vm); + + assertTrue(avoids(LAST_HOST_ID)); } @Test diff --git a/server/src/main/java/org/apache/cloudstack/cluster/ClusterDrsServiceImpl.java b/server/src/main/java/org/apache/cloudstack/cluster/ClusterDrsServiceImpl.java index 5f91666b115b..cbe2cfcd160a 100644 --- a/server/src/main/java/org/apache/cloudstack/cluster/ClusterDrsServiceImpl.java +++ b/server/src/main/java/org/apache/cloudstack/cluster/ClusterDrsServiceImpl.java @@ -25,6 +25,7 @@ import com.cloud.dc.ClusterVO; import com.cloud.dc.dao.ClusterDao; import com.cloud.deploy.DataCenterDeployment; +import com.cloud.deploy.DeploymentPlan; import com.cloud.deploy.DeploymentPlanner.ExcludeList; import com.cloud.domain.Domain; import com.cloud.event.ActionEventUtils; @@ -80,6 +81,7 @@ import org.apache.cloudstack.jobs.JobInfo; import org.apache.cloudstack.managed.context.ManagedContextTimerTask; import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.collections.MapUtils; import org.apache.commons.lang3.time.DateUtils; import javax.inject.Inject; @@ -432,6 +434,29 @@ private List> getMigrationPlans( return migrationPlan; } + /** + * Turns the non-strict affinity preferences recorded on the plan into exclusions. + * + * Non-strict groups express themselves by lowering a host's priority rather than by excluding + * it, and DRS only reads the exclude list - so the preference was being discarded. Rebalancing + * is never a reason to break it: the VM is already running somewhere that satisfies the group, + * and leaving it there is always available to DRS. "Non-strict" means the rule may be broken + * when there is nowhere else to put a VM, which cannot arise while merely rebalancing. + */ + protected void excludeHostsDispreferredByAffinity(DeploymentPlan plan, ExcludeList excludes) { + Map priorities = plan.getHostPriorities(); + if (MapUtils.isEmpty(priorities)) { + return; + } + for (Map.Entry entry : priorities.entrySet()) { + if (entry.getValue() != null && entry.getValue() < DeploymentPlan.DEFAULT_HOST_PRIORITY) { + excludes.addHost(entry.getKey()); + logger.debug("Host {} is dispreferred by a non-strict affinity group, so DRS will not migrate onto it", + entry.getKey()); + } + } + } + private Map getVmToExcludesMap(List vmList, Map hostMap, Set vmsWithAffinityGroups, Map> vmToCompatibleHostsCache, Map vmIdServiceOfferingMap) { @@ -452,6 +477,7 @@ private Map getVmToExcludesMap(List vmList, M excludes = managementServer.applyAffinityConstraints( vm, vmProfile, plan, vmList); + excludeHostsDispreferredByAffinity(plan, excludes); } else { // VM has no affinity groups - create minimal ExcludeList (just source host) excludes = new ExcludeList(); @@ -776,18 +802,35 @@ void processPlans() { * against current placements, and nothing downstream re-checks it - migrateVirtualMachine does * not enforce affinity groups. * + * @param vm + * the VM the plan wants to move + * @param destHost + * where the plan wants to move it * @param dispatched * migrations already queued by this run, which the database does not reflect yet + * @param dispatchedSourceHosts + * hosts those queued migrations have not actually left yet + * @return true when the migration should not go ahead */ - protected boolean destinationViolatesAffinity(VirtualMachine vm, Host destHost, List dispatched) { + protected boolean destinationViolatesAffinity(VirtualMachine vm, Host destHost, List dispatched, + List dispatchedSourceHosts) { + if (vm.getHostId() == null) { + logger.debug("VM {} is no longer running, so its planned migration is out of date", vm); + return true; + } if (CollectionUtils.isEmpty(affinityGroupVMMapDao.listByInstanceId(vm.getId()))) { return false; } + if (dispatchedSourceHosts.contains(destHost.getId())) { + logger.debug("Host {} is still occupied by a VM whose migration away from it is only queued", destHost); + return true; + } DataCenterDeployment plan = new DataCenterDeployment(destHost.getDataCenterId(), destHost.getPodId(), destHost.getClusterId(), null, null, null); VirtualMachineProfile vmProfile = new VirtualMachineProfileImpl(vm, null, serviceOfferingDao.findByIdIncludingRemoved(vm.getId(), vm.getServiceOfferingId()), null, null); ExcludeList excludes = managementServer.applyAffinityConstraints(vm, vmProfile, plan, dispatched); + excludeHostsDispreferredByAffinity(plan, excludes); return excludes.shouldAvoid(destHost); } @@ -806,7 +849,9 @@ void executeDrsPlan(ClusterDrsPlanVO plan) { plan.setStatus(ClusterDrsPlan.Status.IN_PROGRESS); drsPlanDao.update(plan.getId(), plan); + // a queued migration occupies both ends until it completes, and it may not complete at all List dispatched = new ArrayList<>(); + List dispatchedSourceHosts = new ArrayList<>(); for (ClusterDrsPlanMigrationVO migration : planMigrations) { try { @@ -817,7 +862,7 @@ void executeDrsPlan(ClusterDrsPlanVO plan) { migration.getDestHostId())); } - if (destinationViolatesAffinity(vm, host, dispatched)) { + if (destinationViolatesAffinity(vm, host, dispatched, dispatchedSourceHosts)) { logger.warn("Skipping DRS migration of vm {} to host {}: it no longer satisfies the affinity " + "rules for that VM. The plan was generated against older state.", vm, host); migration.setStatus(JobInfo.Status.FAILED); @@ -833,11 +878,16 @@ void executeDrsPlan(ClusterDrsPlanVO plan) { drsPlanMigrationDao.update(migration.getId(), migration); // the migration job has only been queued, so the database still shows the old host. - // record where it is headed so later migrations in this plan see it. + // record both ends: the VM is headed for the destination but has not left the + // source, and if the job fails it never will. + Long sourceHostId = vm.getHostId(); + if (sourceHostId != null) { + dispatchedSourceHosts.add(sourceHostId); + } vm.setHostId(host.getId()); dispatched.add(vm); } catch (Exception e) { - logger.warn("Unable to execute DRS plan {} due to {}", plan, e.getMessage()); + logger.warn("Unable to execute DRS plan {}", plan, e); migration.setStatus(JobInfo.Status.FAILED); drsPlanMigrationDao.update(migration.getId(), migration); } diff --git a/server/src/test/java/org/apache/cloudstack/cluster/ClusterDrsServiceImplTest.java b/server/src/test/java/org/apache/cloudstack/cluster/ClusterDrsServiceImplTest.java index a6b461724c45..4557df32d87e 100644 --- a/server/src/test/java/org/apache/cloudstack/cluster/ClusterDrsServiceImplTest.java +++ b/server/src/test/java/org/apache/cloudstack/cluster/ClusterDrsServiceImplTest.java @@ -30,6 +30,8 @@ import com.cloud.host.Host; import com.cloud.host.HostVO; import com.cloud.deploy.DeploymentPlanner.ExcludeList; +import com.cloud.deploy.DeploymentPlan; +import com.cloud.deploy.DataCenterDeployment; import com.cloud.host.dao.HostDao; import com.cloud.offering.ServiceOffering; import com.cloud.org.Cluster; @@ -63,6 +65,7 @@ import org.mockito.Mock; import org.mockito.MockedStatic; import org.mockito.Mockito; +import org.mockito.ArgumentCaptor; import org.mockito.MockitoAnnotations; import org.mockito.Spy; import org.mockito.junit.MockitoJUnitRunner; @@ -72,7 +75,10 @@ import java.lang.reflect.Field; import java.util.ArrayList; import java.util.Collections; +import java.util.Arrays; import org.apache.cloudstack.jobs.JobInfo; +import org.apache.cloudstack.framework.jobs.AsyncJobManager; +import org.apache.cloudstack.framework.jobs.impl.AsyncJobVO; import java.util.Date; import java.util.HashMap; import java.util.List; @@ -127,6 +133,9 @@ public class ClusterDrsServiceImplTest { @Mock private AffinityGroupVMMapDao affinityGroupVMMapDao; + @Mock + private AsyncJobManager asyncJobManager; + @Mock private VMInstanceDetailsDao vmInstanceDetailsDao; @@ -956,59 +965,127 @@ public void testProcessPlans() { Mockito.verify(clusterDrsService, Mockito.times(2)).executeDrsPlan(Mockito.any(ClusterDrsPlanVO.class)); } + private VMInstanceVO vmWithAffinityGroup(long id, Long hostId) { + VMInstanceVO vm = Mockito.mock(VMInstanceVO.class); + Mockito.lenient().when(vm.getId()).thenReturn(id); + Mockito.lenient().when(vm.getHostId()).thenReturn(hostId); + Mockito.lenient().when(vm.getServiceOfferingId()).thenReturn(5L); + Mockito.lenient().when(affinityGroupVMMapDao.listByInstanceId(id)) + .thenReturn(Collections.singletonList(Mockito.mock(AffinityGroupVMMapVO.class))); + Mockito.lenient().when(serviceOfferingDao.findByIdIncludingRemoved(id, 5L)) + .thenReturn(Mockito.mock(ServiceOfferingVO.class)); + return vm; + } + + private HostVO host(long id) { + HostVO host = Mockito.mock(HostVO.class); + Mockito.lenient().when(host.getId()).thenReturn(id); + return host; + } + + private void affinityExcludes(Long... hostIds) { + ExcludeList excludes = new ExcludeList(); + for (Long hostId : hostIds) { + excludes.addHost(hostId); + } + Mockito.when(managementServer.applyAffinityConstraints(Mockito.any(), Mockito.any(), Mockito.any(), Mockito.any())) + .thenReturn(excludes); + } + @Test - public void testDestinationViolatesAffinityWhenVmHasNoAffinityGroups() { + public void testDestinationAllowedWhenVmHasNoAffinityGroups() { VMInstanceVO vm = Mockito.mock(VMInstanceVO.class); Mockito.when(vm.getId()).thenReturn(1L); + Mockito.when(vm.getHostId()).thenReturn(10L); Mockito.when(affinityGroupVMMapDao.listByInstanceId(1L)).thenReturn(Collections.emptyList()); - HostVO destHost = Mockito.mock(HostVO.class); - - assertFalse(clusterDrsService.destinationViolatesAffinity(vm, destHost, Collections.emptyList())); + assertFalse(clusterDrsService.destinationViolatesAffinity(vm, host(20L), + Collections.emptyList(), Collections.emptyList())); Mockito.verify(managementServer, Mockito.never()) .applyAffinityConstraints(Mockito.any(), Mockito.any(), Mockito.any(), Mockito.any()); } @Test - public void testDestinationViolatesAffinityWhenDestHostIsExcluded() { - VMInstanceVO vm = Mockito.mock(VMInstanceVO.class); - Mockito.when(vm.getId()).thenReturn(1L); - Mockito.when(vm.getServiceOfferingId()).thenReturn(5L); - Mockito.when(affinityGroupVMMapDao.listByInstanceId(1L)) - .thenReturn(Collections.singletonList(Mockito.mock(AffinityGroupVMMapVO.class))); - Mockito.when(serviceOfferingDao.findByIdIncludingRemoved(1L, 5L)) - .thenReturn(Mockito.mock(ServiceOfferingVO.class)); + public void testDestinationRefusedWhenAffinityExcludesIt() { + VMInstanceVO vm = vmWithAffinityGroup(1L, 10L); + affinityExcludes(20L); - HostVO destHost = Mockito.mock(HostVO.class); - Mockito.when(destHost.getId()).thenReturn(20L); + assertTrue(clusterDrsService.destinationViolatesAffinity(vm, host(20L), + Collections.emptyList(), Collections.emptyList())); + } - ExcludeList excludes = new ExcludeList(); - excludes.addHost(20L); - Mockito.when(managementServer.applyAffinityConstraints(Mockito.any(), Mockito.any(), Mockito.any(), Mockito.any())) - .thenReturn(excludes); + @Test + public void testDestinationAllowedWhenAffinityExcludesSomewhereElse() { + VMInstanceVO vm = vmWithAffinityGroup(1L, 10L); + affinityExcludes(21L); - assertTrue(clusterDrsService.destinationViolatesAffinity(vm, destHost, Collections.emptyList())); + assertFalse(clusterDrsService.destinationViolatesAffinity(vm, host(20L), + Collections.emptyList(), Collections.emptyList())); } @Test - public void testDestinationViolatesAffinityWhenDestHostIsAllowed() { + public void testDestinationRefusedWhenVmIsNoLongerRunning() { VMInstanceVO vm = Mockito.mock(VMInstanceVO.class); - Mockito.when(vm.getId()).thenReturn(1L); - Mockito.when(vm.getServiceOfferingId()).thenReturn(5L); - Mockito.when(affinityGroupVMMapDao.listByInstanceId(1L)) - .thenReturn(Collections.singletonList(Mockito.mock(AffinityGroupVMMapVO.class))); - Mockito.when(serviceOfferingDao.findByIdIncludingRemoved(1L, 5L)) - .thenReturn(Mockito.mock(ServiceOfferingVO.class)); + Mockito.when(vm.getHostId()).thenReturn(null); - HostVO destHost = Mockito.mock(HostVO.class); - Mockito.when(destHost.getId()).thenReturn(20L); + assertTrue("a plan for a VM that has since stopped is out of date", + clusterDrsService.destinationViolatesAffinity(vm, host(20L), + Collections.emptyList(), Collections.emptyList())); + } - ExcludeList excludes = new ExcludeList(); - excludes.addHost(21L); - Mockito.when(managementServer.applyAffinityConstraints(Mockito.any(), Mockito.any(), Mockito.any(), Mockito.any())) - .thenReturn(excludes); + @Test + public void testDestinationRefusedWhileItIsStillOccupiedByAQueuedMigration() { + // the swap case: A is queued to leave host1, so host1 is not free for B yet - and if A's + // migration fails, A never leaves at all + VMInstanceVO b = vmWithAffinityGroup(2L, 11L); + + assertTrue("a host is not free until the VM leaving it has actually gone", + clusterDrsService.destinationViolatesAffinity(b, host(10L), + Collections.emptyList(), Collections.singletonList(10L))); + Mockito.verify(managementServer, Mockito.never()) + .applyAffinityConstraints(Mockito.any(), Mockito.any(), Mockito.any(), Mockito.any()); + } + + @Test + public void testExecuteDrsPlanKeepsSourceHostsOccupiedAcrossMigrations() { + // plan: A host10 -> host12, B host11 -> host10. B must not be sent to host10. + ClusterDrsPlanVO plan = Mockito.mock(ClusterDrsPlanVO.class); + Mockito.when(plan.getId()).thenReturn(1L); + + ClusterDrsPlanMigrationVO first = Mockito.mock(ClusterDrsPlanMigrationVO.class); + Mockito.when(first.getVmId()).thenReturn(1L); + Mockito.when(first.getDestHostId()).thenReturn(12L); + ClusterDrsPlanMigrationVO second = Mockito.mock(ClusterDrsPlanMigrationVO.class); + Mockito.when(second.getId()).thenReturn(8L); + Mockito.when(second.getVmId()).thenReturn(2L); + Mockito.when(second.getDestHostId()).thenReturn(10L); + Mockito.when(drsPlanMigrationDao.listPlanMigrationsToExecute(1L)) + .thenReturn(Arrays.asList(first, second)); + + VMInstanceVO a = Mockito.mock(VMInstanceVO.class); + Mockito.when(a.getHostId()).thenReturn(10L); + VMInstanceVO b = Mockito.mock(VMInstanceVO.class); + Mockito.when(vmInstanceDao.findById(1L)).thenReturn(a); + Mockito.when(vmInstanceDao.findById(2L)).thenReturn(b); + HostVO host12 = host(12L); + HostVO host10 = host(10L); + Mockito.when(hostDao.findById(12L)).thenReturn(host12); + Mockito.when(hostDao.findById(10L)).thenReturn(host10); + + Mockito.doReturn(false).when(clusterDrsService).destinationViolatesAffinity( + Mockito.eq(a), Mockito.any(), Mockito.any(), Mockito.any()); + Mockito.doReturn(1L).when(clusterDrsService) + .createMigrateVMAsyncJob(Mockito.any(), Mockito.any(), Mockito.anyLong()); + Mockito.when(asyncJobManager.getAsyncJob(1L)).thenReturn(Mockito.mock(AsyncJobVO.class)); + + clusterDrsService.executeDrsPlan(plan); - assertFalse(clusterDrsService.destinationViolatesAffinity(vm, destHost, Collections.emptyList())); + // A's source host must have been carried into the check for B + ArgumentCaptor sources = ArgumentCaptor.forClass(List.class); + Mockito.verify(clusterDrsService).destinationViolatesAffinity( + Mockito.eq(b), Mockito.any(), Mockito.any(), sources.capture()); + assertTrue("the host A is leaving must still count as occupied", + sources.getValue().contains(10L)); } @Test @@ -1024,12 +1101,12 @@ public void testExecuteDrsPlanSkipsMigrationThatViolatesAffinity() { .thenReturn(Collections.singletonList(migration)); VMInstanceVO vm = Mockito.mock(VMInstanceVO.class); - HostVO destHost = Mockito.mock(HostVO.class); + HostVO host20 = host(20L); Mockito.when(vmInstanceDao.findById(1L)).thenReturn(vm); - Mockito.when(hostDao.findById(20L)).thenReturn(destHost); + Mockito.when(hostDao.findById(20L)).thenReturn(host20); Mockito.doReturn(true).when(clusterDrsService) - .destinationViolatesAffinity(Mockito.any(), Mockito.any(), Mockito.any()); + .destinationViolatesAffinity(Mockito.any(), Mockito.any(), Mockito.any(), Mockito.any()); clusterDrsService.executeDrsPlan(plan); @@ -1037,4 +1114,21 @@ public void testExecuteDrsPlanSkipsMigrationThatViolatesAffinity() { Mockito.verify(clusterDrsService, Mockito.never()) .createMigrateVMAsyncJob(Mockito.any(), Mockito.any(), Mockito.anyLong()); } + + @Test + public void testNonStrictAntiAffinityIsHonoured() { + // a non-strict group lowers a host's priority rather than excluding it. rebalancing is + // never a reason to break it, since not migrating is always available. + DataCenterDeployment plan = new DataCenterDeployment(1L, 1L, 1L, null, null, null); + plan.adjustHostPriority(30L, DeploymentPlan.HostPriorityAdjustment.LOWER); + plan.adjustHostPriority(31L, DeploymentPlan.HostPriorityAdjustment.HIGHER); + + ExcludeList excludes = new ExcludeList(); + clusterDrsService.excludeHostsDispreferredByAffinity(plan, excludes); + + assertTrue("a dispreferred host must not be a DRS destination", + excludes.getHostsToAvoid().contains(30L)); + assertFalse("a preferred host must stay available", + excludes.getHostsToAvoid().contains(31L)); + } } From 6482992a1d1334103826a40f6304a6111791b43f Mon Sep 17 00:00:00 2001 From: Brad House Date: Wed, 9 Sep 2026 22:36:04 +0000 Subject: [PATCH 11/18] Address the rest of the DRS anti-affinity review - a skipped migration is CANCELLED, not FAILED. Nothing went wrong; the plan went out of date. FAILED was indistinguishable from a migration that genuinely broke, and left no record of why - record an event when one is skipped, so it is visible rather than a silent no-op in an otherwise successful plan - the processors also cover dedicated resources and DPDK, so the refusal message no longer claims every skip is about an affinity group - hoist the third copy of getVmIdVmMap into AffinityProcessorBase Signed-off-by: Brad House --- .../affinity/AffinityProcessorBase.java | 19 +++++++++++++++++++ .../affinity/HostAntiAffinityProcessor.java | 9 --------- .../NonStrictHostAffinityProcessor.java | 9 --------- .../cluster/ClusterDrsServiceImpl.java | 17 +++++++++++++---- .../cluster/ClusterDrsServiceImplTest.java | 2 +- 5 files changed, 33 insertions(+), 23 deletions(-) diff --git a/api/src/main/java/org/apache/cloudstack/affinity/AffinityProcessorBase.java b/api/src/main/java/org/apache/cloudstack/affinity/AffinityProcessorBase.java index 96ca35f264ca..b330b6aa321c 100644 --- a/api/src/main/java/org/apache/cloudstack/affinity/AffinityProcessorBase.java +++ b/api/src/main/java/org/apache/cloudstack/affinity/AffinityProcessorBase.java @@ -25,7 +25,9 @@ import com.cloud.vm.VirtualMachineProfile; import java.util.Collections; +import java.util.HashMap; import java.util.List; +import java.util.Map; public class AffinityProcessorBase extends AdapterBase implements AffinityGroupProcessor { @@ -44,6 +46,23 @@ public void process(VirtualMachineProfile vm, DeploymentPlan plan, ExcludeList a } + + /** + * Indexes placements supplied by the caller. Callers such as DRS build a plan of several moves + * in memory and persist it only at the end, so during planning the database still shows the old + * host for every VM the plan has already moved. + */ + protected Map getVmIdVmMap(List vmList) { + Map vmIdVmMap = new HashMap<>(); + if (vmList == null) { + return vmIdVmMap; + } + for (VirtualMachine vm : vmList) { + vmIdVmMap.put(vm.getId(), vm); + } + return vmIdVmMap; + } + @Override public String getType() { return _type; diff --git a/plugins/affinity-group-processors/host-anti-affinity/src/main/java/org/apache/cloudstack/affinity/HostAntiAffinityProcessor.java b/plugins/affinity-group-processors/host-anti-affinity/src/main/java/org/apache/cloudstack/affinity/HostAntiAffinityProcessor.java index 9c69fbd7053c..b71142d0a9e5 100644 --- a/plugins/affinity-group-processors/host-anti-affinity/src/main/java/org/apache/cloudstack/affinity/HostAntiAffinityProcessor.java +++ b/plugins/affinity-group-processors/host-anti-affinity/src/main/java/org/apache/cloudstack/affinity/HostAntiAffinityProcessor.java @@ -18,7 +18,6 @@ import java.util.Arrays; import java.util.Collections; -import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.stream.Collectors; @@ -133,14 +132,6 @@ protected void processAffinityGroup(AffinityGroupVMMapVO vmGroupMapping, Exclude } } - protected Map getVmIdVmMap(List vmList) { - Map vmIdVmMap = new HashMap<>(); - for (VirtualMachine vm : vmList) { - vmIdVmMap.put(vm.getId(), vm); - } - return vmIdVmMap; - } - protected void avoidHostOfVmInAffinityGroup(ExcludeList avoid, VMInstanceVO groupVM) { if (groupVM.getHostId() != null) { avoid.addHost(groupVM.getHostId()); diff --git a/plugins/affinity-group-processors/non-strict-host-affinity/src/main/java/org/apache/cloudstack/affinity/NonStrictHostAffinityProcessor.java b/plugins/affinity-group-processors/non-strict-host-affinity/src/main/java/org/apache/cloudstack/affinity/NonStrictHostAffinityProcessor.java index 8cb92ce6fa18..c79efca5e7de 100644 --- a/plugins/affinity-group-processors/non-strict-host-affinity/src/main/java/org/apache/cloudstack/affinity/NonStrictHostAffinityProcessor.java +++ b/plugins/affinity-group-processors/non-strict-host-affinity/src/main/java/org/apache/cloudstack/affinity/NonStrictHostAffinityProcessor.java @@ -18,7 +18,6 @@ import java.util.Arrays; import java.util.Collections; -import java.util.HashMap; import java.util.List; import java.util.Map; @@ -113,14 +112,6 @@ protected void processAffinityGroup(AffinityGroupVMMapVO vmGroupMapping, Deploym } } - protected Map getVmIdVmMap(List vmList) { - Map vmIdVmMap = new HashMap<>(); - for (VirtualMachine vm : vmList) { - vmIdVmMap.put(vm.getId(), vm); - } - return vmIdVmMap; - } - protected void processVmInAffinityGroup(DeploymentPlan plan, VMInstanceVO groupVM) { if (groupVM.getHostId() != null) { Integer priority = adjustHostPriority(plan, groupVM.getHostId()); diff --git a/server/src/main/java/org/apache/cloudstack/cluster/ClusterDrsServiceImpl.java b/server/src/main/java/org/apache/cloudstack/cluster/ClusterDrsServiceImpl.java index cbe2cfcd160a..e495e048a815 100644 --- a/server/src/main/java/org/apache/cloudstack/cluster/ClusterDrsServiceImpl.java +++ b/server/src/main/java/org/apache/cloudstack/cluster/ClusterDrsServiceImpl.java @@ -795,13 +795,16 @@ void processPlans() { * the DRS plan to be executed */ /** - * Checks a planned migration against the affinity rules as they stand now. + * Checks a planned migration against the placement rules as they stand now. * * A plan is generated once and executed later, so state can have moved on: VMs may have been * created, migrated or destroyed in between. Anti-affinity in particular is only meaningful * against current placements, and nothing downstream re-checks it - migrateVirtualMachine does * not enforce affinity groups. * + * The processors also cover dedicated resources and DPDK, so a refusal is not necessarily about + * an affinity group. + * * @param vm * the VM the plan wants to move * @param destHost @@ -863,10 +866,16 @@ void executeDrsPlan(ClusterDrsPlanVO plan) { } if (destinationViolatesAffinity(vm, host, dispatched, dispatchedSourceHosts)) { - logger.warn("Skipping DRS migration of vm {} to host {}: it no longer satisfies the affinity " + - "rules for that VM. The plan was generated against older state.", vm, host); - migration.setStatus(JobInfo.Status.FAILED); + String reason = String.format("Skipped DRS migration of %s to %s: the destination no longer " + + "satisfies the placement rules for that VM. The plan was generated against older " + + "state.", vm, host); + logger.warn(reason); + // cancelled rather than failed: nothing went wrong, the plan went out of date + migration.setStatus(JobInfo.Status.CANCELLED); drsPlanMigrationDao.update(migration.getId(), migration); + ActionEventUtils.onCompletedActionEvent(User.UID_SYSTEM, Account.ACCOUNT_ID_SYSTEM, + EventVO.LEVEL_WARN, EventTypes.EVENT_CLUSTER_DRS, false, reason, + plan.getClusterId(), ApiCommandResourceType.Cluster.toString(), plan.getEventId()); continue; } diff --git a/server/src/test/java/org/apache/cloudstack/cluster/ClusterDrsServiceImplTest.java b/server/src/test/java/org/apache/cloudstack/cluster/ClusterDrsServiceImplTest.java index 4557df32d87e..6c43a4ae7951 100644 --- a/server/src/test/java/org/apache/cloudstack/cluster/ClusterDrsServiceImplTest.java +++ b/server/src/test/java/org/apache/cloudstack/cluster/ClusterDrsServiceImplTest.java @@ -1110,7 +1110,7 @@ public void testExecuteDrsPlanSkipsMigrationThatViolatesAffinity() { clusterDrsService.executeDrsPlan(plan); - Mockito.verify(migration).setStatus(JobInfo.Status.FAILED); + Mockito.verify(migration).setStatus(JobInfo.Status.CANCELLED); Mockito.verify(clusterDrsService, Mockito.never()) .createMigrateVMAsyncJob(Mockito.any(), Mockito.any(), Mockito.anyLong()); } From 5375b3b2017563935a5ca604f574c1cdbda183dc Mon Sep 17 00:00:00 2001 From: Brad House Date: Wed, 9 Sep 2026 22:52:29 +0000 Subject: [PATCH 12/18] Stop DRS planning from redoing the same work Plan generation took minutes on a busy cluster. Two causes, both about repeating work rather than about the algorithm. Working out where a VM could go runs the host allocators and inspects every volume, and it was done once per VM. VMs that would get the same answer - same offering, template, current host, storage and affinity groups - now share one pass. A cluster of similar VMs goes from one pass per VM to one per handful. Affinity was re-evaluated for every VM in the cluster on every iteration, up to drs.max.migrations times. Only VMs sharing a group with the one just moved can have changed, so only those are re-evaluated after the first pass. Also: a VM whose grouping key cannot be worked out is now considered on its own rather than silently dropped from the plan. Signed-off-by: Brad House --- .../cluster/ClusterDrsServiceImpl.java | 98 ++++++++++++-- .../cluster/ClusterDrsEquivalenceTest.java | 121 ++++++++++++++++++ .../cluster/ClusterDrsServiceImplTest.java | 4 + 3 files changed, 214 insertions(+), 9 deletions(-) create mode 100644 server/src/test/java/org/apache/cloudstack/cluster/ClusterDrsEquivalenceTest.java diff --git a/server/src/main/java/org/apache/cloudstack/cluster/ClusterDrsServiceImpl.java b/server/src/main/java/org/apache/cloudstack/cluster/ClusterDrsServiceImpl.java index e495e048a815..ebc3364f5111 100644 --- a/server/src/main/java/org/apache/cloudstack/cluster/ClusterDrsServiceImpl.java +++ b/server/src/main/java/org/apache/cloudstack/cluster/ClusterDrsServiceImpl.java @@ -44,6 +44,8 @@ import com.cloud.user.User; import com.cloud.utils.DateUtil; import com.cloud.utils.Pair; +import com.cloud.storage.VolumeVO; +import com.cloud.storage.dao.VolumeDao; import com.cloud.utils.Ternary; import com.cloud.utils.component.ComponentContext; import com.cloud.utils.component.ManagerBase; @@ -94,6 +96,7 @@ import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Set; import java.util.Timer; import java.util.TimerTask; @@ -146,6 +149,9 @@ public class ClusterDrsServiceImpl extends ManagerBase implements ClusterDrsServ @Inject AffinityGroupVMMapDao affinityGroupVMMapDao; + @Inject + VolumeDao volumeDao; + List drsAlgorithms = new ArrayList<>(); Map drsAlgorithmMap = new HashMap<>(); @@ -393,13 +399,22 @@ private List> getMigrationPlans( ClusterDrsAlgorithm algorithm = getDrsAlgorithm(ClusterDrsAlgorithm.valueIn(cluster.getId())); int iteration = 0; List> migrationPlan = new ArrayList<>(); + Map vmToExcludesMap = null; + Set staleAffinityVmIds = new HashSet<>(); while (iteration < maxIterations && algorithm.needsDrs(cluster, new ArrayList<>(hostCpuMap.values()), new ArrayList<>(hostMemoryMap.values()))) { logger.debug("Starting DRS iteration {} for cluster {}", iteration + 1, cluster); - // Re-evaluate affinity constraints with current (simulated) VM placements - Map vmToExcludesMap = getVmToExcludesMap(vmList, hostMap, vmsWithAffinityGroups, - vmToCompatibleHostsCache, vmIdServiceOfferingMap); + // Affinity only changes for VMs that share a group with the one just moved, so after + // the first pass only those are re-evaluated rather than every VM in the cluster + if (vmToExcludesMap == null) { + vmToExcludesMap = getVmToExcludesMap(vmList, hostMap, vmsWithAffinityGroups, + vmToCompatibleHostsCache, vmIdServiceOfferingMap, null); + } else if (!staleAffinityVmIds.isEmpty()) { + vmToExcludesMap.putAll(getVmToExcludesMap(vmList, hostMap, vmsWithAffinityGroups, + vmToCompatibleHostsCache, vmIdServiceOfferingMap, staleAffinityVmIds)); + staleAffinityVmIds = new HashSet<>(); + } logger.debug("Completed affinity evaluation for DRS iteration {} for cluster {}", iteration + 1, cluster); @@ -429,6 +444,7 @@ private List> getMigrationPlans( hostMemoryMap.get(vm.getHostId()).first(hostMemoryMap.get(vm.getHostId()).first() - vmMemory); hostMemoryMap.get(destHost.getId()).first(hostMemoryMap.get(destHost.getId()).first() + vmMemory); vm.setHostId(destHost.getId()); + staleAffinityVmIds = affinityPeersOf(vm, vmsWithAffinityGroups); iteration++; } return migrationPlan; @@ -457,11 +473,34 @@ protected void excludeHostsDispreferredByAffinity(DeploymentPlan plan, ExcludeLi } } + /** + * The VMs whose affinity picture changes when the given VM moves: the VM itself, and everything + * sharing an affinity group with it. + */ + protected Set affinityPeersOf(VirtualMachine vm, Set vmsWithAffinityGroups) { + Set peers = new HashSet<>(); + peers.add(vm.getId()); + if (!vmsWithAffinityGroups.contains(vm.getId())) { + return peers; + } + for (AffinityGroupVMMapVO mapping : affinityGroupVMMapDao.listByInstanceId(vm.getId())) { + peers.addAll(affinityGroupVMMapDao.listVmIdsByAffinityGroup(mapping.getAffinityGroupId())); + } + return peers; + } + + /** + * @param onlyVmIds + * when set, re-evaluates just these VMs rather than the whole cluster + */ private Map getVmToExcludesMap(List vmList, Map hostMap, Set vmsWithAffinityGroups, Map> vmToCompatibleHostsCache, - Map vmIdServiceOfferingMap) { + Map vmIdServiceOfferingMap, Set onlyVmIds) { Map vmToExcludesMap = new HashMap<>(); for (VirtualMachine vm : vmList) { + if (onlyVmIds != null && !onlyVmIds.contains(vm.getId())) { + continue; + } if (vmToCompatibleHostsCache.containsKey(vm.getId())) { Host srcHost = hostMap.get(vm.getHostId()); if (srcHost != null) { @@ -511,20 +550,39 @@ private Pair>, Map>> get .map(VMInstanceDetailVO::getResourceId) .collect(Collectors.toSet()); + // Working out where a VM could go is the expensive part of planning: it runs the host + // allocators and inspects every volume. VMs that would get the same answer are grouped and + // the work is done once for the group. In a cluster of similar VMs this is the difference + // between one pass per VM and one pass per handful. + Map, Integer>, List, Map>> byEquivalence = + new HashMap<>(); + int computed = 0; + for (VirtualMachine vm : vmList) { - // Skip ineligible VMs if (shouldSkipVMForDRS(vm, skipDrsVmIds)) { logger.debug("Skipping VM {} for DRS as it is ineligible.", vm); continue; } + // a VM whose key cannot be worked out is treated as one of a kind rather than dropped + String key; + try { + key = migrationEquivalenceKey(vm); + } catch (Exception e) { + logger.debug("Could not group VM {} with others, considering it on its own", vm, e); + key = "vm-" + vm.getId(); + } + try { - // Use listHostsForMigrationOfVM to get suitable hosts (validated by getCapableSuitableHosts) - // This ensures the same validation as the "find host for migration" command Ternary, Integer>, List, Map> hostsForMigration = - managementServer.listHostsForMigrationOfVM(vm, 0L, 500L, null, vmList); + byEquivalence.get(key); + if (hostsForMigration == null) { + hostsForMigration = managementServer.listHostsForMigrationOfVM(vm, 0L, 500L, null, vmList); + byEquivalence.put(key, hostsForMigration); + computed++; + } - List suitableHosts = hostsForMigration.second(); // Get suitable hosts (validated by HostAllocator) + List suitableHosts = hostsForMigration.second(); Map requiresStorageMotion = hostsForMigration.third(); if (suitableHosts != null && !suitableHosts.isEmpty()) { @@ -535,9 +593,31 @@ private Pair>, Map>> get logger.debug("Could not get suitable hosts for VM {}: {}", vm, e.getMessage()); } } + logger.debug("Worked out candidate hosts for {} VMs in {} passes", vmToCompatibleHostsCache.size(), computed); return new Pair<>(vmToCompatibleHostsCache, vmToStorageMotionCache); } + /** + * Identifies VMs that would get the same answer from listHostsForMigrationOfVM. + * + * The answer depends on what the VM asks for (its offering and template), where it is now, the + * volumes that would have to follow it, and the affinity groups it belongs to. Two VMs matching + * on all of those are interchangeable for the purpose of finding candidate hosts. + */ + protected String migrationEquivalenceKey(VirtualMachine vm) { + List poolIds = volumeDao.findCreatedByInstance(vm.getId()).stream() + .map(VolumeVO::getPoolId) + .filter(Objects::nonNull) + .sorted() + .collect(Collectors.toList()); + List groupIds = affinityGroupVMMapDao.listByInstanceId(vm.getId()).stream() + .map(AffinityGroupVMMapVO::getAffinityGroupId) + .sorted() + .collect(Collectors.toList()); + return String.format("%s|%s|%s|%s|%s|%s", vm.getServiceOfferingId(), vm.getTemplateId(), vm.getHostId(), + vm.getHypervisorType(), poolIds, groupIds); + } + /** * Pre-fetch affinity group mappings for all eligible VMs (once, before iterations) * This allows us to skip expensive affinity processing for VMs without affinity groups diff --git a/server/src/test/java/org/apache/cloudstack/cluster/ClusterDrsEquivalenceTest.java b/server/src/test/java/org/apache/cloudstack/cluster/ClusterDrsEquivalenceTest.java new file mode 100644 index 000000000000..f2725acb9ca8 --- /dev/null +++ b/server/src/test/java/org/apache/cloudstack/cluster/ClusterDrsEquivalenceTest.java @@ -0,0 +1,121 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.cluster; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +import org.apache.cloudstack.affinity.AffinityGroupVMMapVO; +import org.apache.cloudstack.affinity.dao.AffinityGroupVMMapDao; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; + +import com.cloud.hypervisor.Hypervisor; +import com.cloud.storage.VolumeVO; +import com.cloud.storage.dao.VolumeDao; +import com.cloud.vm.VMInstanceVO; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; + +/** + * Working out where a VM could go is the expensive part of DRS planning, so VMs that would get the + * same answer share the work. These check what counts as the same answer. + */ +@RunWith(MockitoJUnitRunner.class) +public class ClusterDrsEquivalenceTest { + + @Mock + private VolumeDao volumeDao; + + @Mock + private AffinityGroupVMMapDao affinityGroupVMMapDao; + + @InjectMocks + private ClusterDrsServiceImpl service = new ClusterDrsServiceImpl(); + + private VMInstanceVO vm(long id, long offeringId, long templateId, Long hostId, Long... poolIds) { + VMInstanceVO vm = Mockito.mock(VMInstanceVO.class); + Mockito.lenient().when(vm.getId()).thenReturn(id); + Mockito.lenient().when(vm.getServiceOfferingId()).thenReturn(offeringId); + Mockito.lenient().when(vm.getTemplateId()).thenReturn(templateId); + Mockito.lenient().when(vm.getHostId()).thenReturn(hostId); + Mockito.lenient().when(vm.getHypervisorType()).thenReturn(Hypervisor.HypervisorType.KVM); + + List volumes = Arrays.stream(poolIds).map(poolId -> { + VolumeVO volume = Mockito.mock(VolumeVO.class); + Mockito.lenient().when(volume.getPoolId()).thenReturn(poolId); + return volume; + }).collect(java.util.stream.Collectors.toList()); + Mockito.lenient().when(volumeDao.findCreatedByInstance(id)).thenReturn(volumes); + Mockito.lenient().when(affinityGroupVMMapDao.listByInstanceId(id)).thenReturn(Collections.emptyList()); + return vm; + } + + @Test + public void testIdenticalVmsShareOnePass() { + assertEquals(service.migrationEquivalenceKey(vm(1L, 10L, 20L, 30L, 40L)), + service.migrationEquivalenceKey(vm(2L, 10L, 20L, 30L, 40L))); + } + + @Test + public void testVolumeOrderDoesNotMatter() { + assertEquals(service.migrationEquivalenceKey(vm(1L, 10L, 20L, 30L, 40L, 41L)), + service.migrationEquivalenceKey(vm(2L, 10L, 20L, 30L, 41L, 40L))); + } + + @Test + public void testDifferentOfferingIsADifferentAnswer() { + assertNotEquals(service.migrationEquivalenceKey(vm(1L, 10L, 20L, 30L, 40L)), + service.migrationEquivalenceKey(vm(2L, 11L, 20L, 30L, 40L))); + } + + @Test + public void testDifferentCurrentHostIsADifferentAnswer() { + assertNotEquals(service.migrationEquivalenceKey(vm(1L, 10L, 20L, 30L, 40L)), + service.migrationEquivalenceKey(vm(2L, 10L, 20L, 31L, 40L))); + } + + @Test + public void testDifferentStorageIsADifferentAnswer() { + assertNotEquals(service.migrationEquivalenceKey(vm(1L, 10L, 20L, 30L, 40L)), + service.migrationEquivalenceKey(vm(2L, 10L, 20L, 30L, 42L))); + } + + @Test + public void testDifferentTemplateIsADifferentAnswer() { + assertNotEquals(service.migrationEquivalenceKey(vm(1L, 10L, 20L, 30L, 40L)), + service.migrationEquivalenceKey(vm(2L, 10L, 21L, 30L, 40L))); + } + + @Test + public void testDifferentAffinityGroupsIsADifferentAnswer() { + VMInstanceVO grouped = vm(1L, 10L, 20L, 30L, 40L); + AffinityGroupVMMapVO mapping = Mockito.mock(AffinityGroupVMMapVO.class); + Mockito.when(mapping.getAffinityGroupId()).thenReturn(99L); + Mockito.when(affinityGroupVMMapDao.listByInstanceId(1L)).thenReturn(Collections.singletonList(mapping)); + + assertNotEquals("a VM in an affinity group cannot reuse an ungrouped VM's candidate hosts", + service.migrationEquivalenceKey(grouped), + service.migrationEquivalenceKey(vm(2L, 10L, 20L, 30L, 40L))); + } +} diff --git a/server/src/test/java/org/apache/cloudstack/cluster/ClusterDrsServiceImplTest.java b/server/src/test/java/org/apache/cloudstack/cluster/ClusterDrsServiceImplTest.java index 6c43a4ae7951..c80fcb70f33a 100644 --- a/server/src/test/java/org/apache/cloudstack/cluster/ClusterDrsServiceImplTest.java +++ b/server/src/test/java/org/apache/cloudstack/cluster/ClusterDrsServiceImplTest.java @@ -38,6 +38,7 @@ import com.cloud.org.Grouping; import com.cloud.server.ManagementServer; import com.cloud.service.ServiceOfferingVO; +import com.cloud.storage.dao.VolumeDao; import org.apache.cloudstack.affinity.AffinityGroupVMMapVO; import com.cloud.service.dao.ServiceOfferingDao; import com.cloud.utils.Pair; @@ -136,6 +137,9 @@ public class ClusterDrsServiceImplTest { @Mock private AsyncJobManager asyncJobManager; + @Mock + private VolumeDao volumeDao; + @Mock private VMInstanceDetailsDao vmInstanceDetailsDao; From 073371f1a734d6a0f507cd4b269f29f5048ffaff Mon Sep 17 00:00:00 2001 From: Brad House Date: Wed, 9 Sep 2026 23:24:12 +0000 Subject: [PATCH 13/18] Add 'weighted' DRS algorithm Opt-in per cluster via drs.algorithm. balanced and condensed are untouched and the default is unchanged. The existing algorithms balance one metric chosen by drs.metric, so whichever is not chosen goes unwatched: a cluster can sit inside its imbalance threshold on memory while CPU load varies several fold, and DRS correctly does nothing. Allocation is also a poor stand-in for load under overprovisioning, where a saturated host still reports a small percentage allocated. 'weighted' blends four figures per host - CPU and memory allocated, CPU and memory in use - and balances the result. Imbalance keeps its existing definition, standard deviation over the mean, so drs.imbalance still means what it did. - allocation is measured against what a host can hand out, so the overcommit ratio is applied - a migration is only worth making if the cluster ends up more even, and one that has to move storage costs more - hosts without utilisation samples fall back to allocation figures needsDrs decides whether to look at a cluster at all, and it was handed lists with no host identity, so measured load could not be attributed to a host. Added a map form alongside it; algorithms that only need the values keep the default and are unaffected. Signed-off-by: Brad House --- PendingReleaseNotes | 18 ++ .../main/java/com/cloud/host}/HostLoad.java | 2 +- .../java/com/cloud/host/HostLoadService.java | 32 +++ .../cluster/ClusterDrsAlgorithm.java | 17 ++ .../cloudstack/cluster/ClusterDrsService.java | 4 +- client/pom.xml | 5 + plugins/drs/cluster/weighted/pom.xml | 33 +++ .../apache/cloudstack/cluster/Weighted.java | 245 ++++++++++++++++++ .../weighted/spring-weighted-context.xml | 33 +++ .../cloudstack/cluster/WeightedTest.java | 192 ++++++++++++++ plugins/pom.xml | 1 + .../allocator/impl/HostLoadTracker.java | 5 +- .../allocator/impl/WeightedHostScorer.java | 1 + .../cluster/ClusterDrsServiceImpl.java | 3 +- .../allocator/impl/HostLoadTrackerTest.java | 2 + .../impl/WeightedHostScorerRankTest.java | 1 + .../impl/WeightedHostScorerTest.java | 1 + .../WeightedPlacementDistributionTest.java | 2 + .../cluster/ClusterDrsServiceImplTest.java | 24 +- 19 files changed, 603 insertions(+), 18 deletions(-) rename {server/src/main/java/com/cloud/agent/manager/allocator/impl => api/src/main/java/com/cloud/host}/HostLoad.java (97%) create mode 100644 api/src/main/java/com/cloud/host/HostLoadService.java create mode 100644 plugins/drs/cluster/weighted/pom.xml create mode 100644 plugins/drs/cluster/weighted/src/main/java/org/apache/cloudstack/cluster/Weighted.java create mode 100644 plugins/drs/cluster/weighted/src/main/resources/META-INF/cloudstack/weighted/spring-weighted-context.xml create mode 100644 plugins/drs/cluster/weighted/src/test/java/org/apache/cloudstack/cluster/WeightedTest.java diff --git a/PendingReleaseNotes b/PendingReleaseNotes index 6156cc3ae1b1..7bf6337376b0 100644 --- a/PendingReleaseNotes +++ b/PendingReleaseNotes @@ -72,3 +72,21 @@ example.ver.1 > example.ver.2: utilisation comes from a moving average configured with host.load.sample.interval and host.load.half.life. Until a management server has collected samples, ranking falls back to allocation figures alone. + + * New DRS algorithm 'weighted' for drs.algorithm, alongside 'balanced' and 'condensed'. The + default is unchanged; the new one is opt-in per cluster. + + The existing algorithms balance a single metric chosen by drs.metric, so choosing one leaves + the other unwatched: a cluster can be even on memory while its CPU load varies several fold and + nothing moves. Allocation is also a poor stand-in for load under overprovisioning, where a + saturated host can still report a small percentage allocated. + + 'weighted' blends CPU and memory allocated with CPU and memory in use, and balances the result. + Imbalance keeps its existing meaning - standard deviation over the mean - so drs.imbalance + still means what it did. Tuned with the drs.weighted.* settings, all cluster scoped. Where no + utilisation samples are available it falls back to allocation figures. + + * DRS plan generation is considerably faster on large clusters. Working out where a VM could go + is now done once for each group of VMs that would get the same answer rather than once per VM, + and affinity is re-evaluated only for VMs affected by the migration just planned rather than + for every VM on every iteration. diff --git a/server/src/main/java/com/cloud/agent/manager/allocator/impl/HostLoad.java b/api/src/main/java/com/cloud/host/HostLoad.java similarity index 97% rename from server/src/main/java/com/cloud/agent/manager/allocator/impl/HostLoad.java rename to api/src/main/java/com/cloud/host/HostLoad.java index 2e3d14ba8384..a2257c51bccb 100644 --- a/server/src/main/java/com/cloud/agent/manager/allocator/impl/HostLoad.java +++ b/api/src/main/java/com/cloud/host/HostLoad.java @@ -14,7 +14,7 @@ // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. -package com.cloud.agent.manager.allocator.impl; +package com.cloud.host; /** * Smoothed view of what a host is actually doing, as opposed to what has been allocated on it. diff --git a/api/src/main/java/com/cloud/host/HostLoadService.java b/api/src/main/java/com/cloud/host/HostLoadService.java new file mode 100644 index 000000000000..d8e9e88cb27a --- /dev/null +++ b/api/src/main/java/com/cloud/host/HostLoadService.java @@ -0,0 +1,32 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.host; + +/** + * A smoothed view of what each host is actually doing, as opposed to what has been allocated on it. + * + * Allocation figures say what was promised; under overprovisioning they can be far from what a host + * is really carrying. Placement and rebalancing both need the second view. + */ +public interface HostLoadService { + + /** + * @return the host's smoothed load, or a value reporting itself unusable when the host has not + * been sampled recently enough to rank on + */ + HostLoad getLoad(long hostId); +} diff --git a/api/src/main/java/org/apache/cloudstack/cluster/ClusterDrsAlgorithm.java b/api/src/main/java/org/apache/cloudstack/cluster/ClusterDrsAlgorithm.java index 368487c2b9b0..722247e6fc51 100644 --- a/api/src/main/java/org/apache/cloudstack/cluster/ClusterDrsAlgorithm.java +++ b/api/src/main/java/org/apache/cloudstack/cluster/ClusterDrsAlgorithm.java @@ -62,6 +62,23 @@ public interface ClusterDrsAlgorithm extends Adapter { boolean needsDrs(Cluster cluster, List> cpuList, List> memoryList) throws ConfigurationException; + /** + * As above, but keeping host identity. + * + * An algorithm that considers anything beyond the figures in the two maps - measured load, for + * instance - cannot attribute it without knowing which host each entry belongs to. Algorithms + * that only need the values keep the default. + * + * @param hostCpuMap + * host id to a Ternary of used, reserved and total CPU + * @param hostMemoryMap + * host id to a Ternary of used, reserved and total memory + */ + default boolean needsDrs(Cluster cluster, Map> hostCpuMap, + Map> hostMemoryMap) throws ConfigurationException { + return needsDrs(cluster, new ArrayList<>(hostCpuMap.values()), new ArrayList<>(hostMemoryMap.values())); + } + /** * Calculates the metrics (improvement, cost, benefit) for migrating a VM to a destination host. Improvement is * calculated based on the change in cluster imbalance before and after the migration. diff --git a/api/src/main/java/org/apache/cloudstack/cluster/ClusterDrsService.java b/api/src/main/java/org/apache/cloudstack/cluster/ClusterDrsService.java index ba6a6464fc20..7323f7fb4916 100644 --- a/api/src/main/java/org/apache/cloudstack/cluster/ClusterDrsService.java +++ b/api/src/main/java/org/apache/cloudstack/cluster/ClusterDrsService.java @@ -51,9 +51,9 @@ public interface ClusterDrsService extends Manager, Configurable, Scheduler { true, ConfigKey.Scope.Cluster, null, "Maximum number of migrations for DRS", null, null, null); ConfigKey ClusterDrsAlgorithm = new ConfigKey<>(String.class, "drs.algorithm", - ConfigKey.CATEGORY_ADVANCED, "balanced", "The DRS algorithm to be executed on the cluster. Possible values are condensed, balanced.", + ConfigKey.CATEGORY_ADVANCED, "balanced", "The DRS algorithm to be executed on the cluster. Possible values are condensed, balanced, weighted. 'weighted' balances CPU and memory together, and on measured load as well as allocation, tuned with the drs.weighted.* settings.", true, ConfigKey.Scope.Cluster, null, "DRS algorithm", null, null, - null, ConfigKey.Kind.Select, "condensed,balanced"); + null, ConfigKey.Kind.Select, "condensed,balanced,weighted"); ConfigKey ClusterDrsImbalanceThreshold = new ConfigKey<>(Float.class, "drs.imbalance", ConfigKey.CATEGORY_ADVANCED, "0.4", diff --git a/client/pom.xml b/client/pom.xml index cc031a4912b1..ebbf812823ca 100644 --- a/client/pom.xml +++ b/client/pom.xml @@ -592,6 +592,11 @@ cloud-plugin-cluster-drs-condensed ${project.version} + + org.apache.cloudstack + cloud-plugin-cluster-drs-weighted + ${project.version} + org.apache.cloudstack cloud-plugin-database-quota diff --git a/plugins/drs/cluster/weighted/pom.xml b/plugins/drs/cluster/weighted/pom.xml new file mode 100644 index 000000000000..4c0fa3e6a902 --- /dev/null +++ b/plugins/drs/cluster/weighted/pom.xml @@ -0,0 +1,33 @@ + + + + + 4.0.0 + Apache CloudStack Plugin - Cluster DRS Algorithm - Weighted + cloud-plugin-cluster-drs-weighted + + org.apache.cloudstack + cloudstack-plugins + 24.0.0-SNAPSHOT + ../../../pom.xml + + diff --git a/plugins/drs/cluster/weighted/src/main/java/org/apache/cloudstack/cluster/Weighted.java b/plugins/drs/cluster/weighted/src/main/java/org/apache/cloudstack/cluster/Weighted.java new file mode 100644 index 000000000000..f136bfa1ad9d --- /dev/null +++ b/plugins/drs/cluster/weighted/src/main/java/org/apache/cloudstack/cluster/Weighted.java @@ -0,0 +1,245 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.cluster; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import javax.inject.Inject; +import javax.naming.ConfigurationException; + +import org.apache.cloudstack.framework.config.ConfigKey; +import org.apache.cloudstack.framework.config.Configurable; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import com.cloud.dc.ClusterDetailsDao; +import com.cloud.dc.ClusterDetailsVO; +import com.cloud.host.Host; +import com.cloud.host.HostLoad; +import com.cloud.host.HostScoringWeights; +import com.cloud.host.HostLoadService; +import com.cloud.offering.ServiceOffering; +import com.cloud.org.Cluster; +import com.cloud.utils.Ternary; +import com.cloud.utils.component.AdapterBase; +import com.cloud.vm.VirtualMachine; +import com.cloud.vm.VmDetailConstants; + +/** + * Balances a cluster on CPU and memory together, and on what hosts are really doing rather than + * only on what has been allocated to them. + * + * The existing algorithms balance a single metric chosen by drs.metric. Choosing one leaves the + * other unwatched: a cluster can be even on memory while its CPU load varies several fold, and + * nothing moves. Allocation is also a poor stand-in for load under overprovisioning, where a + * saturated host can still report a small percentage allocated. + * + * This blends four figures per host - CPU and memory allocated, CPU and memory in use - and + * balances the result. The weights are the same host.weighted.* settings initial placement uses, on + * purpose: if the two weighted them differently they would disagree about which host is the better + * one, and rebalancing could move VMs off hosts that placement had just chosen. Imbalance keeps the same meaning as the other algorithms: the standard + * deviation of the per-host figure over its mean, so drs.imbalance still means what it did. + */ +public class Weighted extends AdapterBase implements ClusterDrsAlgorithm, Configurable { + + private static final Logger LOGGER = LogManager.getLogger(Weighted.class); + + @Inject + private HostLoadService hostLoadService; + + @Inject + private ClusterDetailsDao clusterDetailsDao; + + @Override + public String getName() { + return "weighted"; + } + + @Override + public boolean needsDrs(Cluster cluster, List> cpuList, + List> memoryList) throws ConfigurationException { + // without host identity, measured load cannot be attributed; the map form is what DRS calls + Map> cpuMap = new HashMap<>(); + Map> memoryMap = new HashMap<>(); + for (int i = 0; i < cpuList.size() && i < memoryList.size(); i++) { + cpuMap.put((long) -(i + 1), cpuList.get(i)); + memoryMap.put((long) -(i + 1), memoryList.get(i)); + } + return needsDrs(cluster, cpuMap, memoryMap); + } + + @Override + public boolean needsDrs(Cluster cluster, Map> hostCpuMap, + Map> hostMemoryMap) throws ConfigurationException { + double threshold = 1.0 - ClusterDrsService.ClusterDrsImbalanceThreshold.valueIn(cluster.getId()); + double imbalance = imbalanceOf(blendByHost(cluster, hostCpuMap, hostMemoryMap).values()); + boolean needed = imbalance > threshold; + LOGGER.debug("Cluster {} {} DRS. Imbalance: {} Threshold: {} Algorithm: {}", + cluster, needed ? "needs" : "does not need", imbalance, threshold, getName()); + return needed; + } + + @Override + public Ternary getMetrics(Cluster cluster, VirtualMachine vm, + ServiceOffering serviceOffering, Host destHost, + Map> hostCpuMap, Map> hostMemoryMap, + Boolean requiresStorageMotion, Double preImbalance, + double[] baseMetricsArray, Map hostIdToIndexMap) throws ConfigurationException { + + double before = imbalanceOf(blendByHost(cluster, hostCpuMap, hostMemoryMap).values()); + + long vmCpu = (long) serviceOffering.getCpu() * serviceOffering.getSpeed(); + long vmMemory = serviceOffering.getRamSize() * 1024L * 1024L; + Map> cpuAfter = withVmMoved(hostCpuMap, vm.getHostId(), destHost.getId(), vmCpu); + Map> memoryAfter = withVmMoved(hostMemoryMap, vm.getHostId(), destHost.getId(), vmMemory); + + double after = imbalanceOf(blendByHost(cluster, cpuAfter, memoryAfter).values()); + + double improvement = before - after; + // moving a VM costs something and buys nothing unless the cluster ends up more even, so a + // migration is only worth making when it measurably helps + double cost = Boolean.TRUE.equals(requiresStorageMotion) ? 1.0 : 0.0; + double benefit = improvement > 0 ? 1.0 + improvement : 0.0; + + LOGGER.trace("Cluster {} imbalance {} -> {} moving {} to {}", cluster, before, after, vm, destHost); + return new Ternary<>(improvement, cost, benefit); + } + + /** + * One figure per host, blending what is allocated with what is in use. + */ + protected Map blendByHost(Cluster cluster, Map> hostCpuMap, + Map> hostMemoryMap) { + float cpuOvercommit = overcommitRatio(cluster.getId(), VmDetailConstants.CPU_OVER_COMMIT_RATIO); + float memoryOvercommit = overcommitRatio(cluster.getId(), VmDetailConstants.MEMORY_OVER_COMMIT_RATIO); + + double cpuAllocatedWeight = weight(HostScoringWeights.CpuAllocatedWeight, cluster.getId()); + double memoryAllocatedWeight = weight(HostScoringWeights.MemoryAllocatedWeight, cluster.getId()); + double cpuUsedWeight = weight(HostScoringWeights.CpuUsedWeight, cluster.getId()); + double memoryUsedWeight = weight(HostScoringWeights.MemoryUsedWeight, cluster.getId()); + + Map blended = new HashMap<>(); + for (Map.Entry> entry : hostCpuMap.entrySet()) { + long hostId = entry.getKey(); + Ternary memory = hostMemoryMap.get(hostId); + if (memory == null) { + continue; + } + double cpuAllocated = fractionOf(entry.getValue(), cpuOvercommit); + double memoryAllocated = fractionOf(memory, memoryOvercommit); + + HostLoad load = hostLoadService == null ? HostLoad.UNKNOWN : hostLoadService.getLoad(hostId); + double usedCpuWeight = load.isUsable() ? cpuUsedWeight : 0; + double usedMemoryWeight = load.isUsable() ? memoryUsedWeight : 0; + + double sum = cpuAllocatedWeight + memoryAllocatedWeight + usedCpuWeight + usedMemoryWeight; + if (sum <= 0) { + blended.put(hostId, 0.0); + continue; + } + blended.put(hostId, (cpuAllocatedWeight * cpuAllocated + + memoryAllocatedWeight * memoryAllocated + + usedCpuWeight * load.getCpuUtilisation() + + usedMemoryWeight * load.getMemoryUtilisation()) / sum); + } + return blended; + } + + private Map> withVmMoved(Map> original, + Long sourceHostId, long destHostId, long amount) { + Map> copy = new HashMap<>(); + for (Map.Entry> entry : original.entrySet()) { + Ternary value = entry.getValue(); + long used = value.first(); + if (entry.getKey().equals(sourceHostId)) { + used -= amount; + } else if (entry.getKey() == destHostId) { + used += amount; + } + copy.put(entry.getKey(), new Ternary<>(used, value.second(), value.third())); + } + return copy; + } + + /** + * Used over what the host can hand out, which is its real total scaled by the overcommit ratio. + */ + private double fractionOf(Ternary capacity, float overcommit) { + double allocatable = (capacity.third() - capacity.second()) * (double) overcommit; + if (allocatable <= 0) { + return 0; + } + return clamp(capacity.first() / allocatable); + } + + protected float overcommitRatio(long clusterId, String key) { + ClusterDetailsVO detail = clusterDetailsDao.findDetail(clusterId, key); + if (detail == null || detail.getValue() == null) { + return 1f; + } + try { + float ratio = Float.parseFloat(detail.getValue()); + return ratio > 0 ? ratio : 1f; + } catch (NumberFormatException e) { + return 1f; + } + } + + private double weight(ConfigKey key, long clusterId) { + Double value = key.valueIn(clusterId); + if (value == null || value < 0) { + return 0; + } + return value; + } + + /** + * Standard deviation over the mean, the same definition the other algorithms use, so that + * drs.imbalance keeps its meaning. + */ + protected double imbalanceOf(java.util.Collection values) { + if (values == null || values.isEmpty()) { + return 0; + } + double[] array = values.stream().mapToDouble(Double::doubleValue).toArray(); + double mean = MEAN_CALCULATOR.evaluate(array); + if (mean == 0) { + return 0; + } + return STDDEV_CALCULATOR.evaluate(array, mean) / mean; + } + + private static double clamp(double value) { + if (Double.isNaN(value) || value < 0) { + return 0; + } + return Math.min(value, 1); + } + + @Override + public String getConfigComponentName() { + return Weighted.class.getSimpleName(); + } + + @Override + public ConfigKey[] getConfigKeys() { + // the four host.weighted.* weights are shared with initial placement, which registers them + return new ConfigKey[] {StorageMotionCost}; + } +} diff --git a/plugins/drs/cluster/weighted/src/main/resources/META-INF/cloudstack/weighted/spring-weighted-context.xml b/plugins/drs/cluster/weighted/src/main/resources/META-INF/cloudstack/weighted/spring-weighted-context.xml new file mode 100644 index 000000000000..903ae08087a1 --- /dev/null +++ b/plugins/drs/cluster/weighted/src/main/resources/META-INF/cloudstack/weighted/spring-weighted-context.xml @@ -0,0 +1,33 @@ + + + + + + + diff --git a/plugins/drs/cluster/weighted/src/test/java/org/apache/cloudstack/cluster/WeightedTest.java b/plugins/drs/cluster/weighted/src/test/java/org/apache/cloudstack/cluster/WeightedTest.java new file mode 100644 index 000000000000..f3c0fafbbec5 --- /dev/null +++ b/plugins/drs/cluster/weighted/src/test/java/org/apache/cloudstack/cluster/WeightedTest.java @@ -0,0 +1,192 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.cluster; + +import java.util.HashMap; +import java.util.Map; + +import javax.naming.ConfigurationException; + +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; + +import com.cloud.dc.ClusterDetailsDao; +import com.cloud.dc.ClusterDetailsVO; +import com.cloud.host.Host; +import com.cloud.host.HostLoad; +import com.cloud.host.HostLoadService; +import com.cloud.offering.ServiceOffering; +import com.cloud.org.Cluster; +import com.cloud.utils.Ternary; +import com.cloud.vm.VirtualMachine; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +@RunWith(MockitoJUnitRunner.class) +public class WeightedTest { + + private static final long CLUSTER_ID = 1L; + private static final long CPU_TOTAL = 192L * 2400L; + private static final long MEMORY_TOTAL = 1_132_000L * 1024L * 1024L; + private static final int CPU_OVERCOMMIT = 10; + private static final int MEMORY_OVERCOMMIT = 4; + + @Mock + private HostLoadService hostLoadService; + + @Mock + private ClusterDetailsDao clusterDetailsDao; + + @InjectMocks + private Weighted weighted = new Weighted(); + + private Cluster cluster; + private final Map> cpu = new HashMap<>(); + private final Map> memory = new HashMap<>(); + + @Before + public void setUp() { + cluster = Mockito.mock(Cluster.class); + Mockito.lenient().when(cluster.getId()).thenReturn(CLUSTER_ID); + Mockito.lenient().when(clusterDetailsDao.findDetail(CLUSTER_ID, "cpuOvercommitRatio")) + .thenReturn(new ClusterDetailsVO(CLUSTER_ID, "cpuOvercommitRatio", String.valueOf(CPU_OVERCOMMIT))); + Mockito.lenient().when(clusterDetailsDao.findDetail(CLUSTER_ID, "memoryOvercommitRatio")) + .thenReturn(new ClusterDetailsVO(CLUSTER_ID, "memoryOvercommitRatio", String.valueOf(MEMORY_OVERCOMMIT))); + } + + private void host(long id, double cpuAllocatedFraction, double memoryAllocatedFraction, HostLoad load) { + cpu.put(id, new Ternary<>((long) (CPU_TOTAL * CPU_OVERCOMMIT * cpuAllocatedFraction), 0L, CPU_TOTAL)); + memory.put(id, new Ternary<>((long) (MEMORY_TOTAL * MEMORY_OVERCOMMIT * memoryAllocatedFraction), 0L, MEMORY_TOTAL)); + Mockito.lenient().when(hostLoadService.getLoad(id)).thenReturn(load); + } + + @Test + public void testAnEvenClusterHasNoImbalance() { + host(1L, 0.4, 0.4, new HostLoad(0.4, 0.4, 10)); + host(2L, 0.4, 0.4, new HostLoad(0.4, 0.4, 10)); + + assertEquals(0.0, weighted.imbalanceOf(weighted.blendByHost(cluster, cpu, memory).values()), 1e-9); + } + + @Test + public void testCpuLoadImbalanceIsSeenWhenMemoryIsEven() { + // the case a memory-only metric misses entirely: memory even, CPU load three fold apart + host(1L, 0.30, 0.40, new HostLoad(0.90, 0.40, 10)); + host(2L, 0.30, 0.40, new HostLoad(0.30, 0.40, 10)); + + assertTrue("balancing on memory alone would call this cluster even", + weighted.imbalanceOf(weighted.blendByHost(cluster, cpu, memory).values()) > 0.15); + } + + @Test + public void testMemoryImbalanceIsSeenWhenCpuIsEven() { + host(1L, 0.30, 0.90, new HostLoad(0.30, 0.90, 10)); + host(2L, 0.30, 0.10, new HostLoad(0.30, 0.10, 10)); + + assertTrue(weighted.imbalanceOf(weighted.blendByHost(cluster, cpu, memory).values()) > 0.15); + } + + @Test + public void testAllocationBeyondPhysicalSizeStillDiscriminates() { + // at a factor of 10 both hosts are past their physical CPU; they must not both read as full + host(1L, 0.20, 0.20, HostLoad.UNKNOWN); + host(2L, 0.80, 0.20, HostLoad.UNKNOWN); + + Map blended = weighted.blendByHost(cluster, cpu, memory); + assertTrue(blended.get(2L) > blended.get(1L)); + } + + @Test + public void testUnmeasuredHostsFallBackToAllocation() { + host(1L, 0.20, 0.20, HostLoad.UNKNOWN); + host(2L, 0.60, 0.60, HostLoad.UNKNOWN); + + Map blended = weighted.blendByHost(cluster, cpu, memory); + assertEquals(0.20, blended.get(1L), 0.01); + assertEquals(0.60, blended.get(2L), 0.01); + } + + @Test + public void testMovingAVmOffTheBusyHostIsAnImprovement() throws ConfigurationException { + host(1L, 0.60, 0.60, new HostLoad(0.80, 0.60, 10)); + host(2L, 0.20, 0.20, new HostLoad(0.20, 0.20, 10)); + + VirtualMachine vm = Mockito.mock(VirtualMachine.class); + Mockito.when(vm.getHostId()).thenReturn(1L); + Host dest = Mockito.mock(Host.class); + Mockito.when(dest.getId()).thenReturn(2L); + ServiceOffering offering = Mockito.mock(ServiceOffering.class); + Mockito.when(offering.getCpu()).thenReturn(8); + Mockito.when(offering.getSpeed()).thenReturn(2400); + Mockito.when(offering.getRamSize()).thenReturn(32768); + + Ternary metrics = weighted.getMetrics(cluster, vm, offering, dest, + cpu, memory, false, null, new double[0], new HashMap<>()); + + assertTrue("moving away from the busier host should even the cluster out", metrics.first() > 0); + assertTrue("and should be considered worth doing", metrics.third() > metrics.second()); + } + + @Test + public void testMovingAVmOntoTheBusyHostIsNotAnImprovement() throws ConfigurationException { + host(1L, 0.60, 0.60, new HostLoad(0.80, 0.60, 10)); + host(2L, 0.20, 0.20, new HostLoad(0.20, 0.20, 10)); + + VirtualMachine vm = Mockito.mock(VirtualMachine.class); + Mockito.when(vm.getHostId()).thenReturn(2L); + Host dest = Mockito.mock(Host.class); + Mockito.when(dest.getId()).thenReturn(1L); + ServiceOffering offering = Mockito.mock(ServiceOffering.class); + Mockito.when(offering.getCpu()).thenReturn(8); + Mockito.when(offering.getSpeed()).thenReturn(2400); + Mockito.when(offering.getRamSize()).thenReturn(32768); + + Ternary metrics = weighted.getMetrics(cluster, vm, offering, dest, + cpu, memory, false, null, new double[0], new HashMap<>()); + + assertTrue("piling onto the busier host makes the cluster less even", metrics.first() < 0); + assertEquals("and must not be considered worth doing", 0.0, metrics.third(), 1e-9); + } + + @Test + public void testStorageMotionCostsMore() throws ConfigurationException { + host(1L, 0.60, 0.60, new HostLoad(0.80, 0.60, 10)); + host(2L, 0.20, 0.20, new HostLoad(0.20, 0.20, 10)); + + VirtualMachine vm = Mockito.mock(VirtualMachine.class); + Mockito.when(vm.getHostId()).thenReturn(1L); + Host dest = Mockito.mock(Host.class); + Mockito.when(dest.getId()).thenReturn(2L); + ServiceOffering offering = Mockito.mock(ServiceOffering.class); + Mockito.when(offering.getCpu()).thenReturn(8); + Mockito.when(offering.getSpeed()).thenReturn(2400); + Mockito.when(offering.getRamSize()).thenReturn(32768); + + double withoutStorage = weighted.getMetrics(cluster, vm, offering, dest, cpu, memory, false, + null, new double[0], new HashMap<>()).second(); + double withStorage = weighted.getMetrics(cluster, vm, offering, dest, cpu, memory, true, + null, new double[0], new HashMap<>()).second(); + + assertTrue("a migration that has to move storage should cost more", withStorage > withoutStorage); + } +} diff --git a/plugins/pom.xml b/plugins/pom.xml index 92768827f658..08b906e9a6b6 100755 --- a/plugins/pom.xml +++ b/plugins/pom.xml @@ -77,6 +77,7 @@ drs/cluster/balanced drs/cluster/condensed + drs/cluster/weighted event-bus/inmemory event-bus/kafka diff --git a/server/src/main/java/com/cloud/agent/manager/allocator/impl/HostLoadTracker.java b/server/src/main/java/com/cloud/agent/manager/allocator/impl/HostLoadTracker.java index 4543af9b8b14..cd51ae2d8686 100644 --- a/server/src/main/java/com/cloud/agent/manager/allocator/impl/HostLoadTracker.java +++ b/server/src/main/java/com/cloud/agent/manager/allocator/impl/HostLoadTracker.java @@ -28,6 +28,8 @@ import org.apache.cloudstack.framework.config.Configurable; import org.apache.cloudstack.managed.context.ManagedContextRunnable; +import com.cloud.host.HostLoad; +import com.cloud.host.HostLoadService; import com.cloud.host.HostStats; import com.cloud.host.HostVO; import com.cloud.host.Status; @@ -63,7 +65,7 @@ * * Memory is taken as used over total and is sound everywhere. */ -public class HostLoadTracker extends ManagerBase implements Configurable { +public class HostLoadTracker extends ManagerBase implements HostLoadService, Configurable { public static final ConfigKey HostLoadSampleInterval = new ConfigKey<>(ConfigKey.CATEGORY_ADVANCED, Integer.class, "host.load.sample.interval", "60", @@ -175,6 +177,7 @@ protected void record(long hostId, HostStats stats, long now) { : current.fold(cpu, memory, now, halfLife, stats)); } + @Override public HostLoad getLoad(long hostId) { return getLoad(hostId, System.currentTimeMillis()); } diff --git a/server/src/main/java/com/cloud/agent/manager/allocator/impl/WeightedHostScorer.java b/server/src/main/java/com/cloud/agent/manager/allocator/impl/WeightedHostScorer.java index ab50172fec12..1b537f9068d9 100644 --- a/server/src/main/java/com/cloud/agent/manager/allocator/impl/WeightedHostScorer.java +++ b/server/src/main/java/com/cloud/agent/manager/allocator/impl/WeightedHostScorer.java @@ -38,6 +38,7 @@ import com.cloud.dc.ClusterDetailsDao; import com.cloud.dc.ClusterDetailsVO; import com.cloud.host.Host; +import com.cloud.host.HostLoad; import com.cloud.host.HostScoringWeights; import com.cloud.utils.Pair; import com.cloud.utils.component.AdapterBase; diff --git a/server/src/main/java/org/apache/cloudstack/cluster/ClusterDrsServiceImpl.java b/server/src/main/java/org/apache/cloudstack/cluster/ClusterDrsServiceImpl.java index ebc3364f5111..30f90f7bd263 100644 --- a/server/src/main/java/org/apache/cloudstack/cluster/ClusterDrsServiceImpl.java +++ b/server/src/main/java/org/apache/cloudstack/cluster/ClusterDrsServiceImpl.java @@ -401,8 +401,7 @@ private List> getMigrationPlans( List> migrationPlan = new ArrayList<>(); Map vmToExcludesMap = null; Set staleAffinityVmIds = new HashSet<>(); - while (iteration < maxIterations && algorithm.needsDrs(cluster, new ArrayList<>(hostCpuMap.values()), - new ArrayList<>(hostMemoryMap.values()))) { + while (iteration < maxIterations && algorithm.needsDrs(cluster, hostCpuMap, hostMemoryMap)) { logger.debug("Starting DRS iteration {} for cluster {}", iteration + 1, cluster); // Affinity only changes for VMs that share a group with the one just moved, so after diff --git a/server/src/test/java/com/cloud/agent/manager/allocator/impl/HostLoadTrackerTest.java b/server/src/test/java/com/cloud/agent/manager/allocator/impl/HostLoadTrackerTest.java index 0be9b1310f04..0f8b38f695ea 100644 --- a/server/src/test/java/com/cloud/agent/manager/allocator/impl/HostLoadTrackerTest.java +++ b/server/src/test/java/com/cloud/agent/manager/allocator/impl/HostLoadTrackerTest.java @@ -17,6 +17,8 @@ package com.cloud.agent.manager.allocator.impl; import org.junit.Before; +import com.cloud.host.HostLoad; + import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.InjectMocks; diff --git a/server/src/test/java/com/cloud/agent/manager/allocator/impl/WeightedHostScorerRankTest.java b/server/src/test/java/com/cloud/agent/manager/allocator/impl/WeightedHostScorerRankTest.java index 4521d5652e02..b06c6dedbcc6 100644 --- a/server/src/test/java/com/cloud/agent/manager/allocator/impl/WeightedHostScorerRankTest.java +++ b/server/src/test/java/com/cloud/agent/manager/allocator/impl/WeightedHostScorerRankTest.java @@ -40,6 +40,7 @@ import com.cloud.dc.ClusterDetailsDao; import com.cloud.dc.ClusterDetailsVO; import com.cloud.host.Host; +import com.cloud.host.HostLoad; import com.cloud.utils.Pair; import com.cloud.vm.dao.VMInstanceDao; diff --git a/server/src/test/java/com/cloud/agent/manager/allocator/impl/WeightedHostScorerTest.java b/server/src/test/java/com/cloud/agent/manager/allocator/impl/WeightedHostScorerTest.java index d5097d79d21a..ecbf36d831e5 100644 --- a/server/src/test/java/com/cloud/agent/manager/allocator/impl/WeightedHostScorerTest.java +++ b/server/src/test/java/com/cloud/agent/manager/allocator/impl/WeightedHostScorerTest.java @@ -30,6 +30,7 @@ import org.mockito.junit.MockitoJUnitRunner; import com.cloud.host.Host; +import com.cloud.host.HostLoad; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotEquals; diff --git a/server/src/test/java/com/cloud/agent/manager/allocator/impl/WeightedPlacementDistributionTest.java b/server/src/test/java/com/cloud/agent/manager/allocator/impl/WeightedPlacementDistributionTest.java index 22fa6c565839..de057c0b3831 100644 --- a/server/src/test/java/com/cloud/agent/manager/allocator/impl/WeightedPlacementDistributionTest.java +++ b/server/src/test/java/com/cloud/agent/manager/allocator/impl/WeightedPlacementDistributionTest.java @@ -22,6 +22,8 @@ import java.util.List; import java.util.Random; +import com.cloud.host.HostLoad; + import org.junit.Test; import static org.junit.Assert.assertTrue; diff --git a/server/src/test/java/org/apache/cloudstack/cluster/ClusterDrsServiceImplTest.java b/server/src/test/java/org/apache/cloudstack/cluster/ClusterDrsServiceImplTest.java index c80fcb70f33a..c400c9ec46ed 100644 --- a/server/src/test/java/org/apache/cloudstack/cluster/ClusterDrsServiceImplTest.java +++ b/server/src/test/java/org/apache/cloudstack/cluster/ClusterDrsServiceImplTest.java @@ -231,7 +231,7 @@ public void testGetDrsPlan() throws ConfigurationException { Mockito.when(hostDao.findByClusterId(1L)).thenReturn(hostList); Mockito.when(vmInstanceDao.listByClusterId(1L)).thenReturn(vmList); - Mockito.when(balancedAlgorithm.needsDrs(Mockito.any(), Mockito.anyList(), Mockito.anyList())).thenReturn( + Mockito.when(balancedAlgorithm.needsDrs(Mockito.any(), Mockito.anyMap(), Mockito.anyMap())).thenReturn( true, false); Mockito.doReturn(new Pair<>(vm1, host2)).when(clusterDrsService).getBestMigration( @@ -246,8 +246,8 @@ public void testGetDrsPlan() throws ConfigurationException { Mockito.verify(hostDao, Mockito.times(1)).findByClusterId(1L); Mockito.verify(vmInstanceDao, Mockito.times(1)).listByClusterId(1L); - Mockito.verify(balancedAlgorithm, Mockito.times(2)).needsDrs(Mockito.any(), Mockito.anyList(), - Mockito.anyList()); + Mockito.verify(balancedAlgorithm, Mockito.times(2)).needsDrs(Mockito.any(), Mockito.anyMap(), + Mockito.anyMap()); assertEquals(1, iterations.size()); } @@ -311,7 +311,7 @@ public void testGetDrsPlanWithSystemVMs() throws ConfigurationException { Mockito.when(hostDao.findByClusterId(1L)).thenReturn(hostList); Mockito.when(vmInstanceDao.listByClusterId(1L)).thenReturn(vmList); - Mockito.when(balancedAlgorithm.needsDrs(Mockito.any(), Mockito.anyList(), Mockito.anyList())).thenReturn(true); + Mockito.when(balancedAlgorithm.needsDrs(Mockito.any(), Mockito.anyMap(), Mockito.anyMap())).thenReturn(true); Mockito.when(hostJoinDao.searchByIds(Mockito.any())).thenReturn(List.of(hostJoin1)); List> result = clusterDrsService.getDrsPlan(cluster, 5); @@ -353,7 +353,7 @@ public void testGetDrsPlanWithNonRunningVMs() throws ConfigurationException { Mockito.when(hostDao.findByClusterId(1L)).thenReturn(hostList); Mockito.when(vmInstanceDao.listByClusterId(1L)).thenReturn(vmList); - Mockito.when(balancedAlgorithm.needsDrs(Mockito.any(), Mockito.anyList(), Mockito.anyList())).thenReturn(true); + Mockito.when(balancedAlgorithm.needsDrs(Mockito.any(), Mockito.anyMap(), Mockito.anyMap())).thenReturn(true); Mockito.when(hostJoinDao.searchByIds(Mockito.any())).thenReturn(List.of(hostJoin1)); List> result = clusterDrsService.getDrsPlan(cluster, 5); @@ -400,7 +400,7 @@ public void testGetDrsPlanWithSkipDrsFlag() throws ConfigurationException { Mockito.when(hostDao.findByClusterId(1L)).thenReturn(hostList); Mockito.when(vmInstanceDao.listByClusterId(1L)).thenReturn(vmList); - Mockito.when(balancedAlgorithm.needsDrs(Mockito.any(), Mockito.anyList(), Mockito.anyList())).thenReturn(true); + Mockito.when(balancedAlgorithm.needsDrs(Mockito.any(), Mockito.anyMap(), Mockito.anyMap())).thenReturn(true); Mockito.when(hostJoinDao.searchByIds(Mockito.any())).thenReturn(List.of(hostJoin1)); List> result = clusterDrsService.getDrsPlan(cluster, 5); @@ -445,7 +445,7 @@ public void testGetDrsPlanWithNoCompatibleHosts() throws ConfigurationException Mockito.when(hostDao.findByClusterId(1L)).thenReturn(hostList); Mockito.when(vmInstanceDao.listByClusterId(1L)).thenReturn(vmList); - Mockito.when(balancedAlgorithm.needsDrs(Mockito.any(), Mockito.anyList(), Mockito.anyList())).thenReturn(true); + Mockito.when(balancedAlgorithm.needsDrs(Mockito.any(), Mockito.anyMap(), Mockito.anyMap())).thenReturn(true); Mockito.when(serviceOfferingDao.findByIdIncludingRemoved(Mockito.anyLong(), Mockito.anyLong())).thenReturn(serviceOffering); Mockito.when(hostJoinDao.searchByIds(Mockito.any())).thenReturn(List.of(hostJoin1)); // Return a Ternary with an empty suitable-hosts list to exercise the "no compatible hosts" path @@ -493,7 +493,7 @@ public void testGetDrsPlanWithExceptionInCompatibilityCheck() throws Configurati Mockito.when(hostDao.findByClusterId(1L)).thenReturn(hostList); Mockito.when(vmInstanceDao.listByClusterId(1L)).thenReturn(vmList); - Mockito.when(balancedAlgorithm.needsDrs(Mockito.any(), Mockito.anyList(), Mockito.anyList())).thenReturn(true); + Mockito.when(balancedAlgorithm.needsDrs(Mockito.any(), Mockito.anyMap(), Mockito.anyMap())).thenReturn(true); Mockito.when(serviceOfferingDao.findByIdIncludingRemoved(Mockito.anyLong(), Mockito.anyLong())).thenReturn(serviceOffering); Mockito.when(hostJoinDao.searchByIds(Mockito.any())).thenReturn(List.of(hostJoin1)); // Throw an explicit exception so the catch-and-log path is exercised intentionally @@ -542,7 +542,7 @@ public void testGetDrsPlanWithNoBestMigration() throws ConfigurationException { Mockito.when(hostDao.findByClusterId(1L)).thenReturn(hostList); Mockito.when(vmInstanceDao.listByClusterId(1L)).thenReturn(vmList); - Mockito.when(balancedAlgorithm.needsDrs(Mockito.any(), Mockito.anyList(), Mockito.anyList())).thenReturn(true); + Mockito.when(balancedAlgorithm.needsDrs(Mockito.any(), Mockito.anyMap(), Mockito.anyMap())).thenReturn(true); Mockito.when(serviceOfferingDao.findByIdIncludingRemoved(Mockito.anyLong(), Mockito.anyLong())).thenReturn(serviceOffering); Mockito.when(hostJoinDao.searchByIds(Mockito.any())).thenReturn(List.of(hostJoin1)); @@ -617,7 +617,7 @@ public void testGetDrsPlanWithMultipleIterations() throws ConfigurationException Mockito.when(hostDao.findByClusterId(1L)).thenReturn(hostList); Mockito.when(vmInstanceDao.listByClusterId(1L)).thenReturn(vmList); - Mockito.when(balancedAlgorithm.needsDrs(Mockito.any(), Mockito.anyList(), Mockito.anyList())).thenReturn( + Mockito.when(balancedAlgorithm.needsDrs(Mockito.any(), Mockito.anyMap(), Mockito.anyMap())).thenReturn( true, true, false); Mockito.when(serviceOfferingDao.findByIdIncludingRemoved(Mockito.anyLong(), Mockito.anyLong())).thenReturn(serviceOffering); Mockito.when(hostJoinDao.searchByIds(1L, 2L)).thenReturn(List.of(hostJoin1, hostJoin2)); @@ -631,7 +631,7 @@ public void testGetDrsPlanWithMultipleIterations() throws ConfigurationException List> result = clusterDrsService.getDrsPlan(cluster, 5); assertEquals(2, result.size()); - Mockito.verify(balancedAlgorithm, Mockito.times(3)).needsDrs(Mockito.any(), Mockito.anyList(), Mockito.anyList()); + Mockito.verify(balancedAlgorithm, Mockito.times(3)).needsDrs(Mockito.any(), Mockito.anyMap(), Mockito.anyMap()); } @Test @@ -663,7 +663,7 @@ public void testGetDrsPlanWithMigrationToOriginalHost() throws ConfigurationExce Mockito.when(hostDao.findByClusterId(1L)).thenReturn(hostList); Mockito.when(vmInstanceDao.listByClusterId(1L)).thenReturn(vmList); - Mockito.when(balancedAlgorithm.needsDrs(Mockito.any(), Mockito.anyList(), Mockito.anyList())).thenReturn(true); + Mockito.when(balancedAlgorithm.needsDrs(Mockito.any(), Mockito.anyMap(), Mockito.anyMap())).thenReturn(true); Mockito.when(serviceOfferingDao.findByIdIncludingRemoved(Mockito.anyLong(), Mockito.anyLong())).thenReturn(serviceOffering); // Return migration to original host (host1) - should break the loop From 4cc5a9076099b6feafaf3ceb13f8fd9e7365c98c Mon Sep 17 00:00:00 2001 From: Brad House Date: Thu, 10 Sep 2026 00:21:34 +0000 Subject: [PATCH 14/18] Fix defects found in review of the weighted DRS algorithm Two of these meant the algorithm could not run at all. The plugin had no module.properties, so it was never discovered, never instantiated and never registered. Selecting it would have failed every DRS run with "Invalid algorithm configured". The plugin injected a service that lives in a sibling Spring module and is therefore not visible to it, so once the module did load, the management server would fail to start. - the caller now works out the load once per plan and passes it in - which also fixes the cost of getting it: getMetrics runs for every candidate VM and host, and it was reading two cluster details and four settings from the database on each call Grouping VMs to share one candidate-host lookup was unsound. The key listed the inputs it thought mattered, and missed several that are held per VM rather than per offering - a custom offering's size, boot mode, device settings. Two VMs could then share a host list neither of them should have had. - only group VMs that carry none of those, rather than trying to enumerate everything that could matter - key on disk offering as well as pool, since storage tags come from the offering Reserved capacity was multiplied by the overcommit ratio instead of being subtracted from the scaled total, so a host reading 11% fuller than its peer read 67% fuller, and DRS would evacuate hosts merely for holding reserved capacity. Hosts were compared on different bases. A host with no utilisation samples was measured on allocation alone while its peers were measured on a blend, reporting a difference that was an artefact of the monitoring. Utilisation is now used only when every host has it. The cost and benefit terms cancelled out, so storage motion was free despite the claim otherwise. A migration that has to move storage now has to earn more than one that does not, by a configurable margin. Also: allocation beyond what a host can hand out is no longer flattened to a single value, and the settings that shape the other algorithms' single metric are documented as not applying here. Signed-off-by: Brad House --- PendingReleaseNotes | 15 +- .../cluster/ClusterDrsAlgorithm.java | 15 +- .../apache/cloudstack/cluster/Weighted.java | 133 ++++++++++++++---- .../cloudstack/weighted/module.properties | 18 +++ .../cloudstack/cluster/WeightedTest.java | 121 ++++++++++++++-- .../cluster/ClusterDrsServiceImpl.java | 65 +++++++-- .../cluster/ClusterDrsEquivalenceTest.java | 68 ++++++++- .../cluster/ClusterDrsServiceImplTest.java | 26 ++-- 8 files changed, 391 insertions(+), 70 deletions(-) create mode 100644 plugins/drs/cluster/weighted/src/main/resources/META-INF/cloudstack/weighted/module.properties diff --git a/PendingReleaseNotes b/PendingReleaseNotes index 7bf6337376b0..83240f4a359a 100644 --- a/PendingReleaseNotes +++ b/PendingReleaseNotes @@ -82,9 +82,18 @@ example.ver.1 > example.ver.2: saturated host can still report a small percentage allocated. 'weighted' blends CPU and memory allocated with CPU and memory in use, and balances the result. - Imbalance keeps its existing meaning - standard deviation over the mean - so drs.imbalance - still means what it did. Tuned with the drs.weighted.* settings, all cluster scoped. Where no - utilisation samples are available it falls back to allocation figures. + Imbalance keeps its existing shape - standard deviation over the mean - but it is computed over + a blend rather than over one metric, so drs.imbalance is not calibrated the same way and is + worth re-checking after switching. drs.metric, drs.metric.type and drs.metric.use.ratio choose + and shape the single metric the other algorithms balance; they do not apply to 'weighted' and + are ignored. + + Utilisation is only used when every host in the cluster has been sampled. Comparing a host + measured on utilisation against one measured on allocation alone would report a difference that + is an artefact of the monitoring rather than of the load, so the cluster falls back to + allocation figures until every host can be measured. + + Tuned with the drs.weighted.* settings, all cluster scoped. * DRS plan generation is considerably faster on large clusters. Working out where a VM could go is now done once for each group of VMs that would get the same answer rather than once per VM, diff --git a/api/src/main/java/org/apache/cloudstack/cluster/ClusterDrsAlgorithm.java b/api/src/main/java/org/apache/cloudstack/cluster/ClusterDrsAlgorithm.java index 722247e6fc51..3c8d89dad15e 100644 --- a/api/src/main/java/org/apache/cloudstack/cluster/ClusterDrsAlgorithm.java +++ b/api/src/main/java/org/apache/cloudstack/cluster/ClusterDrsAlgorithm.java @@ -20,6 +20,7 @@ package org.apache.cloudstack.cluster; import com.cloud.host.Host; +import com.cloud.host.HostLoad; import com.cloud.offering.ServiceOffering; import com.cloud.org.Cluster; import com.cloud.utils.Ternary; @@ -75,10 +76,22 @@ boolean needsDrs(Cluster cluster, List> cpuList, * host id to a Ternary of used, reserved and total memory */ default boolean needsDrs(Cluster cluster, Map> hostCpuMap, - Map> hostMemoryMap) throws ConfigurationException { + Map> hostMemoryMap, Map hostLoadMap) + throws ConfigurationException { return needsDrs(cluster, new ArrayList<>(hostCpuMap.values()), new ArrayList<>(hostMemoryMap.values())); } + /** + * Called once per plan, before any migration is considered, so that an algorithm can do work + * that would otherwise be repeated for every candidate VM and host. + * + * @param hostLoadMap + * measured load per host, empty when nothing has been sampled + */ + default void prepare(Cluster cluster, Map> hostCpuMap, + Map> hostMemoryMap, Map hostLoadMap) { + } + /** * Calculates the metrics (improvement, cost, benefit) for migrating a VM to a destination host. Improvement is * calculated based on the change in cluster imbalance before and after the migration. diff --git a/plugins/drs/cluster/weighted/src/main/java/org/apache/cloudstack/cluster/Weighted.java b/plugins/drs/cluster/weighted/src/main/java/org/apache/cloudstack/cluster/Weighted.java index f136bfa1ad9d..19feb328ea87 100644 --- a/plugins/drs/cluster/weighted/src/main/java/org/apache/cloudstack/cluster/Weighted.java +++ b/plugins/drs/cluster/weighted/src/main/java/org/apache/cloudstack/cluster/Weighted.java @@ -33,7 +33,6 @@ import com.cloud.host.Host; import com.cloud.host.HostLoad; import com.cloud.host.HostScoringWeights; -import com.cloud.host.HostLoadService; import com.cloud.offering.ServiceOffering; import com.cloud.org.Cluster; import com.cloud.utils.Ternary; @@ -53,19 +52,92 @@ * This blends four figures per host - CPU and memory allocated, CPU and memory in use - and * balances the result. The weights are the same host.weighted.* settings initial placement uses, on * purpose: if the two weighted them differently they would disagree about which host is the better - * one, and rebalancing could move VMs off hosts that placement had just chosen. Imbalance keeps the same meaning as the other algorithms: the standard - * deviation of the per-host figure over its mean, so drs.imbalance still means what it did. + * one, and rebalancing could move VMs off hosts that placement had just chosen. Imbalance keeps the same shape as the other algorithms - the standard + * deviation of the per-host figure over its mean - but it is computed over a blend rather than over + * one metric, so drs.imbalance is not calibrated the same way and is worth re-checking after + * switching. + * + * drs.metric, drs.metric.type and drs.metric.use.ratio choose and shape the single metric the other + * algorithms balance. They do not apply here and are ignored. */ public class Weighted extends AdapterBase implements ClusterDrsAlgorithm, Configurable { private static final Logger LOGGER = LogManager.getLogger(Weighted.class); - @Inject - private HostLoadService hostLoadService; + public static final ConfigKey StorageMotionCost = new ConfigKey<>(ConfigKey.CATEGORY_ADVANCED, + Double.class, "drs.weighted.storage.motion.cost", "0.02", + "How much a migration must improve the cluster's imbalance to be worth also moving the VM's " + + "storage. Migrations that do not need storage moved only have to improve it at all.", + true, ConfigKey.Scope.Cluster); @Inject private ClusterDetailsDao clusterDetailsDao; + /** + * Everything that is constant for one plan. getMetrics is called for every candidate VM and + * host - up to hundreds of thousands of times for a large cluster - so nothing in that path may + * hit the database or re-read settings. + */ + private static final class PlanContext { + private final float cpuOvercommit; + private final float memoryOvercommit; + private final double cpuAllocatedWeight; + private final double memoryAllocatedWeight; + private final double cpuUsedWeight; + private final double memoryUsedWeight; + private final Map hostLoadMap; + private final boolean everyHostMeasured; + + private PlanContext(float cpuOvercommit, float memoryOvercommit, double cpuAllocatedWeight, + double memoryAllocatedWeight, double cpuUsedWeight, double memoryUsedWeight, + Map hostLoadMap, boolean everyHostMeasured) { + this.everyHostMeasured = everyHostMeasured; + this.cpuOvercommit = cpuOvercommit; + this.memoryOvercommit = memoryOvercommit; + this.cpuAllocatedWeight = cpuAllocatedWeight; + this.memoryAllocatedWeight = memoryAllocatedWeight; + this.cpuUsedWeight = cpuUsedWeight; + this.memoryUsedWeight = memoryUsedWeight; + this.hostLoadMap = hostLoadMap; + } + + private HostLoad loadOf(long hostId) { + HostLoad load = hostLoadMap.get(hostId); + return load == null ? HostLoad.UNKNOWN : load; + } + } + + private final ThreadLocal context = new ThreadLocal<>(); + + @Override + public void prepare(Cluster cluster, Map> hostCpuMap, + Map> hostMemoryMap, Map hostLoadMap) { + long clusterId = cluster.getId(); + context.set(new PlanContext( + overcommitRatio(clusterId, VmDetailConstants.CPU_OVER_COMMIT_RATIO), + overcommitRatio(clusterId, VmDetailConstants.MEMORY_OVER_COMMIT_RATIO), + weight(HostScoringWeights.CpuAllocatedWeight, clusterId), + weight(HostScoringWeights.MemoryAllocatedWeight, clusterId), + weight(HostScoringWeights.CpuUsedWeight, clusterId), + weight(HostScoringWeights.MemoryUsedWeight, clusterId), + hostLoadMap == null ? new HashMap<>() : hostLoadMap, + hostLoadMap != null && !hostLoadMap.isEmpty() + && hostLoadMap.values().stream().allMatch(HostLoad::isUsable))); + } + + /** + * Falls back to reading everything when prepare has not been called, so the algorithm still + * works for a caller that does not know about it. + */ + private PlanContext contextFor(Cluster cluster) { + PlanContext prepared = context.get(); + if (prepared != null) { + return prepared; + } + prepare(cluster, null, null, null); + return context.get(); + } + @Override public String getName() { return "weighted"; @@ -81,12 +153,13 @@ public boolean needsDrs(Cluster cluster, List> cpuList cpuMap.put((long) -(i + 1), cpuList.get(i)); memoryMap.put((long) -(i + 1), memoryList.get(i)); } - return needsDrs(cluster, cpuMap, memoryMap); + return needsDrs(cluster, cpuMap, memoryMap, new HashMap<>()); } @Override public boolean needsDrs(Cluster cluster, Map> hostCpuMap, - Map> hostMemoryMap) throws ConfigurationException { + Map> hostMemoryMap, Map hostLoadMap) + throws ConfigurationException { double threshold = 1.0 - ClusterDrsService.ClusterDrsImbalanceThreshold.valueIn(cluster.getId()); double imbalance = imbalanceOf(blendByHost(cluster, hostCpuMap, hostMemoryMap).values()); boolean needed = imbalance > threshold; @@ -111,11 +184,12 @@ public Ternary getMetrics(Cluster cluster, VirtualMachin double after = imbalanceOf(blendByHost(cluster, cpuAfter, memoryAfter).values()); + // the caller migrates when benefit > cost, so expressing both in units of imbalance makes + // that comparison mean "is this worth what it costs". A migration that has to move storage + // has to earn more than one that does not. double improvement = before - after; - // moving a VM costs something and buys nothing unless the cluster ends up more even, so a - // migration is only worth making when it measurably helps - double cost = Boolean.TRUE.equals(requiresStorageMotion) ? 1.0 : 0.0; - double benefit = improvement > 0 ? 1.0 + improvement : 0.0; + double cost = Boolean.TRUE.equals(requiresStorageMotion) ? weight(StorageMotionCost, cluster.getId()) : 0.0; + double benefit = improvement; LOGGER.trace("Cluster {} imbalance {} -> {} moving {} to {}", cluster, before, after, vm, destHost); return new Ternary<>(improvement, cost, benefit); @@ -126,13 +200,7 @@ public Ternary getMetrics(Cluster cluster, VirtualMachin */ protected Map blendByHost(Cluster cluster, Map> hostCpuMap, Map> hostMemoryMap) { - float cpuOvercommit = overcommitRatio(cluster.getId(), VmDetailConstants.CPU_OVER_COMMIT_RATIO); - float memoryOvercommit = overcommitRatio(cluster.getId(), VmDetailConstants.MEMORY_OVER_COMMIT_RATIO); - - double cpuAllocatedWeight = weight(HostScoringWeights.CpuAllocatedWeight, cluster.getId()); - double memoryAllocatedWeight = weight(HostScoringWeights.MemoryAllocatedWeight, cluster.getId()); - double cpuUsedWeight = weight(HostScoringWeights.CpuUsedWeight, cluster.getId()); - double memoryUsedWeight = weight(HostScoringWeights.MemoryUsedWeight, cluster.getId()); + PlanContext ctx = contextFor(cluster); Map blended = new HashMap<>(); for (Map.Entry> entry : hostCpuMap.entrySet()) { @@ -141,20 +209,24 @@ protected Map blendByHost(Cluster cluster, Map> withVmMoved(Map capacity, float overcommit) { - double allocatable = (capacity.third() - capacity.second()) * (double) overcommit; + // overcommit scales the host's total; reserved is then taken off that, which is how + // CapacityManager computes free capacity everywhere else. Multiplying reserved by the ratio + // instead would make a host look fuller the more capacity it merely has reserved. + double allocatable = capacity.third() * (double) overcommit - capacity.second(); if (allocatable <= 0) { return 0; } - return clamp(capacity.first() / allocatable); + return capacity.first() / allocatable; } protected float overcommitRatio(long clusterId, String key) { @@ -239,7 +314,7 @@ public String getConfigComponentName() { @Override public ConfigKey[] getConfigKeys() { - // the four host.weighted.* weights are shared with initial placement, which registers them + // the four host.weighted.* weights are shared with initial placement and registered there return new ConfigKey[] {StorageMotionCost}; } } diff --git a/plugins/drs/cluster/weighted/src/main/resources/META-INF/cloudstack/weighted/module.properties b/plugins/drs/cluster/weighted/src/main/resources/META-INF/cloudstack/weighted/module.properties new file mode 100644 index 000000000000..636ecd1700e5 --- /dev/null +++ b/plugins/drs/cluster/weighted/src/main/resources/META-INF/cloudstack/weighted/module.properties @@ -0,0 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +name=weighted +parent=cluster diff --git a/plugins/drs/cluster/weighted/src/test/java/org/apache/cloudstack/cluster/WeightedTest.java b/plugins/drs/cluster/weighted/src/test/java/org/apache/cloudstack/cluster/WeightedTest.java index f3c0fafbbec5..e45dba7d3103 100644 --- a/plugins/drs/cluster/weighted/src/test/java/org/apache/cloudstack/cluster/WeightedTest.java +++ b/plugins/drs/cluster/weighted/src/test/java/org/apache/cloudstack/cluster/WeightedTest.java @@ -33,13 +33,13 @@ import com.cloud.dc.ClusterDetailsVO; import com.cloud.host.Host; import com.cloud.host.HostLoad; -import com.cloud.host.HostLoadService; import com.cloud.offering.ServiceOffering; import com.cloud.org.Cluster; import com.cloud.utils.Ternary; import com.cloud.vm.VirtualMachine; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @RunWith(MockitoJUnitRunner.class) @@ -51,9 +51,6 @@ public class WeightedTest { private static final int CPU_OVERCOMMIT = 10; private static final int MEMORY_OVERCOMMIT = 4; - @Mock - private HostLoadService hostLoadService; - @Mock private ClusterDetailsDao clusterDetailsDao; @@ -63,6 +60,7 @@ public class WeightedTest { private Cluster cluster; private final Map> cpu = new HashMap<>(); private final Map> memory = new HashMap<>(); + private final Map load = new HashMap<>(); @Before public void setUp() { @@ -74,10 +72,22 @@ public void setUp() { .thenReturn(new ClusterDetailsVO(CLUSTER_ID, "memoryOvercommitRatio", String.valueOf(MEMORY_OVERCOMMIT))); } - private void host(long id, double cpuAllocatedFraction, double memoryAllocatedFraction, HostLoad load) { - cpu.put(id, new Ternary<>((long) (CPU_TOTAL * CPU_OVERCOMMIT * cpuAllocatedFraction), 0L, CPU_TOTAL)); - memory.put(id, new Ternary<>((long) (MEMORY_TOTAL * MEMORY_OVERCOMMIT * memoryAllocatedFraction), 0L, MEMORY_TOTAL)); - Mockito.lenient().when(hostLoadService.getLoad(id)).thenReturn(load); + private void host(long id, double cpuAllocatedFraction, double memoryAllocatedFraction, HostLoad hostLoad) { + host(id, cpuAllocatedFraction, memoryAllocatedFraction, hostLoad, 0L, 0L); + } + + private void host(long id, double cpuAllocatedFraction, double memoryAllocatedFraction, HostLoad hostLoad, + long reservedCpu, long reservedMemory) { + cpu.put(id, new Ternary<>((long) (CPU_TOTAL * CPU_OVERCOMMIT * cpuAllocatedFraction), reservedCpu, CPU_TOTAL)); + memory.put(id, new Ternary<>((long) (MEMORY_TOTAL * MEMORY_OVERCOMMIT * memoryAllocatedFraction), + reservedMemory, MEMORY_TOTAL)); + load.put(id, hostLoad); + } + + /** DRS calls prepare once per plan; every test must do the same. */ + private Map blend() { + weighted.prepare(cluster, cpu, memory, load); + return weighted.blendByHost(cluster, cpu, memory); } @Test @@ -85,7 +95,7 @@ public void testAnEvenClusterHasNoImbalance() { host(1L, 0.4, 0.4, new HostLoad(0.4, 0.4, 10)); host(2L, 0.4, 0.4, new HostLoad(0.4, 0.4, 10)); - assertEquals(0.0, weighted.imbalanceOf(weighted.blendByHost(cluster, cpu, memory).values()), 1e-9); + assertEquals(0.0, weighted.imbalanceOf(blend().values()), 1e-9); } @Test @@ -95,7 +105,7 @@ public void testCpuLoadImbalanceIsSeenWhenMemoryIsEven() { host(2L, 0.30, 0.40, new HostLoad(0.30, 0.40, 10)); assertTrue("balancing on memory alone would call this cluster even", - weighted.imbalanceOf(weighted.blendByHost(cluster, cpu, memory).values()) > 0.15); + weighted.imbalanceOf(blend().values()) > 0.15); } @Test @@ -103,7 +113,7 @@ public void testMemoryImbalanceIsSeenWhenCpuIsEven() { host(1L, 0.30, 0.90, new HostLoad(0.30, 0.90, 10)); host(2L, 0.30, 0.10, new HostLoad(0.30, 0.10, 10)); - assertTrue(weighted.imbalanceOf(weighted.blendByHost(cluster, cpu, memory).values()) > 0.15); + assertTrue(weighted.imbalanceOf(blend().values()) > 0.15); } @Test @@ -112,7 +122,7 @@ public void testAllocationBeyondPhysicalSizeStillDiscriminates() { host(1L, 0.20, 0.20, HostLoad.UNKNOWN); host(2L, 0.80, 0.20, HostLoad.UNKNOWN); - Map blended = weighted.blendByHost(cluster, cpu, memory); + Map blended = blend(); assertTrue(blended.get(2L) > blended.get(1L)); } @@ -121,7 +131,7 @@ public void testUnmeasuredHostsFallBackToAllocation() { host(1L, 0.20, 0.20, HostLoad.UNKNOWN); host(2L, 0.60, 0.60, HostLoad.UNKNOWN); - Map blended = weighted.blendByHost(cluster, cpu, memory); + Map blended = blend(); assertEquals(0.20, blended.get(1L), 0.01); assertEquals(0.60, blended.get(2L), 0.01); } @@ -140,6 +150,7 @@ public void testMovingAVmOffTheBusyHostIsAnImprovement() throws ConfigurationExc Mockito.when(offering.getSpeed()).thenReturn(2400); Mockito.when(offering.getRamSize()).thenReturn(32768); + weighted.prepare(cluster, cpu, memory, load); Ternary metrics = weighted.getMetrics(cluster, vm, offering, dest, cpu, memory, false, null, new double[0], new HashMap<>()); @@ -161,11 +172,13 @@ public void testMovingAVmOntoTheBusyHostIsNotAnImprovement() throws Configuratio Mockito.when(offering.getSpeed()).thenReturn(2400); Mockito.when(offering.getRamSize()).thenReturn(32768); + weighted.prepare(cluster, cpu, memory, load); Ternary metrics = weighted.getMetrics(cluster, vm, offering, dest, cpu, memory, false, null, new double[0], new HashMap<>()); assertTrue("piling onto the busier host makes the cluster less even", metrics.first() < 0); - assertEquals("and must not be considered worth doing", 0.0, metrics.third(), 1e-9); + // the caller migrates when benefit > cost + assertFalse("and must not be considered worth doing", metrics.third() > metrics.second()); } @Test @@ -182,6 +195,7 @@ public void testStorageMotionCostsMore() throws ConfigurationException { Mockito.when(offering.getSpeed()).thenReturn(2400); Mockito.when(offering.getRamSize()).thenReturn(32768); + weighted.prepare(cluster, cpu, memory, load); double withoutStorage = weighted.getMetrics(cluster, vm, offering, dest, cpu, memory, false, null, new double[0], new HashMap<>()).second(); double withStorage = weighted.getMetrics(cluster, vm, offering, dest, cpu, memory, true, @@ -189,4 +203,81 @@ public void testStorageMotionCostsMore() throws ConfigurationException { assertTrue("a migration that has to move storage should cost more", withStorage > withoutStorage); } -} + + @Test + public void testReservedCapacityDoesNotMakeAHostLookFuller() { + // reserved is taken off the overcommitted total, not multiplied by the ratio. Getting this + // backwards made a host with reserved capacity read far fuller than an identical one + // without, and DRS would evacuate it for no reason. + host(1L, 0.10, 0.10, HostLoad.UNKNOWN, 0L, 0L); + host(2L, 0.10, 0.10, HostLoad.UNKNOWN, CPU_TOTAL / 10, MEMORY_TOTAL / 10); + + Map blended = blend(); + + assertEquals("a little reserved capacity should barely move the figure", + blended.get(1L), blended.get(2L), 0.02); + } + + @Test + public void testAllocationBeyondTheOvercommittedTotalIsNotFlattened() { + // hosts past what they can hand out must stay distinguishable, not both read as full + host(1L, 1.20, 0.20, HostLoad.UNKNOWN); + host(2L, 2.40, 0.20, HostLoad.UNKNOWN); + + Map blended = blend(); + + assertTrue("two oversubscribed hosts must not read identically", + blended.get(2L) > blended.get(1L)); + } + + @Test + public void testUtilisationIsDroppedWhenAnyHostCannotBeMeasured() { + // one host with broken telemetry must not read as loaded simply because it is measured on a + // different basis to its peers - that would evacuate whichever host stopped reporting + host(1L, 0.50, 0.50, new HostLoad(0.10, 0.10, 10)); + host(2L, 0.50, 0.50, HostLoad.UNKNOWN); + + Map blended = blend(); + + assertEquals("identical hosts must read identically when one cannot be measured", + blended.get(1L), blended.get(2L), 1e-9); + assertEquals(0.0, weighted.imbalanceOf(blended.values()), 1e-9); + } + + @Test + public void testUtilisationIsUsedWhenEveryHostIsMeasured() { + host(1L, 0.50, 0.50, new HostLoad(0.90, 0.50, 10)); + host(2L, 0.50, 0.50, new HostLoad(0.10, 0.50, 10)); + + assertTrue("with every host measured, load must separate them", + weighted.imbalanceOf(blend().values()) > 0.05); + } + + @Test + public void testStorageMotionMustEarnMoreThanAPlainMigration() throws ConfigurationException { + host(1L, 0.52, 0.50, new HostLoad(0.52, 0.50, 10)); + host(2L, 0.48, 0.50, new HostLoad(0.48, 0.50, 10)); + + VirtualMachine vm = Mockito.mock(VirtualMachine.class); + Mockito.when(vm.getHostId()).thenReturn(1L); + Host dest = Mockito.mock(Host.class); + Mockito.when(dest.getId()).thenReturn(2L); + ServiceOffering offering = Mockito.mock(ServiceOffering.class); + Mockito.when(offering.getCpu()).thenReturn(1); + Mockito.when(offering.getSpeed()).thenReturn(500); + Mockito.when(offering.getRamSize()).thenReturn(512); + + weighted.prepare(cluster, cpu, memory, load); + Ternary plain = weighted.getMetrics(cluster, vm, offering, dest, + cpu, memory, false, null, new double[0], new HashMap<>()); + weighted.prepare(cluster, cpu, memory, load); + Ternary withStorage = weighted.getMetrics(cluster, vm, offering, dest, + cpu, memory, true, null, new double[0], new HashMap<>()); + + // the caller migrates when benefit > cost + assertTrue("a small gain is worth taking without moving storage", + plain.third() > plain.second()); + assertFalse("the same small gain is not worth moving storage for", + withStorage.third() > withStorage.second()); + } +} \ No newline at end of file diff --git a/server/src/main/java/org/apache/cloudstack/cluster/ClusterDrsServiceImpl.java b/server/src/main/java/org/apache/cloudstack/cluster/ClusterDrsServiceImpl.java index 30f90f7bd263..b7aaf96809f1 100644 --- a/server/src/main/java/org/apache/cloudstack/cluster/ClusterDrsServiceImpl.java +++ b/server/src/main/java/org/apache/cloudstack/cluster/ClusterDrsServiceImpl.java @@ -34,6 +34,8 @@ import com.cloud.event.dao.EventDao; import com.cloud.exception.InvalidParameterValueException; import com.cloud.host.Host; +import com.cloud.host.HostLoad; +import com.cloud.host.HostLoadService; import com.cloud.host.HostVO; import com.cloud.host.dao.HostDao; import com.cloud.offering.ServiceOffering; @@ -44,7 +46,6 @@ import com.cloud.user.User; import com.cloud.utils.DateUtil; import com.cloud.utils.Pair; -import com.cloud.storage.VolumeVO; import com.cloud.storage.dao.VolumeDao; import com.cloud.utils.Ternary; import com.cloud.utils.component.ComponentContext; @@ -96,7 +97,6 @@ import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.Objects; import java.util.Set; import java.util.Timer; import java.util.TimerTask; @@ -152,6 +152,9 @@ public class ClusterDrsServiceImpl extends ManagerBase implements ClusterDrsServ @Inject VolumeDao volumeDao; + @Inject + HostLoadService hostLoadService; + List drsAlgorithms = new ArrayList<>(); Map drsAlgorithmMap = new HashMap<>(); @@ -401,7 +404,13 @@ private List> getMigrationPlans( List> migrationPlan = new ArrayList<>(); Map vmToExcludesMap = null; Set staleAffinityVmIds = new HashSet<>(); - while (iteration < maxIterations && algorithm.needsDrs(cluster, hostCpuMap, hostMemoryMap)) { + Map hostLoadMap = new HashMap<>(); + for (Long hostId : hostCpuMap.keySet()) { + hostLoadMap.put(hostId, hostLoadService.getLoad(hostId)); + } + algorithm.prepare(cluster, hostCpuMap, hostMemoryMap, hostLoadMap); + + while (iteration < maxIterations && algorithm.needsDrs(cluster, hostCpuMap, hostMemoryMap, hostLoadMap)) { logger.debug("Starting DRS iteration {} for cluster {}", iteration + 1, cluster); // Affinity only changes for VMs that share a group with the one just moved, so after @@ -596,17 +605,35 @@ private Pair>, Map>> get return new Pair<>(vmToCompatibleHostsCache, vmToStorageMotionCache); } + /** + * VM details that narrow which hosts a VM can run on. Matched as prefixes and case + * insensitively, so that related keys are covered without listing each one. + */ + private static final List PLACEMENT_AFFECTING_DETAIL_PREFIXES = List.of( + "uefi", "boot", "dpdk", "cpunumber", "cpuspeed", "memory", "rootdisk", "nic", "gpu", "vgpu", + "extraconfig", "hypervisortoolsversion", "kvm", "vmware", "hyperv"); + /** * Identifies VMs that would get the same answer from listHostsForMigrationOfVM. * - * The answer depends on what the VM asks for (its offering and template), where it is now, the - * volumes that would have to follow it, and the affinity groups it belongs to. Two VMs matching - * on all of those are interchangeable for the purpose of finding candidate hosts. + * The answer depends on what the VM asks for, where it is now, the volumes that would have to + * follow it, and its affinity groups. Rather than enumerate everything that could possibly + * matter and risk missing one, a VM is only grouped when it has none of the per-VM inputs that + * are known to change the answer - a custom offering whose size comes from the VM rather than + * the offering, a boot mode or device setting, and so on. Anything else is worked out on its + * own, which costs what it always did. + * + * @return a key shared with equivalent VMs, or one unique to this VM when it cannot be grouped */ protected String migrationEquivalenceKey(VirtualMachine vm) { - List poolIds = volumeDao.findCreatedByInstance(vm.getId()).stream() - .map(VolumeVO::getPoolId) - .filter(Objects::nonNull) + ServiceOffering offering = serviceOfferingDao.findByIdIncludingRemoved(vm.getId(), vm.getServiceOfferingId()); + if (offering == null || offering.isDynamic() || hasPlacementAffectingDetails(vm)) { + return "vm-" + vm.getId(); + } + + List volumes = volumeDao.findCreatedByInstance(vm.getId()).stream() + .filter(volume -> volume.getPoolId() != null) + .map(volume -> volume.getPoolId() + ":" + volume.getDiskOfferingId()) .sorted() .collect(Collectors.toList()); List groupIds = affinityGroupVMMapDao.listByInstanceId(vm.getId()).stream() @@ -614,7 +641,25 @@ protected String migrationEquivalenceKey(VirtualMachine vm) { .sorted() .collect(Collectors.toList()); return String.format("%s|%s|%s|%s|%s|%s", vm.getServiceOfferingId(), vm.getTemplateId(), vm.getHostId(), - vm.getHypervisorType(), poolIds, groupIds); + vm.getHypervisorType(), volumes, groupIds); + } + + /** + * Whether the VM carries any detail that narrows the hosts it can run on. Such a VM is never + * grouped with another, because the details are per VM and two VMs on the same offering can + * differ entirely. + */ + protected boolean hasPlacementAffectingDetails(VirtualMachine vm) { + Map details = vmInstanceDetailsDao.listDetailsKeyPairs(vm.getId()); + if (MapUtils.isEmpty(details)) { + return false; + } + for (String key : details.keySet()) { + if (PLACEMENT_AFFECTING_DETAIL_PREFIXES.stream().anyMatch(prefix -> key.toLowerCase().startsWith(prefix))) { + return true; + } + } + return false; } /** diff --git a/server/src/test/java/org/apache/cloudstack/cluster/ClusterDrsEquivalenceTest.java b/server/src/test/java/org/apache/cloudstack/cluster/ClusterDrsEquivalenceTest.java index f2725acb9ca8..0d596db0a387 100644 --- a/server/src/test/java/org/apache/cloudstack/cluster/ClusterDrsEquivalenceTest.java +++ b/server/src/test/java/org/apache/cloudstack/cluster/ClusterDrsEquivalenceTest.java @@ -30,6 +30,9 @@ import org.mockito.junit.MockitoJUnitRunner; import com.cloud.hypervisor.Hypervisor; +import com.cloud.service.ServiceOfferingVO; +import com.cloud.service.dao.ServiceOfferingDao; +import com.cloud.vm.dao.VMInstanceDetailsDao; import com.cloud.storage.VolumeVO; import com.cloud.storage.dao.VolumeDao; import com.cloud.vm.VMInstanceVO; @@ -50,6 +53,12 @@ public class ClusterDrsEquivalenceTest { @Mock private AffinityGroupVMMapDao affinityGroupVMMapDao; + @Mock + private ServiceOfferingDao serviceOfferingDao; + + @Mock + private VMInstanceDetailsDao vmInstanceDetailsDao; + @InjectMocks private ClusterDrsServiceImpl service = new ClusterDrsServiceImpl(); @@ -68,9 +77,25 @@ private VMInstanceVO vm(long id, long offeringId, long templateId, Long hostId, }).collect(java.util.stream.Collectors.toList()); Mockito.lenient().when(volumeDao.findCreatedByInstance(id)).thenReturn(volumes); Mockito.lenient().when(affinityGroupVMMapDao.listByInstanceId(id)).thenReturn(Collections.emptyList()); + + ServiceOfferingVO offering = Mockito.mock(ServiceOfferingVO.class); + Mockito.lenient().when(offering.isDynamic()).thenReturn(false); + Mockito.lenient().when(serviceOfferingDao.findByIdIncludingRemoved(id, offeringId)).thenReturn(offering); + Mockito.lenient().when(vmInstanceDetailsDao.listDetailsKeyPairs(id)).thenReturn(Collections.emptyMap()); return vm; } + private void offeringIsDynamic(long vmId, long offeringId) { + ServiceOfferingVO offering = Mockito.mock(ServiceOfferingVO.class); + Mockito.when(offering.isDynamic()).thenReturn(true); + Mockito.when(serviceOfferingDao.findByIdIncludingRemoved(vmId, offeringId)).thenReturn(offering); + } + + private void vmHasDetail(long vmId, String key, String value) { + Mockito.when(vmInstanceDetailsDao.listDetailsKeyPairs(vmId)) + .thenReturn(java.util.Map.of(key, value)); + } + @Test public void testIdenticalVmsShareOnePass() { assertEquals(service.migrationEquivalenceKey(vm(1L, 10L, 20L, 30L, 40L)), @@ -118,4 +143,45 @@ public void testDifferentAffinityGroupsIsADifferentAnswer() { service.migrationEquivalenceKey(grouped), service.migrationEquivalenceKey(vm(2L, 10L, 20L, 30L, 40L))); } -} + + @Test + public void testVolumesOnTheSamePoolButDifferentDiskOfferingsAreNotGrouped() { + VMInstanceVO a = vm(1L, 10L, 20L, 30L, 40L); + VMInstanceVO b = vm(2L, 10L, 20L, 30L, 40L); + VolumeVO differentOffering = Mockito.mock(VolumeVO.class); + Mockito.when(differentOffering.getPoolId()).thenReturn(40L); + Mockito.when(differentOffering.getDiskOfferingId()).thenReturn(99L); + Mockito.when(volumeDao.findCreatedByInstance(2L)).thenReturn(List.of(differentOffering)); + + assertNotEquals("storage tags come from the disk offering, so it changes the answer", + service.migrationEquivalenceKey(a), service.migrationEquivalenceKey(b)); + } + + @Test + public void testACustomOfferingIsNeverGrouped() { + // a dynamic offering takes its size from the VM, so two VMs on the same offering can be + // wildly different and must not share a candidate host list + VMInstanceVO a = vm(1L, 10L, 20L, 30L, 40L); + VMInstanceVO b = vm(2L, 10L, 20L, 30L, 40L); + offeringIsDynamic(1L, 10L); + offeringIsDynamic(2L, 10L); + + assertNotEquals(service.migrationEquivalenceKey(a), service.migrationEquivalenceKey(b)); + } + + @Test + public void testAVmWithABootModeIsNeverGrouped() { + VMInstanceVO a = vm(1L, 10L, 20L, 30L, 40L); + VMInstanceVO b = vm(2L, 10L, 20L, 30L, 40L); + vmHasDetail(1L, "UEFI", "SECURE"); + + assertNotEquals("a UEFI VM cannot run everywhere a BIOS VM can", + service.migrationEquivalenceKey(a), service.migrationEquivalenceKey(b)); + } + + @Test + public void testAVmWithNoDetailsIsStillGrouped() { + assertEquals(service.migrationEquivalenceKey(vm(1L, 10L, 20L, 30L, 40L)), + service.migrationEquivalenceKey(vm(2L, 10L, 20L, 30L, 40L))); + } +} \ No newline at end of file diff --git a/server/src/test/java/org/apache/cloudstack/cluster/ClusterDrsServiceImplTest.java b/server/src/test/java/org/apache/cloudstack/cluster/ClusterDrsServiceImplTest.java index c400c9ec46ed..92e1c45599e2 100644 --- a/server/src/test/java/org/apache/cloudstack/cluster/ClusterDrsServiceImplTest.java +++ b/server/src/test/java/org/apache/cloudstack/cluster/ClusterDrsServiceImplTest.java @@ -28,6 +28,7 @@ import com.cloud.event.dao.EventDao; import com.cloud.exception.InvalidParameterValueException; import com.cloud.host.Host; +import com.cloud.host.HostLoadService; import com.cloud.host.HostVO; import com.cloud.deploy.DeploymentPlanner.ExcludeList; import com.cloud.deploy.DeploymentPlan; @@ -140,6 +141,9 @@ public class ClusterDrsServiceImplTest { @Mock private VolumeDao volumeDao; + @Mock + private HostLoadService hostLoadService; + @Mock private VMInstanceDetailsDao vmInstanceDetailsDao; @@ -231,7 +235,7 @@ public void testGetDrsPlan() throws ConfigurationException { Mockito.when(hostDao.findByClusterId(1L)).thenReturn(hostList); Mockito.when(vmInstanceDao.listByClusterId(1L)).thenReturn(vmList); - Mockito.when(balancedAlgorithm.needsDrs(Mockito.any(), Mockito.anyMap(), Mockito.anyMap())).thenReturn( + Mockito.when(balancedAlgorithm.needsDrs(Mockito.any(), Mockito.anyMap(), Mockito.anyMap(), Mockito.anyMap())).thenReturn( true, false); Mockito.doReturn(new Pair<>(vm1, host2)).when(clusterDrsService).getBestMigration( @@ -247,7 +251,7 @@ public void testGetDrsPlan() throws ConfigurationException { Mockito.verify(hostDao, Mockito.times(1)).findByClusterId(1L); Mockito.verify(vmInstanceDao, Mockito.times(1)).listByClusterId(1L); Mockito.verify(balancedAlgorithm, Mockito.times(2)).needsDrs(Mockito.any(), Mockito.anyMap(), - Mockito.anyMap()); + Mockito.anyMap(), Mockito.anyMap()); assertEquals(1, iterations.size()); } @@ -311,7 +315,7 @@ public void testGetDrsPlanWithSystemVMs() throws ConfigurationException { Mockito.when(hostDao.findByClusterId(1L)).thenReturn(hostList); Mockito.when(vmInstanceDao.listByClusterId(1L)).thenReturn(vmList); - Mockito.when(balancedAlgorithm.needsDrs(Mockito.any(), Mockito.anyMap(), Mockito.anyMap())).thenReturn(true); + Mockito.when(balancedAlgorithm.needsDrs(Mockito.any(), Mockito.anyMap(), Mockito.anyMap(), Mockito.anyMap())).thenReturn(true); Mockito.when(hostJoinDao.searchByIds(Mockito.any())).thenReturn(List.of(hostJoin1)); List> result = clusterDrsService.getDrsPlan(cluster, 5); @@ -353,7 +357,7 @@ public void testGetDrsPlanWithNonRunningVMs() throws ConfigurationException { Mockito.when(hostDao.findByClusterId(1L)).thenReturn(hostList); Mockito.when(vmInstanceDao.listByClusterId(1L)).thenReturn(vmList); - Mockito.when(balancedAlgorithm.needsDrs(Mockito.any(), Mockito.anyMap(), Mockito.anyMap())).thenReturn(true); + Mockito.when(balancedAlgorithm.needsDrs(Mockito.any(), Mockito.anyMap(), Mockito.anyMap(), Mockito.anyMap())).thenReturn(true); Mockito.when(hostJoinDao.searchByIds(Mockito.any())).thenReturn(List.of(hostJoin1)); List> result = clusterDrsService.getDrsPlan(cluster, 5); @@ -400,7 +404,7 @@ public void testGetDrsPlanWithSkipDrsFlag() throws ConfigurationException { Mockito.when(hostDao.findByClusterId(1L)).thenReturn(hostList); Mockito.when(vmInstanceDao.listByClusterId(1L)).thenReturn(vmList); - Mockito.when(balancedAlgorithm.needsDrs(Mockito.any(), Mockito.anyMap(), Mockito.anyMap())).thenReturn(true); + Mockito.when(balancedAlgorithm.needsDrs(Mockito.any(), Mockito.anyMap(), Mockito.anyMap(), Mockito.anyMap())).thenReturn(true); Mockito.when(hostJoinDao.searchByIds(Mockito.any())).thenReturn(List.of(hostJoin1)); List> result = clusterDrsService.getDrsPlan(cluster, 5); @@ -445,7 +449,7 @@ public void testGetDrsPlanWithNoCompatibleHosts() throws ConfigurationException Mockito.when(hostDao.findByClusterId(1L)).thenReturn(hostList); Mockito.when(vmInstanceDao.listByClusterId(1L)).thenReturn(vmList); - Mockito.when(balancedAlgorithm.needsDrs(Mockito.any(), Mockito.anyMap(), Mockito.anyMap())).thenReturn(true); + Mockito.when(balancedAlgorithm.needsDrs(Mockito.any(), Mockito.anyMap(), Mockito.anyMap(), Mockito.anyMap())).thenReturn(true); Mockito.when(serviceOfferingDao.findByIdIncludingRemoved(Mockito.anyLong(), Mockito.anyLong())).thenReturn(serviceOffering); Mockito.when(hostJoinDao.searchByIds(Mockito.any())).thenReturn(List.of(hostJoin1)); // Return a Ternary with an empty suitable-hosts list to exercise the "no compatible hosts" path @@ -493,7 +497,7 @@ public void testGetDrsPlanWithExceptionInCompatibilityCheck() throws Configurati Mockito.when(hostDao.findByClusterId(1L)).thenReturn(hostList); Mockito.when(vmInstanceDao.listByClusterId(1L)).thenReturn(vmList); - Mockito.when(balancedAlgorithm.needsDrs(Mockito.any(), Mockito.anyMap(), Mockito.anyMap())).thenReturn(true); + Mockito.when(balancedAlgorithm.needsDrs(Mockito.any(), Mockito.anyMap(), Mockito.anyMap(), Mockito.anyMap())).thenReturn(true); Mockito.when(serviceOfferingDao.findByIdIncludingRemoved(Mockito.anyLong(), Mockito.anyLong())).thenReturn(serviceOffering); Mockito.when(hostJoinDao.searchByIds(Mockito.any())).thenReturn(List.of(hostJoin1)); // Throw an explicit exception so the catch-and-log path is exercised intentionally @@ -542,7 +546,7 @@ public void testGetDrsPlanWithNoBestMigration() throws ConfigurationException { Mockito.when(hostDao.findByClusterId(1L)).thenReturn(hostList); Mockito.when(vmInstanceDao.listByClusterId(1L)).thenReturn(vmList); - Mockito.when(balancedAlgorithm.needsDrs(Mockito.any(), Mockito.anyMap(), Mockito.anyMap())).thenReturn(true); + Mockito.when(balancedAlgorithm.needsDrs(Mockito.any(), Mockito.anyMap(), Mockito.anyMap(), Mockito.anyMap())).thenReturn(true); Mockito.when(serviceOfferingDao.findByIdIncludingRemoved(Mockito.anyLong(), Mockito.anyLong())).thenReturn(serviceOffering); Mockito.when(hostJoinDao.searchByIds(Mockito.any())).thenReturn(List.of(hostJoin1)); @@ -617,7 +621,7 @@ public void testGetDrsPlanWithMultipleIterations() throws ConfigurationException Mockito.when(hostDao.findByClusterId(1L)).thenReturn(hostList); Mockito.when(vmInstanceDao.listByClusterId(1L)).thenReturn(vmList); - Mockito.when(balancedAlgorithm.needsDrs(Mockito.any(), Mockito.anyMap(), Mockito.anyMap())).thenReturn( + Mockito.when(balancedAlgorithm.needsDrs(Mockito.any(), Mockito.anyMap(), Mockito.anyMap(), Mockito.anyMap())).thenReturn( true, true, false); Mockito.when(serviceOfferingDao.findByIdIncludingRemoved(Mockito.anyLong(), Mockito.anyLong())).thenReturn(serviceOffering); Mockito.when(hostJoinDao.searchByIds(1L, 2L)).thenReturn(List.of(hostJoin1, hostJoin2)); @@ -631,7 +635,7 @@ public void testGetDrsPlanWithMultipleIterations() throws ConfigurationException List> result = clusterDrsService.getDrsPlan(cluster, 5); assertEquals(2, result.size()); - Mockito.verify(balancedAlgorithm, Mockito.times(3)).needsDrs(Mockito.any(), Mockito.anyMap(), Mockito.anyMap()); + Mockito.verify(balancedAlgorithm, Mockito.times(3)).needsDrs(Mockito.any(), Mockito.anyMap(), Mockito.anyMap(), Mockito.anyMap()); } @Test @@ -663,7 +667,7 @@ public void testGetDrsPlanWithMigrationToOriginalHost() throws ConfigurationExce Mockito.when(hostDao.findByClusterId(1L)).thenReturn(hostList); Mockito.when(vmInstanceDao.listByClusterId(1L)).thenReturn(vmList); - Mockito.when(balancedAlgorithm.needsDrs(Mockito.any(), Mockito.anyMap(), Mockito.anyMap())).thenReturn(true); + Mockito.when(balancedAlgorithm.needsDrs(Mockito.any(), Mockito.anyMap(), Mockito.anyMap(), Mockito.anyMap())).thenReturn(true); Mockito.when(serviceOfferingDao.findByIdIncludingRemoved(Mockito.anyLong(), Mockito.anyLong())).thenReturn(serviceOffering); // Return migration to original host (host1) - should break the loop From b5b1c1dd5aad908d67c16f37a44898d2e71a0e6a Mon Sep 17 00:00:00 2001 From: Brad House Date: Thu, 10 Sep 2026 00:25:42 +0000 Subject: [PATCH 15/18] Verify equivalent VMs share one candidate host lookup The grouping could have been disabled and every test would still have passed. Asserts the expensive call is made once for two interchangeable VMs. Signed-off-by: Brad House --- .../cluster/ClusterDrsServiceImplTest.java | 55 ++++++++++++++++++- 1 file changed, 54 insertions(+), 1 deletion(-) diff --git a/server/src/test/java/org/apache/cloudstack/cluster/ClusterDrsServiceImplTest.java b/server/src/test/java/org/apache/cloudstack/cluster/ClusterDrsServiceImplTest.java index 92e1c45599e2..8b3eb1c2efc9 100644 --- a/server/src/test/java/org/apache/cloudstack/cluster/ClusterDrsServiceImplTest.java +++ b/server/src/test/java/org/apache/cloudstack/cluster/ClusterDrsServiceImplTest.java @@ -1139,4 +1139,57 @@ public void testNonStrictAntiAffinityIsHonoured() { assertFalse("a preferred host must stay available", excludes.getHostsToAvoid().contains(31L)); } -} + + @Test + public void testEquivalentVmsShareOneCandidateHostLookup() throws ConfigurationException { + // the expensive call is listHostsForMigrationOfVM. Two interchangeable VMs must cost one + // pass, not two - otherwise the grouping is not actually doing anything. + ClusterVO cluster = Mockito.mock(ClusterVO.class); + Mockito.when(cluster.getId()).thenReturn(1L); + Mockito.when(cluster.getAllocationState()).thenReturn(Grouping.AllocationState.Enabled); + + HostVO host1 = Mockito.mock(HostVO.class); + Mockito.when(host1.getId()).thenReturn(1L); + + List vmList = new ArrayList<>(); + for (long id : new long[] {1L, 2L}) { + VMInstanceVO vm = Mockito.mock(VMInstanceVO.class); + Mockito.when(vm.getId()).thenReturn(id); + Mockito.when(vm.getHostId()).thenReturn(1L); + Mockito.when(vm.getType()).thenReturn(VirtualMachine.Type.User); + Mockito.when(vm.getState()).thenReturn(VirtualMachine.State.Running); + Mockito.when(vm.getServiceOfferingId()).thenReturn(9L); + Mockito.lenient().when(vm.getTemplateId()).thenReturn(8L); + vmList.add(vm); + } + + ServiceOfferingVO offering = Mockito.mock(ServiceOfferingVO.class); + Mockito.when(offering.isDynamic()).thenReturn(false); + Mockito.when(serviceOfferingDao.findByIdIncludingRemoved(Mockito.anyLong(), Mockito.anyLong())) + .thenReturn(offering); + Mockito.when(vmInstanceDetailsDao.listDetailsKeyPairs(Mockito.anyLong())) + .thenReturn(Collections.emptyMap()); + Mockito.when(volumeDao.findCreatedByInstance(Mockito.anyLong())).thenReturn(Collections.emptyList()); + + HostJoinVO hostJoin1 = Mockito.mock(HostJoinVO.class); + Mockito.when(hostJoin1.getId()).thenReturn(1L); + Mockito.when(hostJoin1.getCpus()).thenReturn(4); + Mockito.when(hostJoin1.getSpeed()).thenReturn(1000L); + Mockito.when(hostJoin1.getTotalMemory()).thenReturn(8192L); + + Mockito.when(hostDao.findByClusterId(1L)).thenReturn(List.of(host1)); + Mockito.when(vmInstanceDao.listByClusterId(1L)).thenReturn(vmList); + Mockito.when(hostJoinDao.searchByIds(Mockito.any())).thenReturn(List.of(hostJoin1)); + Mockito.when(balancedAlgorithm.needsDrs(Mockito.any(), Mockito.anyMap(), Mockito.anyMap(), Mockito.anyMap())) + .thenReturn(false); + Mockito.when(managementServer.listHostsForMigrationOfVM(Mockito.any(), Mockito.anyLong(), + Mockito.anyLong(), Mockito.any(), Mockito.anyList())) + .thenReturn(new Ternary<>(new Pair<>(Collections.emptyList(), 0), + List.of(host1), Collections.emptyMap())); + + clusterDrsService.getDrsPlan(cluster, 5); + + Mockito.verify(managementServer, Mockito.times(1)).listHostsForMigrationOfVM( + Mockito.any(), Mockito.anyLong(), Mockito.anyLong(), Mockito.any(), Mockito.anyList()); + } +} \ No newline at end of file From a6853a0bc8d445f0f80355514123fb6b1cdaa8ed Mon Sep 17 00:00:00 2001 From: Brad House Date: Thu, 10 Sep 2026 12:21:28 +0000 Subject: [PATCH 16/18] Declare the host load tracker where both its consumers can see it It was declared alongside the allocators, in a module context that is a descendant of the one the DRS service lives in. A bean in a descendant context is not visible to its parent, so injecting it into the DRS service failed and the management server did not start. Unit tests cannot catch this - the field is a mock there. It showed up as every integration test job running to its six hour limit waiting for a management server that was never coming up. Declared in the core context instead, which is an ancestor of the allocator module and the same context the DRS service is declared in. Also add the trailing newlines pre-commit asked for. Signed-off-by: Brad House --- .../java/org/apache/cloudstack/cluster/WeightedTest.java | 2 +- .../core/spring-server-core-managers-context.xml | 7 +++++++ .../server-allocator/spring-server-allocator-context.xml | 3 --- .../cloudstack/cluster/ClusterDrsEquivalenceTest.java | 2 +- .../cloudstack/cluster/ClusterDrsServiceImplTest.java | 2 +- 5 files changed, 10 insertions(+), 6 deletions(-) diff --git a/plugins/drs/cluster/weighted/src/test/java/org/apache/cloudstack/cluster/WeightedTest.java b/plugins/drs/cluster/weighted/src/test/java/org/apache/cloudstack/cluster/WeightedTest.java index e45dba7d3103..9a72df3ea51e 100644 --- a/plugins/drs/cluster/weighted/src/test/java/org/apache/cloudstack/cluster/WeightedTest.java +++ b/plugins/drs/cluster/weighted/src/test/java/org/apache/cloudstack/cluster/WeightedTest.java @@ -280,4 +280,4 @@ public void testStorageMotionMustEarnMoreThanAPlainMigration() throws Configurat assertFalse("the same small gain is not worth moving storage for", withStorage.third() > withStorage.second()); } -} \ No newline at end of file +} diff --git a/server/src/main/resources/META-INF/cloudstack/core/spring-server-core-managers-context.xml b/server/src/main/resources/META-INF/cloudstack/core/spring-server-core-managers-context.xml index c0bcba44c642..60c837f0afd9 100644 --- a/server/src/main/resources/META-INF/cloudstack/core/spring-server-core-managers-context.xml +++ b/server/src/main/resources/META-INF/cloudstack/core/spring-server-core-managers-context.xml @@ -370,6 +370,13 @@ value="#{affinityProcessorsRegistry.registered}" /> + + + diff --git a/server/src/main/resources/META-INF/cloudstack/server-allocator/spring-server-allocator-context.xml b/server/src/main/resources/META-INF/cloudstack/server-allocator/spring-server-allocator-context.xml index 69aea2cf8451..3eaf9bb90bc5 100644 --- a/server/src/main/resources/META-INF/cloudstack/server-allocator/spring-server-allocator-context.xml +++ b/server/src/main/resources/META-INF/cloudstack/server-allocator/spring-server-allocator-context.xml @@ -31,9 +31,6 @@ - - diff --git a/server/src/test/java/org/apache/cloudstack/cluster/ClusterDrsEquivalenceTest.java b/server/src/test/java/org/apache/cloudstack/cluster/ClusterDrsEquivalenceTest.java index 0d596db0a387..7a3dc0fdae2f 100644 --- a/server/src/test/java/org/apache/cloudstack/cluster/ClusterDrsEquivalenceTest.java +++ b/server/src/test/java/org/apache/cloudstack/cluster/ClusterDrsEquivalenceTest.java @@ -184,4 +184,4 @@ public void testAVmWithNoDetailsIsStillGrouped() { assertEquals(service.migrationEquivalenceKey(vm(1L, 10L, 20L, 30L, 40L)), service.migrationEquivalenceKey(vm(2L, 10L, 20L, 30L, 40L))); } -} \ No newline at end of file +} diff --git a/server/src/test/java/org/apache/cloudstack/cluster/ClusterDrsServiceImplTest.java b/server/src/test/java/org/apache/cloudstack/cluster/ClusterDrsServiceImplTest.java index 8b3eb1c2efc9..e78f489eab63 100644 --- a/server/src/test/java/org/apache/cloudstack/cluster/ClusterDrsServiceImplTest.java +++ b/server/src/test/java/org/apache/cloudstack/cluster/ClusterDrsServiceImplTest.java @@ -1192,4 +1192,4 @@ public void testEquivalentVmsShareOneCandidateHostLookup() throws ConfigurationE Mockito.verify(managementServer, Mockito.times(1)).listHostsForMigrationOfVM( Mockito.any(), Mockito.anyLong(), Mockito.anyLong(), Mockito.any(), Mockito.anyList()); } -} \ No newline at end of file +} From 685538bf6b260eae85954a6b1e4666523b41f2d2 Mon Sep 17 00:00:00 2001 From: Brad House Date: Thu, 10 Sep 2026 12:30:52 +0000 Subject: [PATCH 17/18] Re-check every migration, not only VMs in an affinity group Review feedback on the PR. applyAffinityConstraints also applies DPDK and dedicated-resource exclusions, and neither depends on affinity group membership. Returning early for VMs with no groups therefore skipped those checks, which the method's own documentation said it covered. The cost is one call per planned migration, bounded by drs.max.migrations, at execution time rather than in any hot loop. Also move executeDrsPlan's javadoc back onto executeDrsPlan; it was left stranded above the method inserted before it. Signed-off-by: Brad House --- .../cluster/ClusterDrsServiceImpl.java | 22 +++++++++---------- .../cluster/ClusterDrsServiceImplTest.java | 11 +++++++--- 2 files changed, 18 insertions(+), 15 deletions(-) diff --git a/server/src/main/java/org/apache/cloudstack/cluster/ClusterDrsServiceImpl.java b/server/src/main/java/org/apache/cloudstack/cluster/ClusterDrsServiceImpl.java index b7aaf96809f1..7862de7551ed 100644 --- a/server/src/main/java/org/apache/cloudstack/cluster/ClusterDrsServiceImpl.java +++ b/server/src/main/java/org/apache/cloudstack/cluster/ClusterDrsServiceImpl.java @@ -911,13 +911,6 @@ void processPlans() { } } - /** - * Executes the DRS plan by migrating virtual machines to their destination hosts. - * If there are no migrations to be executed, the plan is marked as completed. - * - * @param plan - * the DRS plan to be executed - */ /** * Checks a planned migration against the placement rules as they stand now. * @@ -926,8 +919,9 @@ void processPlans() { * against current placements, and nothing downstream re-checks it - migrateVirtualMachine does * not enforce affinity groups. * - * The processors also cover dedicated resources and DPDK, so a refusal is not necessarily about - * an affinity group. + * Checked for every VM, not only those in an affinity group: applyAffinityConstraints also + * applies DPDK and dedicated-resource exclusions, which apply regardless of group membership. + * A refusal here is therefore not necessarily about an affinity group. * * @param vm * the VM the plan wants to move @@ -945,9 +939,6 @@ protected boolean destinationViolatesAffinity(VirtualMachine vm, Host destHost, logger.debug("VM {} is no longer running, so its planned migration is out of date", vm); return true; } - if (CollectionUtils.isEmpty(affinityGroupVMMapDao.listByInstanceId(vm.getId()))) { - return false; - } if (dispatchedSourceHosts.contains(destHost.getId())) { logger.debug("Host {} is still occupied by a VM whose migration away from it is only queued", destHost); return true; @@ -961,6 +952,13 @@ protected boolean destinationViolatesAffinity(VirtualMachine vm, Host destHost, return excludes.shouldAvoid(destHost); } + /** + * Executes the DRS plan by migrating virtual machines to their destination hosts. + * If there are no migrations to be executed, the plan is marked as completed. + * + * @param plan + * the DRS plan to be executed + */ void executeDrsPlan(ClusterDrsPlanVO plan) { List planMigrations = drsPlanMigrationDao.listPlanMigrationsToExecute(plan.getId()); if (planMigrations == null || planMigrations.isEmpty()) { diff --git a/server/src/test/java/org/apache/cloudstack/cluster/ClusterDrsServiceImplTest.java b/server/src/test/java/org/apache/cloudstack/cluster/ClusterDrsServiceImplTest.java index e78f489eab63..a0084ae47a1c 100644 --- a/server/src/test/java/org/apache/cloudstack/cluster/ClusterDrsServiceImplTest.java +++ b/server/src/test/java/org/apache/cloudstack/cluster/ClusterDrsServiceImplTest.java @@ -1001,15 +1001,20 @@ private void affinityExcludes(Long... hostIds) { } @Test - public void testDestinationAllowedWhenVmHasNoAffinityGroups() { + public void testAVmWithNoAffinityGroupsIsStillRechecked() { + // applyAffinityConstraints also applies DPDK and dedicated-resource exclusions, which do + // not depend on affinity group membership, so every VM has to go through it VMInstanceVO vm = Mockito.mock(VMInstanceVO.class); Mockito.when(vm.getId()).thenReturn(1L); Mockito.when(vm.getHostId()).thenReturn(10L); - Mockito.when(affinityGroupVMMapDao.listByInstanceId(1L)).thenReturn(Collections.emptyList()); + Mockito.when(vm.getServiceOfferingId()).thenReturn(5L); + Mockito.when(serviceOfferingDao.findByIdIncludingRemoved(1L, 5L)) + .thenReturn(Mockito.mock(ServiceOfferingVO.class)); + affinityExcludes(21L); assertFalse(clusterDrsService.destinationViolatesAffinity(vm, host(20L), Collections.emptyList(), Collections.emptyList())); - Mockito.verify(managementServer, Mockito.never()) + Mockito.verify(managementServer) .applyAffinityConstraints(Mockito.any(), Mockito.any(), Mockito.any(), Mockito.any()); } From 63f5931e266fa1e46d9295e1769c57a7d86b4515 Mon Sep 17 00:00:00 2001 From: Brad House Date: Thu, 10 Sep 2026 12:37:22 +0000 Subject: [PATCH 18/18] Address review feedback on the weighted allocation algorithm vm_instance.update_time is a TIMESTAMP column, so there is no date that reliably means "never" - anything past 2038 is out of range. Counting with no cut-off used Long.MAX_VALUE, which is roughly year 292 million. The query now leaves the timestamp out entirely in that case rather than binding an impossible one. Also catch Exception rather than Throwable in the new query, and put the config key array one entry per line. Signed-off-by: Brad House --- .../com/cloud/vm/dao/VMInstanceDaoImpl.java | 15 +++++++++------ .../allocator/impl/WeightedHostScorer.java | 17 ++++++++++++++--- 2 files changed, 23 insertions(+), 9 deletions(-) diff --git a/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDaoImpl.java b/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDaoImpl.java index f553c3f6dca9..87135c035410 100755 --- a/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDaoImpl.java @@ -155,7 +155,10 @@ public class VMInstanceDaoImpl extends GenericDaoBase implem private static final String COUNT_VMS_BASED_ON_VGPU_TYPES2 = "GROUP BY gpu_card.name, vgpu_profile.name"; - private static final String COUNT_VMS_BY_HOST = "SELECT host.id, COUNT(vm.id), SUM(IF(vm.update_time > ?, 1, 0)) " + + // %s is the "changed state recently" test, or a constant 0 when no cut-off is given. It is not + // a bound parameter because there is no timestamp that reliably means "never" - update_time is + // a TIMESTAMP column, so anything past 2038 is out of range. + private static final String COUNT_VMS_BY_HOST = "SELECT host.id, COUNT(vm.id), SUM(IF(%s, 1, 0)) " + "FROM `cloud`.`host` host LEFT JOIN `cloud`.`vm_instance` vm " + "ON vm.host_id = host.id AND vm.state IN ('Running', 'Starting', 'Stopping', 'Migrating') " + "AND vm.removed IS NULL WHERE host.type = 'Routing' AND host.removed IS NULL AND host.data_center_id = ? "; @@ -807,7 +810,7 @@ public Pair, Map> listPodIdsInZoneByVmCount(long dataCe public Map> countVmsByHost(long dcId, Long podId, Long clusterId, Date changedStateAfter) { TransactionLegacy txn = TransactionLegacy.currentTxn(); Map> result = new HashMap<>(); - String sql = COUNT_VMS_BY_HOST; + String sql = String.format(COUNT_VMS_BY_HOST, changedStateAfter != null ? "vm.update_time > ?" : "0"); if (podId != null) { sql = sql + " AND host.pod_id = ? "; } @@ -818,9 +821,9 @@ public Map> countVmsByHost(long dcId, Long podId, Long cl try { PreparedStatement pstmt = txn.prepareAutoCloseStatement(sql); int index = 1; - // a cut-off in the future counts nothing as recent, which is what a null asks for - long cutOff = changedStateAfter != null ? changedStateAfter.getTime() : Long.MAX_VALUE; - pstmt.setTimestamp(index++, new Timestamp(cutOff)); + if (changedStateAfter != null) { + pstmt.setTimestamp(index++, new Timestamp(changedStateAfter.getTime())); + } pstmt.setLong(index++, dcId); if (podId != null) { pstmt.setLong(index++, podId); @@ -835,7 +838,7 @@ public Map> countVmsByHost(long dcId, Long podId, Long cl return result; } catch (SQLException e) { throw new CloudRuntimeException("DB Exception on: " + sql, e); - } catch (Throwable e) { + } catch (Exception e) { throw new CloudRuntimeException("Caught: " + sql, e); } } diff --git a/server/src/main/java/com/cloud/agent/manager/allocator/impl/WeightedHostScorer.java b/server/src/main/java/com/cloud/agent/manager/allocator/impl/WeightedHostScorer.java index 1b537f9068d9..44d3b300caa7 100644 --- a/server/src/main/java/com/cloud/agent/manager/allocator/impl/WeightedHostScorer.java +++ b/server/src/main/java/com/cloud/agent/manager/allocator/impl/WeightedHostScorer.java @@ -400,8 +400,19 @@ public String getConfigComponentName() { @Override public ConfigKey[] getConfigKeys() { - return new ConfigKey[] {HostScoringWeights.CpuAllocatedWeight, HostScoringWeights.CpuUsedWeight, HostScoringWeights.MemoryAllocatedWeight, HostScoringWeights.MemoryUsedWeight, - VmCountWeight, RecentStartWeight, DominantResourceWeight, RecentStartWindow, ExpectedVmsPerHost, - CpuUtilisationThreshold, MemoryUtilisationThreshold, SelectionSpread}; + return new ConfigKey[] { + HostScoringWeights.CpuAllocatedWeight, + HostScoringWeights.CpuUsedWeight, + HostScoringWeights.MemoryAllocatedWeight, + HostScoringWeights.MemoryUsedWeight, + VmCountWeight, + RecentStartWeight, + DominantResourceWeight, + RecentStartWindow, + ExpectedVmsPerHost, + CpuUtilisationThreshold, + MemoryUtilisationThreshold, + SelectionSpread + }; } }