diff --git a/PendingReleaseNotes b/PendingReleaseNotes index 02d63811e36e..83240f4a359a 100644 --- a/PendingReleaseNotes +++ b/PendingReleaseNotes @@ -53,3 +53,49 @@ example.ver.1 > example.ver.2: the guest while the NVRAM sidecar is copied, so that the captured firmware state is consistent with the disk snapshot. Non-UEFI VMs are unaffected and continue to snapshot live. + + * New VM allocation algorithm 'balancedweighted' for vm.allocation.algorithm. Existing + algorithms and the default are unchanged; the new one is opt-in. + + Existing algorithms rank hosts on allocated capacity alone. Under a large overprovisioning + factor that reads badly: allocation is measured against a total that has already been + multiplied by the factor, so a host under real strain can still report a low percentage and + keep attracting new VMs. Concurrent deployments make it worse, since they all read the same + figures before any of them is accounted for. + + 'balancedweighted' ranks hosts on a blend of allocated CPU and memory, measured CPU and + memory utilisation, VM count, and how many VMs started on the host recently. It holds back + hosts that are measurably too busy, and chooses at random from among the best scoring hosts + so that simultaneous deployments do not all pick the same one. + + Tuned with the host.weighted.* settings, most of which are cluster scoped. Measured + utilisation comes from a moving average configured with host.load.sample.interval and + host.load.half.life. Until a management server has collected samples, ranking falls back to + allocation figures alone. + + * New DRS algorithm 'weighted' for drs.algorithm, alongside 'balanced' and 'condensed'. The + default is unchanged; the new one is opt-in per cluster. + + The existing algorithms balance a single metric chosen by drs.metric, so choosing one leaves + the other unwatched: a cluster can be even on memory while its CPU load varies several fold and + nothing moves. Allocation is also a poor stand-in for load under overprovisioning, where a + saturated host can still report a small percentage allocated. + + 'weighted' blends CPU and memory allocated with CPU and memory in use, and balances the result. + Imbalance keeps its existing shape - standard deviation over the mean - but it is computed over + a blend rather than over one metric, so drs.imbalance is not calibrated the same way and is + worth re-checking after switching. drs.metric, drs.metric.type and drs.metric.use.ratio choose + and shape the single metric the other algorithms balance; they do not apply to 'weighted' and + are ignored. + + Utilisation is only used when every host in the cluster has been sampled. Comparing a host + measured on utilisation against one measured on allocation alone would report a difference that + is an artefact of the monitoring rather than of the load, so the cluster falls back to + allocation figures until every host can be measured. + + Tuned with the drs.weighted.* settings, all cluster scoped. + + * DRS plan generation is considerably faster on large clusters. Working out where a VM could go + is now done once for each group of VMs that would get the same answer rather than once per VM, + and affinity is re-evaluated only for VMs affected by the migration just planned rather than + for every VM on every iteration. diff --git a/api/src/main/java/com/cloud/deploy/DeploymentClusterPlanner.java b/api/src/main/java/com/cloud/deploy/DeploymentClusterPlanner.java index 9471c3d5c84c..5bf7dffbbf9b 100644 --- a/api/src/main/java/com/cloud/deploy/DeploymentClusterPlanner.java +++ b/api/src/main/java/com/cloud/deploy/DeploymentClusterPlanner.java @@ -62,11 +62,14 @@ public interface DeploymentClusterPlanner extends DeploymentPlanner { "vm.allocation.algorithm", "Advanced", "random", - "Order in which hosts within a cluster will be considered for VM allocation. The value can be 'random', 'firstfit', 'userdispersing', or 'firstfitleastconsumed'.", + "Order in which hosts within a cluster will be considered for VM allocation. The value can be 'random', " + + "'firstfit', 'userdispersing', 'firstfitleastconsumed', or 'balancedweighted'. 'balancedweighted' " + + "ranks hosts on a blend of allocated capacity, measured utilisation, VM count and how many VMs " + + "started recently, and is tuned with the host.weighted.* settings.", true, ConfigKey.Scope.Global, null, null, null, null, null, ConfigKey.Kind.Select, - "random,firstfit,userdispersing,firstfitleastconsumed"); + "random,firstfit,userdispersing,firstfitleastconsumed,balancedweighted"); /** * This is called to determine list of possible clusters where a virtual diff --git a/api/src/main/java/com/cloud/deploy/DeploymentPlanner.java b/api/src/main/java/com/cloud/deploy/DeploymentPlanner.java index 22d796d4a775..2d2964916eed 100644 --- a/api/src/main/java/com/cloud/deploy/DeploymentPlanner.java +++ b/api/src/main/java/com/cloud/deploy/DeploymentPlanner.java @@ -70,7 +70,7 @@ public interface DeploymentPlanner extends Adapter { boolean canHandle(VirtualMachineProfile vm, DeploymentPlan plan, ExcludeList avoid); public enum AllocationAlgorithm { - random, firstfit, userdispersing, firstfitleastconsumed; + random, firstfit, userdispersing, firstfitleastconsumed, balancedweighted; } public enum PlannerResourceUsage { diff --git a/api/src/main/java/com/cloud/host/HostLoad.java b/api/src/main/java/com/cloud/host/HostLoad.java new file mode 100644 index 000000000000..a2257c51bccb --- /dev/null +++ b/api/src/main/java/com/cloud/host/HostLoad.java @@ -0,0 +1,60 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.host; + +/** + * Smoothed view of what a host is actually doing, as opposed to what has been allocated on it. + * Fractions are of the host's real capacity and ignore overprovisioning. + */ +public class HostLoad { + + public static final HostLoad UNKNOWN = new HostLoad(0, 0, 0); + + private final double cpuUtilisation; + private final double memoryUtilisation; + private final long samples; + + public HostLoad(double cpuUtilisation, double memoryUtilisation, long samples) { + this.cpuUtilisation = cpuUtilisation; + this.memoryUtilisation = memoryUtilisation; + this.samples = samples; + } + + public double getCpuUtilisation() { + return cpuUtilisation; + } + + public double getMemoryUtilisation() { + return memoryUtilisation; + } + + public long getSamples() { + return samples; + } + + /** + * False until enough has been observed to rank on. Callers fall back to allocation figures. + */ + public boolean isUsable() { + return samples > 0; + } + + @Override + public String toString() { + return String.format("HostLoad[cpu=%.3f, memory=%.3f, samples=%d]", cpuUtilisation, memoryUtilisation, samples); + } +} diff --git a/api/src/main/java/com/cloud/host/HostLoadService.java b/api/src/main/java/com/cloud/host/HostLoadService.java new file mode 100644 index 000000000000..d8e9e88cb27a --- /dev/null +++ b/api/src/main/java/com/cloud/host/HostLoadService.java @@ -0,0 +1,32 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.host; + +/** + * A smoothed view of what each host is actually doing, as opposed to what has been allocated on it. + * + * Allocation figures say what was promised; under overprovisioning they can be far from what a host + * is really carrying. Placement and rebalancing both need the second view. + */ +public interface HostLoadService { + + /** + * @return the host's smoothed load, or a value reporting itself unusable when the host has not + * been sampled recently enough to rank on + */ + HostLoad getLoad(long hostId); +} diff --git a/api/src/main/java/com/cloud/host/HostScoringWeights.java b/api/src/main/java/com/cloud/host/HostScoringWeights.java new file mode 100644 index 000000000000..886487aeedf6 --- /dev/null +++ b/api/src/main/java/com/cloud/host/HostScoringWeights.java @@ -0,0 +1,57 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.host; + +import org.apache.cloudstack.framework.config.ConfigKey; + +/** + * How much each signal counts when ranking hosts by how loaded they are. + * + * Shared by initial placement and by rebalancing on purpose. If the two weighted these differently + * they would disagree about which host is the better one, and rebalancing could move VMs off hosts + * that placement had just chosen, only for placement to put them back. + * + * Weights are relative to each other; only their ratios matter, and zero disables a term. Terms that + * only make sense for one of the two - how many VMs a host carries, how many started recently - stay + * with whichever uses them. + */ +public interface HostScoringWeights { + + String WEIGHT_DESCRIPTION_SUFFIX = " Relative weight, only meaningful compared with the other " + + "host.weighted.* weights. Zero disables the term. Used by both the 'balancedweighted' " + + "allocation algorithm and the 'weighted' DRS algorithm."; + + ConfigKey CpuAllocatedWeight = new ConfigKey<>(ConfigKey.CATEGORY_ADVANCED, + Double.class, "host.weighted.cpu.allocated.weight", "1.0", + "How much CPU allocated on a host counts against it." + WEIGHT_DESCRIPTION_SUFFIX, + true, ConfigKey.Scope.Cluster); + + ConfigKey CpuUsedWeight = new ConfigKey<>(ConfigKey.CATEGORY_ADVANCED, + Double.class, "host.weighted.cpu.used.weight", "2.0", + "How much measured CPU utilisation counts against a host." + WEIGHT_DESCRIPTION_SUFFIX, + true, ConfigKey.Scope.Cluster); + + ConfigKey MemoryAllocatedWeight = new ConfigKey<>(ConfigKey.CATEGORY_ADVANCED, + Double.class, "host.weighted.memory.allocated.weight", "1.0", + "How much memory allocated on a host counts against it." + WEIGHT_DESCRIPTION_SUFFIX, + true, ConfigKey.Scope.Cluster); + + ConfigKey MemoryUsedWeight = new ConfigKey<>(ConfigKey.CATEGORY_ADVANCED, + Double.class, "host.weighted.memory.used.weight", "2.0", + "How much measured memory utilisation counts against a host." + WEIGHT_DESCRIPTION_SUFFIX, + true, ConfigKey.Scope.Cluster); +} diff --git a/api/src/main/java/org/apache/cloudstack/affinity/AffinityProcessorBase.java b/api/src/main/java/org/apache/cloudstack/affinity/AffinityProcessorBase.java index 96ca35f264ca..b330b6aa321c 100644 --- a/api/src/main/java/org/apache/cloudstack/affinity/AffinityProcessorBase.java +++ b/api/src/main/java/org/apache/cloudstack/affinity/AffinityProcessorBase.java @@ -25,7 +25,9 @@ import com.cloud.vm.VirtualMachineProfile; import java.util.Collections; +import java.util.HashMap; import java.util.List; +import java.util.Map; public class AffinityProcessorBase extends AdapterBase implements AffinityGroupProcessor { @@ -44,6 +46,23 @@ public void process(VirtualMachineProfile vm, DeploymentPlan plan, ExcludeList a } + + /** + * Indexes placements supplied by the caller. Callers such as DRS build a plan of several moves + * in memory and persist it only at the end, so during planning the database still shows the old + * host for every VM the plan has already moved. + */ + protected Map getVmIdVmMap(List vmList) { + Map vmIdVmMap = new HashMap<>(); + if (vmList == null) { + return vmIdVmMap; + } + for (VirtualMachine vm : vmList) { + vmIdVmMap.put(vm.getId(), vm); + } + return vmIdVmMap; + } + @Override public String getType() { return _type; diff --git a/api/src/main/java/org/apache/cloudstack/cluster/ClusterDrsAlgorithm.java b/api/src/main/java/org/apache/cloudstack/cluster/ClusterDrsAlgorithm.java index 368487c2b9b0..3c8d89dad15e 100644 --- a/api/src/main/java/org/apache/cloudstack/cluster/ClusterDrsAlgorithm.java +++ b/api/src/main/java/org/apache/cloudstack/cluster/ClusterDrsAlgorithm.java @@ -20,6 +20,7 @@ package org.apache.cloudstack.cluster; import com.cloud.host.Host; +import com.cloud.host.HostLoad; import com.cloud.offering.ServiceOffering; import com.cloud.org.Cluster; import com.cloud.utils.Ternary; @@ -62,6 +63,35 @@ public interface ClusterDrsAlgorithm extends Adapter { boolean needsDrs(Cluster cluster, List> cpuList, List> memoryList) throws ConfigurationException; + /** + * As above, but keeping host identity. + * + * An algorithm that considers anything beyond the figures in the two maps - measured load, for + * instance - cannot attribute it without knowing which host each entry belongs to. Algorithms + * that only need the values keep the default. + * + * @param hostCpuMap + * host id to a Ternary of used, reserved and total CPU + * @param hostMemoryMap + * host id to a Ternary of used, reserved and total memory + */ + default boolean needsDrs(Cluster cluster, Map> hostCpuMap, + Map> hostMemoryMap, Map hostLoadMap) + throws ConfigurationException { + return needsDrs(cluster, new ArrayList<>(hostCpuMap.values()), new ArrayList<>(hostMemoryMap.values())); + } + + /** + * Called once per plan, before any migration is considered, so that an algorithm can do work + * that would otherwise be repeated for every candidate VM and host. + * + * @param hostLoadMap + * measured load per host, empty when nothing has been sampled + */ + default void prepare(Cluster cluster, Map> hostCpuMap, + Map> hostMemoryMap, Map hostLoadMap) { + } + /** * Calculates the metrics (improvement, cost, benefit) for migrating a VM to a destination host. Improvement is * calculated based on the change in cluster imbalance before and after the migration. diff --git a/api/src/main/java/org/apache/cloudstack/cluster/ClusterDrsService.java b/api/src/main/java/org/apache/cloudstack/cluster/ClusterDrsService.java index ba6a6464fc20..7323f7fb4916 100644 --- a/api/src/main/java/org/apache/cloudstack/cluster/ClusterDrsService.java +++ b/api/src/main/java/org/apache/cloudstack/cluster/ClusterDrsService.java @@ -51,9 +51,9 @@ public interface ClusterDrsService extends Manager, Configurable, Scheduler { true, ConfigKey.Scope.Cluster, null, "Maximum number of migrations for DRS", null, null, null); ConfigKey ClusterDrsAlgorithm = new ConfigKey<>(String.class, "drs.algorithm", - ConfigKey.CATEGORY_ADVANCED, "balanced", "The DRS algorithm to be executed on the cluster. Possible values are condensed, balanced.", + ConfigKey.CATEGORY_ADVANCED, "balanced", "The DRS algorithm to be executed on the cluster. Possible values are condensed, balanced, weighted. 'weighted' balances CPU and memory together, and on measured load as well as allocation, tuned with the drs.weighted.* settings.", true, ConfigKey.Scope.Cluster, null, "DRS algorithm", null, null, - null, ConfigKey.Kind.Select, "condensed,balanced"); + null, ConfigKey.Kind.Select, "condensed,balanced,weighted"); ConfigKey ClusterDrsImbalanceThreshold = new ConfigKey<>(Float.class, "drs.imbalance", ConfigKey.CATEGORY_ADVANCED, "0.4", diff --git a/client/pom.xml b/client/pom.xml index cc031a4912b1..ebbf812823ca 100644 --- a/client/pom.xml +++ b/client/pom.xml @@ -592,6 +592,11 @@ cloud-plugin-cluster-drs-condensed ${project.version} + + org.apache.cloudstack + cloud-plugin-cluster-drs-weighted + ${project.version} + org.apache.cloudstack cloud-plugin-database-quota diff --git a/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDao.java b/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDao.java index 1a5b8cedd9ea..0b5fb4c64f9b 100755 --- a/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDao.java +++ b/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDao.java @@ -131,6 +131,18 @@ public interface VMInstanceDao extends GenericDao, StateDao< List listHostIdsByVmCount(long dcId, Long podId, Long clusterId, long accountId); + /** + * Counts the VMs occupying each host in a zone, pod or cluster, in one query. + * + * @param changedStateAfter + * cut-off for the second count: VMs whose state last changed after this. Approximates + * the VMs still working through their startup load, which neither allocation nor a + * utilisation average has caught up with yet. + * @return host id to {total VMs, VMs that changed state recently}. Every host in scope appears, + * including hosts with no VMs. + */ + Map> countVmsByHost(long dcId, Long podId, Long clusterId, Date changedStateAfter); + Long countRunningAndStartingByAccount(long accountId); Long countByZoneAndState(long zoneId, State state); diff --git a/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDaoImpl.java b/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDaoImpl.java index ae1e838649ba..87135c035410 100755 --- a/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/vm/dao/VMInstanceDaoImpl.java @@ -18,6 +18,7 @@ import java.sql.PreparedStatement; import java.sql.ResultSet; +import java.sql.Timestamp; import java.sql.SQLException; import java.util.ArrayList; import java.util.Collection; @@ -154,6 +155,15 @@ public class VMInstanceDaoImpl extends GenericDaoBase implem private static final String COUNT_VMS_BASED_ON_VGPU_TYPES2 = "GROUP BY gpu_card.name, vgpu_profile.name"; + // %s is the "changed state recently" test, or a constant 0 when no cut-off is given. It is not + // a bound parameter because there is no timestamp that reliably means "never" - update_time is + // a TIMESTAMP column, so anything past 2038 is out of range. + private static final String COUNT_VMS_BY_HOST = "SELECT host.id, COUNT(vm.id), SUM(IF(%s, 1, 0)) " + + "FROM `cloud`.`host` host LEFT JOIN `cloud`.`vm_instance` vm " + + "ON vm.host_id = host.id AND vm.state IN ('Running', 'Starting', 'Stopping', 'Migrating') " + + "AND vm.removed IS NULL WHERE host.type = 'Routing' AND host.removed IS NULL AND host.data_center_id = ? "; + private static final String COUNT_VMS_BY_HOST_PART2 = " GROUP BY host.id "; + private static final String UPDATE_SYSTEM_VM_TEMPLATE_ID_FOR_HYPERVISOR = "UPDATE `cloud`.`vm_instance` SET vm_template_id = ? WHERE type <> 'User' AND hypervisor_type = ? AND removed is NULL"; private static final String COUNT_VMS_BY_ZONE_AND_STATE_AND_HOST_TAG = "SELECT COUNT(1) FROM vm_instance vi JOIN service_offering so ON vi.service_offering_id=so.id " + @@ -795,6 +805,44 @@ public Pair, Map> listPodIdsInZoneByVmCount(long dataCe } } + + @Override + public Map> countVmsByHost(long dcId, Long podId, Long clusterId, Date changedStateAfter) { + TransactionLegacy txn = TransactionLegacy.currentTxn(); + Map> result = new HashMap<>(); + String sql = String.format(COUNT_VMS_BY_HOST, changedStateAfter != null ? "vm.update_time > ?" : "0"); + if (podId != null) { + sql = sql + " AND host.pod_id = ? "; + } + if (clusterId != null) { + sql = sql + " AND host.cluster_id = ? "; + } + sql = sql + COUNT_VMS_BY_HOST_PART2; + try { + PreparedStatement pstmt = txn.prepareAutoCloseStatement(sql); + int index = 1; + if (changedStateAfter != null) { + pstmt.setTimestamp(index++, new Timestamp(changedStateAfter.getTime())); + } + pstmt.setLong(index++, dcId); + if (podId != null) { + pstmt.setLong(index++, podId); + } + if (clusterId != null) { + pstmt.setLong(index++, clusterId); + } + ResultSet rs = pstmt.executeQuery(); + while (rs.next()) { + result.put(rs.getLong(1), new Pair<>(rs.getLong(2), rs.getLong(3))); + } + return result; + } catch (SQLException e) { + throw new CloudRuntimeException("DB Exception on: " + sql, e); + } catch (Exception e) { + throw new CloudRuntimeException("Caught: " + sql, e); + } + } + @Override public List listHostIdsByVmCount(long dcId, Long podId, Long clusterId, long accountId) { TransactionLegacy txn = TransactionLegacy.currentTxn(); diff --git a/plugins/affinity-group-processors/host-anti-affinity/src/main/java/org/apache/cloudstack/affinity/HostAntiAffinityProcessor.java b/plugins/affinity-group-processors/host-anti-affinity/src/main/java/org/apache/cloudstack/affinity/HostAntiAffinityProcessor.java index bd29a48f2588..b71142d0a9e5 100644 --- a/plugins/affinity-group-processors/host-anti-affinity/src/main/java/org/apache/cloudstack/affinity/HostAntiAffinityProcessor.java +++ b/plugins/affinity-group-processors/host-anti-affinity/src/main/java/org/apache/cloudstack/affinity/HostAntiAffinityProcessor.java @@ -17,6 +17,7 @@ package org.apache.cloudstack.affinity; import java.util.Arrays; +import java.util.Collections; import java.util.List; import java.util.Map; import java.util.stream.Collectors; @@ -59,7 +60,7 @@ public class HostAntiAffinityProcessor extends AffinityProcessorBase implements protected AffinityGroupDao _affinityGroupDao; @Inject protected AffinityGroupVMMapDao _affinityGroupVMMapDao; - private int _vmCapacityReleaseInterval; + protected int _vmCapacityReleaseInterval; @Inject protected ConfigurationDao _configDao; @@ -82,7 +83,7 @@ public void doInTransactionWithoutResult(TransactionStatus status) { _affinityGroupDao.listByIds(affinityGroupIds, true); } for (AffinityGroupVMMapVO vmGroupMapping : vmGroupMappings) { - processAffinityGroup(vmGroupMapping, avoid, vm); + processAffinityGroup(vmGroupMapping, avoid, vm, vmList); } } }); @@ -90,6 +91,18 @@ public void doInTransactionWithoutResult(TransactionStatus status) { } protected void processAffinityGroup(AffinityGroupVMMapVO vmGroupMapping, ExcludeList avoid, VirtualMachine vm) { + processAffinityGroup(vmGroupMapping, avoid, vm, Collections.emptyList()); + } + + /** + * Applies anti-affinity for one group. + * + * @param vmList + * placements to honour in preference to what the database says. DRS builds a plan of + * several migrations in memory and only persists it later, so during plan generation + * the database still shows the old host for every VM the plan has already moved. + */ + protected void processAffinityGroup(AffinityGroupVMMapVO vmGroupMapping, ExcludeList avoid, VirtualMachine vm, List vmList) { if (vmGroupMapping != null) { AffinityGroupVO group = _affinityGroupDao.findById(vmGroupMapping.getAffinityGroupId()); @@ -100,24 +113,35 @@ protected void processAffinityGroup(AffinityGroupVMMapVO vmGroupMapping, Exclude List groupVMIds = _affinityGroupVMMapDao.listVmIdsByAffinityGroup(group.getId()); groupVMIds.remove(vm.getId()); + Map plannedVms = getVmIdVmMap(vmList); + for (Long groupVMId : groupVMIds) { + VirtualMachine plannedVm = plannedVms.get(groupVMId); + if (plannedVm != null && plannedVm.getHostId() != null) { + avoid.addHost(plannedVm.getHostId()); + logger.debug("Added host {} to avoid set, since VM {} is placed on the host by the plan being built", + plannedVm.getHostId(), plannedVm); + continue; + } VMInstanceVO groupVM = _vmInstanceDao.findById(groupVMId); - if (groupVM != null && !groupVM.isRemoved()) { - if (groupVM.getHostId() != null) { - avoid.addHost(groupVM.getHostId()); - if (logger.isDebugEnabled()) { - logger.debug("Added host {} to avoid set, since VM {} is present on the host", groupVM.getHostId(), groupVM); - } - } - } else if (Arrays.asList(VirtualMachine.State.Starting, VirtualMachine.State.Stopped).contains(groupVM.getState()) && groupVM.getLastHostId() != null) { - long secondsSinceLastUpdate = (DateUtil.currentGMTTime().getTime() - groupVM.getUpdateTime().getTime()) / 1000; - if (secondsSinceLastUpdate < _vmCapacityReleaseInterval) { - avoid.addHost(groupVM.getLastHostId()); - if (logger.isDebugEnabled()) { - logger.debug("Added host {} to avoid set, since VM {} is present on the host, in Stopped state but has reserved capacity", groupVM.getLastHostId(), groupVM); - } - } + if (groupVM == null || groupVM.isRemoved()) { + continue; } + avoidHostOfVmInAffinityGroup(avoid, groupVM); + } + } + } + + protected void avoidHostOfVmInAffinityGroup(ExcludeList avoid, VMInstanceVO groupVM) { + if (groupVM.getHostId() != null) { + avoid.addHost(groupVM.getHostId()); + logger.debug("Added host {} to avoid set, since VM {} is present on the host", groupVM.getHostId(), groupVM); + } else if (Arrays.asList(VirtualMachine.State.Starting, VirtualMachine.State.Stopped).contains(groupVM.getState()) && groupVM.getLastHostId() != null) { + long secondsSinceLastUpdate = (DateUtil.currentGMTTime().getTime() - groupVM.getUpdateTime().getTime()) / 1000; + if (secondsSinceLastUpdate < _vmCapacityReleaseInterval) { + avoid.addHost(groupVM.getLastHostId()); + logger.debug("Added host {} to avoid set, since VM {} is in {} state on the host but still has reserved capacity", + groupVM.getLastHostId(), groupVM, groupVM.getState()); } } } diff --git a/plugins/affinity-group-processors/host-anti-affinity/src/test/java/org/apache/cloudstack/affinity/HostAntiAffinityProcessorTest.java b/plugins/affinity-group-processors/host-anti-affinity/src/test/java/org/apache/cloudstack/affinity/HostAntiAffinityProcessorTest.java new file mode 100644 index 000000000000..3ef05ce07bfb --- /dev/null +++ b/plugins/affinity-group-processors/host-anti-affinity/src/test/java/org/apache/cloudstack/affinity/HostAntiAffinityProcessorTest.java @@ -0,0 +1,233 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.affinity; + +import com.cloud.deploy.DeploymentPlanner.ExcludeList; +import com.cloud.utils.DateUtil; +import com.cloud.vm.VMInstanceVO; +import com.cloud.vm.VirtualMachine; +import com.cloud.vm.dao.VMInstanceDao; +import org.apache.cloudstack.affinity.dao.AffinityGroupDao; +import org.apache.cloudstack.affinity.dao.AffinityGroupVMMapDao; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; +import org.mockito.Spy; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Date; +import java.util.List; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.lenient; +import static org.mockito.Mockito.when; + +@RunWith(JUnit4.class) +public class HostAntiAffinityProcessorTest { + + private static final long AFFINITY_GROUP_ID = 2L; + private static final long VM_ID = 3L; + private static final long GROUP_VM_ID = 1L; + private static final long HOST_ID = 10L; + private static final long LAST_HOST_ID = 11L; + private static final long PLANNED_HOST_ID = 12L; + private static final int CAPACITY_RELEASE_INTERVAL = 3600; + + @Mock + AffinityGroupDao _affinityGroupDao; + + @Mock + AffinityGroupVMMapDao _affinityGroupVMMapDao; + + @Mock + VMInstanceDao _vmInstanceDao; + + @Spy + @InjectMocks + HostAntiAffinityProcessor processor = new HostAntiAffinityProcessor(); + + @Mock + VirtualMachine vm; + + @Mock + VMInstanceVO groupVM; + + @Mock + AffinityGroupVO affinityGroupVO; + + @Mock + AffinityGroupVMMapVO vmGroupMapping; + + private ExcludeList avoid; + + @Before + public void setUp() { + MockitoAnnotations.openMocks(this); + processor._vmCapacityReleaseInterval = CAPACITY_RELEASE_INTERVAL; + avoid = new ExcludeList(); + + when(vm.getId()).thenReturn(VM_ID); + when(vmGroupMapping.getAffinityGroupId()).thenReturn(AFFINITY_GROUP_ID); + when(_affinityGroupDao.findById(AFFINITY_GROUP_ID)).thenReturn(affinityGroupVO); + when(affinityGroupVO.getId()).thenReturn(AFFINITY_GROUP_ID); + when(_affinityGroupVMMapDao.listVmIdsByAffinityGroup(AFFINITY_GROUP_ID)) + .thenReturn(new ArrayList<>(Arrays.asList(GROUP_VM_ID, VM_ID))); + } + + private boolean avoids(long hostId) { + return avoid.getHostsToAvoid() != null && avoid.getHostsToAvoid().contains(hostId); + } + + @Test + public void testRunningGroupVmHostIsAvoided() { + when(_vmInstanceDao.findById(GROUP_VM_ID)).thenReturn(groupVM); + when(groupVM.isRemoved()).thenReturn(false); + when(groupVM.getHostId()).thenReturn(HOST_ID); + + processor.processAffinityGroup(vmGroupMapping, avoid, vm); + + assertTrue(avoids(HOST_ID)); + } + + @Test + public void testStoppedGroupVmWithReservedCapacityLastHostIsAvoided() { + when(_vmInstanceDao.findById(GROUP_VM_ID)).thenReturn(groupVM); + when(groupVM.isRemoved()).thenReturn(false); + when(groupVM.getHostId()).thenReturn(null); + when(groupVM.getState()).thenReturn(VirtualMachine.State.Stopped); + when(groupVM.getLastHostId()).thenReturn(LAST_HOST_ID); + when(groupVM.getUpdateTime()).thenReturn(DateUtil.currentGMTTime()); + + processor.processAffinityGroup(vmGroupMapping, avoid, vm); + + assertTrue(avoids(LAST_HOST_ID)); + } + + @Test + public void testStoppedGroupVmPastReleaseIntervalIsNotAvoided() { + Date wellPast = new Date(DateUtil.currentGMTTime().getTime() - (CAPACITY_RELEASE_INTERVAL + 60) * 1000L); + when(_vmInstanceDao.findById(GROUP_VM_ID)).thenReturn(groupVM); + when(groupVM.isRemoved()).thenReturn(false); + when(groupVM.getHostId()).thenReturn(null); + when(groupVM.getState()).thenReturn(VirtualMachine.State.Stopped); + when(groupVM.getLastHostId()).thenReturn(LAST_HOST_ID); + when(groupVM.getUpdateTime()).thenReturn(wellPast); + + processor.processAffinityGroup(vmGroupMapping, avoid, vm); + + assertFalse(avoids(LAST_HOST_ID)); + } + + @Test + public void testMissingGroupVmIsSkipped() { + when(_vmInstanceDao.findById(GROUP_VM_ID)).thenReturn(null); + + processor.processAffinityGroup(vmGroupMapping, avoid, vm); + + assertTrue(avoid.getHostsToAvoid() == null || avoid.getHostsToAvoid().isEmpty()); + } + + @Test + public void testRemovedGroupVmIsSkipped() { + when(_vmInstanceDao.findById(GROUP_VM_ID)).thenReturn(groupVM); + when(groupVM.isRemoved()).thenReturn(true); + // a removed VM is not running anywhere, so neither of its hosts should be avoided + lenient().when(groupVM.getHostId()).thenReturn(HOST_ID); + lenient().when(groupVM.getState()).thenReturn(VirtualMachine.State.Stopped); + lenient().when(groupVM.getLastHostId()).thenReturn(LAST_HOST_ID); + lenient().when(groupVM.getUpdateTime()).thenReturn(DateUtil.currentGMTTime()); + + processor.processAffinityGroup(vmGroupMapping, avoid, vm); + + assertFalse(avoids(HOST_ID)); + assertFalse(avoids(LAST_HOST_ID)); + } + + @Test + public void testStartingGroupVmWithReservedCapacityLastHostIsAvoided() { + when(_vmInstanceDao.findById(GROUP_VM_ID)).thenReturn(groupVM); + when(groupVM.isRemoved()).thenReturn(false); + when(groupVM.getHostId()).thenReturn(null); + when(groupVM.getState()).thenReturn(VirtualMachine.State.Starting); + when(groupVM.getLastHostId()).thenReturn(LAST_HOST_ID); + when(groupVM.getUpdateTime()).thenReturn(DateUtil.currentGMTTime()); + + processor.processAffinityGroup(vmGroupMapping, avoid, vm); + + assertTrue(avoids(LAST_HOST_ID)); + } + + @Test + public void testPlannedHostWinsOverStaleDatabaseHost() { + VMInstanceVO plannedVm = org.mockito.Mockito.mock(VMInstanceVO.class); + when(plannedVm.getId()).thenReturn(GROUP_VM_ID); + when(plannedVm.getHostId()).thenReturn(PLANNED_HOST_ID); + + // the database still shows the pre-migration host + when(_vmInstanceDao.findById(GROUP_VM_ID)).thenReturn(groupVM); + when(groupVM.isRemoved()).thenReturn(false); + when(groupVM.getHostId()).thenReturn(HOST_ID); + + processor.processAffinityGroup(vmGroupMapping, avoid, vm, Arrays.asList(plannedVm)); + + assertTrue(avoids(PLANNED_HOST_ID)); + assertFalse(avoids(HOST_ID)); + } + + @Test + public void testPlannedVmWithoutHostFallsBackToDatabase() { + VMInstanceVO plannedVm = org.mockito.Mockito.mock(VMInstanceVO.class); + when(plannedVm.getId()).thenReturn(GROUP_VM_ID); + when(plannedVm.getHostId()).thenReturn(null); + + when(_vmInstanceDao.findById(GROUP_VM_ID)).thenReturn(groupVM); + when(groupVM.isRemoved()).thenReturn(false); + when(groupVM.getHostId()).thenReturn(HOST_ID); + + processor.processAffinityGroup(vmGroupMapping, avoid, vm, Arrays.asList(plannedVm)); + + assertTrue(avoids(HOST_ID)); + } + + @Test + public void testEmptyVmListBehavesAsBefore() { + when(_vmInstanceDao.findById(GROUP_VM_ID)).thenReturn(groupVM); + when(groupVM.isRemoved()).thenReturn(false); + when(groupVM.getHostId()).thenReturn(HOST_ID); + + processor.processAffinityGroup(vmGroupMapping, avoid, vm, Collections.emptyList()); + + assertTrue(avoids(HOST_ID)); + } + + @Test + public void testVmIsNotAvoidedAgainstItself() { + List ids = new ArrayList<>(Arrays.asList(VM_ID)); + when(_affinityGroupVMMapDao.listVmIdsByAffinityGroup(AFFINITY_GROUP_ID)).thenReturn(ids); + + processor.processAffinityGroup(vmGroupMapping, avoid, vm); + + assertTrue(avoid.getHostsToAvoid() == null || avoid.getHostsToAvoid().isEmpty()); + } +} diff --git a/plugins/affinity-group-processors/non-strict-host-affinity/src/main/java/org/apache/cloudstack/affinity/NonStrictHostAffinityProcessor.java b/plugins/affinity-group-processors/non-strict-host-affinity/src/main/java/org/apache/cloudstack/affinity/NonStrictHostAffinityProcessor.java index 49e3f60ed5d0..c79efca5e7de 100644 --- a/plugins/affinity-group-processors/non-strict-host-affinity/src/main/java/org/apache/cloudstack/affinity/NonStrictHostAffinityProcessor.java +++ b/plugins/affinity-group-processors/non-strict-host-affinity/src/main/java/org/apache/cloudstack/affinity/NonStrictHostAffinityProcessor.java @@ -17,6 +17,7 @@ package org.apache.cloudstack.affinity; import java.util.Arrays; +import java.util.Collections; import java.util.List; import java.util.Map; @@ -67,13 +68,24 @@ public void process(VirtualMachineProfile vmProfile, DeploymentPlan plan, Exclud for (AffinityGroupVMMapVO vmGroupMapping : vmGroupMappings) { if (vmGroupMapping != null) { - processAffinityGroup(vmGroupMapping, plan, vm); + processAffinityGroup(vmGroupMapping, plan, vm, vmList); } } } protected void processAffinityGroup(AffinityGroupVMMapVO vmGroupMapping, DeploymentPlan plan, VirtualMachine vm) { + processAffinityGroup(vmGroupMapping, plan, vm, Collections.emptyList()); + } + + /** + * Adjusts host priorities for one group. + * + * @param vmList + * placements to honour in preference to what the database says, for callers such as DRS + * that build a plan in memory before persisting it. + */ + protected void processAffinityGroup(AffinityGroupVMMapVO vmGroupMapping, DeploymentPlan plan, VirtualMachine vm, List vmList) { AffinityGroupVO group = affinityGroupDao.findById(vmGroupMapping.getAffinityGroupId()); if (logger.isDebugEnabled()) { @@ -83,7 +95,16 @@ protected void processAffinityGroup(AffinityGroupVMMapVO vmGroupMapping, Deploym List groupVMIds = affinityGroupVMMapDao.listVmIdsByAffinityGroup(group.getId()); groupVMIds.remove(vm.getId()); + Map plannedVms = getVmIdVmMap(vmList); + for (Long groupVMId : groupVMIds) { + VirtualMachine plannedVm = plannedVms.get(groupVMId); + if (plannedVm != null && plannedVm.getHostId() != null) { + Integer priority = adjustHostPriority(plan, plannedVm.getHostId()); + logger.debug("Updated host {} priority to {}, since VM {} is placed on the host by the plan being built", + plannedVm.getHostId(), priority, plannedVm); + continue; + } VMInstanceVO groupVM = vmInstanceDao.findById(groupVMId); if (groupVM != null && !groupVM.isRemoved()) { processVmInAffinityGroup(plan, groupVM); diff --git a/plugins/affinity-group-processors/non-strict-host-affinity/src/test/java/org/apache/cloudstack/affinity/NonStrictHostAffinityProcessorTest.java b/plugins/affinity-group-processors/non-strict-host-affinity/src/test/java/org/apache/cloudstack/affinity/NonStrictHostAffinityProcessorTest.java index cb91870cbd39..6e14b26cb690 100644 --- a/plugins/affinity-group-processors/non-strict-host-affinity/src/test/java/org/apache/cloudstack/affinity/NonStrictHostAffinityProcessorTest.java +++ b/plugins/affinity-group-processors/non-strict-host-affinity/src/test/java/org/apache/cloudstack/affinity/NonStrictHostAffinityProcessorTest.java @@ -170,4 +170,37 @@ public void testProcessWithNotRunningVM() { Assert.assertNotNull(plan.getHostPriorities().get(host2Id)); Assert.assertEquals(Integer.valueOf(1), plan.getHostPriorities().get(host2Id)); } + + @Test + public void testPlannedHostWinsOverStaleDatabaseHost() { + VirtualMachine vm = Mockito.mock(VirtualMachine.class); + when(vm.getId()).thenReturn(vmId); + VirtualMachineProfile vmProfile = Mockito.mock(VirtualMachineProfile.class); + when(vmProfile.getVirtualMachine()).thenReturn(vm); + + List vmGroupMappings = new ArrayList<>(); + vmGroupMappings.add(new AffinityGroupVMMapVO(affinityGroupId, vmId)); + when(_affinityGroupVMMapDao.findByVmIdType(eq(vmId), nullable(String.class))).thenReturn(vmGroupMappings); + + DataCenterDeployment plan = new DataCenterDeployment(zoneId); + ExcludeList avoid = new ExcludeList(); + + AffinityGroupVO affinityGroupVO = Mockito.mock(AffinityGroupVO.class); + when(affinityGroupDao.findById(affinityGroupId)).thenReturn(affinityGroupVO); + when(affinityGroupVO.getId()).thenReturn(affinityGroupId); + when(_affinityGroupVMMapDao.listVmIdsByAffinityGroup(affinityGroupId)) + .thenReturn(new ArrayList<>(Arrays.asList(vmId, vm2Id))); + + // the plan being built has already moved vm2 to host3; the database is not consulted + VMInstanceVO planned = Mockito.mock(VMInstanceVO.class); + when(planned.getId()).thenReturn(vm2Id); + when(planned.getHostId()).thenReturn(host3Id); + + processor.process(vmProfile, plan, avoid, Arrays.asList(planned)); + + Assert.assertEquals(1, plan.getHostPriorities().size()); + Assert.assertNotNull(plan.getHostPriorities().get(host3Id)); + Assert.assertNull(plan.getHostPriorities().get(host2Id)); + Mockito.verify(vmInstanceDao, Mockito.never()).findById(vm2Id); + } } diff --git a/plugins/drs/cluster/weighted/pom.xml b/plugins/drs/cluster/weighted/pom.xml new file mode 100644 index 000000000000..4c0fa3e6a902 --- /dev/null +++ b/plugins/drs/cluster/weighted/pom.xml @@ -0,0 +1,33 @@ + + + + + 4.0.0 + Apache CloudStack Plugin - Cluster DRS Algorithm - Weighted + cloud-plugin-cluster-drs-weighted + + org.apache.cloudstack + cloudstack-plugins + 24.0.0-SNAPSHOT + ../../../pom.xml + + diff --git a/plugins/drs/cluster/weighted/src/main/java/org/apache/cloudstack/cluster/Weighted.java b/plugins/drs/cluster/weighted/src/main/java/org/apache/cloudstack/cluster/Weighted.java new file mode 100644 index 000000000000..19feb328ea87 --- /dev/null +++ b/plugins/drs/cluster/weighted/src/main/java/org/apache/cloudstack/cluster/Weighted.java @@ -0,0 +1,320 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.cluster; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import javax.inject.Inject; +import javax.naming.ConfigurationException; + +import org.apache.cloudstack.framework.config.ConfigKey; +import org.apache.cloudstack.framework.config.Configurable; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import com.cloud.dc.ClusterDetailsDao; +import com.cloud.dc.ClusterDetailsVO; +import com.cloud.host.Host; +import com.cloud.host.HostLoad; +import com.cloud.host.HostScoringWeights; +import com.cloud.offering.ServiceOffering; +import com.cloud.org.Cluster; +import com.cloud.utils.Ternary; +import com.cloud.utils.component.AdapterBase; +import com.cloud.vm.VirtualMachine; +import com.cloud.vm.VmDetailConstants; + +/** + * Balances a cluster on CPU and memory together, and on what hosts are really doing rather than + * only on what has been allocated to them. + * + * The existing algorithms balance a single metric chosen by drs.metric. Choosing one leaves the + * other unwatched: a cluster can be even on memory while its CPU load varies several fold, and + * nothing moves. Allocation is also a poor stand-in for load under overprovisioning, where a + * saturated host can still report a small percentage allocated. + * + * This blends four figures per host - CPU and memory allocated, CPU and memory in use - and + * balances the result. The weights are the same host.weighted.* settings initial placement uses, on + * purpose: if the two weighted them differently they would disagree about which host is the better + * one, and rebalancing could move VMs off hosts that placement had just chosen. Imbalance keeps the same shape as the other algorithms - the standard + * deviation of the per-host figure over its mean - but it is computed over a blend rather than over + * one metric, so drs.imbalance is not calibrated the same way and is worth re-checking after + * switching. + * + * drs.metric, drs.metric.type and drs.metric.use.ratio choose and shape the single metric the other + * algorithms balance. They do not apply here and are ignored. + */ +public class Weighted extends AdapterBase implements ClusterDrsAlgorithm, Configurable { + + private static final Logger LOGGER = LogManager.getLogger(Weighted.class); + + public static final ConfigKey StorageMotionCost = new ConfigKey<>(ConfigKey.CATEGORY_ADVANCED, + Double.class, "drs.weighted.storage.motion.cost", "0.02", + "How much a migration must improve the cluster's imbalance to be worth also moving the VM's " + + "storage. Migrations that do not need storage moved only have to improve it at all.", + true, ConfigKey.Scope.Cluster); + + @Inject + private ClusterDetailsDao clusterDetailsDao; + + /** + * Everything that is constant for one plan. getMetrics is called for every candidate VM and + * host - up to hundreds of thousands of times for a large cluster - so nothing in that path may + * hit the database or re-read settings. + */ + private static final class PlanContext { + private final float cpuOvercommit; + private final float memoryOvercommit; + private final double cpuAllocatedWeight; + private final double memoryAllocatedWeight; + private final double cpuUsedWeight; + private final double memoryUsedWeight; + private final Map hostLoadMap; + private final boolean everyHostMeasured; + + private PlanContext(float cpuOvercommit, float memoryOvercommit, double cpuAllocatedWeight, + double memoryAllocatedWeight, double cpuUsedWeight, double memoryUsedWeight, + Map hostLoadMap, boolean everyHostMeasured) { + this.everyHostMeasured = everyHostMeasured; + this.cpuOvercommit = cpuOvercommit; + this.memoryOvercommit = memoryOvercommit; + this.cpuAllocatedWeight = cpuAllocatedWeight; + this.memoryAllocatedWeight = memoryAllocatedWeight; + this.cpuUsedWeight = cpuUsedWeight; + this.memoryUsedWeight = memoryUsedWeight; + this.hostLoadMap = hostLoadMap; + } + + private HostLoad loadOf(long hostId) { + HostLoad load = hostLoadMap.get(hostId); + return load == null ? HostLoad.UNKNOWN : load; + } + } + + private final ThreadLocal context = new ThreadLocal<>(); + + @Override + public void prepare(Cluster cluster, Map> hostCpuMap, + Map> hostMemoryMap, Map hostLoadMap) { + long clusterId = cluster.getId(); + context.set(new PlanContext( + overcommitRatio(clusterId, VmDetailConstants.CPU_OVER_COMMIT_RATIO), + overcommitRatio(clusterId, VmDetailConstants.MEMORY_OVER_COMMIT_RATIO), + weight(HostScoringWeights.CpuAllocatedWeight, clusterId), + weight(HostScoringWeights.MemoryAllocatedWeight, clusterId), + weight(HostScoringWeights.CpuUsedWeight, clusterId), + weight(HostScoringWeights.MemoryUsedWeight, clusterId), + hostLoadMap == null ? new HashMap<>() : hostLoadMap, + hostLoadMap != null && !hostLoadMap.isEmpty() + && hostLoadMap.values().stream().allMatch(HostLoad::isUsable))); + } + + /** + * Falls back to reading everything when prepare has not been called, so the algorithm still + * works for a caller that does not know about it. + */ + private PlanContext contextFor(Cluster cluster) { + PlanContext prepared = context.get(); + if (prepared != null) { + return prepared; + } + prepare(cluster, null, null, null); + return context.get(); + } + + @Override + public String getName() { + return "weighted"; + } + + @Override + public boolean needsDrs(Cluster cluster, List> cpuList, + List> memoryList) throws ConfigurationException { + // without host identity, measured load cannot be attributed; the map form is what DRS calls + Map> cpuMap = new HashMap<>(); + Map> memoryMap = new HashMap<>(); + for (int i = 0; i < cpuList.size() && i < memoryList.size(); i++) { + cpuMap.put((long) -(i + 1), cpuList.get(i)); + memoryMap.put((long) -(i + 1), memoryList.get(i)); + } + return needsDrs(cluster, cpuMap, memoryMap, new HashMap<>()); + } + + @Override + public boolean needsDrs(Cluster cluster, Map> hostCpuMap, + Map> hostMemoryMap, Map hostLoadMap) + throws ConfigurationException { + double threshold = 1.0 - ClusterDrsService.ClusterDrsImbalanceThreshold.valueIn(cluster.getId()); + double imbalance = imbalanceOf(blendByHost(cluster, hostCpuMap, hostMemoryMap).values()); + boolean needed = imbalance > threshold; + LOGGER.debug("Cluster {} {} DRS. Imbalance: {} Threshold: {} Algorithm: {}", + cluster, needed ? "needs" : "does not need", imbalance, threshold, getName()); + return needed; + } + + @Override + public Ternary getMetrics(Cluster cluster, VirtualMachine vm, + ServiceOffering serviceOffering, Host destHost, + Map> hostCpuMap, Map> hostMemoryMap, + Boolean requiresStorageMotion, Double preImbalance, + double[] baseMetricsArray, Map hostIdToIndexMap) throws ConfigurationException { + + double before = imbalanceOf(blendByHost(cluster, hostCpuMap, hostMemoryMap).values()); + + long vmCpu = (long) serviceOffering.getCpu() * serviceOffering.getSpeed(); + long vmMemory = serviceOffering.getRamSize() * 1024L * 1024L; + Map> cpuAfter = withVmMoved(hostCpuMap, vm.getHostId(), destHost.getId(), vmCpu); + Map> memoryAfter = withVmMoved(hostMemoryMap, vm.getHostId(), destHost.getId(), vmMemory); + + double after = imbalanceOf(blendByHost(cluster, cpuAfter, memoryAfter).values()); + + // the caller migrates when benefit > cost, so expressing both in units of imbalance makes + // that comparison mean "is this worth what it costs". A migration that has to move storage + // has to earn more than one that does not. + double improvement = before - after; + double cost = Boolean.TRUE.equals(requiresStorageMotion) ? weight(StorageMotionCost, cluster.getId()) : 0.0; + double benefit = improvement; + + LOGGER.trace("Cluster {} imbalance {} -> {} moving {} to {}", cluster, before, after, vm, destHost); + return new Ternary<>(improvement, cost, benefit); + } + + /** + * One figure per host, blending what is allocated with what is in use. + */ + protected Map blendByHost(Cluster cluster, Map> hostCpuMap, + Map> hostMemoryMap) { + PlanContext ctx = contextFor(cluster); + + Map blended = new HashMap<>(); + for (Map.Entry> entry : hostCpuMap.entrySet()) { + long hostId = entry.getKey(); + Ternary memory = hostMemoryMap.get(hostId); + if (memory == null) { + continue; + } + double cpuAllocated = fractionOf(entry.getValue(), ctx.cpuOvercommit); + double memoryAllocated = fractionOf(memory, ctx.memoryOvercommit); + + // utilisation is only used when every host has it. Imbalance compares hosts against + // each other, so mixing hosts measured on utilisation with hosts measured on allocation + // alone would report a difference that is an artefact of the monitoring, not the load - + // and would evacuate whichever host stopped reporting. + HostLoad load = ctx.loadOf(hostId); + double usedCpuWeight = ctx.everyHostMeasured ? ctx.cpuUsedWeight : 0; + double usedMemoryWeight = ctx.everyHostMeasured ? ctx.memoryUsedWeight : 0; + + double sum = ctx.cpuAllocatedWeight + ctx.memoryAllocatedWeight + usedCpuWeight + usedMemoryWeight; + if (sum <= 0) { + blended.put(hostId, 0.0); + continue; + } + blended.put(hostId, (ctx.cpuAllocatedWeight * cpuAllocated + + ctx.memoryAllocatedWeight * memoryAllocated + + usedCpuWeight * load.getCpuUtilisation() + + usedMemoryWeight * load.getMemoryUtilisation()) / sum); + } + return blended; + } + + private Map> withVmMoved(Map> original, + Long sourceHostId, long destHostId, long amount) { + Map> copy = new HashMap<>(); + for (Map.Entry> entry : original.entrySet()) { + Ternary value = entry.getValue(); + long used = value.first(); + if (entry.getKey().equals(sourceHostId)) { + used -= amount; + } else if (entry.getKey() == destHostId) { + used += amount; + } + copy.put(entry.getKey(), new Ternary<>(used, value.second(), value.third())); + } + return copy; + } + + /** + * Used over what the host can hand out, which is its real total scaled by the overcommit ratio. + */ + private double fractionOf(Ternary capacity, float overcommit) { + // overcommit scales the host's total; reserved is then taken off that, which is how + // CapacityManager computes free capacity everywhere else. Multiplying reserved by the ratio + // instead would make a host look fuller the more capacity it merely has reserved. + double allocatable = capacity.third() * (double) overcommit - capacity.second(); + if (allocatable <= 0) { + return 0; + } + return capacity.first() / allocatable; + } + + protected float overcommitRatio(long clusterId, String key) { + ClusterDetailsVO detail = clusterDetailsDao.findDetail(clusterId, key); + if (detail == null || detail.getValue() == null) { + return 1f; + } + try { + float ratio = Float.parseFloat(detail.getValue()); + return ratio > 0 ? ratio : 1f; + } catch (NumberFormatException e) { + return 1f; + } + } + + private double weight(ConfigKey key, long clusterId) { + Double value = key.valueIn(clusterId); + if (value == null || value < 0) { + return 0; + } + return value; + } + + /** + * Standard deviation over the mean, the same definition the other algorithms use, so that + * drs.imbalance keeps its meaning. + */ + protected double imbalanceOf(java.util.Collection values) { + if (values == null || values.isEmpty()) { + return 0; + } + double[] array = values.stream().mapToDouble(Double::doubleValue).toArray(); + double mean = MEAN_CALCULATOR.evaluate(array); + if (mean == 0) { + return 0; + } + return STDDEV_CALCULATOR.evaluate(array, mean) / mean; + } + + private static double clamp(double value) { + if (Double.isNaN(value) || value < 0) { + return 0; + } + return Math.min(value, 1); + } + + @Override + public String getConfigComponentName() { + return Weighted.class.getSimpleName(); + } + + @Override + public ConfigKey[] getConfigKeys() { + // the four host.weighted.* weights are shared with initial placement and registered there + return new ConfigKey[] {StorageMotionCost}; + } +} diff --git a/plugins/drs/cluster/weighted/src/main/resources/META-INF/cloudstack/weighted/module.properties b/plugins/drs/cluster/weighted/src/main/resources/META-INF/cloudstack/weighted/module.properties new file mode 100644 index 000000000000..636ecd1700e5 --- /dev/null +++ b/plugins/drs/cluster/weighted/src/main/resources/META-INF/cloudstack/weighted/module.properties @@ -0,0 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +name=weighted +parent=cluster diff --git a/plugins/drs/cluster/weighted/src/main/resources/META-INF/cloudstack/weighted/spring-weighted-context.xml b/plugins/drs/cluster/weighted/src/main/resources/META-INF/cloudstack/weighted/spring-weighted-context.xml new file mode 100644 index 000000000000..903ae08087a1 --- /dev/null +++ b/plugins/drs/cluster/weighted/src/main/resources/META-INF/cloudstack/weighted/spring-weighted-context.xml @@ -0,0 +1,33 @@ + + + + + + + diff --git a/plugins/drs/cluster/weighted/src/test/java/org/apache/cloudstack/cluster/WeightedTest.java b/plugins/drs/cluster/weighted/src/test/java/org/apache/cloudstack/cluster/WeightedTest.java new file mode 100644 index 000000000000..9a72df3ea51e --- /dev/null +++ b/plugins/drs/cluster/weighted/src/test/java/org/apache/cloudstack/cluster/WeightedTest.java @@ -0,0 +1,283 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.cluster; + +import java.util.HashMap; +import java.util.Map; + +import javax.naming.ConfigurationException; + +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; + +import com.cloud.dc.ClusterDetailsDao; +import com.cloud.dc.ClusterDetailsVO; +import com.cloud.host.Host; +import com.cloud.host.HostLoad; +import com.cloud.offering.ServiceOffering; +import com.cloud.org.Cluster; +import com.cloud.utils.Ternary; +import com.cloud.vm.VirtualMachine; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +@RunWith(MockitoJUnitRunner.class) +public class WeightedTest { + + private static final long CLUSTER_ID = 1L; + private static final long CPU_TOTAL = 192L * 2400L; + private static final long MEMORY_TOTAL = 1_132_000L * 1024L * 1024L; + private static final int CPU_OVERCOMMIT = 10; + private static final int MEMORY_OVERCOMMIT = 4; + + @Mock + private ClusterDetailsDao clusterDetailsDao; + + @InjectMocks + private Weighted weighted = new Weighted(); + + private Cluster cluster; + private final Map> cpu = new HashMap<>(); + private final Map> memory = new HashMap<>(); + private final Map load = new HashMap<>(); + + @Before + public void setUp() { + cluster = Mockito.mock(Cluster.class); + Mockito.lenient().when(cluster.getId()).thenReturn(CLUSTER_ID); + Mockito.lenient().when(clusterDetailsDao.findDetail(CLUSTER_ID, "cpuOvercommitRatio")) + .thenReturn(new ClusterDetailsVO(CLUSTER_ID, "cpuOvercommitRatio", String.valueOf(CPU_OVERCOMMIT))); + Mockito.lenient().when(clusterDetailsDao.findDetail(CLUSTER_ID, "memoryOvercommitRatio")) + .thenReturn(new ClusterDetailsVO(CLUSTER_ID, "memoryOvercommitRatio", String.valueOf(MEMORY_OVERCOMMIT))); + } + + private void host(long id, double cpuAllocatedFraction, double memoryAllocatedFraction, HostLoad hostLoad) { + host(id, cpuAllocatedFraction, memoryAllocatedFraction, hostLoad, 0L, 0L); + } + + private void host(long id, double cpuAllocatedFraction, double memoryAllocatedFraction, HostLoad hostLoad, + long reservedCpu, long reservedMemory) { + cpu.put(id, new Ternary<>((long) (CPU_TOTAL * CPU_OVERCOMMIT * cpuAllocatedFraction), reservedCpu, CPU_TOTAL)); + memory.put(id, new Ternary<>((long) (MEMORY_TOTAL * MEMORY_OVERCOMMIT * memoryAllocatedFraction), + reservedMemory, MEMORY_TOTAL)); + load.put(id, hostLoad); + } + + /** DRS calls prepare once per plan; every test must do the same. */ + private Map blend() { + weighted.prepare(cluster, cpu, memory, load); + return weighted.blendByHost(cluster, cpu, memory); + } + + @Test + public void testAnEvenClusterHasNoImbalance() { + host(1L, 0.4, 0.4, new HostLoad(0.4, 0.4, 10)); + host(2L, 0.4, 0.4, new HostLoad(0.4, 0.4, 10)); + + assertEquals(0.0, weighted.imbalanceOf(blend().values()), 1e-9); + } + + @Test + public void testCpuLoadImbalanceIsSeenWhenMemoryIsEven() { + // the case a memory-only metric misses entirely: memory even, CPU load three fold apart + host(1L, 0.30, 0.40, new HostLoad(0.90, 0.40, 10)); + host(2L, 0.30, 0.40, new HostLoad(0.30, 0.40, 10)); + + assertTrue("balancing on memory alone would call this cluster even", + weighted.imbalanceOf(blend().values()) > 0.15); + } + + @Test + public void testMemoryImbalanceIsSeenWhenCpuIsEven() { + host(1L, 0.30, 0.90, new HostLoad(0.30, 0.90, 10)); + host(2L, 0.30, 0.10, new HostLoad(0.30, 0.10, 10)); + + assertTrue(weighted.imbalanceOf(blend().values()) > 0.15); + } + + @Test + public void testAllocationBeyondPhysicalSizeStillDiscriminates() { + // at a factor of 10 both hosts are past their physical CPU; they must not both read as full + host(1L, 0.20, 0.20, HostLoad.UNKNOWN); + host(2L, 0.80, 0.20, HostLoad.UNKNOWN); + + Map blended = blend(); + assertTrue(blended.get(2L) > blended.get(1L)); + } + + @Test + public void testUnmeasuredHostsFallBackToAllocation() { + host(1L, 0.20, 0.20, HostLoad.UNKNOWN); + host(2L, 0.60, 0.60, HostLoad.UNKNOWN); + + Map blended = blend(); + assertEquals(0.20, blended.get(1L), 0.01); + assertEquals(0.60, blended.get(2L), 0.01); + } + + @Test + public void testMovingAVmOffTheBusyHostIsAnImprovement() throws ConfigurationException { + host(1L, 0.60, 0.60, new HostLoad(0.80, 0.60, 10)); + host(2L, 0.20, 0.20, new HostLoad(0.20, 0.20, 10)); + + VirtualMachine vm = Mockito.mock(VirtualMachine.class); + Mockito.when(vm.getHostId()).thenReturn(1L); + Host dest = Mockito.mock(Host.class); + Mockito.when(dest.getId()).thenReturn(2L); + ServiceOffering offering = Mockito.mock(ServiceOffering.class); + Mockito.when(offering.getCpu()).thenReturn(8); + Mockito.when(offering.getSpeed()).thenReturn(2400); + Mockito.when(offering.getRamSize()).thenReturn(32768); + + weighted.prepare(cluster, cpu, memory, load); + Ternary metrics = weighted.getMetrics(cluster, vm, offering, dest, + cpu, memory, false, null, new double[0], new HashMap<>()); + + assertTrue("moving away from the busier host should even the cluster out", metrics.first() > 0); + assertTrue("and should be considered worth doing", metrics.third() > metrics.second()); + } + + @Test + public void testMovingAVmOntoTheBusyHostIsNotAnImprovement() throws ConfigurationException { + host(1L, 0.60, 0.60, new HostLoad(0.80, 0.60, 10)); + host(2L, 0.20, 0.20, new HostLoad(0.20, 0.20, 10)); + + VirtualMachine vm = Mockito.mock(VirtualMachine.class); + Mockito.when(vm.getHostId()).thenReturn(2L); + Host dest = Mockito.mock(Host.class); + Mockito.when(dest.getId()).thenReturn(1L); + ServiceOffering offering = Mockito.mock(ServiceOffering.class); + Mockito.when(offering.getCpu()).thenReturn(8); + Mockito.when(offering.getSpeed()).thenReturn(2400); + Mockito.when(offering.getRamSize()).thenReturn(32768); + + weighted.prepare(cluster, cpu, memory, load); + Ternary metrics = weighted.getMetrics(cluster, vm, offering, dest, + cpu, memory, false, null, new double[0], new HashMap<>()); + + assertTrue("piling onto the busier host makes the cluster less even", metrics.first() < 0); + // the caller migrates when benefit > cost + assertFalse("and must not be considered worth doing", metrics.third() > metrics.second()); + } + + @Test + public void testStorageMotionCostsMore() throws ConfigurationException { + host(1L, 0.60, 0.60, new HostLoad(0.80, 0.60, 10)); + host(2L, 0.20, 0.20, new HostLoad(0.20, 0.20, 10)); + + VirtualMachine vm = Mockito.mock(VirtualMachine.class); + Mockito.when(vm.getHostId()).thenReturn(1L); + Host dest = Mockito.mock(Host.class); + Mockito.when(dest.getId()).thenReturn(2L); + ServiceOffering offering = Mockito.mock(ServiceOffering.class); + Mockito.when(offering.getCpu()).thenReturn(8); + Mockito.when(offering.getSpeed()).thenReturn(2400); + Mockito.when(offering.getRamSize()).thenReturn(32768); + + weighted.prepare(cluster, cpu, memory, load); + double withoutStorage = weighted.getMetrics(cluster, vm, offering, dest, cpu, memory, false, + null, new double[0], new HashMap<>()).second(); + double withStorage = weighted.getMetrics(cluster, vm, offering, dest, cpu, memory, true, + null, new double[0], new HashMap<>()).second(); + + assertTrue("a migration that has to move storage should cost more", withStorage > withoutStorage); + } + + @Test + public void testReservedCapacityDoesNotMakeAHostLookFuller() { + // reserved is taken off the overcommitted total, not multiplied by the ratio. Getting this + // backwards made a host with reserved capacity read far fuller than an identical one + // without, and DRS would evacuate it for no reason. + host(1L, 0.10, 0.10, HostLoad.UNKNOWN, 0L, 0L); + host(2L, 0.10, 0.10, HostLoad.UNKNOWN, CPU_TOTAL / 10, MEMORY_TOTAL / 10); + + Map blended = blend(); + + assertEquals("a little reserved capacity should barely move the figure", + blended.get(1L), blended.get(2L), 0.02); + } + + @Test + public void testAllocationBeyondTheOvercommittedTotalIsNotFlattened() { + // hosts past what they can hand out must stay distinguishable, not both read as full + host(1L, 1.20, 0.20, HostLoad.UNKNOWN); + host(2L, 2.40, 0.20, HostLoad.UNKNOWN); + + Map blended = blend(); + + assertTrue("two oversubscribed hosts must not read identically", + blended.get(2L) > blended.get(1L)); + } + + @Test + public void testUtilisationIsDroppedWhenAnyHostCannotBeMeasured() { + // one host with broken telemetry must not read as loaded simply because it is measured on a + // different basis to its peers - that would evacuate whichever host stopped reporting + host(1L, 0.50, 0.50, new HostLoad(0.10, 0.10, 10)); + host(2L, 0.50, 0.50, HostLoad.UNKNOWN); + + Map blended = blend(); + + assertEquals("identical hosts must read identically when one cannot be measured", + blended.get(1L), blended.get(2L), 1e-9); + assertEquals(0.0, weighted.imbalanceOf(blended.values()), 1e-9); + } + + @Test + public void testUtilisationIsUsedWhenEveryHostIsMeasured() { + host(1L, 0.50, 0.50, new HostLoad(0.90, 0.50, 10)); + host(2L, 0.50, 0.50, new HostLoad(0.10, 0.50, 10)); + + assertTrue("with every host measured, load must separate them", + weighted.imbalanceOf(blend().values()) > 0.05); + } + + @Test + public void testStorageMotionMustEarnMoreThanAPlainMigration() throws ConfigurationException { + host(1L, 0.52, 0.50, new HostLoad(0.52, 0.50, 10)); + host(2L, 0.48, 0.50, new HostLoad(0.48, 0.50, 10)); + + VirtualMachine vm = Mockito.mock(VirtualMachine.class); + Mockito.when(vm.getHostId()).thenReturn(1L); + Host dest = Mockito.mock(Host.class); + Mockito.when(dest.getId()).thenReturn(2L); + ServiceOffering offering = Mockito.mock(ServiceOffering.class); + Mockito.when(offering.getCpu()).thenReturn(1); + Mockito.when(offering.getSpeed()).thenReturn(500); + Mockito.when(offering.getRamSize()).thenReturn(512); + + weighted.prepare(cluster, cpu, memory, load); + Ternary plain = weighted.getMetrics(cluster, vm, offering, dest, + cpu, memory, false, null, new double[0], new HashMap<>()); + weighted.prepare(cluster, cpu, memory, load); + Ternary withStorage = weighted.getMetrics(cluster, vm, offering, dest, + cpu, memory, true, null, new double[0], new HashMap<>()); + + // the caller migrates when benefit > cost + assertTrue("a small gain is worth taking without moving storage", + plain.third() > plain.second()); + assertFalse("the same small gain is not worth moving storage for", + withStorage.third() > withStorage.second()); + } +} diff --git a/plugins/pom.xml b/plugins/pom.xml index 92768827f658..08b906e9a6b6 100755 --- a/plugins/pom.xml +++ b/plugins/pom.xml @@ -77,6 +77,7 @@ drs/cluster/balanced drs/cluster/condensed + drs/cluster/weighted event-bus/inmemory event-bus/kafka diff --git a/server/src/main/java/com/cloud/agent/manager/allocator/impl/FirstFitAllocator.java b/server/src/main/java/com/cloud/agent/manager/allocator/impl/FirstFitAllocator.java index 4bc34d8a5c60..6a5669d17975 100644 --- a/server/src/main/java/com/cloud/agent/manager/allocator/impl/FirstFitAllocator.java +++ b/server/src/main/java/com/cloud/agent/manager/allocator/impl/FirstFitAllocator.java @@ -16,6 +16,7 @@ // under the License. package com.cloud.agent.manager.allocator.impl; +import static com.cloud.deploy.DeploymentPlanner.AllocationAlgorithm.balancedweighted; import static com.cloud.deploy.DeploymentPlanner.AllocationAlgorithm.firstfitleastconsumed; import static com.cloud.deploy.DeploymentPlanner.AllocationAlgorithm.random; import static com.cloud.deploy.DeploymentPlanner.AllocationAlgorithm.userdispersing; @@ -95,6 +96,8 @@ public class FirstFitAllocator extends BaseAllocator { CapacityDao _capacityDao; @Inject VMInstanceDetailsDao vmInstanceDetailsDao; + @Inject + WeightedHostScorer weightedHostScorer; boolean _checkHvm = true; @@ -209,6 +212,8 @@ protected List allocateTo(VirtualMachineProfile vmProfile, DeploymentPlan hosts = reorderHostsByNumberOfVms(plan, hosts, account); } else if (firstfitleastconsumed.toString().equals(vmAllocationAlgorithm)) { hosts = reorderHostsByCapacity(plan, hosts); + } else if (balancedweighted.toString().equals(vmAllocationAlgorithm)) { + hosts = weightedHostScorer.rank(plan.getDataCenterId(), plan.getPodId(), plan.getClusterId(), hosts); } logger.debug("FirstFitAllocator has {} hosts to check for allocation {}.", hosts.size(), hosts); diff --git a/server/src/main/java/com/cloud/agent/manager/allocator/impl/HostLoadTracker.java b/server/src/main/java/com/cloud/agent/manager/allocator/impl/HostLoadTracker.java new file mode 100644 index 000000000000..cd51ae2d8686 --- /dev/null +++ b/server/src/main/java/com/cloud/agent/manager/allocator/impl/HostLoadTracker.java @@ -0,0 +1,263 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.agent.manager.allocator.impl; + +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + +import javax.inject.Inject; + +import org.apache.cloudstack.framework.config.ConfigKey; +import org.apache.cloudstack.framework.config.Configurable; +import org.apache.cloudstack.managed.context.ManagedContextRunnable; + +import com.cloud.host.HostLoad; +import com.cloud.host.HostLoadService; +import com.cloud.host.HostStats; +import com.cloud.host.HostVO; +import com.cloud.host.Status; +import com.cloud.host.dao.HostDao; +import com.cloud.server.StatsCollector; +import com.cloud.deploy.DeploymentClusterPlanner; +import com.cloud.deploy.DeploymentPlanner.AllocationAlgorithm; +import com.cloud.utils.component.ManagerBase; +import com.cloud.utils.concurrency.NamedThreadFactory; + +/** + * Keeps a smoothed view of how hard each host is actually working. + * + * StatsCollector already polls every host, but it keeps only the newest sample and nothing uses it + * for placement. A single sample is too noisy to rank on: a host can look idle moments before a + * batch of VMs starts work. This folds those samples into an exponentially weighted moving average + * so ranking reflects a trend rather than an instant. + * + * The average is per management server and is not persisted. Every management server polls every + * host, so all of them converge on the same picture, and a restarted server simply reports nothing + * usable until it has sampled - callers then fall back to allocation figures. + * + * What getCpuUtilization means depends on the hypervisor, and only KVM reports what this class + * assumes: + * + *
    + *
  • KVM reports busy time as a percentage of the host's cores, which is what is wanted.
    • + *
  • VMware reports the share of CPU that is reserved rather than the share that is busy, so + * the CPU term becomes a second allocation signal there rather than a load signal.
    • + *
  • XenServer sums per-core averages without dividing by core count, so the value ranges up to + * the number of cores and is under-reported here by roughly that factor.
    • + *
+ * + * Memory is taken as used over total and is sound everywhere. + */ +public class HostLoadTracker extends ManagerBase implements HostLoadService, Configurable { + + public static final ConfigKey HostLoadSampleInterval = new ConfigKey<>(ConfigKey.CATEGORY_ADVANCED, + Integer.class, "host.load.sample.interval", "60", + "Seconds between samples of host CPU and memory utilisation, for placement algorithms that " + + "consider actual load. Should not be shorter than host.stats.interval.", + false, ConfigKey.Scope.Global); + + public static final ConfigKey HostLoadStaleAfter = new ConfigKey<>(ConfigKey.CATEGORY_ADVANCED, + Integer.class, "host.load.stale.after", "600", + "Seconds after which a host's utilisation average is considered out of date and stops being used " + + "for placement. A host whose agent stops reporting would otherwise keep vouching for itself " + + "with figures that never change.", + true, ConfigKey.Scope.Global); + + public static final ConfigKey HostLoadHalfLife = new ConfigKey<>(ConfigKey.CATEGORY_ADVANCED, + Integer.class, "host.load.half.life", "300", + "Half life in seconds of the moving average of host utilisation. Larger values react more " + + "slowly and are less affected by short spikes or by guests periodically releasing memory.", + true, ConfigKey.Scope.Global); + + @Inject + private HostDao hostDao; + + @Inject + private StatsCollector statsCollector; + + private final Map samples = new ConcurrentHashMap<>(); + + private ScheduledExecutorService executor; + + @Override + public boolean start() { + int interval = Math.max(1, HostLoadSampleInterval.value()); + executor = Executors.newSingleThreadScheduledExecutor( + new NamedThreadFactory("HostLoadTracker")); + // catch Throwable: an escaping error would cancel all future runs, and the failure would be + // silent - placement would quietly go back to ranking on allocation alone + executor.scheduleWithFixedDelay(new ManagedContextRunnable() { + @Override + protected void runInContext() { + try { + sampleAllHosts(); + } catch (Throwable t) { + logger.warn("Unable to sample host load", t); + } + } + }, interval, interval, TimeUnit.SECONDS); + return true; + } + + @Override + public boolean stop() { + if (executor != null) { + executor.shutdownNow(); + } + return true; + } + + protected void sampleAllHosts() { + if (!isInUse()) { + samples.clear(); + return; + } + for (HostVO host : hostDao.listByType(com.cloud.host.Host.Type.Routing)) { + if (host.getStatus() != Status.Up) { + samples.remove(host.getId()); + continue; + } + record(host.getId(), statsCollector.getHostStats(host.getId())); + } + } + + /** + * Only the placement algorithms that read these figures pay for collecting them. + */ + protected boolean isInUse() { + return AllocationAlgorithm.balancedweighted.toString() + .equals(DeploymentClusterPlanner.VmAllocationAlgorithm.value()); + } + + protected void record(long hostId, HostStats stats) { + record(hostId, stats, System.currentTimeMillis()); + } + + protected void record(long hostId, HostStats stats, long now) { + if (stats == null) { + return; + } + double totalMemory = stats.getTotalMemoryKBs(); + if (totalMemory <= 0) { + return; + } + + Sample previous = samples.get(hostId); + if (previous != null && previous.isSameReadingAs(stats)) { + // StatsCollector keeps the previous entry when a poll fails, so an unchanged object is + // a reading we have already folded, not a fresh measurement + return; + } + + // getCpuUtilization is a percentage of the host's real cores. That holds for KVM; see the + // class javadoc for what it means on other hypervisors. + double cpu = clamp(stats.getCpuUtilization() / 100.0); + double memory = clamp((totalMemory - stats.getFreeMemoryKBs()) / totalMemory); + int halfLife = HostLoadHalfLife.value(); + + samples.compute(hostId, (id, current) -> current == null + ? new Sample(cpu, memory, now, stats) + : current.fold(cpu, memory, now, halfLife, stats)); + } + + @Override + public HostLoad getLoad(long hostId) { + return getLoad(hostId, System.currentTimeMillis()); + } + + protected HostLoad getLoad(long hostId, long now) { + Sample sample = samples.get(hostId); + if (sample == null) { + return HostLoad.UNKNOWN; + } + long staleAfter = Math.max(1, HostLoadStaleAfter.value()) * 1000L; + if (now - sample.updatedAt > staleAfter) { + // the host has stopped reporting; stop letting its last known figures speak for it + return HostLoad.UNKNOWN; + } + return sample.toHostLoad(); + } + + protected void clear() { + samples.clear(); + } + + private static double clamp(double value) { + if (Double.isNaN(value) || value < 0) { + return 0; + } + return Math.min(value, 1); + } + + @Override + public String getConfigComponentName() { + return HostLoadTracker.class.getSimpleName(); + } + + @Override + public ConfigKey[] getConfigKeys() { + return new ConfigKey[] {HostLoadSampleInterval, HostLoadHalfLife, HostLoadStaleAfter}; + } + + /** + * One host's running average. Weighting is by elapsed time rather than by sample count, so a + * missed poll decays the old value by the right amount instead of over-weighting it. + */ + private static final class Sample { + private final double cpu; + private final double memory; + private final long updatedAt; + private final long count; + private final HostStats reading; + + private Sample(double cpu, double memory, long updatedAt, HostStats reading) { + this(cpu, memory, updatedAt, 1, reading); + } + + private Sample(double cpu, double memory, long updatedAt, long count, HostStats reading) { + this.cpu = cpu; + this.memory = memory; + this.updatedAt = updatedAt; + this.count = count; + this.reading = reading; + } + + private boolean isSameReadingAs(HostStats stats) { + return reading == stats; + } + + private Sample fold(double newCpu, double newMemory, long now, int halfLifeSeconds, HostStats reading) { + double alpha = alpha(now - updatedAt, halfLifeSeconds); + return new Sample(cpu + alpha * (newCpu - cpu), memory + alpha * (newMemory - memory), now, + count + 1, reading); + } + + private static double alpha(long elapsedMillis, int halfLifeSeconds) { + if (halfLifeSeconds <= 0 || elapsedMillis <= 0) { + return 1; + } + return 1 - Math.exp(-(elapsedMillis / 1000.0) * Math.log(2) / halfLifeSeconds); + } + + private HostLoad toHostLoad() { + return new HostLoad(cpu, memory, count); + } + } +} diff --git a/server/src/main/java/com/cloud/agent/manager/allocator/impl/WeightedHostScorer.java b/server/src/main/java/com/cloud/agent/manager/allocator/impl/WeightedHostScorer.java new file mode 100644 index 000000000000..44d3b300caa7 --- /dev/null +++ b/server/src/main/java/com/cloud/agent/manager/allocator/impl/WeightedHostScorer.java @@ -0,0 +1,418 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.agent.manager.allocator.impl; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.stream.Collectors; + +import javax.inject.Inject; + +import org.apache.cloudstack.framework.config.ConfigKey; +import org.apache.cloudstack.framework.config.Configurable; + +import com.cloud.capacity.Capacity; +import com.cloud.capacity.CapacityManager; +import com.cloud.capacity.CapacityVO; +import com.cloud.capacity.dao.CapacityDao; +import com.cloud.dc.ClusterDetailsDao; +import com.cloud.dc.ClusterDetailsVO; +import com.cloud.host.Host; +import com.cloud.host.HostLoad; +import com.cloud.host.HostScoringWeights; +import com.cloud.utils.Pair; +import com.cloud.utils.component.AdapterBase; +import com.cloud.vm.VmDetailConstants; +import com.cloud.vm.dao.VMInstanceDao; + +/** + * Ranks hosts on a blend of what has been allocated on them and what they are actually doing. + * + * Ordering purely by allocated capacity misreads a heavily overprovisioned cluster: allocation is + * measured against a total that has been multiplied by the overprovisioning factor, so hosts under + * real strain can still look close to empty and keep attracting new VMs. This blends allocation + * with measured utilisation, VM count, and how many VMs started on the host recently, since a VM + * that has just started is usually working harder than its long run average. + * + * Scores run from 0 (idle) upwards and lower is better. Hosts are then chosen from among the best + * rather than strictly in order - see {@link #applySelectionSpread}. + */ +public class WeightedHostScorer extends AdapterBase implements Configurable { + + private static final String WEIGHT_DESCRIPTION_SUFFIX = + " Relative weight, only meaningful compared with the other host.weighted.* weights. Zero disables the term."; + + public static final ConfigKey VmCountWeight = new ConfigKey<>(ConfigKey.CATEGORY_ADVANCED, + Double.class, "host.weighted.vm.count.weight", "1.0", + "How much the number of VMs already on a host counts against it, regardless of how busy they are." + + WEIGHT_DESCRIPTION_SUFFIX, + true, ConfigKey.Scope.Cluster); + + public static final ConfigKey RecentStartWeight = new ConfigKey<>(ConfigKey.CATEGORY_ADVANCED, + Double.class, "host.weighted.recent.start.weight", "2.0", + "How much VMs started recently on a host count against it. Guards against sending a burst of new " + + "VMs to one host, since neither allocation nor utilisation has caught up with them yet." + + WEIGHT_DESCRIPTION_SUFFIX, + true, ConfigKey.Scope.Cluster); + + public static final ConfigKey DominantResourceWeight = new ConfigKey<>(ConfigKey.CATEGORY_ADVANCED, + Double.class, "host.weighted.dominant.resource.weight", "1.0", + "How much a host's single most stressed resource counts against it, on top of the average across " + + "resources. Keeps a host that is fine on average but nearly out of one resource from ranking well." + + WEIGHT_DESCRIPTION_SUFFIX, + true, ConfigKey.Scope.Cluster); + + public static final ConfigKey RecentStartWindow = new ConfigKey<>(ConfigKey.CATEGORY_ADVANCED, + Integer.class, "host.weighted.recent.start.window", "300", + "Seconds for which a newly started VM counts as recently started.", + true, ConfigKey.Scope.Global); + + public static final ConfigKey ExpectedVmsPerHost = new ConfigKey<>(ConfigKey.CATEGORY_ADVANCED, + Integer.class, "host.weighted.expected.vms.per.host", "50", + "Roughly how many VMs a host is expected to carry. Used only to bring VM counts onto the same " + + "0 to 1 scale as the other terms; it is not a limit and is never enforced.", + true, ConfigKey.Scope.Cluster); + + public static final ConfigKey CpuUtilisationThreshold = new ConfigKey<>(ConfigKey.CATEGORY_ADVANCED, + Double.class, "host.weighted.cpu.utilisation.threshold", "0.85", + "Hosts whose measured CPU utilisation is above this fraction are held back from new VMs. " + + "Ignored if it would leave nowhere to deploy. Set to 1 to disable.", + true, ConfigKey.Scope.Cluster); + + public static final ConfigKey MemoryUtilisationThreshold = new ConfigKey<>(ConfigKey.CATEGORY_ADVANCED, + Double.class, "host.weighted.memory.utilisation.threshold", "0.90", + "Hosts whose measured memory utilisation is above this fraction are held back from new VMs. " + + "Ignored if it would leave nowhere to deploy. Set to 1 to disable.", + true, ConfigKey.Scope.Cluster); + + public static final ConfigKey SelectionSpread = new ConfigKey<>(ConfigKey.CATEGORY_ADVANCED, + Integer.class, "host.weighted.selection.spread", "3", + "How many of the best scoring hosts to choose between at random. Ranking strictly by score sends " + + "concurrent deployments to the same host, because they all read the same figures before any " + + "of them is accounted for. 1 restores strict ordering.", + true, ConfigKey.Scope.Cluster); + + private static final Pair NO_VMS = new Pair<>(0L, 0L); + + @Inject + private CapacityDao capacityDao; + + @Inject + private ClusterDetailsDao clusterDetailsDao; + + @Inject + private VMInstanceDao vmInstanceDao; + + @Inject + private HostLoadTracker hostLoadTracker; + + protected Random random = new Random(); + + /** + * Orders hosts best first. Hosts absent from the capacity tables keep their original relative + * order at the end of the list rather than being dropped. + */ + public List rank(long zoneId, Long podId, Long clusterId, List hosts) { + if (hosts == null || hosts.size() <= 1) { + return hosts == null ? new ArrayList<>() : new ArrayList<>(hosts); + } + + Map scores = score(zoneId, podId, clusterId, hosts); + + List unscored = new ArrayList<>(); + List measured = new ArrayList<>(); + List unmeasured = new ArrayList<>(); + for (Host host : hosts) { + if (!scores.containsKey(host.getId())) { + unscored.add(host); + } else if (hostLoadTracker.getLoad(host.getId()).isUsable()) { + measured.add(host); + } else { + unmeasured.add(host); + } + } + + Comparator byScore = Comparator.comparingDouble(h -> scores.get(h.getId())); + measured.sort(byScore); + unmeasured.sort(byScore); + + List healthy = new ArrayList<>(); + List tooBusy = new ArrayList<>(); + partitionByUtilisation(clusterId, measured, healthy, tooBusy); + + List result = new ArrayList<>(); + if (healthy.isEmpty() && unmeasured.isEmpty()) { + logger.warn("Every candidate host is above its utilisation threshold, so the thresholds are being " + + "ignored for this deployment. The cluster is short of capacity."); + result.addAll(tooBusy); + applySelectionSpread(clusterId, result); + } else { + result.addAll(healthy); + // spread only over hosts known to be healthy, before anything else is appended, + // otherwise a busy or unmeasured host can be shuffled into the lead + applySelectionSpread(clusterId, result); + // a host we cannot measure is not assumed to be idle: it ranks behind every host we can + result.addAll(unmeasured); + result.addAll(tooBusy); + } + + if (!tooBusy.isEmpty()) { + logger.debug("Holding back {} host(s) above their utilisation threshold: {}", tooBusy.size(), tooBusy); + } + logger.debug("Weighted host ranking: {}", () -> result.stream() + .filter(h -> scores.containsKey(h.getId())) + .map(h -> String.format("%s=%.4f", h.getName(), scores.get(h.getId()))) + .collect(Collectors.joining(", "))); + + result.addAll(unscored); + return result; + } + + protected Map score(long zoneId, Long podId, Long clusterId, List hosts) { + List capacities = capacityDao.listHostCapacityByCapacityTypes(zoneId, clusterId, + List.of(Capacity.CAPACITY_TYPE_CPU, Capacity.CAPACITY_TYPE_MEMORY)); + Map> vmCounts = vmInstanceDao.countVmsByHost(zoneId, podId, clusterId, + new Date(System.currentTimeMillis() - RecentStartWindow.value() * 1000L)); + + Map allocated = allocatedFractions(capacities); + + Weights weights = new Weights(clusterId); + Map scores = new HashMap<>(); + for (Host host : hosts) { + Double[] alloc = allocated.get(host.getId()); + if (alloc == null) { + continue; + } + Pair counts = vmCounts.getOrDefault(host.getId(), NO_VMS); + scores.put(host.getId(), scoreHost(weights, alloc[0], alloc[1], hostLoadTracker.getLoad(host.getId()), + counts.first(), counts.second())); + } + return scores; + } + + /** + * Allocated CPU and memory as a fraction of what a host can hand out. + * + * op_host_capacity stores totals raw; overprovisioning is applied when they are read, so the + * cluster's ratio has to be applied here too. Without it the fraction reaches 1 at the host's + * physical size and every host on an overcommitted cluster clamps to 1, which is where this + * algorithm is most needed. + * + * Only hosts with both a CPU and a memory row are returned. A host missing one would otherwise + * score as if that resource were untouched, making it the most attractive host in the cluster. + */ + protected Map allocatedFractions(List capacities) { + Map fractions = new HashMap<>(); + Map seen = new HashMap<>(); + for (CapacityVO capacity : capacities) { + long total = capacity.getTotalCapacity(); + if (total <= 0) { + continue; + } + boolean isCpu = capacity.getCapacityType() == Capacity.CAPACITY_TYPE_CPU; + float overcommit = overcommitRatio(capacity.getClusterId(), isCpu); + double allocatable = total * overcommit; + double used = (double) (capacity.getUsedCapacity() + capacity.getReservedCapacity()) / allocatable; + + Double[] entry = fractions.computeIfAbsent(capacity.getHostOrPoolId(), id -> new Double[] {0.0, 0.0}); + entry[isCpu ? 0 : 1] = clamp(used); + seen.merge(capacity.getHostOrPoolId(), isCpu ? 1 : 2, Integer::sum); + } + fractions.keySet().removeIf(hostId -> seen.getOrDefault(hostId, 0) != 3); + return fractions; + } + + /** + * The cluster's overprovisioning factor, defaulting to none if it cannot be read. + */ + protected float overcommitRatio(Long clusterId, boolean forCpu) { + if (clusterId == null) { + return 1f; + } + String key = forCpu ? VmDetailConstants.CPU_OVER_COMMIT_RATIO : VmDetailConstants.MEMORY_OVER_COMMIT_RATIO; + ClusterDetailsVO detail = clusterDetailsDao.findDetail(clusterId, key); + if (detail == null || detail.getValue() == null) { + return 1f; + } + try { + float ratio = Float.parseFloat(detail.getValue()); + return ratio > 0 ? ratio : 1f; + } catch (NumberFormatException e) { + logger.warn("Cluster {} has an unreadable {} of [{}], treating it as 1.", clusterId, key, detail.getValue()); + return 1f; + } + } + + /** + * The blend. Every term is a fraction of the host's capacity for that resource so the weights + * are directly comparable, and the dominant resource term is added on top of the weighted mean + * so that being nearly out of any one resource is penalised even when the average looks fine. + */ + protected double scoreHostIn(Long clusterId, double cpuAllocated, double memoryAllocated, HostLoad load, + long vmCount, long recentStarts) { + return scoreHost(new Weights(clusterId), cpuAllocated, memoryAllocated, load, vmCount, recentStarts); + } + + protected double scoreHost(Weights weights, double cpuAllocated, double memoryAllocated, HostLoad load, + long vmCount, long recentStarts) { + // a host with no usable load figures is ranked on allocation alone, and is placed behind + // every measured host by the caller rather than being assumed idle + double cpuUsedWeight = load.isUsable() ? weights.cpuUsed : 0; + double memoryUsedWeight = load.isUsable() ? weights.memoryUsed : 0; + + double vmCountTerm = clamp(vmCount / weights.vmScale); + double recentStartTerm = clamp(recentStarts / weights.vmScale); + + double weightSum = weights.cpuAllocated + cpuUsedWeight + weights.memoryAllocated + memoryUsedWeight + + weights.vmCount + weights.recentStart; + + double mean = 0; + if (weightSum > 0) { + mean = (weights.cpuAllocated * cpuAllocated + + cpuUsedWeight * load.getCpuUtilisation() + + weights.memoryAllocated * memoryAllocated + + memoryUsedWeight * load.getMemoryUtilisation() + + weights.vmCount * vmCountTerm + + weights.recentStart * recentStartTerm) / weightSum; + } + + if (weights.dominant <= 0) { + return mean; + } + return (mean + weights.dominant * dominantResource(cpuAllocated, memoryAllocated, load)) / (1 + weights.dominant); + } + + /** + * The most stressed resource on the host. Allocation and utilisation are both considered for + * each resource and the larger is taken, because memory that has been reclaimed from idle + * guests can be taken back as soon as those guests get busy. + */ + protected double dominantResource(double cpuAllocated, double memoryAllocated, HostLoad load) { + double cpu = load.isUsable() ? Math.max(cpuAllocated, load.getCpuUtilisation()) : cpuAllocated; + double memory = load.isUsable() ? Math.max(memoryAllocated, load.getMemoryUtilisation()) : memoryAllocated; + return Math.max(cpu, memory); + } + + /** + * Splits measurably busy hosts out from the rest. Only hosts with load samples can be held + * back; a host that cannot be measured is dealt with by the caller. + */ + protected void partitionByUtilisation(Long clusterId, List measured, List healthy, List tooBusy) { + double cpuThreshold = valueIn(CpuUtilisationThreshold, clusterId); + double memoryThreshold = valueIn(MemoryUtilisationThreshold, clusterId); + + for (Host host : measured) { + HostLoad load = hostLoadTracker.getLoad(host.getId()); + if (load.getCpuUtilisation() > cpuThreshold || load.getMemoryUtilisation() > memoryThreshold) { + tooBusy.add(host); + } else { + healthy.add(host); + } + } + } + + /** + * Shuffles the best few hosts so that deployments made at the same moment do not all pick the + * same one. Capacity is only charged once a VM starts, so until then every concurrent decision + * sees the same figures and strict ordering makes them agree. + */ + protected void applySelectionSpread(Long clusterId, List ranked) { + int spread = Math.min((int) valueIn(SelectionSpread, clusterId), ranked.size()); + if (spread > 1) { + Collections.shuffle(ranked.subList(0, spread), random); + } + } + + /** + * The weights for one ranking, read once rather than per host. + * + * A negative weight would invert the ranking and make the most loaded host the best, so they + * are floored at zero and the bad value is reported. + */ + protected final class Weights { + private final double cpuAllocated; + private final double cpuUsed; + private final double memoryAllocated; + private final double memoryUsed; + private final double vmCount; + private final double recentStart; + private final double dominant; + private final double vmScale; + + protected Weights(Long clusterId) { + cpuAllocated = nonNegative(HostScoringWeights.CpuAllocatedWeight, clusterId); + cpuUsed = nonNegative(HostScoringWeights.CpuUsedWeight, clusterId); + memoryAllocated = nonNegative(HostScoringWeights.MemoryAllocatedWeight, clusterId); + memoryUsed = nonNegative(HostScoringWeights.MemoryUsedWeight, clusterId); + vmCount = nonNegative(VmCountWeight, clusterId); + recentStart = nonNegative(RecentStartWeight, clusterId); + dominant = nonNegative(DominantResourceWeight, clusterId); + vmScale = Math.max(1, valueIn(ExpectedVmsPerHost, clusterId)); + } + + private double nonNegative(ConfigKey key, Long clusterId) { + double value = valueIn(key, clusterId); + if (value < 0) { + logger.warn("{} is set to {}, which would rank the most loaded host first. Treating it as 0.", + key.key(), value); + return 0; + } + return value; + } + } + + private double valueIn(ConfigKey key, Long clusterId) { + T value = clusterId == null ? key.value() : key.valueIn(clusterId); + return value == null ? 0 : value.doubleValue(); + } + + private static double clamp(double value) { + if (Double.isNaN(value) || value < 0) { + return 0; + } + return Math.min(value, 1); + } + + @Override + public String getConfigComponentName() { + return CapacityManager.class.getSimpleName(); + } + + @Override + public ConfigKey[] getConfigKeys() { + return new ConfigKey[] { + HostScoringWeights.CpuAllocatedWeight, + HostScoringWeights.CpuUsedWeight, + HostScoringWeights.MemoryAllocatedWeight, + HostScoringWeights.MemoryUsedWeight, + VmCountWeight, + RecentStartWeight, + DominantResourceWeight, + RecentStartWindow, + ExpectedVmsPerHost, + CpuUtilisationThreshold, + MemoryUtilisationThreshold, + SelectionSpread + }; + } +} diff --git a/server/src/main/java/org/apache/cloudstack/cluster/ClusterDrsServiceImpl.java b/server/src/main/java/org/apache/cloudstack/cluster/ClusterDrsServiceImpl.java index 62075aae596e..7862de7551ed 100644 --- a/server/src/main/java/org/apache/cloudstack/cluster/ClusterDrsServiceImpl.java +++ b/server/src/main/java/org/apache/cloudstack/cluster/ClusterDrsServiceImpl.java @@ -25,6 +25,7 @@ import com.cloud.dc.ClusterVO; import com.cloud.dc.dao.ClusterDao; import com.cloud.deploy.DataCenterDeployment; +import com.cloud.deploy.DeploymentPlan; import com.cloud.deploy.DeploymentPlanner.ExcludeList; import com.cloud.domain.Domain; import com.cloud.event.ActionEventUtils; @@ -33,6 +34,8 @@ import com.cloud.event.dao.EventDao; import com.cloud.exception.InvalidParameterValueException; import com.cloud.host.Host; +import com.cloud.host.HostLoad; +import com.cloud.host.HostLoadService; import com.cloud.host.HostVO; import com.cloud.host.dao.HostDao; import com.cloud.offering.ServiceOffering; @@ -43,6 +46,7 @@ import com.cloud.user.User; import com.cloud.utils.DateUtil; import com.cloud.utils.Pair; +import com.cloud.storage.dao.VolumeDao; import com.cloud.utils.Ternary; import com.cloud.utils.component.ComponentContext; import com.cloud.utils.component.ManagerBase; @@ -80,6 +84,7 @@ import org.apache.cloudstack.jobs.JobInfo; import org.apache.cloudstack.managed.context.ManagedContextTimerTask; import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.collections.MapUtils; import org.apache.commons.lang3.time.DateUtils; import javax.inject.Inject; @@ -144,6 +149,12 @@ public class ClusterDrsServiceImpl extends ManagerBase implements ClusterDrsServ @Inject AffinityGroupVMMapDao affinityGroupVMMapDao; + @Inject + VolumeDao volumeDao; + + @Inject + HostLoadService hostLoadService; + List drsAlgorithms = new ArrayList<>(); Map drsAlgorithmMap = new HashMap<>(); @@ -391,13 +402,27 @@ private List> getMigrationPlans( ClusterDrsAlgorithm algorithm = getDrsAlgorithm(ClusterDrsAlgorithm.valueIn(cluster.getId())); int iteration = 0; List> migrationPlan = new ArrayList<>(); - while (iteration < maxIterations && algorithm.needsDrs(cluster, new ArrayList<>(hostCpuMap.values()), - new ArrayList<>(hostMemoryMap.values()))) { + Map vmToExcludesMap = null; + Set staleAffinityVmIds = new HashSet<>(); + Map hostLoadMap = new HashMap<>(); + for (Long hostId : hostCpuMap.keySet()) { + hostLoadMap.put(hostId, hostLoadService.getLoad(hostId)); + } + algorithm.prepare(cluster, hostCpuMap, hostMemoryMap, hostLoadMap); + + while (iteration < maxIterations && algorithm.needsDrs(cluster, hostCpuMap, hostMemoryMap, hostLoadMap)) { logger.debug("Starting DRS iteration {} for cluster {}", iteration + 1, cluster); - // Re-evaluate affinity constraints with current (simulated) VM placements - Map vmToExcludesMap = getVmToExcludesMap(vmList, hostMap, vmsWithAffinityGroups, - vmToCompatibleHostsCache, vmIdServiceOfferingMap); + // Affinity only changes for VMs that share a group with the one just moved, so after + // the first pass only those are re-evaluated rather than every VM in the cluster + if (vmToExcludesMap == null) { + vmToExcludesMap = getVmToExcludesMap(vmList, hostMap, vmsWithAffinityGroups, + vmToCompatibleHostsCache, vmIdServiceOfferingMap, null); + } else if (!staleAffinityVmIds.isEmpty()) { + vmToExcludesMap.putAll(getVmToExcludesMap(vmList, hostMap, vmsWithAffinityGroups, + vmToCompatibleHostsCache, vmIdServiceOfferingMap, staleAffinityVmIds)); + staleAffinityVmIds = new HashSet<>(); + } logger.debug("Completed affinity evaluation for DRS iteration {} for cluster {}", iteration + 1, cluster); @@ -427,16 +452,63 @@ private List> getMigrationPlans( hostMemoryMap.get(vm.getHostId()).first(hostMemoryMap.get(vm.getHostId()).first() - vmMemory); hostMemoryMap.get(destHost.getId()).first(hostMemoryMap.get(destHost.getId()).first() + vmMemory); vm.setHostId(destHost.getId()); + staleAffinityVmIds = affinityPeersOf(vm, vmsWithAffinityGroups); iteration++; } return migrationPlan; } + /** + * Turns the non-strict affinity preferences recorded on the plan into exclusions. + * + * Non-strict groups express themselves by lowering a host's priority rather than by excluding + * it, and DRS only reads the exclude list - so the preference was being discarded. Rebalancing + * is never a reason to break it: the VM is already running somewhere that satisfies the group, + * and leaving it there is always available to DRS. "Non-strict" means the rule may be broken + * when there is nowhere else to put a VM, which cannot arise while merely rebalancing. + */ + protected void excludeHostsDispreferredByAffinity(DeploymentPlan plan, ExcludeList excludes) { + Map priorities = plan.getHostPriorities(); + if (MapUtils.isEmpty(priorities)) { + return; + } + for (Map.Entry entry : priorities.entrySet()) { + if (entry.getValue() != null && entry.getValue() < DeploymentPlan.DEFAULT_HOST_PRIORITY) { + excludes.addHost(entry.getKey()); + logger.debug("Host {} is dispreferred by a non-strict affinity group, so DRS will not migrate onto it", + entry.getKey()); + } + } + } + + /** + * The VMs whose affinity picture changes when the given VM moves: the VM itself, and everything + * sharing an affinity group with it. + */ + protected Set affinityPeersOf(VirtualMachine vm, Set vmsWithAffinityGroups) { + Set peers = new HashSet<>(); + peers.add(vm.getId()); + if (!vmsWithAffinityGroups.contains(vm.getId())) { + return peers; + } + for (AffinityGroupVMMapVO mapping : affinityGroupVMMapDao.listByInstanceId(vm.getId())) { + peers.addAll(affinityGroupVMMapDao.listVmIdsByAffinityGroup(mapping.getAffinityGroupId())); + } + return peers; + } + + /** + * @param onlyVmIds + * when set, re-evaluates just these VMs rather than the whole cluster + */ private Map getVmToExcludesMap(List vmList, Map hostMap, Set vmsWithAffinityGroups, Map> vmToCompatibleHostsCache, - Map vmIdServiceOfferingMap) { + Map vmIdServiceOfferingMap, Set onlyVmIds) { Map vmToExcludesMap = new HashMap<>(); for (VirtualMachine vm : vmList) { + if (onlyVmIds != null && !onlyVmIds.contains(vm.getId())) { + continue; + } if (vmToCompatibleHostsCache.containsKey(vm.getId())) { Host srcHost = hostMap.get(vm.getHostId()); if (srcHost != null) { @@ -452,6 +524,7 @@ private Map getVmToExcludesMap(List vmList, M excludes = managementServer.applyAffinityConstraints( vm, vmProfile, plan, vmList); + excludeHostsDispreferredByAffinity(plan, excludes); } else { // VM has no affinity groups - create minimal ExcludeList (just source host) excludes = new ExcludeList(); @@ -485,20 +558,39 @@ private Pair>, Map>> get .map(VMInstanceDetailVO::getResourceId) .collect(Collectors.toSet()); + // Working out where a VM could go is the expensive part of planning: it runs the host + // allocators and inspects every volume. VMs that would get the same answer are grouped and + // the work is done once for the group. In a cluster of similar VMs this is the difference + // between one pass per VM and one pass per handful. + Map, Integer>, List, Map>> byEquivalence = + new HashMap<>(); + int computed = 0; + for (VirtualMachine vm : vmList) { - // Skip ineligible VMs if (shouldSkipVMForDRS(vm, skipDrsVmIds)) { logger.debug("Skipping VM {} for DRS as it is ineligible.", vm); continue; } + // a VM whose key cannot be worked out is treated as one of a kind rather than dropped + String key; + try { + key = migrationEquivalenceKey(vm); + } catch (Exception e) { + logger.debug("Could not group VM {} with others, considering it on its own", vm, e); + key = "vm-" + vm.getId(); + } + try { - // Use listHostsForMigrationOfVM to get suitable hosts (validated by getCapableSuitableHosts) - // This ensures the same validation as the "find host for migration" command Ternary, Integer>, List, Map> hostsForMigration = - managementServer.listHostsForMigrationOfVM(vm, 0L, 500L, null, vmList); + byEquivalence.get(key); + if (hostsForMigration == null) { + hostsForMigration = managementServer.listHostsForMigrationOfVM(vm, 0L, 500L, null, vmList); + byEquivalence.put(key, hostsForMigration); + computed++; + } - List suitableHosts = hostsForMigration.second(); // Get suitable hosts (validated by HostAllocator) + List suitableHosts = hostsForMigration.second(); Map requiresStorageMotion = hostsForMigration.third(); if (suitableHosts != null && !suitableHosts.isEmpty()) { @@ -509,9 +601,67 @@ private Pair>, Map>> get logger.debug("Could not get suitable hosts for VM {}: {}", vm, e.getMessage()); } } + logger.debug("Worked out candidate hosts for {} VMs in {} passes", vmToCompatibleHostsCache.size(), computed); return new Pair<>(vmToCompatibleHostsCache, vmToStorageMotionCache); } + /** + * VM details that narrow which hosts a VM can run on. Matched as prefixes and case + * insensitively, so that related keys are covered without listing each one. + */ + private static final List PLACEMENT_AFFECTING_DETAIL_PREFIXES = List.of( + "uefi", "boot", "dpdk", "cpunumber", "cpuspeed", "memory", "rootdisk", "nic", "gpu", "vgpu", + "extraconfig", "hypervisortoolsversion", "kvm", "vmware", "hyperv"); + + /** + * Identifies VMs that would get the same answer from listHostsForMigrationOfVM. + * + * The answer depends on what the VM asks for, where it is now, the volumes that would have to + * follow it, and its affinity groups. Rather than enumerate everything that could possibly + * matter and risk missing one, a VM is only grouped when it has none of the per-VM inputs that + * are known to change the answer - a custom offering whose size comes from the VM rather than + * the offering, a boot mode or device setting, and so on. Anything else is worked out on its + * own, which costs what it always did. + * + * @return a key shared with equivalent VMs, or one unique to this VM when it cannot be grouped + */ + protected String migrationEquivalenceKey(VirtualMachine vm) { + ServiceOffering offering = serviceOfferingDao.findByIdIncludingRemoved(vm.getId(), vm.getServiceOfferingId()); + if (offering == null || offering.isDynamic() || hasPlacementAffectingDetails(vm)) { + return "vm-" + vm.getId(); + } + + List volumes = volumeDao.findCreatedByInstance(vm.getId()).stream() + .filter(volume -> volume.getPoolId() != null) + .map(volume -> volume.getPoolId() + ":" + volume.getDiskOfferingId()) + .sorted() + .collect(Collectors.toList()); + List groupIds = affinityGroupVMMapDao.listByInstanceId(vm.getId()).stream() + .map(AffinityGroupVMMapVO::getAffinityGroupId) + .sorted() + .collect(Collectors.toList()); + return String.format("%s|%s|%s|%s|%s|%s", vm.getServiceOfferingId(), vm.getTemplateId(), vm.getHostId(), + vm.getHypervisorType(), volumes, groupIds); + } + + /** + * Whether the VM carries any detail that narrows the hosts it can run on. Such a VM is never + * grouped with another, because the details are per VM and two VMs on the same offering can + * differ entirely. + */ + protected boolean hasPlacementAffectingDetails(VirtualMachine vm) { + Map details = vmInstanceDetailsDao.listDetailsKeyPairs(vm.getId()); + if (MapUtils.isEmpty(details)) { + return false; + } + for (String key : details.keySet()) { + if (PLACEMENT_AFFECTING_DETAIL_PREFIXES.stream().anyMatch(prefix -> key.toLowerCase().startsWith(prefix))) { + return true; + } + } + return false; + } + /** * Pre-fetch affinity group mappings for all eligible VMs (once, before iterations) * This allows us to skip expensive affinity processing for VMs without affinity groups @@ -761,6 +911,47 @@ void processPlans() { } } + /** + * Checks a planned migration against the placement rules as they stand now. + * + * A plan is generated once and executed later, so state can have moved on: VMs may have been + * created, migrated or destroyed in between. Anti-affinity in particular is only meaningful + * against current placements, and nothing downstream re-checks it - migrateVirtualMachine does + * not enforce affinity groups. + * + * Checked for every VM, not only those in an affinity group: applyAffinityConstraints also + * applies DPDK and dedicated-resource exclusions, which apply regardless of group membership. + * A refusal here is therefore not necessarily about an affinity group. + * + * @param vm + * the VM the plan wants to move + * @param destHost + * where the plan wants to move it + * @param dispatched + * migrations already queued by this run, which the database does not reflect yet + * @param dispatchedSourceHosts + * hosts those queued migrations have not actually left yet + * @return true when the migration should not go ahead + */ + protected boolean destinationViolatesAffinity(VirtualMachine vm, Host destHost, List dispatched, + List dispatchedSourceHosts) { + if (vm.getHostId() == null) { + logger.debug("VM {} is no longer running, so its planned migration is out of date", vm); + return true; + } + if (dispatchedSourceHosts.contains(destHost.getId())) { + logger.debug("Host {} is still occupied by a VM whose migration away from it is only queued", destHost); + return true; + } + DataCenterDeployment plan = new DataCenterDeployment(destHost.getDataCenterId(), destHost.getPodId(), + destHost.getClusterId(), null, null, null); + VirtualMachineProfile vmProfile = new VirtualMachineProfileImpl(vm, null, + serviceOfferingDao.findByIdIncludingRemoved(vm.getId(), vm.getServiceOfferingId()), null, null); + ExcludeList excludes = managementServer.applyAffinityConstraints(vm, vmProfile, plan, dispatched); + excludeHostsDispreferredByAffinity(plan, excludes); + return excludes.shouldAvoid(destHost); + } + /** * Executes the DRS plan by migrating virtual machines to their destination hosts. * If there are no migrations to be executed, the plan is marked as completed. @@ -783,23 +974,51 @@ void executeDrsPlan(ClusterDrsPlanVO plan) { plan.setStatus(ClusterDrsPlan.Status.IN_PROGRESS); drsPlanDao.update(plan.getId(), plan); + // a queued migration occupies both ends until it completes, and it may not complete at all + List dispatched = new ArrayList<>(); + List dispatchedSourceHosts = new ArrayList<>(); + for (ClusterDrsPlanMigrationVO migration : planMigrations) { try { - VirtualMachine vm = vmInstanceDao.findById(migration.getVmId()); + VMInstanceVO vm = vmInstanceDao.findById(migration.getVmId()); Host host = hostDao.findById(migration.getDestHostId()); if (vm == null || host == null) { throw new CloudRuntimeException(String.format("vm %s or host %s is not found", migration.getVmId(), migration.getDestHostId())); } + if (destinationViolatesAffinity(vm, host, dispatched, dispatchedSourceHosts)) { + String reason = String.format("Skipped DRS migration of %s to %s: the destination no longer " + + "satisfies the placement rules for that VM. The plan was generated against older " + + "state.", vm, host); + logger.warn(reason); + // cancelled rather than failed: nothing went wrong, the plan went out of date + migration.setStatus(JobInfo.Status.CANCELLED); + drsPlanMigrationDao.update(migration.getId(), migration); + ActionEventUtils.onCompletedActionEvent(User.UID_SYSTEM, Account.ACCOUNT_ID_SYSTEM, + EventVO.LEVEL_WARN, EventTypes.EVENT_CLUSTER_DRS, false, reason, + plan.getClusterId(), ApiCommandResourceType.Cluster.toString(), plan.getEventId()); + continue; + } + logger.debug("Executing DRS plan {} for vm {} to host {}", plan, vm, host); long jobId = createMigrateVMAsyncJob(vm, host, plan.getEventId()); AsyncJobVO job = asyncJobManager.getAsyncJob(jobId); migration.setJobId(jobId); migration.setStatus(job.getStatus()); drsPlanMigrationDao.update(migration.getId(), migration); + + // the migration job has only been queued, so the database still shows the old host. + // record both ends: the VM is headed for the destination but has not left the + // source, and if the job fails it never will. + Long sourceHostId = vm.getHostId(); + if (sourceHostId != null) { + dispatchedSourceHosts.add(sourceHostId); + } + vm.setHostId(host.getId()); + dispatched.add(vm); } catch (Exception e) { - logger.warn("Unable to execute DRS plan {} due to {}", plan, e.getMessage()); + logger.warn("Unable to execute DRS plan {}", plan, e); migration.setStatus(JobInfo.Status.FAILED); drsPlanMigrationDao.update(migration.getId(), migration); } diff --git a/server/src/main/resources/META-INF/cloudstack/core/spring-server-core-managers-context.xml b/server/src/main/resources/META-INF/cloudstack/core/spring-server-core-managers-context.xml index c0bcba44c642..60c837f0afd9 100644 --- a/server/src/main/resources/META-INF/cloudstack/core/spring-server-core-managers-context.xml +++ b/server/src/main/resources/META-INF/cloudstack/core/spring-server-core-managers-context.xml @@ -370,6 +370,13 @@ value="#{affinityProcessorsRegistry.registered}" /> + + + diff --git a/server/src/main/resources/META-INF/cloudstack/server-allocator/spring-server-allocator-context.xml b/server/src/main/resources/META-INF/cloudstack/server-allocator/spring-server-allocator-context.xml index 28b96b8d194b..3eaf9bb90bc5 100644 --- a/server/src/main/resources/META-INF/cloudstack/server-allocator/spring-server-allocator-context.xml +++ b/server/src/main/resources/META-INF/cloudstack/server-allocator/spring-server-allocator-context.xml @@ -31,6 +31,9 @@ + + diff --git a/server/src/test/java/com/cloud/agent/manager/allocator/impl/HostLoadTrackerTest.java b/server/src/test/java/com/cloud/agent/manager/allocator/impl/HostLoadTrackerTest.java new file mode 100644 index 000000000000..0f8b38f695ea --- /dev/null +++ b/server/src/test/java/com/cloud/agent/manager/allocator/impl/HostLoadTrackerTest.java @@ -0,0 +1,184 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.agent.manager.allocator.impl; + +import org.junit.Before; +import com.cloud.host.HostLoad; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; + +import com.cloud.host.HostStats; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +@RunWith(MockitoJUnitRunner.class) +public class HostLoadTrackerTest { + + private static final long HOST_ID = 1L; + private static final long HALF_LIFE_MS = 300 * 1000L; + + @InjectMocks + private HostLoadTracker tracker = new HostLoadTracker(); + + private long now; + + @Before + public void setUp() { + tracker.clear(); + now = 1_000_000L; + } + + private HostStats stats(double cpuPercent, double usedMemoryFraction) { + HostStats stats = Mockito.mock(HostStats.class); + Mockito.lenient().when(stats.getCpuUtilization()).thenReturn(cpuPercent); + Mockito.lenient().when(stats.getTotalMemoryKBs()).thenReturn(1000.0); + Mockito.lenient().when(stats.getFreeMemoryKBs()).thenReturn(1000.0 * (1 - usedMemoryFraction)); + return stats; + } + + private void sample(double cpuPercent, double usedMemoryFraction, long advanceMs) { + now += advanceMs; + tracker.record(HOST_ID, stats(cpuPercent, usedMemoryFraction), now); + } + + private HostLoad load() { + return tracker.getLoad(HOST_ID, now); + } + + @Test + public void testUnknownHostIsNotUsable() { + assertFalse(tracker.getLoad(999L, now).isUsable()); + } + + @Test + public void testFirstSampleIsTakenAsIs() { + sample(40, 0.6, 0); + + HostLoad load = load(); + assertTrue(load.isUsable()); + assertEquals(0.40, load.getCpuUtilisation(), 1e-6); + assertEquals(0.60, load.getMemoryUtilisation(), 1e-6); + } + + @Test + public void testSingleSpikeDoesNotDominateTheAverage() { + sample(10, 0.1, 0); + sample(100, 0.1, 60 * 1000L); + + // one sample a fifth of a half life in should move the average part of the way, not all of it + double cpu = load().getCpuUtilisation(); + assertTrue("a single spike must not take over the average: " + cpu, cpu < 0.30); + assertTrue("but it must move it: " + cpu, cpu > 0.10); + } + + @Test + public void testSustainedLoadConvergesOnTheNewValue() { + sample(10, 0.1, 0); + for (int i = 0; i < 40; i++) { + sample(90, 0.9, 60 * 1000L); + } + + HostLoad load = load(); + assertEquals(0.90, load.getCpuUtilisation(), 0.01); + assertEquals(0.90, load.getMemoryUtilisation(), 0.01); + } + + @Test + public void testHalfLifeMovesAverageHalfWay() { + sample(0, 0, 0); + sample(100, 1.0, HALF_LIFE_MS); + + assertEquals(0.5, load().getCpuUtilisation(), 0.01); + } + + @Test + public void testMissedSamplesDecayByElapsedTimeNotSampleCount() { + sample(0, 0, 0); + sample(100, 1.0, 4 * HALF_LIFE_MS); + + // four half lives of catching up in one sample, so almost all the way there + assertTrue(load().getCpuUtilisation() > 0.9); + } + + @Test + public void testNullStatsAreIgnored() { + tracker.record(HOST_ID, null, now); + assertFalse(load().isUsable()); + } + + @Test + public void testHostReportingNoMemoryIsIgnored() { + HostStats broken = Mockito.mock(HostStats.class); + Mockito.lenient().when(broken.getTotalMemoryKBs()).thenReturn(0.0); + + tracker.record(HOST_ID, broken, now); + + assertFalse(load().isUsable()); + } + + @Test + public void testUnchangedReadingIsNotFoldedAgain() { + // StatsCollector keeps the previous entry when a poll fails, so the same object comes back + HostStats reading = stats(10, 0.1); + tracker.record(HOST_ID, reading, now); + // stay inside the staleness window so this tests folding, not expiry + for (int i = 0; i < 5; i++) { + now += 60 * 1000L; + tracker.record(HOST_ID, reading, now); + } + + assertEquals("re-reading one measurement must not count as six", 1, load().getSamples()); + } + + @Test + public void testAHostThatStopsReportingBecomesUnusable() { + HostStats reading = stats(10, 0.1); + tracker.record(HOST_ID, reading, now); + assertTrue(load().isUsable()); + + // the agent stops updating; StatsCollector keeps handing back the same stale entry + for (int i = 0; i < 20; i++) { + now += 60 * 1000L; + tracker.record(HOST_ID, reading, now); + } + + assertFalse("a host that stopped reporting must not keep vouching for itself", load().isUsable()); + } + + @Test + public void testFreshReadingsKeepAHostUsable() { + for (int i = 0; i < 20; i++) { + sample(10 + i, 0.1, 60 * 1000L); + } + assertTrue(load().isUsable()); + } + + @Test + public void testOutOfRangeValuesAreClamped() { + sample(250, 2.0, 0); + + HostLoad load = load(); + assertEquals(1.0, load.getCpuUtilisation(), 1e-6); + assertEquals(1.0, load.getMemoryUtilisation(), 1e-6); + } +} diff --git a/server/src/test/java/com/cloud/agent/manager/allocator/impl/WeightedHostScorerRankTest.java b/server/src/test/java/com/cloud/agent/manager/allocator/impl/WeightedHostScorerRankTest.java new file mode 100644 index 000000000000..b06c6dedbcc6 --- /dev/null +++ b/server/src/test/java/com/cloud/agent/manager/allocator/impl/WeightedHostScorerRankTest.java @@ -0,0 +1,220 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.agent.manager.allocator.impl; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.Set; +import java.util.stream.Collectors; + +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; + +import com.cloud.capacity.Capacity; +import com.cloud.capacity.CapacityVO; +import com.cloud.capacity.dao.CapacityDao; +import com.cloud.dc.ClusterDetailsDao; +import com.cloud.dc.ClusterDetailsVO; +import com.cloud.host.Host; +import com.cloud.host.HostLoad; +import com.cloud.utils.Pair; +import com.cloud.vm.dao.VMInstanceDao; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +/** + * Exercises rank() as the allocator calls it, rather than the scoring function alone, so the + * capacity denominator, the utilisation thresholds and the random spread are all covered. + */ +@RunWith(MockitoJUnitRunner.class) +public class WeightedHostScorerRankTest { + + private static final long ZONE = 1L; + private static final long CLUSTER = 7L; + private static final long CORES = 192L * 2400L; + private static final long MEMORY = 1_132_000L * 1024L * 1024L; + private static final int CPU_OVERCOMMIT = 10; + private static final int MEMORY_OVERCOMMIT = 4; + + @Mock + private CapacityDao capacityDao; + + @Mock + private VMInstanceDao vmInstanceDao; + + @Mock + private ClusterDetailsDao clusterDetailsDao; + + @Mock + private HostLoadTracker hostLoadTracker; + + @InjectMocks + private WeightedHostScorer scorer = new WeightedHostScorer(); + + private final List capacities = new ArrayList<>(); + private final Map> vmCounts = new HashMap<>(); + private final Map hosts = new HashMap<>(); + + @Before + public void setUp() { + scorer.random = new Random(1L); + Mockito.lenient().when(clusterDetailsDao.findDetail(Mockito.eq(CLUSTER), Mockito.contains("cpu"))) + .thenReturn(new ClusterDetailsVO(CLUSTER, "cpuOvercommitRatio", String.valueOf(CPU_OVERCOMMIT))); + Mockito.lenient().when(clusterDetailsDao.findDetail(Mockito.eq(CLUSTER), Mockito.contains("memory"))) + .thenReturn(new ClusterDetailsVO(CLUSTER, "memoryOvercommitRatio", String.valueOf(MEMORY_OVERCOMMIT))); + Mockito.lenient().when(capacityDao.listHostCapacityByCapacityTypes(Mockito.eq(ZONE), Mockito.eq(CLUSTER), Mockito.any())) + .thenReturn(capacities); + Mockito.lenient().when(vmInstanceDao.countVmsByHost(Mockito.eq(ZONE), Mockito.any(), Mockito.eq(CLUSTER), Mockito.any())) + .thenReturn(vmCounts); + } + + private CapacityVO capacity(long hostId, short type, long used, long total) { + CapacityVO capacity = new CapacityVO(hostId, ZONE, 1L, CLUSTER, used, total, type); + capacity.setReservedCapacity(0L); + return capacity; + } + + /** Registers a host with a share of its allocatable CPU and memory already committed. */ + private Host host(long id, String name, double cpuAllocatedFraction, double memoryAllocatedFraction, + HostLoad load, long vms) { + Host host = Mockito.mock(Host.class); + Mockito.lenient().when(host.getId()).thenReturn(id); + Mockito.lenient().when(host.getName()).thenReturn(name); + capacities.add(capacity(id, Capacity.CAPACITY_TYPE_CPU, + (long) (CORES * CPU_OVERCOMMIT * cpuAllocatedFraction), CORES)); + capacities.add(capacity(id, Capacity.CAPACITY_TYPE_MEMORY, + (long) (MEMORY * MEMORY_OVERCOMMIT * memoryAllocatedFraction), MEMORY)); + vmCounts.put(id, new Pair<>(vms, 0L)); + Mockito.lenient().when(hostLoadTracker.getLoad(id)).thenReturn(load); + hosts.put(id, host); + return host; + } + + private List rankedNames(List input) { + return scorer.rank(ZONE, 1L, CLUSTER, input).stream().map(Host::getName).collect(Collectors.toList()); + } + + @Test + public void testAllocationIsMeasuredAgainstTheOvercommittedTotal() { + // both hosts are well past their physical CPU, which is normal at a factor of 10. + // if the denominator ignored the factor both would clamp to 1.0 and rank equal. + Host light = host(1L, "light", 0.20, 0.20, new HostLoad(0.1, 0.1, 5), 20); + Host heavy = host(2L, "heavy", 0.80, 0.20, new HostLoad(0.1, 0.1, 5), 20); + + Map scores = scorer.score(ZONE, 1L, CLUSTER, Arrays.asList(light, heavy)); + + assertTrue("hosts past their physical size must still be distinguishable", + scores.get(heavy.getId()) > scores.get(light.getId())); + } + + @Test + public void testBusyHostRanksBehindQuietOneAtEqualAllocation() { + Host quiet = host(1L, "quiet", 0.30, 0.30, new HostLoad(0.05, 0.05, 10), 30); + Host busy = host(2L, "busy", 0.30, 0.30, new HostLoad(0.70, 0.30, 10), 30); + + assertEquals("quiet", rankedNames(Arrays.asList(busy, quiet)).get(0)); + } + + @Test + public void testHostOverThresholdIsNotChosenWhileAHealthyOneExists() { + // one healthy host and five over threshold: the spread must not shuffle a busy host in front + List input = new ArrayList<>(); + input.add(host(1L, "healthy", 0.30, 0.30, new HostLoad(0.50, 0.50, 10), 30)); + for (long id = 2; id <= 6; id++) { + input.add(host(id, "busy" + id, 0.30, 0.30, new HostLoad(0.95, 0.50, 10), 30)); + } + + for (int attempt = 0; attempt < 50; attempt++) { + assertEquals("the only healthy host must always lead", "healthy", rankedNames(input).get(0)); + } + } + + @Test + public void testUnmeasuredHostRanksBehindEveryMeasuredHost() { + // a host whose stats have stopped must not look idle and collect the deployments + Host measured = host(1L, "measured", 0.60, 0.60, new HostLoad(0.40, 0.40, 10), 60); + Host unmeasured = host(2L, "unmeasured", 0.05, 0.05, HostLoad.UNKNOWN, 2); + + List ranked = rankedNames(Arrays.asList(unmeasured, measured)); + + assertEquals("measured", ranked.get(0)); + assertEquals("unmeasured", ranked.get(1)); + } + + @Test + public void testRankingFallsBackToAllocationWhenNothingIsMeasured() { + Host light = host(1L, "light", 0.10, 0.10, HostLoad.UNKNOWN, 5); + Host heavy = host(2L, "heavy", 0.90, 0.90, HostLoad.UNKNOWN, 90); + + assertEquals("light", rankedNames(Arrays.asList(heavy, light)).get(0)); + } + + @Test + public void testEveryHostIsStillOfferedWhenTheWholeClusterIsBusy() { + List input = Arrays.asList( + host(1L, "a", 0.30, 0.30, new HostLoad(0.95, 0.50, 10), 30), + host(2L, "b", 0.40, 0.30, new HostLoad(0.97, 0.50, 10), 40)); + + List ranked = rankedNames(input); + + assertEquals("deployment must remain possible", 2, ranked.size()); + } + + @Test + public void testSpreadVariesTheLeadAmongHealthyHosts() { + List input = Arrays.asList( + host(1L, "a", 0.30, 0.30, new HostLoad(0.10, 0.10, 10), 30), + host(2L, "b", 0.31, 0.30, new HostLoad(0.10, 0.10, 10), 30), + host(3L, "c", 0.32, 0.30, new HostLoad(0.10, 0.10, 10), 30), + host(4L, "d", 0.90, 0.30, new HostLoad(0.10, 0.10, 10), 90)); + + Set leaders = new HashSet<>(); + for (int attempt = 0; attempt < 200; attempt++) { + leaders.add(rankedNames(input).get(0)); + } + + assertTrue("concurrent deployments must not all pick one host", leaders.size() > 1); + assertFalse("the clearly worst host must never lead", leaders.contains("d")); + } + + @Test + public void testHostMissingACapacityRowIsNotRankedFirst() { + Host complete = host(1L, "complete", 0.60, 0.60, new HostLoad(0.40, 0.40, 10), 60); + Host partial = Mockito.mock(Host.class); + Mockito.lenient().when(partial.getId()).thenReturn(2L); + Mockito.lenient().when(partial.getName()).thenReturn("partial"); + // only a CPU row: memory must not be treated as untouched + capacities.add(capacity(2L, Capacity.CAPACITY_TYPE_CPU, 0L, CORES)); + + List ranked = rankedNames(Arrays.asList(partial, complete)); + + assertEquals("complete", ranked.get(0)); + assertEquals("partial", ranked.get(1)); + } +} diff --git a/server/src/test/java/com/cloud/agent/manager/allocator/impl/WeightedHostScorerTest.java b/server/src/test/java/com/cloud/agent/manager/allocator/impl/WeightedHostScorerTest.java new file mode 100644 index 000000000000..ecbf36d831e5 --- /dev/null +++ b/server/src/test/java/com/cloud/agent/manager/allocator/impl/WeightedHostScorerTest.java @@ -0,0 +1,176 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.agent.manager.allocator.impl; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; + +import com.cloud.host.Host; +import com.cloud.host.HostLoad; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertTrue; + +@RunWith(MockitoJUnitRunner.class) +public class WeightedHostScorerTest { + + private static final HostLoad IDLE = new HostLoad(0.0, 0.0, 10); + + @Mock + private HostLoadTracker hostLoadTracker; + + @InjectMocks + private WeightedHostScorer scorer = new WeightedHostScorer(); + + private long nextHostId; + + @Before + public void setUp() { + nextHostId = 1; + } + + private Host host(String name) { + Host host = Mockito.mock(Host.class); + Mockito.lenient().when(host.getId()).thenReturn(nextHostId++); + Mockito.lenient().when(host.getName()).thenReturn(name); + return host; + } + + private double score(double cpuAllocated, double memAllocated, HostLoad load, long vms, long recentStarts) { + return scorer.scoreHostIn(null, cpuAllocated, memAllocated, load, vms, recentStarts); + } + + @Test + public void testIdleHostScoresZero() { + assertEquals(0.0, score(0, 0, IDLE, 0, 0), 1e-9); + } + + @Test + public void testMoreAllocationScoresHigher() { + assertTrue(score(0.5, 0.5, IDLE, 0, 0) > score(0.1, 0.1, IDLE, 0, 0)); + } + + @Test + public void testBusyHostScoresHigherThanIdleHostWithSameAllocation() { + HostLoad busy = new HostLoad(0.9, 0.5, 10); + assertTrue("measured load must separate hosts that look identical by allocation", + score(0.05, 0.05, busy, 20, 0) > score(0.05, 0.05, IDLE, 20, 0)); + } + + @Test + public void testUtilisationIgnoredUntilThereAreSamples() { + // a host with no samples must not be treated as idle, it must rank on allocation alone + double noSamples = score(0.4, 0.4, HostLoad.UNKNOWN, 10, 0); + double idleSamples = score(0.4, 0.4, IDLE, 10, 0); + assertTrue("a host with no load samples should not outrank a measurably idle one", + noSamples >= idleSamples); + } + + @Test + public void testDominantResourcePenalisesLopsidedHost() { + // same mean across resources, but one host is nearly out of memory + double balanced = score(0.5, 0.5, IDLE, 0, 0); + double lopsided = score(0.05, 0.95, IDLE, 0, 0); + assertTrue("a host nearly out of one resource must not rank as well as an evenly loaded one", + lopsided > balanced); + } + + @Test + public void testDominantResourceUsesUtilisationWhenHigherThanAllocation() { + HostLoad reclaimedButBusy = new HostLoad(0.95, 0.95, 10); + assertTrue(score(0.05, 0.05, reclaimedButBusy, 0, 0) > score(0.05, 0.05, IDLE, 0, 0)); + } + + @Test + public void testVmCountPenalisesHost() { + assertTrue(score(0.1, 0.1, IDLE, 120, 0) > score(0.1, 0.1, IDLE, 5, 0)); + } + + @Test + public void testRecentStartsPenaliseHost() { + assertTrue("VMs that just started are not yet visible in allocation or utilisation", + score(0.1, 0.1, IDLE, 20, 20) > score(0.1, 0.1, IDLE, 20, 0)); + } + + @Test + public void testScoreStaysWithinUnitRange() { + assertTrue(score(1, 1, new HostLoad(1, 1, 10), 1000, 1000) <= 1.0); + assertTrue(score(0, 0, IDLE, 0, 0) >= 0.0); + } + + @Test + public void testBusyHostIsHeldBackByThreshold() { + Host quiet = host("quiet"); + Host busy = host("busy"); + Mockito.when(hostLoadTracker.getLoad(quiet.getId())).thenReturn(new HostLoad(0.10, 0.10, 10)); + Mockito.when(hostLoadTracker.getLoad(busy.getId())).thenReturn(new HostLoad(0.99, 0.10, 10)); + + List healthy = new ArrayList<>(); + List tooBusy = new ArrayList<>(); + scorer.partitionByUtilisation(null, new ArrayList<>(List.of(quiet, busy)), healthy, tooBusy); + + assertEquals(1, healthy.size()); + assertSame(quiet, healthy.get(0)); + assertSame("host over the CPU threshold must be held back", busy, tooBusy.get(0)); + } + + @Test + public void testThresholdIsIgnoredWhenEveryHostIsBusy() { + Host a = host("a"); + Host b = host("b"); + Mockito.when(hostLoadTracker.getLoad(Mockito.anyLong())).thenReturn(new HostLoad(0.99, 0.99, 10)); + + List healthy = new ArrayList<>(); + List tooBusy = new ArrayList<>(); + scorer.partitionByUtilisation(null, new ArrayList<>(List.of(a, b)), healthy, tooBusy); + + assertEquals("both hosts are over threshold", 2, tooBusy.size()); + assertTrue(healthy.isEmpty()); + } + + @Test + public void testSelectionSpreadVariesTheChosenHost() { + Set chosen = new HashSet<>(); + List hosts = List.of(host("a"), host("b"), host("c"), host("d"), host("e")); + for (int i = 0; i < 200; i++) { + List ranked = new ArrayList<>(hosts); + scorer.applySelectionSpread(null, ranked); + chosen.add(ranked.get(0).getName()); + } + assertNotEquals("strict ordering sends every concurrent deployment to the same host", 1, chosen.size()); + assertTrue("only the best scoring hosts should be candidates", chosen.size() <= 3); + } + + @Test + public void testSelectionSpreadLeavesShortListsAlone() { + List ranked = new ArrayList<>(List.of(host("only"))); + scorer.applySelectionSpread(null, ranked); + assertEquals(1, ranked.size()); + } +} diff --git a/server/src/test/java/com/cloud/agent/manager/allocator/impl/WeightedPlacementDistributionTest.java b/server/src/test/java/com/cloud/agent/manager/allocator/impl/WeightedPlacementDistributionTest.java new file mode 100644 index 000000000000..de057c0b3831 --- /dev/null +++ b/server/src/test/java/com/cloud/agent/manager/allocator/impl/WeightedPlacementDistributionTest.java @@ -0,0 +1,321 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.agent.manager.allocator.impl; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Comparator; +import java.util.List; +import java.util.Random; + +import com.cloud.host.HostLoad; + +import org.junit.Test; + +import static org.junit.Assert.assertTrue; + +/** + * Simulates placement over a churning, heavily overprovisioned fleet. + * + * This is the regression fixture for the failure the weighted scoring exists to prevent. Ranking + * hosts on allocated capacity alone is blind to what a host is really doing, and under a large + * overprovisioning factor the gap between the two can be enormous: a host can report a few percent + * allocated while its cores are saturated. Anything the scheduler is not told about - VMs it has + * lost track of, guests using far more than their share - is invisible, so the emptiest looking + * host keeps being chosen no matter how hard it is working. + * + * Three things are modelled that a single-shot unit test cannot show: + * + * - deployments arrive in concurrent batches and every decision in a batch reads the same figures, + * because capacity is only charged once a VM starts + * - real load that allocation cannot account for + * - VMs are short lived and their lifetimes vary, so hosts empty unevenly + * + * Deterministic: fixed seeds, no wall clock. + */ +public class WeightedPlacementDistributionTest { + + private static final int HOSTS = 9; + private static final int CORES_PER_HOST = 192; + private static final int MEMORY_MB_PER_HOST = 1_132_000; + private static final int CPU_OVERCOMMIT = 10; + private static final int MEMORY_OVERCOMMIT = 4; + + private static final int VM_CORES = 4; + private static final int VM_MEMORY_MB = 8_192; + + private static final int BATCHES = 300; + private static final int VMS_PER_BATCH = 8; + private static final int SPREAD = 3; + + /** Share of VMs that peg their cores for their whole life, as build runners do. */ + private static final double BUSY_FRACTION = 0.35; + + /** + * Cores in use on some hosts that allocation knows nothing about. Stands in for anything the + * scheduler cannot see - VMs it believes are gone, or guests far exceeding their request. + */ + private static final int UNACCOUNTED_CORES = 60; + + private static final class SimHost { + final long id; + final double unaccountedCores; + int vms; + int recentStarts; + double busyCores; + double memoryMb; + + SimHost(long id, double unaccountedCores) { + this.id = id; + this.unaccountedCores = unaccountedCores; + } + + /** What the capacity tables would report: uniform per VM, against an inflated total. */ + double cpuAllocated() { + return (double) vms * VM_CORES / (CORES_PER_HOST * CPU_OVERCOMMIT); + } + + double memoryAllocated() { + return (double) vms * VM_MEMORY_MB / ((double) MEMORY_MB_PER_HOST * MEMORY_OVERCOMMIT); + } + + /** What the host is really doing, including what allocation cannot see. */ + double realCores() { + return busyCores + unaccountedCores; + } + + HostLoad load() { + return new HostLoad(Math.min(1, realCores() / CORES_PER_HOST), + Math.min(1, memoryMb / MEMORY_MB_PER_HOST), 10); + } + } + + /** Frozen figures for one batch, so every decision in the batch sees the same thing. */ + private static final class HostView { + final SimHost host; + final double cpuAllocated; + final double memoryAllocated; + final HostLoad load; + final int vms; + final int recentStarts; + + HostView(SimHost host) { + this.host = host; + this.cpuAllocated = host.cpuAllocated(); + this.memoryAllocated = host.memoryAllocated(); + this.load = host.load(); + this.vms = host.vms; + this.recentStarts = host.recentStarts; + } + } + + private interface Placement { + SimHost choose(List snapshot, Random random); + } + + /** What firstfitleastconsumed does today: strictly the lowest allocated fraction. */ + private static final Placement LEAST_ALLOCATED = (snapshot, random) -> + snapshot.stream().min(Comparator.comparingDouble(v -> v.cpuAllocated)).orElseThrow().host; + + /** + * Allocation-only ranking with the same random spread as the weighted arm. Isolates what the + * scoring contributes from what the spread alone contributes. + */ + private static final Placement LEAST_ALLOCATED_WITH_SPREAD = (snapshot, random) -> { + List ranked = new ArrayList<>(snapshot); + ranked.sort(Comparator.comparingDouble(v -> v.cpuAllocated)); + return ranked.get(random.nextInt(Math.min(SPREAD, ranked.size()))).host; + }; + + private Placement weighted() { + WeightedHostScorer scorer = new WeightedHostScorer(); + return (snapshot, random) -> { + List ranked = new ArrayList<>(snapshot); + ranked.sort(Comparator.comparingDouble(v -> + scorer.scoreHostIn(null, v.cpuAllocated, v.memoryAllocated, v.load, v.vms, v.recentStarts))); + return ranked.get(random.nextInt(Math.min(SPREAD, ranked.size()))).host; + }; + } + + private static final class Result { + final double[] vmCounts; + final double[] realCores; + + Result(double[] vmCounts, double[] realCores) { + this.vmCounts = vmCounts; + this.realCores = realCores; + } + + double vmSkew() { + return max(vmCounts) / mean(vmCounts); + } + + double loadSkew() { + return max(realCores) / mean(realCores); + } + } + + /** The VMs to be placed, fixed before any arm runs so all arms see the same workload. */ + private List workload(long seed) { + Random random = new Random(seed); + List vms = new ArrayList<>(); + for (int i = 0; i < BATCHES * VMS_PER_BATCH; i++) { + vms.add(new int[] {random.nextDouble() < BUSY_FRACTION ? 1 : 0, 10 + random.nextInt(50)}); + } + return vms; + } + + private Result run(Placement placement, long seed) { + List workload = workload(seed); + // a separate stream for placement decisions, so arms that consult it differently still see + // the same workload + Random random = new Random(seed ^ 0x5DEECE66DL); + int next = 0; + List hosts = new ArrayList<>(); + for (int i = 0; i < HOSTS; i++) { + // a third of the fleet carries load the scheduler cannot account for + hosts.add(new SimHost(i + 1, i % 3 == 0 ? UNACCOUNTED_CORES : 0)); + } + List live = new ArrayList<>(); // {hostIndex, busy, batchesLeft} + + for (int batch = 0; batch < BATCHES; batch++) { + live.removeIf(vm -> { + if (--vm[2] > 0) { + return false; + } + SimHost host = hosts.get(vm[0]); + host.vms--; + host.memoryMb -= VM_MEMORY_MB; + if (vm[1] == 1) { + host.busyCores -= VM_CORES; + } + return true; + }); + + hosts.forEach(h -> h.recentStarts = 0); + + List snapshot = new ArrayList<>(); + for (SimHost host : hosts) { + snapshot.add(new HostView(host)); + } + + for (int i = 0; i < VMS_PER_BATCH; i++) { + SimHost chosen = placement.choose(snapshot, random); + int[] vm = workload.get(next++); + boolean busy = vm[0] == 1; + chosen.vms++; + chosen.recentStarts++; + chosen.memoryMb += VM_MEMORY_MB; + if (busy) { + chosen.busyCores += VM_CORES; + } + // lifetimes vary, so hosts do not empty in the order they filled + live.add(new int[] {hosts.indexOf(chosen), vm[0], vm[1]}); + } + } + + return new Result(hosts.stream().mapToDouble(h -> h.vms).toArray(), + hosts.stream().mapToDouble(SimHost::realCores).toArray()); + } + + private static double max(double[] values) { + return Arrays.stream(values).max().orElse(0); + } + + private static double mean(double[] values) { + return Arrays.stream(values).average().orElse(0); + } + + @Test + public void testAllocationOnlyOrderingPilesRealLoadOntoTheBusiestHosts() { + Result result = run(LEAST_ALLOCATED, 42L); + + assertTrue(String.format("expected allocation-only ranking to be blind to real load, " + + "got cores %s (max/mean %.2f)", Arrays.toString(result.realCores), result.loadSkew()), + result.loadSkew() > 1.4); + } + + @Test + public void testWeightedScoringKeepsRealLoadEven() { + Result result = run(weighted(), 42L); + + // a third of the fleet carries a fixed handicap the scheduler can only stop adding to, not + // remove, so some residual skew is expected. Measured across seeds: allocation-only ranking + // lands at 1.84 to 2.01, weighted at 1.27 to 1.40. + assertTrue(String.format("real load should be spread, got cores %s (max/mean %.2f)", + Arrays.toString(result.realCores), result.loadSkew()), + result.loadSkew() < 1.5); + } + + @Test + public void testWeightedScoringBeatsAllocationOnlyOnEverySeed() { + for (long seed : new long[] {1L, 7L, 42L, 99L, 12345L}) { + double baseline = run(LEAST_ALLOCATED, seed).loadSkew(); + double improved = run(weighted(), seed).loadSkew(); + assertTrue(String.format("seed %d: weighted %.2f should beat allocation-only %.2f", + seed, improved, baseline), + improved < baseline); + } + } + + @Test + public void testTheScoringNotJustTheSpreadIsWhatEvensOutRealLoad() { + // the control: same random spread, ranking still blind to real load. If the spread alone + // were doing the work, this arm would do as well as the weighted one. + for (long seed : new long[] {1L, 42L, 12345L}) { + double spreadOnly = run(LEAST_ALLOCATED_WITH_SPREAD, seed).loadSkew(); + double weighted = run(weighted(), seed).loadSkew(); + assertTrue(String.format("seed %d: weighted %.2f should beat spread-only %.2f", + seed, weighted, spreadOnly), + weighted < spreadOnly); + } + } + + @Test + public void testWeightedScoringGivesFewerVmsToHostsCarryingHiddenLoad() { + Result result = run(weighted(), 42L); + + // hosts 0, 3 and 6 carry load that allocation cannot see, so they should get fewer VMs. + // uneven VM counts are the right answer here - it is real load that should come out even. + double withHiddenLoad = mean(new double[] {result.vmCounts[0], result.vmCounts[3], result.vmCounts[6]}); + double withoutHiddenLoad = mean(new double[] {result.vmCounts[1], result.vmCounts[2], result.vmCounts[4], + result.vmCounts[5], result.vmCounts[7], result.vmCounts[8]}); + + assertTrue(String.format("hosts with hidden load should take fewer VMs: %.1f vs %.1f (counts %s)", + withHiddenLoad, withoutHiddenLoad, Arrays.toString(result.vmCounts)), + withHiddenLoad < withoutHiddenLoad * 0.75); + assertTrue("no host should be left completely unused: " + Arrays.toString(result.vmCounts), + min(result.vmCounts) > 0); + } + + @Test + public void testAllocationOnlyOrderingIgnoresHiddenLoadEntirely() { + Result result = run(LEAST_ALLOCATED, 42L); + + double withHiddenLoad = mean(new double[] {result.vmCounts[0], result.vmCounts[3], result.vmCounts[6]}); + double withoutHiddenLoad = mean(new double[] {result.vmCounts[1], result.vmCounts[2], result.vmCounts[4], + result.vmCounts[5], result.vmCounts[7], result.vmCounts[8]}); + + assertTrue(String.format("allocation-only ranking should treat loaded and idle hosts alike, got %.1f vs %.1f", + withHiddenLoad, withoutHiddenLoad), + withHiddenLoad > withoutHiddenLoad * 0.9); + } + + private static double min(double[] values) { + return Arrays.stream(values).min().orElse(0); + } +} diff --git a/server/src/test/java/org/apache/cloudstack/cluster/ClusterDrsEquivalenceTest.java b/server/src/test/java/org/apache/cloudstack/cluster/ClusterDrsEquivalenceTest.java new file mode 100644 index 000000000000..7a3dc0fdae2f --- /dev/null +++ b/server/src/test/java/org/apache/cloudstack/cluster/ClusterDrsEquivalenceTest.java @@ -0,0 +1,187 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.cluster; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +import org.apache.cloudstack.affinity.AffinityGroupVMMapVO; +import org.apache.cloudstack.affinity.dao.AffinityGroupVMMapDao; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; + +import com.cloud.hypervisor.Hypervisor; +import com.cloud.service.ServiceOfferingVO; +import com.cloud.service.dao.ServiceOfferingDao; +import com.cloud.vm.dao.VMInstanceDetailsDao; +import com.cloud.storage.VolumeVO; +import com.cloud.storage.dao.VolumeDao; +import com.cloud.vm.VMInstanceVO; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; + +/** + * Working out where a VM could go is the expensive part of DRS planning, so VMs that would get the + * same answer share the work. These check what counts as the same answer. + */ +@RunWith(MockitoJUnitRunner.class) +public class ClusterDrsEquivalenceTest { + + @Mock + private VolumeDao volumeDao; + + @Mock + private AffinityGroupVMMapDao affinityGroupVMMapDao; + + @Mock + private ServiceOfferingDao serviceOfferingDao; + + @Mock + private VMInstanceDetailsDao vmInstanceDetailsDao; + + @InjectMocks + private ClusterDrsServiceImpl service = new ClusterDrsServiceImpl(); + + private VMInstanceVO vm(long id, long offeringId, long templateId, Long hostId, Long... poolIds) { + VMInstanceVO vm = Mockito.mock(VMInstanceVO.class); + Mockito.lenient().when(vm.getId()).thenReturn(id); + Mockito.lenient().when(vm.getServiceOfferingId()).thenReturn(offeringId); + Mockito.lenient().when(vm.getTemplateId()).thenReturn(templateId); + Mockito.lenient().when(vm.getHostId()).thenReturn(hostId); + Mockito.lenient().when(vm.getHypervisorType()).thenReturn(Hypervisor.HypervisorType.KVM); + + List volumes = Arrays.stream(poolIds).map(poolId -> { + VolumeVO volume = Mockito.mock(VolumeVO.class); + Mockito.lenient().when(volume.getPoolId()).thenReturn(poolId); + return volume; + }).collect(java.util.stream.Collectors.toList()); + Mockito.lenient().when(volumeDao.findCreatedByInstance(id)).thenReturn(volumes); + Mockito.lenient().when(affinityGroupVMMapDao.listByInstanceId(id)).thenReturn(Collections.emptyList()); + + ServiceOfferingVO offering = Mockito.mock(ServiceOfferingVO.class); + Mockito.lenient().when(offering.isDynamic()).thenReturn(false); + Mockito.lenient().when(serviceOfferingDao.findByIdIncludingRemoved(id, offeringId)).thenReturn(offering); + Mockito.lenient().when(vmInstanceDetailsDao.listDetailsKeyPairs(id)).thenReturn(Collections.emptyMap()); + return vm; + } + + private void offeringIsDynamic(long vmId, long offeringId) { + ServiceOfferingVO offering = Mockito.mock(ServiceOfferingVO.class); + Mockito.when(offering.isDynamic()).thenReturn(true); + Mockito.when(serviceOfferingDao.findByIdIncludingRemoved(vmId, offeringId)).thenReturn(offering); + } + + private void vmHasDetail(long vmId, String key, String value) { + Mockito.when(vmInstanceDetailsDao.listDetailsKeyPairs(vmId)) + .thenReturn(java.util.Map.of(key, value)); + } + + @Test + public void testIdenticalVmsShareOnePass() { + assertEquals(service.migrationEquivalenceKey(vm(1L, 10L, 20L, 30L, 40L)), + service.migrationEquivalenceKey(vm(2L, 10L, 20L, 30L, 40L))); + } + + @Test + public void testVolumeOrderDoesNotMatter() { + assertEquals(service.migrationEquivalenceKey(vm(1L, 10L, 20L, 30L, 40L, 41L)), + service.migrationEquivalenceKey(vm(2L, 10L, 20L, 30L, 41L, 40L))); + } + + @Test + public void testDifferentOfferingIsADifferentAnswer() { + assertNotEquals(service.migrationEquivalenceKey(vm(1L, 10L, 20L, 30L, 40L)), + service.migrationEquivalenceKey(vm(2L, 11L, 20L, 30L, 40L))); + } + + @Test + public void testDifferentCurrentHostIsADifferentAnswer() { + assertNotEquals(service.migrationEquivalenceKey(vm(1L, 10L, 20L, 30L, 40L)), + service.migrationEquivalenceKey(vm(2L, 10L, 20L, 31L, 40L))); + } + + @Test + public void testDifferentStorageIsADifferentAnswer() { + assertNotEquals(service.migrationEquivalenceKey(vm(1L, 10L, 20L, 30L, 40L)), + service.migrationEquivalenceKey(vm(2L, 10L, 20L, 30L, 42L))); + } + + @Test + public void testDifferentTemplateIsADifferentAnswer() { + assertNotEquals(service.migrationEquivalenceKey(vm(1L, 10L, 20L, 30L, 40L)), + service.migrationEquivalenceKey(vm(2L, 10L, 21L, 30L, 40L))); + } + + @Test + public void testDifferentAffinityGroupsIsADifferentAnswer() { + VMInstanceVO grouped = vm(1L, 10L, 20L, 30L, 40L); + AffinityGroupVMMapVO mapping = Mockito.mock(AffinityGroupVMMapVO.class); + Mockito.when(mapping.getAffinityGroupId()).thenReturn(99L); + Mockito.when(affinityGroupVMMapDao.listByInstanceId(1L)).thenReturn(Collections.singletonList(mapping)); + + assertNotEquals("a VM in an affinity group cannot reuse an ungrouped VM's candidate hosts", + service.migrationEquivalenceKey(grouped), + service.migrationEquivalenceKey(vm(2L, 10L, 20L, 30L, 40L))); + } + + @Test + public void testVolumesOnTheSamePoolButDifferentDiskOfferingsAreNotGrouped() { + VMInstanceVO a = vm(1L, 10L, 20L, 30L, 40L); + VMInstanceVO b = vm(2L, 10L, 20L, 30L, 40L); + VolumeVO differentOffering = Mockito.mock(VolumeVO.class); + Mockito.when(differentOffering.getPoolId()).thenReturn(40L); + Mockito.when(differentOffering.getDiskOfferingId()).thenReturn(99L); + Mockito.when(volumeDao.findCreatedByInstance(2L)).thenReturn(List.of(differentOffering)); + + assertNotEquals("storage tags come from the disk offering, so it changes the answer", + service.migrationEquivalenceKey(a), service.migrationEquivalenceKey(b)); + } + + @Test + public void testACustomOfferingIsNeverGrouped() { + // a dynamic offering takes its size from the VM, so two VMs on the same offering can be + // wildly different and must not share a candidate host list + VMInstanceVO a = vm(1L, 10L, 20L, 30L, 40L); + VMInstanceVO b = vm(2L, 10L, 20L, 30L, 40L); + offeringIsDynamic(1L, 10L); + offeringIsDynamic(2L, 10L); + + assertNotEquals(service.migrationEquivalenceKey(a), service.migrationEquivalenceKey(b)); + } + + @Test + public void testAVmWithABootModeIsNeverGrouped() { + VMInstanceVO a = vm(1L, 10L, 20L, 30L, 40L); + VMInstanceVO b = vm(2L, 10L, 20L, 30L, 40L); + vmHasDetail(1L, "UEFI", "SECURE"); + + assertNotEquals("a UEFI VM cannot run everywhere a BIOS VM can", + service.migrationEquivalenceKey(a), service.migrationEquivalenceKey(b)); + } + + @Test + public void testAVmWithNoDetailsIsStillGrouped() { + assertEquals(service.migrationEquivalenceKey(vm(1L, 10L, 20L, 30L, 40L)), + service.migrationEquivalenceKey(vm(2L, 10L, 20L, 30L, 40L))); + } +} diff --git a/server/src/test/java/org/apache/cloudstack/cluster/ClusterDrsServiceImplTest.java b/server/src/test/java/org/apache/cloudstack/cluster/ClusterDrsServiceImplTest.java index 6390b29097b5..a0084ae47a1c 100644 --- a/server/src/test/java/org/apache/cloudstack/cluster/ClusterDrsServiceImplTest.java +++ b/server/src/test/java/org/apache/cloudstack/cluster/ClusterDrsServiceImplTest.java @@ -28,13 +28,19 @@ import com.cloud.event.dao.EventDao; import com.cloud.exception.InvalidParameterValueException; import com.cloud.host.Host; +import com.cloud.host.HostLoadService; import com.cloud.host.HostVO; +import com.cloud.deploy.DeploymentPlanner.ExcludeList; +import com.cloud.deploy.DeploymentPlan; +import com.cloud.deploy.DataCenterDeployment; import com.cloud.host.dao.HostDao; import com.cloud.offering.ServiceOffering; import com.cloud.org.Cluster; import com.cloud.org.Grouping; import com.cloud.server.ManagementServer; import com.cloud.service.ServiceOfferingVO; +import com.cloud.storage.dao.VolumeDao; +import org.apache.cloudstack.affinity.AffinityGroupVMMapVO; import com.cloud.service.dao.ServiceOfferingDao; import com.cloud.utils.Pair; import com.cloud.utils.Ternary; @@ -61,6 +67,7 @@ import org.mockito.Mock; import org.mockito.MockedStatic; import org.mockito.Mockito; +import org.mockito.ArgumentCaptor; import org.mockito.MockitoAnnotations; import org.mockito.Spy; import org.mockito.junit.MockitoJUnitRunner; @@ -70,6 +77,10 @@ import java.lang.reflect.Field; import java.util.ArrayList; import java.util.Collections; +import java.util.Arrays; +import org.apache.cloudstack.jobs.JobInfo; +import org.apache.cloudstack.framework.jobs.AsyncJobManager; +import org.apache.cloudstack.framework.jobs.impl.AsyncJobVO; import java.util.Date; import java.util.HashMap; import java.util.List; @@ -77,6 +88,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; import static org.junit.Assert.assertNull; @RunWith(MockitoJUnitRunner.class) @@ -123,6 +135,15 @@ public class ClusterDrsServiceImplTest { @Mock private AffinityGroupVMMapDao affinityGroupVMMapDao; + @Mock + private AsyncJobManager asyncJobManager; + + @Mock + private VolumeDao volumeDao; + + @Mock + private HostLoadService hostLoadService; + @Mock private VMInstanceDetailsDao vmInstanceDetailsDao; @@ -214,7 +235,7 @@ public void testGetDrsPlan() throws ConfigurationException { Mockito.when(hostDao.findByClusterId(1L)).thenReturn(hostList); Mockito.when(vmInstanceDao.listByClusterId(1L)).thenReturn(vmList); - Mockito.when(balancedAlgorithm.needsDrs(Mockito.any(), Mockito.anyList(), Mockito.anyList())).thenReturn( + Mockito.when(balancedAlgorithm.needsDrs(Mockito.any(), Mockito.anyMap(), Mockito.anyMap(), Mockito.anyMap())).thenReturn( true, false); Mockito.doReturn(new Pair<>(vm1, host2)).when(clusterDrsService).getBestMigration( @@ -229,8 +250,8 @@ public void testGetDrsPlan() throws ConfigurationException { Mockito.verify(hostDao, Mockito.times(1)).findByClusterId(1L); Mockito.verify(vmInstanceDao, Mockito.times(1)).listByClusterId(1L); - Mockito.verify(balancedAlgorithm, Mockito.times(2)).needsDrs(Mockito.any(), Mockito.anyList(), - Mockito.anyList()); + Mockito.verify(balancedAlgorithm, Mockito.times(2)).needsDrs(Mockito.any(), Mockito.anyMap(), + Mockito.anyMap(), Mockito.anyMap()); assertEquals(1, iterations.size()); } @@ -294,7 +315,7 @@ public void testGetDrsPlanWithSystemVMs() throws ConfigurationException { Mockito.when(hostDao.findByClusterId(1L)).thenReturn(hostList); Mockito.when(vmInstanceDao.listByClusterId(1L)).thenReturn(vmList); - Mockito.when(balancedAlgorithm.needsDrs(Mockito.any(), Mockito.anyList(), Mockito.anyList())).thenReturn(true); + Mockito.when(balancedAlgorithm.needsDrs(Mockito.any(), Mockito.anyMap(), Mockito.anyMap(), Mockito.anyMap())).thenReturn(true); Mockito.when(hostJoinDao.searchByIds(Mockito.any())).thenReturn(List.of(hostJoin1)); List> result = clusterDrsService.getDrsPlan(cluster, 5); @@ -336,7 +357,7 @@ public void testGetDrsPlanWithNonRunningVMs() throws ConfigurationException { Mockito.when(hostDao.findByClusterId(1L)).thenReturn(hostList); Mockito.when(vmInstanceDao.listByClusterId(1L)).thenReturn(vmList); - Mockito.when(balancedAlgorithm.needsDrs(Mockito.any(), Mockito.anyList(), Mockito.anyList())).thenReturn(true); + Mockito.when(balancedAlgorithm.needsDrs(Mockito.any(), Mockito.anyMap(), Mockito.anyMap(), Mockito.anyMap())).thenReturn(true); Mockito.when(hostJoinDao.searchByIds(Mockito.any())).thenReturn(List.of(hostJoin1)); List> result = clusterDrsService.getDrsPlan(cluster, 5); @@ -383,7 +404,7 @@ public void testGetDrsPlanWithSkipDrsFlag() throws ConfigurationException { Mockito.when(hostDao.findByClusterId(1L)).thenReturn(hostList); Mockito.when(vmInstanceDao.listByClusterId(1L)).thenReturn(vmList); - Mockito.when(balancedAlgorithm.needsDrs(Mockito.any(), Mockito.anyList(), Mockito.anyList())).thenReturn(true); + Mockito.when(balancedAlgorithm.needsDrs(Mockito.any(), Mockito.anyMap(), Mockito.anyMap(), Mockito.anyMap())).thenReturn(true); Mockito.when(hostJoinDao.searchByIds(Mockito.any())).thenReturn(List.of(hostJoin1)); List> result = clusterDrsService.getDrsPlan(cluster, 5); @@ -428,7 +449,7 @@ public void testGetDrsPlanWithNoCompatibleHosts() throws ConfigurationException Mockito.when(hostDao.findByClusterId(1L)).thenReturn(hostList); Mockito.when(vmInstanceDao.listByClusterId(1L)).thenReturn(vmList); - Mockito.when(balancedAlgorithm.needsDrs(Mockito.any(), Mockito.anyList(), Mockito.anyList())).thenReturn(true); + Mockito.when(balancedAlgorithm.needsDrs(Mockito.any(), Mockito.anyMap(), Mockito.anyMap(), Mockito.anyMap())).thenReturn(true); Mockito.when(serviceOfferingDao.findByIdIncludingRemoved(Mockito.anyLong(), Mockito.anyLong())).thenReturn(serviceOffering); Mockito.when(hostJoinDao.searchByIds(Mockito.any())).thenReturn(List.of(hostJoin1)); // Return a Ternary with an empty suitable-hosts list to exercise the "no compatible hosts" path @@ -476,7 +497,7 @@ public void testGetDrsPlanWithExceptionInCompatibilityCheck() throws Configurati Mockito.when(hostDao.findByClusterId(1L)).thenReturn(hostList); Mockito.when(vmInstanceDao.listByClusterId(1L)).thenReturn(vmList); - Mockito.when(balancedAlgorithm.needsDrs(Mockito.any(), Mockito.anyList(), Mockito.anyList())).thenReturn(true); + Mockito.when(balancedAlgorithm.needsDrs(Mockito.any(), Mockito.anyMap(), Mockito.anyMap(), Mockito.anyMap())).thenReturn(true); Mockito.when(serviceOfferingDao.findByIdIncludingRemoved(Mockito.anyLong(), Mockito.anyLong())).thenReturn(serviceOffering); Mockito.when(hostJoinDao.searchByIds(Mockito.any())).thenReturn(List.of(hostJoin1)); // Throw an explicit exception so the catch-and-log path is exercised intentionally @@ -525,7 +546,7 @@ public void testGetDrsPlanWithNoBestMigration() throws ConfigurationException { Mockito.when(hostDao.findByClusterId(1L)).thenReturn(hostList); Mockito.when(vmInstanceDao.listByClusterId(1L)).thenReturn(vmList); - Mockito.when(balancedAlgorithm.needsDrs(Mockito.any(), Mockito.anyList(), Mockito.anyList())).thenReturn(true); + Mockito.when(balancedAlgorithm.needsDrs(Mockito.any(), Mockito.anyMap(), Mockito.anyMap(), Mockito.anyMap())).thenReturn(true); Mockito.when(serviceOfferingDao.findByIdIncludingRemoved(Mockito.anyLong(), Mockito.anyLong())).thenReturn(serviceOffering); Mockito.when(hostJoinDao.searchByIds(Mockito.any())).thenReturn(List.of(hostJoin1)); @@ -600,7 +621,7 @@ public void testGetDrsPlanWithMultipleIterations() throws ConfigurationException Mockito.when(hostDao.findByClusterId(1L)).thenReturn(hostList); Mockito.when(vmInstanceDao.listByClusterId(1L)).thenReturn(vmList); - Mockito.when(balancedAlgorithm.needsDrs(Mockito.any(), Mockito.anyList(), Mockito.anyList())).thenReturn( + Mockito.when(balancedAlgorithm.needsDrs(Mockito.any(), Mockito.anyMap(), Mockito.anyMap(), Mockito.anyMap())).thenReturn( true, true, false); Mockito.when(serviceOfferingDao.findByIdIncludingRemoved(Mockito.anyLong(), Mockito.anyLong())).thenReturn(serviceOffering); Mockito.when(hostJoinDao.searchByIds(1L, 2L)).thenReturn(List.of(hostJoin1, hostJoin2)); @@ -614,7 +635,7 @@ public void testGetDrsPlanWithMultipleIterations() throws ConfigurationException List> result = clusterDrsService.getDrsPlan(cluster, 5); assertEquals(2, result.size()); - Mockito.verify(balancedAlgorithm, Mockito.times(3)).needsDrs(Mockito.any(), Mockito.anyList(), Mockito.anyList()); + Mockito.verify(balancedAlgorithm, Mockito.times(3)).needsDrs(Mockito.any(), Mockito.anyMap(), Mockito.anyMap(), Mockito.anyMap()); } @Test @@ -646,7 +667,7 @@ public void testGetDrsPlanWithMigrationToOriginalHost() throws ConfigurationExce Mockito.when(hostDao.findByClusterId(1L)).thenReturn(hostList); Mockito.when(vmInstanceDao.listByClusterId(1L)).thenReturn(vmList); - Mockito.when(balancedAlgorithm.needsDrs(Mockito.any(), Mockito.anyList(), Mockito.anyList())).thenReturn(true); + Mockito.when(balancedAlgorithm.needsDrs(Mockito.any(), Mockito.anyMap(), Mockito.anyMap(), Mockito.anyMap())).thenReturn(true); Mockito.when(serviceOfferingDao.findByIdIncludingRemoved(Mockito.anyLong(), Mockito.anyLong())).thenReturn(serviceOffering); // Return migration to original host (host1) - should break the loop @@ -951,4 +972,229 @@ public void testProcessPlans() { Mockito.verify(clusterDrsService, Mockito.times(2)).executeDrsPlan(Mockito.any(ClusterDrsPlanVO.class)); } + + private VMInstanceVO vmWithAffinityGroup(long id, Long hostId) { + VMInstanceVO vm = Mockito.mock(VMInstanceVO.class); + Mockito.lenient().when(vm.getId()).thenReturn(id); + Mockito.lenient().when(vm.getHostId()).thenReturn(hostId); + Mockito.lenient().when(vm.getServiceOfferingId()).thenReturn(5L); + Mockito.lenient().when(affinityGroupVMMapDao.listByInstanceId(id)) + .thenReturn(Collections.singletonList(Mockito.mock(AffinityGroupVMMapVO.class))); + Mockito.lenient().when(serviceOfferingDao.findByIdIncludingRemoved(id, 5L)) + .thenReturn(Mockito.mock(ServiceOfferingVO.class)); + return vm; + } + + private HostVO host(long id) { + HostVO host = Mockito.mock(HostVO.class); + Mockito.lenient().when(host.getId()).thenReturn(id); + return host; + } + + private void affinityExcludes(Long... hostIds) { + ExcludeList excludes = new ExcludeList(); + for (Long hostId : hostIds) { + excludes.addHost(hostId); + } + Mockito.when(managementServer.applyAffinityConstraints(Mockito.any(), Mockito.any(), Mockito.any(), Mockito.any())) + .thenReturn(excludes); + } + + @Test + public void testAVmWithNoAffinityGroupsIsStillRechecked() { + // applyAffinityConstraints also applies DPDK and dedicated-resource exclusions, which do + // not depend on affinity group membership, so every VM has to go through it + VMInstanceVO vm = Mockito.mock(VMInstanceVO.class); + Mockito.when(vm.getId()).thenReturn(1L); + Mockito.when(vm.getHostId()).thenReturn(10L); + Mockito.when(vm.getServiceOfferingId()).thenReturn(5L); + Mockito.when(serviceOfferingDao.findByIdIncludingRemoved(1L, 5L)) + .thenReturn(Mockito.mock(ServiceOfferingVO.class)); + affinityExcludes(21L); + + assertFalse(clusterDrsService.destinationViolatesAffinity(vm, host(20L), + Collections.emptyList(), Collections.emptyList())); + Mockito.verify(managementServer) + .applyAffinityConstraints(Mockito.any(), Mockito.any(), Mockito.any(), Mockito.any()); + } + + @Test + public void testDestinationRefusedWhenAffinityExcludesIt() { + VMInstanceVO vm = vmWithAffinityGroup(1L, 10L); + affinityExcludes(20L); + + assertTrue(clusterDrsService.destinationViolatesAffinity(vm, host(20L), + Collections.emptyList(), Collections.emptyList())); + } + + @Test + public void testDestinationAllowedWhenAffinityExcludesSomewhereElse() { + VMInstanceVO vm = vmWithAffinityGroup(1L, 10L); + affinityExcludes(21L); + + assertFalse(clusterDrsService.destinationViolatesAffinity(vm, host(20L), + Collections.emptyList(), Collections.emptyList())); + } + + @Test + public void testDestinationRefusedWhenVmIsNoLongerRunning() { + VMInstanceVO vm = Mockito.mock(VMInstanceVO.class); + Mockito.when(vm.getHostId()).thenReturn(null); + + assertTrue("a plan for a VM that has since stopped is out of date", + clusterDrsService.destinationViolatesAffinity(vm, host(20L), + Collections.emptyList(), Collections.emptyList())); + } + + @Test + public void testDestinationRefusedWhileItIsStillOccupiedByAQueuedMigration() { + // the swap case: A is queued to leave host1, so host1 is not free for B yet - and if A's + // migration fails, A never leaves at all + VMInstanceVO b = vmWithAffinityGroup(2L, 11L); + + assertTrue("a host is not free until the VM leaving it has actually gone", + clusterDrsService.destinationViolatesAffinity(b, host(10L), + Collections.emptyList(), Collections.singletonList(10L))); + Mockito.verify(managementServer, Mockito.never()) + .applyAffinityConstraints(Mockito.any(), Mockito.any(), Mockito.any(), Mockito.any()); + } + + @Test + public void testExecuteDrsPlanKeepsSourceHostsOccupiedAcrossMigrations() { + // plan: A host10 -> host12, B host11 -> host10. B must not be sent to host10. + ClusterDrsPlanVO plan = Mockito.mock(ClusterDrsPlanVO.class); + Mockito.when(plan.getId()).thenReturn(1L); + + ClusterDrsPlanMigrationVO first = Mockito.mock(ClusterDrsPlanMigrationVO.class); + Mockito.when(first.getVmId()).thenReturn(1L); + Mockito.when(first.getDestHostId()).thenReturn(12L); + ClusterDrsPlanMigrationVO second = Mockito.mock(ClusterDrsPlanMigrationVO.class); + Mockito.when(second.getId()).thenReturn(8L); + Mockito.when(second.getVmId()).thenReturn(2L); + Mockito.when(second.getDestHostId()).thenReturn(10L); + Mockito.when(drsPlanMigrationDao.listPlanMigrationsToExecute(1L)) + .thenReturn(Arrays.asList(first, second)); + + VMInstanceVO a = Mockito.mock(VMInstanceVO.class); + Mockito.when(a.getHostId()).thenReturn(10L); + VMInstanceVO b = Mockito.mock(VMInstanceVO.class); + Mockito.when(vmInstanceDao.findById(1L)).thenReturn(a); + Mockito.when(vmInstanceDao.findById(2L)).thenReturn(b); + HostVO host12 = host(12L); + HostVO host10 = host(10L); + Mockito.when(hostDao.findById(12L)).thenReturn(host12); + Mockito.when(hostDao.findById(10L)).thenReturn(host10); + + Mockito.doReturn(false).when(clusterDrsService).destinationViolatesAffinity( + Mockito.eq(a), Mockito.any(), Mockito.any(), Mockito.any()); + Mockito.doReturn(1L).when(clusterDrsService) + .createMigrateVMAsyncJob(Mockito.any(), Mockito.any(), Mockito.anyLong()); + Mockito.when(asyncJobManager.getAsyncJob(1L)).thenReturn(Mockito.mock(AsyncJobVO.class)); + + clusterDrsService.executeDrsPlan(plan); + + // A's source host must have been carried into the check for B + ArgumentCaptor sources = ArgumentCaptor.forClass(List.class); + Mockito.verify(clusterDrsService).destinationViolatesAffinity( + Mockito.eq(b), Mockito.any(), Mockito.any(), sources.capture()); + assertTrue("the host A is leaving must still count as occupied", + sources.getValue().contains(10L)); + } + + @Test + public void testExecuteDrsPlanSkipsMigrationThatViolatesAffinity() { + ClusterDrsPlanVO plan = Mockito.mock(ClusterDrsPlanVO.class); + Mockito.when(plan.getId()).thenReturn(1L); + + ClusterDrsPlanMigrationVO migration = Mockito.mock(ClusterDrsPlanMigrationVO.class); + Mockito.when(migration.getId()).thenReturn(7L); + Mockito.when(migration.getVmId()).thenReturn(1L); + Mockito.when(migration.getDestHostId()).thenReturn(20L); + Mockito.when(drsPlanMigrationDao.listPlanMigrationsToExecute(1L)) + .thenReturn(Collections.singletonList(migration)); + + VMInstanceVO vm = Mockito.mock(VMInstanceVO.class); + HostVO host20 = host(20L); + Mockito.when(vmInstanceDao.findById(1L)).thenReturn(vm); + Mockito.when(hostDao.findById(20L)).thenReturn(host20); + + Mockito.doReturn(true).when(clusterDrsService) + .destinationViolatesAffinity(Mockito.any(), Mockito.any(), Mockito.any(), Mockito.any()); + + clusterDrsService.executeDrsPlan(plan); + + Mockito.verify(migration).setStatus(JobInfo.Status.CANCELLED); + Mockito.verify(clusterDrsService, Mockito.never()) + .createMigrateVMAsyncJob(Mockito.any(), Mockito.any(), Mockito.anyLong()); + } + + @Test + public void testNonStrictAntiAffinityIsHonoured() { + // a non-strict group lowers a host's priority rather than excluding it. rebalancing is + // never a reason to break it, since not migrating is always available. + DataCenterDeployment plan = new DataCenterDeployment(1L, 1L, 1L, null, null, null); + plan.adjustHostPriority(30L, DeploymentPlan.HostPriorityAdjustment.LOWER); + plan.adjustHostPriority(31L, DeploymentPlan.HostPriorityAdjustment.HIGHER); + + ExcludeList excludes = new ExcludeList(); + clusterDrsService.excludeHostsDispreferredByAffinity(plan, excludes); + + assertTrue("a dispreferred host must not be a DRS destination", + excludes.getHostsToAvoid().contains(30L)); + assertFalse("a preferred host must stay available", + excludes.getHostsToAvoid().contains(31L)); + } + + @Test + public void testEquivalentVmsShareOneCandidateHostLookup() throws ConfigurationException { + // the expensive call is listHostsForMigrationOfVM. Two interchangeable VMs must cost one + // pass, not two - otherwise the grouping is not actually doing anything. + ClusterVO cluster = Mockito.mock(ClusterVO.class); + Mockito.when(cluster.getId()).thenReturn(1L); + Mockito.when(cluster.getAllocationState()).thenReturn(Grouping.AllocationState.Enabled); + + HostVO host1 = Mockito.mock(HostVO.class); + Mockito.when(host1.getId()).thenReturn(1L); + + List vmList = new ArrayList<>(); + for (long id : new long[] {1L, 2L}) { + VMInstanceVO vm = Mockito.mock(VMInstanceVO.class); + Mockito.when(vm.getId()).thenReturn(id); + Mockito.when(vm.getHostId()).thenReturn(1L); + Mockito.when(vm.getType()).thenReturn(VirtualMachine.Type.User); + Mockito.when(vm.getState()).thenReturn(VirtualMachine.State.Running); + Mockito.when(vm.getServiceOfferingId()).thenReturn(9L); + Mockito.lenient().when(vm.getTemplateId()).thenReturn(8L); + vmList.add(vm); + } + + ServiceOfferingVO offering = Mockito.mock(ServiceOfferingVO.class); + Mockito.when(offering.isDynamic()).thenReturn(false); + Mockito.when(serviceOfferingDao.findByIdIncludingRemoved(Mockito.anyLong(), Mockito.anyLong())) + .thenReturn(offering); + Mockito.when(vmInstanceDetailsDao.listDetailsKeyPairs(Mockito.anyLong())) + .thenReturn(Collections.emptyMap()); + Mockito.when(volumeDao.findCreatedByInstance(Mockito.anyLong())).thenReturn(Collections.emptyList()); + + HostJoinVO hostJoin1 = Mockito.mock(HostJoinVO.class); + Mockito.when(hostJoin1.getId()).thenReturn(1L); + Mockito.when(hostJoin1.getCpus()).thenReturn(4); + Mockito.when(hostJoin1.getSpeed()).thenReturn(1000L); + Mockito.when(hostJoin1.getTotalMemory()).thenReturn(8192L); + + Mockito.when(hostDao.findByClusterId(1L)).thenReturn(List.of(host1)); + Mockito.when(vmInstanceDao.listByClusterId(1L)).thenReturn(vmList); + Mockito.when(hostJoinDao.searchByIds(Mockito.any())).thenReturn(List.of(hostJoin1)); + Mockito.when(balancedAlgorithm.needsDrs(Mockito.any(), Mockito.anyMap(), Mockito.anyMap(), Mockito.anyMap())) + .thenReturn(false); + Mockito.when(managementServer.listHostsForMigrationOfVM(Mockito.any(), Mockito.anyLong(), + Mockito.anyLong(), Mockito.any(), Mockito.anyList())) + .thenReturn(new Ternary<>(new Pair<>(Collections.emptyList(), 0), + List.of(host1), Collections.emptyMap())); + + clusterDrsService.getDrsPlan(cluster, 5); + + Mockito.verify(managementServer, Mockito.times(1)).listHostsForMigrationOfVM( + Mockito.any(), Mockito.anyLong(), Mockito.anyLong(), Mockito.any(), Mockito.anyList()); + } }