Allow a service offering to set its own overcommit ratio

PendingReleaseNotes

@@ -39,3 +39,24 @@ example.ver.1 > example.ver.2:
     which can now be attached to Instances. This is to prevent the Secondary
     Storage to grow to enormous sizes as Linux Distributions keep growing in
     size while a stripped down Linux should fit on a 2.88MB floppy.
+
+  * A service offering may now set its own overcommit ratio, overriding its cluster's, using the
+    cpuOvercommitRatio and memoryOvercommitRatio service offering details - the same detail keys
+    already used at cluster and VM scope. Absent, the cluster's ratio applies as before, so nothing
+    changes for existing offerings.
+
+    The value means the same thing at every scope: how far a VM's declared size is inflated
+    relative to what it really holds. 1 means not overcommitted at all, which is how an offering is
+    exempted from its cluster's overcommit. Values below 1 would mean reserving more than the VM
+    asked for and are rejected. An offering may declare a ratio higher than its cluster's; this is
+    logged.
+
+    An offering with a memory ratio of 1 on an overcommitted cluster is charged its full request
+    and keeps all of it: free page reporting and balloon auto-deflate are turned off for its VMs,
+    so the guest does not lend unused memory back to the host. The balloon device itself stays, so
+    memory statistics are unaffected. This is intended for infrastructure VMs that should not
+    participate in the overcommit that other workloads rely on.
+
+    Also fixes a latent accounting bug: capacity was charged raw when a VM started and stopped, but
+    recalculated with the VM's overcommit ratio applied. These agreed only while every VM shared
+    its cluster's ratio.

api/src/main/java/com/cloud/vm/VmDetailConstants.java

@@ -70,6 +70,7 @@ public interface VmDetailConstants {
     // Misc details for internal usage (not to be set/changed by user or admin)
     String CPU_OVER_COMMIT_RATIO = "cpuOvercommitRatio";
     String MEMORY_OVER_COMMIT_RATIO = "memoryOvercommitRatio";
+    String MEMORY_RECLAIM_DISABLED = "memoryReclaimDisabled";
     String MESSAGE_RESERVED_CAPACITY_FREED_FLAG = "Message.ReservedCapacityFreed.Flag";
     String DEPLOY_VM = "deployvm";
     String SSH_PUBLIC_KEY = "SSH.PublicKey";

engine/components-api/src/main/java/com/cloud/capacity/CapacityManager.java

@@ -162,6 +162,12 @@ boolean checkIfHostHasCapacity(Host host, Integer cpu, long ram, boolean checkFr
 
     void updateCapacityForHost(Host host);
 
+    /**
+     * Expresses a VM's request in the units capacity is counted in, taking account of any overcommit
+     * ratio its service offering sets in place of its cluster's.
+     */
+    long scaleRequestForOffering(long serviceOfferingId, long clusterId, long requested, boolean forCpu);
+
     /**
      * @param pool storage pool
      * @param templateForVmCreation template that will be used for vm creation

engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java

@@ -248,6 +248,7 @@
 import com.cloud.resource.ResourceManager;
 import com.cloud.resource.ResourceState;
 import com.cloud.service.ServiceOfferingVO;
+import com.cloud.service.dao.ServiceOfferingDetailsDao;
 import com.cloud.service.dao.ServiceOfferingDao;
 import com.cloud.storage.DiskOfferingVO;
 import com.cloud.storage.ScopeType;
@@ -405,6 +406,8 @@ public class VirtualMachineManagerImpl extends ManagerBase implements VirtualMac
     @Inject
     private ClusterDetailsDao _clusterDetailsDao;
     @Inject
+    private ServiceOfferingDetailsDao _serviceOfferingDetailsDao;
+    @Inject
     private VMInstanceDetailsDao vmInstanceDetailsDao;
     @Inject
     private VolumeOrchestrationService volumeMgr;
@@ -1857,22 +1860,107 @@ private void addToNetworkNameMap(long networkId, long dataCenterId, Map<Long, St
     private void updateOverCommitRatioForVmProfile(VirtualMachineProfile vmProfile, long clusterId) {
         final ClusterDetailsVO clusterDetailCpu = _clusterDetailsDao.findDetail(clusterId, VmDetailConstants.CPU_OVER_COMMIT_RATIO);
         final ClusterDetailsVO clusterDetailRam = _clusterDetailsDao.findDetail(clusterId, VmDetailConstants.MEMORY_OVER_COMMIT_RATIO);
-        final float parsedClusterCpuDetailCpu = Float.parseFloat(clusterDetailCpu.getValue());
-        final float parsedClusterDetailRam = Float.parseFloat(clusterDetailRam.getValue());
-        VMInstanceDetailVO vmDetailCpu = vmInstanceDetailsDao.findDetail(vmProfile.getId(), VmDetailConstants.CPU_OVER_COMMIT_RATIO);
-        VMInstanceDetailVO vmDetailRam = vmInstanceDetailsDao.findDetail(vmProfile.getId(), VmDetailConstants.MEMORY_OVER_COMMIT_RATIO);
+        final float clusterCpuRatio = Float.parseFloat(clusterDetailCpu.getValue());
+        final float clusterRamRatio = Float.parseFloat(clusterDetailRam.getValue());
+
+        final Float offeringCpuRatio = offeringOverCommitRatio(vmProfile, VmDetailConstants.CPU_OVER_COMMIT_RATIO, clusterCpuRatio);
+        final Float offeringRamRatio = offeringOverCommitRatio(vmProfile, VmDetailConstants.MEMORY_OVER_COMMIT_RATIO, clusterRamRatio);
+
+        final float cpuRatio = offeringCpuRatio != null ? offeringCpuRatio : clusterCpuRatio;
+        final float ramRatio = offeringRamRatio != null ? offeringRamRatio : clusterRamRatio;
+
+        persistOverCommitRatio(vmProfile.getId(), VmDetailConstants.CPU_OVER_COMMIT_RATIO, cpuRatio, clusterCpuRatio);
+        persistOverCommitRatio(vmProfile.getId(), VmDetailConstants.MEMORY_OVER_COMMIT_RATIO, ramRatio, clusterRamRatio);
+        persistMemoryReclaimFlag(vmProfile.getId(), offeringRamRatio);
+
+        vmProfile.setCpuOvercommitRatio(cpuRatio);
+        vmProfile.setMemoryOvercommitRatio(ramRatio);
+    }
+
+    /**
+     * The overcommit ratio a VM's service offering asks for, or null when it does not ask for one
+     * and should inherit its cluster's.
+     *
+     * An offering setting a ratio of 1 on an overcommitted cluster is how infrastructure VMs are
+     * kept off the overcommit: they are charged their full request and hold all of it.
+     */
+    protected Float offeringOverCommitRatio(VirtualMachineProfile vmProfile, String key, float clusterRatio) {
+        String offeringRatio = _serviceOfferingDetailsDao.getDetail(vmProfile.getServiceOfferingId(), key);
+        if (offeringRatio == null) {
+            return null;
+        }
+        float ratio;
+        try {
+            ratio = Float.parseFloat(offeringRatio);
+        } catch (NumberFormatException e) {
+            logger.warn("Ignoring {} of [{}] on service offering {}: it is not a number.",
+                    key, offeringRatio, vmProfile.getServiceOfferingId());
+            return null;
+        }
+        if (ratio < 1) {
+            // rejected when the offering is created; only reachable by writing the detail directly
+            logger.warn("Ignoring {} of {} on service offering {}: it must be at least 1.",
+                    key, offeringRatio, vmProfile.getServiceOfferingId());
+            return null;
+        }
+        if (ratio > clusterRatio) {
+            logger.info("Service offering {} declares {} of {}, more than its cluster's {}, so its VMs are "
+                            + "oversubscribed further than the cluster default.",
+                    vmProfile.getServiceOfferingId(), key, ratio, clusterRatio);
+        }
+        return ratio;
+    }
 
-        if ((vmDetailCpu == null && parsedClusterCpuDetailCpu > 1f) ||
-                (vmDetailCpu != null && Float.parseFloat(vmDetailCpu.getValue()) != parsedClusterCpuDetailCpu)) {
-            vmInstanceDetailsDao.addDetail(vmProfile.getId(), VmDetailConstants.CPU_OVER_COMMIT_RATIO, clusterDetailCpu.getValue(), true);
+    /**
+     * Turns off returning the guest's unused pages to the host, for a VM whose offering has
+     * explicitly opted out of overcommit. Otherwise the exemption would hold in the books only.
+     *
+     * Only an explicit choice counts. An offering that says nothing inherits its cluster, and a
+     * cluster with no overcommit configured is the default everywhere - reading that as "pin the
+     * memory of every VM" would change behaviour for installations that opted into nothing.
+     *
+     * @param offeringRatio
+     *         the ratio the offering asked for, or null if it asked for nothing
+     */
+    protected void persistMemoryReclaimFlag(long vmId, Float offeringRatio) {
+        boolean disable = offeringRatio != null && offeringRatio <= 1f;
+        VMInstanceDetailVO existing = vmInstanceDetailsDao.findDetail(vmId, VmDetailConstants.MEMORY_RECLAIM_DISABLED);
+        if (disable && existing == null) {
+            vmInstanceDetailsDao.addDetail(vmId, VmDetailConstants.MEMORY_RECLAIM_DISABLED, "true", true);
+        } else if (!disable && existing != null) {
+            vmInstanceDetailsDao.removeDetail(vmId, VmDetailConstants.MEMORY_RECLAIM_DISABLED);
         }
-        if ((vmDetailRam == null && parsedClusterDetailRam > 1f) ||
-                (vmDetailRam != null && Float.parseFloat(vmDetailRam.getValue()) != parsedClusterDetailRam)) {
-            vmInstanceDetailsDao.addDetail(vmProfile.getId(), VmDetailConstants.MEMORY_OVER_COMMIT_RATIO, clusterDetailRam.getValue(), true);
+    }
+
+    /**
+     * Records the ratio the VM is running at.
+     *
+     * Capacity is counted in cluster-overcommitted units, and the charge scales a VM by this detail,
+     * so it has to be present whenever the VM's ratio differs from its cluster's - otherwise the VM
+     * is charged as though it shared the cluster's ratio. It is also kept while the cluster is
+     * overcommitted at all, so that capacity accounting survives the cluster ratio being changed
+     * beneath a running VM.
+     */
+    protected void persistOverCommitRatio(long vmId, String key, float ratio, float clusterRatio) {
+        boolean needed = ratio != clusterRatio || clusterRatio > 1f;
+        VMInstanceDetailVO existing = vmInstanceDetailsDao.findDetail(vmId, key);
+        if (!needed) {
+            if (existing != null) {
+                vmInstanceDetailsDao.removeDetail(vmId, key);
+            }
+            return;
         }
+        if (existing == null || parseRatioOrDefault(existing.getValue(), Float.NaN) != ratio) {
+            vmInstanceDetailsDao.addDetail(vmId, key, String.valueOf(ratio), true);
+        }
+    }
 
-        vmProfile.setCpuOvercommitRatio(Float.parseFloat(clusterDetailCpu.getValue()));
-        vmProfile.setMemoryOvercommitRatio(Float.parseFloat(clusterDetailRam.getValue()));
+    private float parseRatioOrDefault(String value, float fallback) {
+        try {
+            return Float.parseFloat(value);
+        } catch (NumberFormatException | NullPointerException e) {
+            return fallback;
+        }
     }
 
     /**

engine/orchestration/src/test/java/com/cloud/vm/VirtualMachineManagerImplTest.java

@@ -107,6 +107,7 @@
 import com.cloud.api.query.dao.UserVmJoinDao;
 import com.cloud.api.query.vo.UserVmJoinVO;
 import com.cloud.dc.ClusterDetailsDao;
+import com.cloud.service.dao.ServiceOfferingDetailsDao;
 import com.cloud.dc.ClusterDetailsVO;
 import com.cloud.dc.ClusterVO;
 import com.cloud.dc.DataCenter;
@@ -271,6 +272,8 @@ public class VirtualMachineManagerImplTest {
     @Mock
     private ClusterDetailsDao _clusterDetailsDao;
     @Mock
+    private ServiceOfferingDetailsDao _serviceOfferingDetailsDao;
+    @Mock
     private VMInstanceDetailsDao vmInstanceDetailsDao;
     @Mock
     private ItWorkDao _workDao;