Add 'weighted' DRS algorithm, and speed up DRS plan generation

PendingReleaseNotes

@@ -120,6 +120,33 @@ example.ver.1 > example.ver.2:
     host.load.half.life. Until a management server has collected samples, ranking falls back to
     allocation figures alone.
 
+  * New DRS algorithm 'weighted' for drs.algorithm, alongside 'balanced' and 'condensed'. The
+    default is unchanged; the new one is opt-in per cluster.
+
+    The existing algorithms balance a single metric chosen by drs.metric, so choosing one leaves
+    the other unwatched: a cluster can be even on memory while its CPU load varies several fold and
+    nothing moves. Allocation is also a poor stand-in for load under overprovisioning, where a
+    saturated host can still report a small percentage allocated.
+
+    'weighted' blends CPU and memory allocated with CPU and memory in use, and balances the result.
+    Imbalance keeps its existing shape - standard deviation over the mean - but it is computed over
+    a blend rather than over one metric, so drs.imbalance is not calibrated the same way and is
+    worth re-checking after switching. drs.metric, drs.metric.type and drs.metric.use.ratio choose
+    and shape the single metric the other algorithms balance; they do not apply to 'weighted' and
+    are ignored.
+
+    Utilisation is only used when every host in the cluster has been sampled. Comparing a host
+    measured on utilisation against one measured on allocation alone would report a difference that
+    is an artefact of the monitoring rather than of the load, so the cluster falls back to
+    allocation figures until every host can be measured.
+
+    Tuned with the drs.weighted.* settings, all cluster scoped.
+
+  * DRS plan generation is considerably faster on large clusters. Working out where a VM could go
+    is now done once for each group of VMs that would get the same answer rather than once per VM,
+    and affinity is re-evaluated only for VMs affected by the migration just planned rather than
+    for every VM on every iteration.
+
 KVM console:
   * Added the global setting `consoleproxy.kvm.multiple.viewers.enabled` to allow
     multiple users to simultaneously access the console of the same KVM user VM.

server/src/main/java/com/cloud/agent/manager/allocator/impl/HostLoad.java → api/src/main/java/com/cloud/host/HostLoad.java

@@ -14,7 +14,7 @@
 // KIND, either express or implied.  See the License for the
 // specific language governing permissions and limitations
 // under the License.
-package com.cloud.agent.manager.allocator.impl;
+package com.cloud.host;
 
 /**
  * Smoothed view of what a host is actually doing, as opposed to what has been allocated on it.

api/src/main/java/com/cloud/host/HostLoadService.java

@@ -0,0 +1,32 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package com.cloud.host;
+
+/**
+ * A smoothed view of what each host is actually doing, as opposed to what has been allocated on it.
+ *
+ * Allocation figures say what was promised; under overprovisioning they can be far from what a host
+ * is really carrying. Placement and rebalancing both need the second view.
+ */
+public interface HostLoadService {
+
+    /**
+     * @return the host's smoothed load, or a value reporting itself unusable when the host has not
+     *         been sampled recently enough to rank on
+     */
+    HostLoad getLoad(long hostId);
+}

api/src/main/java/org/apache/cloudstack/affinity/AffinityProcessorBase.java

@@ -25,7 +25,9 @@
 import com.cloud.vm.VirtualMachineProfile;
 
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 
 public class AffinityProcessorBase extends AdapterBase implements AffinityGroupProcessor {
 
@@ -44,6 +46,23 @@ public void process(VirtualMachineProfile vm, DeploymentPlan plan, ExcludeList a
 
     }
 
+
+    /**
+     * Indexes placements supplied by the caller. Callers such as DRS build a plan of several moves
+     * in memory and persist it only at the end, so during planning the database still shows the old
+     * host for every VM the plan has already moved.
+     */
+    protected Map<Long, VirtualMachine> getVmIdVmMap(List<VirtualMachine> vmList) {
+        Map<Long, VirtualMachine> vmIdVmMap = new HashMap<>();
+        if (vmList == null) {
+            return vmIdVmMap;
+        }
+        for (VirtualMachine vm : vmList) {
+            vmIdVmMap.put(vm.getId(), vm);
+        }
+        return vmIdVmMap;
+    }
+
     @Override
     public String getType() {
         return _type;

api/src/main/java/org/apache/cloudstack/cluster/ClusterDrsAlgorithm.java

@@ -20,6 +20,7 @@
 package org.apache.cloudstack.cluster;
 
 import com.cloud.host.Host;
+import com.cloud.host.HostLoad;
 import com.cloud.offering.ServiceOffering;
 import com.cloud.org.Cluster;
 import com.cloud.utils.Ternary;
@@ -62,6 +63,35 @@ public interface ClusterDrsAlgorithm extends Adapter {
     boolean needsDrs(Cluster cluster, List<Ternary<Long, Long, Long>> cpuList,
                      List<Ternary<Long, Long, Long>> memoryList) throws ConfigurationException;
 
+    /**
+     * As above, but keeping host identity.
+     *
+     * An algorithm that considers anything beyond the figures in the two maps - measured load, for
+     * instance - cannot attribute it without knowing which host each entry belongs to. Algorithms
+     * that only need the values keep the default.
+     *
+     * @param hostCpuMap
+     *         host id to a Ternary of used, reserved and total CPU
+     * @param hostMemoryMap
+     *         host id to a Ternary of used, reserved and total memory
+     */
+    default boolean needsDrs(Cluster cluster, Map<Long, Ternary<Long, Long, Long>> hostCpuMap,
+            Map<Long, Ternary<Long, Long, Long>> hostMemoryMap, Map<Long, HostLoad> hostLoadMap)
+            throws ConfigurationException {
+        return needsDrs(cluster, new ArrayList<>(hostCpuMap.values()), new ArrayList<>(hostMemoryMap.values()));
+    }
+
+    /**
+     * Called once per plan, before any migration is considered, so that an algorithm can do work
+     * that would otherwise be repeated for every candidate VM and host.
+     *
+     * @param hostLoadMap
+     *         measured load per host, empty when nothing has been sampled
+     */
+    default void prepare(Cluster cluster, Map<Long, Ternary<Long, Long, Long>> hostCpuMap,
+            Map<Long, Ternary<Long, Long, Long>> hostMemoryMap, Map<Long, HostLoad> hostLoadMap) {
+    }
+
     /**
      * Calculates the metrics (improvement, cost, benefit) for migrating a VM to a destination host. Improvement is
      * calculated based on the change in cluster imbalance before and after the migration.

api/src/main/java/org/apache/cloudstack/cluster/ClusterDrsService.java

@@ -51,9 +51,9 @@ public interface ClusterDrsService extends Manager, Configurable, Scheduler {
             true, ConfigKey.Scope.Cluster, null, "Maximum number of migrations for DRS", null, null, null);
 
     ConfigKey<String> ClusterDrsAlgorithm = new ConfigKey<>(String.class, "drs.algorithm",
-            ConfigKey.CATEGORY_ADVANCED, "balanced", "The DRS algorithm to be executed on the cluster. Possible values are condensed, balanced.",
+            ConfigKey.CATEGORY_ADVANCED, "balanced", "The DRS algorithm to be executed on the cluster. Possible values are condensed, balanced, weighted. 'weighted' balances CPU and memory together, and on measured load as well as allocation, tuned with the host.weighted.{cpu,memory}.{allocated,used}.weight settings it shares with the 'balancedweighted' allocation algorithm and with drs.weighted.storage.motion.cost. Hosts are sampled for load while any cluster uses it, whatever vm.allocation.algorithm is.",
             true, ConfigKey.Scope.Cluster, null, "DRS algorithm", null, null,
-            null, ConfigKey.Kind.Select, "condensed,balanced");
+            null, ConfigKey.Kind.Select, "condensed,balanced,weighted");
 
     ConfigKey<Float> ClusterDrsImbalanceThreshold = new ConfigKey<>(Float.class, "drs.imbalance",
             ConfigKey.CATEGORY_ADVANCED, "0.4",

client/pom.xml

@@ -592,6 +592,11 @@
             <artifactId>cloud-plugin-cluster-drs-condensed</artifactId>
             <version>${project.version}</version>
         </dependency>
+<dependency>
+            <groupId>org.apache.cloudstack</groupId>
+            <artifactId>cloud-plugin-cluster-drs-weighted</artifactId>
+            <version>${project.version}</version>
+        </dependency>
         <dependency>
             <groupId>org.apache.cloudstack</groupId>
             <artifactId>cloud-plugin-database-quota</artifactId>

plugins/affinity-group-processors/host-affinity/src/main/java/org/apache/cloudstack/affinity/HostAffinityProcessor.java

@@ -17,7 +17,6 @@
 package org.apache.cloudstack.affinity;
 
 import java.util.Collections;
-import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -121,14 +120,6 @@ protected Set<Long> getHostIdSet(List<Long> vmIds, List<VirtualMachine> vmList)
         return hostIds;
     }
 
-    protected Map<Long, VirtualMachine> getVmIdVmMap(List<VirtualMachine> vmList) {
-        Map<Long, VirtualMachine> vmIdVmMap = new HashMap<>();
-        for (VirtualMachine vm : vmList) {
-            vmIdVmMap.put(vm.getId(), vm);
-        }
-        return vmIdVmMap;
-    }
-
     /**
      * Get preferred host ids list from the affinity group VMs
      */

plugins/affinity-group-processors/host-anti-affinity/src/main/java/org/apache/cloudstack/affinity/HostAntiAffinityProcessor.java

@@ -17,6 +17,7 @@
 package org.apache.cloudstack.affinity;
 
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 import java.util.stream.Collectors;
@@ -59,7 +60,7 @@ public class HostAntiAffinityProcessor extends AffinityProcessorBase implements
     protected AffinityGroupDao _affinityGroupDao;
     @Inject
     protected AffinityGroupVMMapDao _affinityGroupVMMapDao;
-    private int _vmCapacityReleaseInterval;
+    protected int _vmCapacityReleaseInterval;
     @Inject
     protected ConfigurationDao _configDao;
 
@@ -82,14 +83,26 @@ public void doInTransactionWithoutResult(TransactionStatus status) {
                     _affinityGroupDao.listByIds(affinityGroupIds, true);
                 }
                 for (AffinityGroupVMMapVO vmGroupMapping : vmGroupMappings) {
-                    processAffinityGroup(vmGroupMapping, avoid, vm);
+                    processAffinityGroup(vmGroupMapping, avoid, vm, vmList);
                 }
             }
         });
 
     }
 
     protected void processAffinityGroup(AffinityGroupVMMapVO vmGroupMapping, ExcludeList avoid, VirtualMachine vm) {
+        processAffinityGroup(vmGroupMapping, avoid, vm, Collections.emptyList());
+    }
+
+    /**
+     * Applies anti-affinity for one group.
+     *
+     * @param vmList
+     *         placements to honour in preference to what the database says. DRS builds a plan of
+     *         several migrations in memory and only persists it later, so during plan generation
+     *         the database still shows the old host for every VM the plan has already moved.
+     */
+    protected void processAffinityGroup(AffinityGroupVMMapVO vmGroupMapping, ExcludeList avoid, VirtualMachine vm, List<VirtualMachine> vmList) {
         if (vmGroupMapping != null) {
             AffinityGroupVO group = _affinityGroupDao.findById(vmGroupMapping.getAffinityGroupId());
 
@@ -100,24 +113,35 @@ protected void processAffinityGroup(AffinityGroupVMMapVO vmGroupMapping, Exclude
             List<Long> groupVMIds = _affinityGroupVMMapDao.listVmIdsByAffinityGroup(group.getId());
             groupVMIds.remove(vm.getId());
 
+            Map<Long, VirtualMachine> plannedVms = getVmIdVmMap(vmList);
+
             for (Long groupVMId : groupVMIds) {
+                VirtualMachine plannedVm = plannedVms.get(groupVMId);
+                if (plannedVm != null && plannedVm.getHostId() != null) {
+                    avoid.addHost(plannedVm.getHostId());
+                    logger.debug("Added host {} to avoid set, since VM {} is placed on the host by the plan being built",
+                            plannedVm.getHostId(), plannedVm);
+                    continue;
+                }
                 VMInstanceVO groupVM = _vmInstanceDao.findById(groupVMId);
-                if (groupVM != null && !groupVM.isRemoved()) {
-                    if (groupVM.getHostId() != null) {
-                        avoid.addHost(groupVM.getHostId());
-                        if (logger.isDebugEnabled()) {
-                            logger.debug("Added host {} to avoid set, since VM {} is present on the host", groupVM.getHostId(), groupVM);
-                        }
-                    }
-                } else if (Arrays.asList(VirtualMachine.State.Starting, VirtualMachine.State.Stopped).contains(groupVM.getState()) && groupVM.getLastHostId() != null) {
-                    long secondsSinceLastUpdate = (DateUtil.currentGMTTime().getTime() - groupVM.getUpdateTime().getTime()) / 1000;
-                    if (secondsSinceLastUpdate < _vmCapacityReleaseInterval) {
-                        avoid.addHost(groupVM.getLastHostId());
-                        if (logger.isDebugEnabled()) {
-                            logger.debug("Added host {} to avoid set, since VM {} is present on the host, in Stopped state but has reserved capacity", groupVM.getLastHostId(), groupVM);
-                        }
-                    }
+                if (groupVM == null || groupVM.isRemoved()) {
+                    continue;
                 }
+                avoidHostOfVmInAffinityGroup(avoid, groupVM);
+            }
+        }
+    }
+
+    protected void avoidHostOfVmInAffinityGroup(ExcludeList avoid, VMInstanceVO groupVM) {
+        if (groupVM.getHostId() != null) {
+            avoid.addHost(groupVM.getHostId());
+            logger.debug("Added host {} to avoid set, since VM {} is present on the host", groupVM.getHostId(), groupVM);
+        } else if (Arrays.asList(VirtualMachine.State.Starting, VirtualMachine.State.Stopped).contains(groupVM.getState()) && groupVM.getLastHostId() != null) {
+            long secondsSinceLastUpdate = (DateUtil.currentGMTTime().getTime() - groupVM.getUpdateTime().getTime()) / 1000;
+            if (secondsSinceLastUpdate < _vmCapacityReleaseInterval) {
+                avoid.addHost(groupVM.getLastHostId());
+                logger.debug("Added host {} to avoid set, since VM {} is in {} state on the host but still has reserved capacity",
+                        groupVM.getLastHostId(), groupVM, groupVM.getState());
             }
         }
     }