simulator: Migrate frequency scaling governors to OS layer

This change moves the CPU frequency scaling governors from the
bare-metal/firmware layer (SimBareMetalMachine) to the OS/Hypervisor
layer (SimHypervisor) where it can make more informed decisions about
the CPU frequency based on the load of the operating system or
hypervisor.

12 files changed, 95 insertions, 115 deletions

diff --git a/opendc-simulator/opendc-simulator-compute/src/main/kotlin/org/opendc/simulator/compute/SimAbstractHypervisor.kt b/opendc-simulator/opendc-simulator-compute/src/main/kotlin/org/opendc/simulator/compute/SimAbstractHypervisor.kt
index 68ecc49f..57c25b86 100644
--- a/opendc-simulator/opendc-simulator-compute/src/main/kotlin/org/opendc/simulator/compute/SimAbstractHypervisor.kt
+++ b/opendc-simulator/opendc-simulator-compute/src/main/kotlin/org/opendc/simulator/compute/SimAbstractHypervisor.kt
@@ -22,6 +22,7 @@
 
 package org.opendc.simulator.compute
 
+import org.opendc.simulator.compute.cpufreq.ScalingGovernor
 import org.opendc.simulator.compute.interference.PerformanceInterferenceModel
 import org.opendc.simulator.compute.model.ProcessingUnit
 import org.opendc.simulator.resources.*
@@ -29,8 +30,14 @@ import org.opendc.simulator.resources.SimResourceSwitch
 
 /**
  * Abstract implementation of the [SimHypervisor] interface.
+ *
+ * @param interpreter The resource interpreter to use.
+ * @param scalingGovernor The scaling governor to use for scaling the CPU frequency of the underlying hardware.
  */
-public abstract class SimAbstractHypervisor(private val interpreter: SimResourceInterpreter) : SimHypervisor {
+public abstract class SimAbstractHypervisor(
+    private val interpreter: SimResourceInterpreter,
+    private val scalingGovernor: ScalingGovernor?
+) : SimHypervisor {
     /**
      * The machine on which the hypervisor runs.
      */
@@ -49,6 +56,11 @@ public abstract class SimAbstractHypervisor(private val interpreter: SimResource
         get() = _vms
 
     /**
+     * The scaling governors attached to the physical CPUs backing this hypervisor.
+     */
+    private val governors = mutableListOf<ScalingGovernor.Logic>()
+
+    /**
      * Construct the [SimResourceSwitch] implementation that performs the actual scheduling of the CPUs.
      */
     public abstract fun createSwitch(ctx: SimMachineContext): SimResourceSwitch
@@ -58,6 +70,16 @@ public abstract class SimAbstractHypervisor(private val interpreter: SimResource
      */
     public abstract fun canFit(model: SimMachineModel, switch: SimResourceSwitch): Boolean
 
+    /**
+     * Trigger the governors to recompute the scaling limits.
+     */
+    protected fun triggerGovernors(load: Double) {
+        for (governor in governors) {
+            governor.onLimit(load)
+        }
+    }
+
+    /* SimHypervisor */
     override fun canFit(model: SimMachineModel): Boolean {
         return canFit(model, switch)
     }
@@ -72,6 +94,21 @@ public abstract class SimAbstractHypervisor(private val interpreter: SimResource
         return vm
     }
 
+    /* SimWorkload */
+    override fun onStart(ctx: SimMachineContext) {
+        context = ctx
+        switch = createSwitch(ctx)
+
+        for (cpu in ctx.cpus) {
+            val governor = scalingGovernor?.createLogic(cpu)
+            if (governor != null) {
+                governors.add(governor)
+                governor.onStart()
+            }
+            switch.addInput(cpu)
+        }
+    }
+
     /**
      * A virtual machine running on the hypervisor.
      *
@@ -94,15 +131,6 @@ public abstract class SimAbstractHypervisor(private val interpreter: SimResource
         }
     }
 
-    override fun onStart(ctx: SimMachineContext) {
-        context = ctx
-        switch = createSwitch(ctx)
-
-        for (cpu in ctx.cpus) {
-            switch.addInput(cpu)
-        }
-    }
-
     /**
      * A [SimProcessingUnit] of a virtual machine.
      */
@@ -110,6 +138,12 @@ public abstract class SimAbstractHypervisor(private val interpreter: SimResource
         private val source: SimResourceProvider,
         override val model: ProcessingUnit
     ) : SimProcessingUnit, SimResourceProvider by source {
+        override var capacity: Double
+            get() = source.capacity
+            set(_) {
+                // Ignore capacity changes
+            }
+
         override fun toString(): String = "SimAbstractHypervisor.VCpu[model=$model]"
     }
 }
diff --git a/opendc-simulator/opendc-simulator-compute/src/main/kotlin/org/opendc/simulator/compute/SimBareMetalMachine.kt b/opendc-simulator/opendc-simulator-compute/src/main/kotlin/org/opendc/simulator/compute/SimBareMetalMachine.kt
index c453cdf3..5d5d1e5a 100644
--- a/opendc-simulator/opendc-simulator-compute/src/main/kotlin/org/opendc/simulator/compute/SimBareMetalMachine.kt
+++ b/opendc-simulator/opendc-simulator-compute/src/main/kotlin/org/opendc/simulator/compute/SimBareMetalMachine.kt
@@ -69,9 +69,18 @@ public class SimBareMetalMachine(
      * A [SimProcessingUnit] of a bare-metal machine.
      */
     private class Cpu(
-        private val source: SimResourceProvider,
+        private val source: SimResourceSource,
         override val model: ProcessingUnit
     ) : SimProcessingUnit, SimResourceProvider by source {
+        override var capacity: Double
+            get() = source.capacity
+            set(value) {
+                // Clamp the capacity of the CPU between [0.0, maxFreq]
+                if (value >= 0.0 && value <= model.frequency) {
+                    source.capacity = value
+                }
+            }
+
         override fun toString(): String = "SimBareMetalMachine.Cpu[model=$model]"
     }
 }
diff --git a/opendc-simulator/opendc-simulator-compute/src/main/kotlin/org/opendc/simulator/compute/SimFairShareHypervisor.kt b/opendc-simulator/opendc-simulator-compute/src/main/kotlin/org/opendc/simulator/compute/SimFairShareHypervisor.kt
index 3ceb3291..e7776c81 100644
--- a/opendc-simulator/opendc-simulator-compute/src/main/kotlin/org/opendc/simulator/compute/SimFairShareHypervisor.kt
+++ b/opendc-simulator/opendc-simulator-compute/src/main/kotlin/org/opendc/simulator/compute/SimFairShareHypervisor.kt
@@ -22,6 +22,7 @@
 
 package org.opendc.simulator.compute
 
+import org.opendc.simulator.compute.cpufreq.ScalingGovernor
 import org.opendc.simulator.compute.workload.SimWorkload
 import org.opendc.simulator.resources.SimResourceInterpreter
 import org.opendc.simulator.resources.SimResourceSwitch
@@ -34,21 +35,23 @@ import org.opendc.simulator.resources.SimResourceSystem
  *
  * @param interpreter The interpreter to manage the machine's resources.
  * @param parent The parent simulation system.
+ * @param scalingGovernor The CPU frequency scaling governor to use for the hypervisor.
  * @param listener The hypervisor listener to use.
  */
 public class SimFairShareHypervisor(
     private val interpreter: SimResourceInterpreter,
     private val parent: SimResourceSystem? = null,
+    scalingGovernor: ScalingGovernor? = null,
     private val listener: SimHypervisor.Listener? = null
-) : SimAbstractHypervisor(interpreter) {
+) : SimAbstractHypervisor(interpreter, scalingGovernor) {
 
     override fun canFit(model: SimMachineModel, switch: SimResourceSwitch): Boolean = true
 
     override fun createSwitch(ctx: SimMachineContext): SimResourceSwitch {
-        return SwitchSystem().switch
+        return SwitchSystem(ctx).switch
     }
 
-    private inner class SwitchSystem : SimResourceSystem {
+    private inner class SwitchSystem(private val ctx: SimMachineContext) : SimResourceSystem {
         val switch = SimResourceSwitchMaxMin(interpreter, this)
 
         override val parent: SimResourceSystem? = this@SimFairShareHypervisor.parent
@@ -82,6 +85,9 @@ public class SimFairShareHypervisor(
             lastDemand = counters.demand
             lastActual = counters.actual
             lastOvercommit = counters.overcommit
+
+            val load = lastCpuDemand / ctx.cpus.sumOf { it.model.frequency }
+            triggerGovernors(load)
         }
     }
 }
diff --git a/opendc-simulator/opendc-simulator-compute/src/main/kotlin/org/opendc/simulator/compute/SimFairShareHypervisorProvider.kt b/opendc-simulator/opendc-simulator-compute/src/main/kotlin/org/opendc/simulator/compute/SimFairShareHypervisorProvider.kt
index d3206196..94c905b2 100644
--- a/opendc-simulator/opendc-simulator-compute/src/main/kotlin/org/opendc/simulator/compute/SimFairShareHypervisorProvider.kt
+++ b/opendc-simulator/opendc-simulator-compute/src/main/kotlin/org/opendc/simulator/compute/SimFairShareHypervisorProvider.kt
@@ -35,5 +35,5 @@ public class SimFairShareHypervisorProvider : SimHypervisorProvider {
         interpreter: SimResourceInterpreter,
         parent: SimResourceSystem?,
         listener: SimHypervisor.Listener?
-    ): SimHypervisor = SimFairShareHypervisor(interpreter, parent, listener)
+    ): SimHypervisor = SimFairShareHypervisor(interpreter, parent, listener = listener)
 }
diff --git a/opendc-simulator/opendc-simulator-compute/src/main/kotlin/org/opendc/simulator/compute/SimProcessingUnit.kt b/opendc-simulator/opendc-simulator-compute/src/main/kotlin/org/opendc/simulator/compute/SimProcessingUnit.kt
index 136a543a..93c9ddfa 100644
--- a/opendc-simulator/opendc-simulator-compute/src/main/kotlin/org/opendc/simulator/compute/SimProcessingUnit.kt
+++ b/opendc-simulator/opendc-simulator-compute/src/main/kotlin/org/opendc/simulator/compute/SimProcessingUnit.kt
@@ -30,6 +30,11 @@ import org.opendc.simulator.resources.SimResourceProvider
  */
 public interface SimProcessingUnit : SimResourceProvider {
     /**
+     * The capacity of the processing unit, which can be adjusted by the workload if supported by the machine.
+     */
+    public override var capacity: Double
+
+    /**
      *  The model representing the static properties of the processing unit.
      */
     public val model: ProcessingUnit
diff --git a/opendc-simulator/opendc-simulator-compute/src/main/kotlin/org/opendc/simulator/compute/SimSpaceSharedHypervisor.kt b/opendc-simulator/opendc-simulator-compute/src/main/kotlin/org/opendc/simulator/compute/SimSpaceSharedHypervisor.kt
index afb47872..f6ae18f7 100644
--- a/opendc-simulator/opendc-simulator-compute/src/main/kotlin/org/opendc/simulator/compute/SimSpaceSharedHypervisor.kt
+++ b/opendc-simulator/opendc-simulator-compute/src/main/kotlin/org/opendc/simulator/compute/SimSpaceSharedHypervisor.kt
@@ -29,7 +29,7 @@ import org.opendc.simulator.resources.SimResourceSwitchExclusive
 /**
  * A [SimHypervisor] that allocates its sub-resources exclusively for the virtual machine that it hosts.
  */
-public class SimSpaceSharedHypervisor(interpreter: SimResourceInterpreter) : SimAbstractHypervisor(interpreter) {
+public class SimSpaceSharedHypervisor(interpreter: SimResourceInterpreter) : SimAbstractHypervisor(interpreter, null) {
     override fun canFit(model: SimMachineModel, switch: SimResourceSwitch): Boolean {
         return switch.inputs.size - switch.outputs.size >= model.cpus.size
     }
diff --git a/opendc-simulator/opendc-simulator-compute/src/main/kotlin/org/opendc/simulator/compute/cpufreq/DemandScalingGovernor.kt b/opendc-simulator/opendc-simulator-compute/src/main/kotlin/org/opendc/simulator/compute/cpufreq/DemandScalingGovernor.kt
deleted file mode 100644
index ddbe1ca0..00000000
--- a/opendc-simulator/opendc-simulator-compute/src/main/kotlin/org/opendc/simulator/compute/cpufreq/DemandScalingGovernor.kt
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
- * Copyright (c) 2021 AtLarge Research
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to deal
- * in the Software without restriction, including without limitation the rights
- * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- * copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in all
- * copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-
-package org.opendc.simulator.compute.cpufreq
-
-/**
- * A CPUFreq [ScalingGovernor] that requests the frequency based on the utilization of the machine.
- */
-public class DemandScalingGovernor : ScalingGovernor {
-    override fun createLogic(ctx: ScalingContext): ScalingGovernor.Logic = object : ScalingGovernor.Logic {
-        override fun onLimit() {
-            ctx.setTarget(ctx.cpu.speed)
-        }
-
-        override fun toString(): String = "DemandScalingGovernor.Logic"
-    }
-}
diff --git a/opendc-simulator/opendc-simulator-compute/src/main/kotlin/org/opendc/simulator/compute/cpufreq/PerformanceScalingGovernor.kt b/opendc-simulator/opendc-simulator-compute/src/main/kotlin/org/opendc/simulator/compute/cpufreq/PerformanceScalingGovernor.kt
index 96f8775a..245877be 100644
--- a/opendc-simulator/opendc-simulator-compute/src/main/kotlin/org/opendc/simulator/compute/cpufreq/PerformanceScalingGovernor.kt
+++ b/opendc-simulator/opendc-simulator-compute/src/main/kotlin/org/opendc/simulator/compute/cpufreq/PerformanceScalingGovernor.kt
@@ -22,13 +22,15 @@
 
 package org.opendc.simulator.compute.cpufreq
 
+import org.opendc.simulator.compute.SimProcessingUnit
+
 /**
  * A CPUFreq [ScalingGovernor] that causes the highest possible frequency to be requested from the resource.
  */
 public class PerformanceScalingGovernor : ScalingGovernor {
-    override fun createLogic(ctx: ScalingContext): ScalingGovernor.Logic = object : ScalingGovernor.Logic {
-        override fun onLimit() {
-            ctx.setTarget(ctx.cpu.model.frequency)
+    override fun createLogic(cpu: SimProcessingUnit): ScalingGovernor.Logic = object : ScalingGovernor.Logic {
+        override fun onStart() {
+            cpu.capacity = cpu.model.frequency
         }
 
         override fun toString(): String = "PerformanceScalingGovernor.Logic"
diff --git a/opendc-simulator/opendc-simulator-compute/src/main/kotlin/org/opendc/simulator/compute/cpufreq/ScalingContext.kt b/opendc-simulator/opendc-simulator-compute/src/main/kotlin/org/opendc/simulator/compute/cpufreq/ScalingContext.kt
deleted file mode 100644
index 18338079..00000000
--- a/opendc-simulator/opendc-simulator-compute/src/main/kotlin/org/opendc/simulator/compute/cpufreq/ScalingContext.kt
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- * Copyright (c) 2021 AtLarge Research
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to deal
- * in the Software without restriction, including without limitation the rights
- * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- * copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in all
- * copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-
-package org.opendc.simulator.compute.cpufreq
-
-import org.opendc.simulator.compute.SimMachine
-import org.opendc.simulator.compute.SimProcessingUnit
-
-/**
- * A [ScalingContext] is used to communicate frequency scaling changes between the [ScalingGovernor] and driver.
- */
-public interface ScalingContext {
-    /**
-     * The machine the processing unit belongs to.
-     */
-    public val machine: SimMachine
-
-    /**
-     * The processing unit associated with this context.
-     */
-    public val cpu: SimProcessingUnit
-
-    /**
-     * Target the processor to run at the specified target [frequency][freq].
-     */
-    public fun setTarget(freq: Double)
-}
diff --git a/opendc-simulator/opendc-simulator-compute/src/main/kotlin/org/opendc/simulator/compute/cpufreq/ScalingGovernor.kt b/opendc-simulator/opendc-simulator-compute/src/main/kotlin/org/opendc/simulator/compute/cpufreq/ScalingGovernor.kt
index c9aea580..b7e7ffc6 100644
--- a/opendc-simulator/opendc-simulator-compute/src/main/kotlin/org/opendc/simulator/compute/cpufreq/ScalingGovernor.kt
+++ b/opendc-simulator/opendc-simulator-compute/src/main/kotlin/org/opendc/simulator/compute/cpufreq/ScalingGovernor.kt
@@ -22,6 +22,8 @@
 
 package org.opendc.simulator.compute.cpufreq
 
+import org.opendc.simulator.compute.SimProcessingUnit
+
 /**
  * A [ScalingGovernor] in the CPUFreq subsystem of OpenDC is responsible for scaling the frequency of simulated CPUs
  * independent of the particular implementation of the CPU.
@@ -33,9 +35,9 @@ package org.opendc.simulator.compute.cpufreq
  */
 public interface ScalingGovernor {
     /**
-     * Create the scaling logic for the specified [context]
+     * Create the scaling logic for the specified [cpu]
      */
-    public fun createLogic(ctx: ScalingContext): Logic
+    public fun createLogic(cpu: SimProcessingUnit): Logic
 
     /**
      * The logic of the scaling governor.
@@ -48,7 +50,9 @@ public interface ScalingGovernor {
 
         /**
          * This method is invoked when the governor should re-decide the frequency limits.
+         *
+         * @param load The load of the system.
          */
-        public fun onLimit() {}
+        public fun onLimit(load: Double) {}
     }
 }
diff --git a/opendc-simulator/opendc-simulator-compute/src/test/kotlin/org/opendc/simulator/compute/SimHypervisorTest.kt b/opendc-simulator/opendc-simulator-compute/src/test/kotlin/org/opendc/simulator/compute/SimHypervisorTest.kt
index 0bcfd9c6..b15692ec 100644
--- a/opendc-simulator/opendc-simulator-compute/src/test/kotlin/org/opendc/simulator/compute/SimHypervisorTest.kt
+++ b/opendc-simulator/opendc-simulator-compute/src/test/kotlin/org/opendc/simulator/compute/SimHypervisorTest.kt
@@ -32,6 +32,7 @@ import org.junit.jupiter.api.BeforeEach
 import org.junit.jupiter.api.Test
 import org.junit.jupiter.api.assertAll
 import org.junit.jupiter.api.assertDoesNotThrow
+import org.opendc.simulator.compute.cpufreq.PerformanceScalingGovernor
 import org.opendc.simulator.compute.model.MemoryUnit
 import org.opendc.simulator.compute.model.ProcessingNode
 import org.opendc.simulator.compute.model.ProcessingUnit
@@ -95,7 +96,7 @@ internal class SimHypervisorTest {
 
         val platform = SimResourceInterpreter(coroutineContext, clock)
         val machine = SimBareMetalMachine(platform, model, SimplePowerDriver(ConstantPowerModel(0.0)))
-        val hypervisor = SimFairShareHypervisor(platform, null, listener)
+        val hypervisor = SimFairShareHypervisor(platform, scalingGovernor = PerformanceScalingGovernor(), listener = listener)
 
         launch {
             machine.run(hypervisor)
@@ -169,7 +170,7 @@ internal class SimHypervisorTest {
         val machine = SimBareMetalMachine(
             platform, model, SimplePowerDriver(ConstantPowerModel(0.0))
         )
-        val hypervisor = SimFairShareHypervisor(platform, null, listener)
+        val hypervisor = SimFairShareHypervisor(platform, listener = listener)
 
         launch {
             machine.run(hypervisor)
diff --git a/opendc-simulator/opendc-simulator-compute/src/test/kotlin/org/opendc/simulator/compute/cpufreq/DemandScalingGovernorTest.kt b/opendc-simulator/opendc-simulator-compute/src/test/kotlin/org/opendc/simulator/compute/cpufreq/PerformanceScalingGovernorTest.kt
index c482d348..8e8b09c8 100644
--- a/opendc-simulator/opendc-simulator-compute/src/test/kotlin/org/opendc/simulator/compute/cpufreq/DemandScalingGovernorTest.kt
+++ b/opendc-simulator/opendc-simulator-compute/src/test/kotlin/org/opendc/simulator/compute/cpufreq/PerformanceScalingGovernorTest.kt
@@ -26,23 +26,24 @@ import io.mockk.every
 import io.mockk.mockk
 import io.mockk.verify
 import org.junit.jupiter.api.Test
+import org.opendc.simulator.compute.SimProcessingUnit
 
 /**
- * Test suite for the [DemandScalingGovernor]
+ * Test suite for the [PerformanceScalingGovernor]
  */
-internal class DemandScalingGovernorTest {
+internal class PerformanceScalingGovernorTest {
     @Test
-    fun testSetDemandLimit() {
-        val ctx = mockk<ScalingContext>(relaxUnitFun = true)
+    fun testSetStartLimit() {
+        val cpu = mockk<SimProcessingUnit>(relaxUnitFun = true)
 
-        every { ctx.cpu.speed } returns 2100.0
+        every { cpu.model.frequency } returns 4100.0
+        every { cpu.speed } returns 2100.0
 
-        val logic = DemandScalingGovernor().createLogic(ctx)
+        val logic = PerformanceScalingGovernor().createLogic(cpu)
 
         logic.onStart()
-        verify(exactly = 0) { ctx.setTarget(any()) }
+        logic.onLimit(1.0)
 
-        logic.onLimit()
-        verify(exactly = 1) { ctx.setTarget(2100.0) }
+        verify(exactly = 1) { cpu.capacity = 4100.0 }
     }
 }