perf(experiments): Add benchmark for Capelin experiment

5 files changed, 170 insertions, 64 deletions

diff --git a/opendc-experiments/opendc-experiments-capelin/build.gradle.kts b/opendc-experiments/opendc-experiments-capelin/build.gradle.kts
index 23a3e4a7..c20556b5 100644
--- a/opendc-experiments/opendc-experiments-capelin/build.gradle.kts
+++ b/opendc-experiments/opendc-experiments-capelin/build.gradle.kts
@@ -26,6 +26,7 @@ description = "Experiments for the Capelin work"
 plugins {
     `experiment-conventions`
     `testing-conventions`
+    `benchmark-conventions`
 }
 
 dependencies {
diff --git a/opendc-experiments/opendc-experiments-capelin/src/jmh/kotlin/org/opendc/experiments/capelin/CapelinBenchmarks.kt b/opendc-experiments/opendc-experiments-capelin/src/jmh/kotlin/org/opendc/experiments/capelin/CapelinBenchmarks.kt
new file mode 100644
index 00000000..48a90985
--- /dev/null
+++ b/opendc-experiments/opendc-experiments-capelin/src/jmh/kotlin/org/opendc/experiments/capelin/CapelinBenchmarks.kt
@@ -0,0 +1,83 @@
+/*
+ * Copyright (c) 2021 AtLarge Research
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+package org.opendc.experiments.capelin
+
+import kotlinx.coroutines.ExperimentalCoroutinesApi
+import org.opendc.compute.service.scheduler.FilterScheduler
+import org.opendc.compute.service.scheduler.filters.ComputeFilter
+import org.opendc.compute.service.scheduler.filters.RamFilter
+import org.opendc.compute.service.scheduler.filters.VCpuFilter
+import org.opendc.compute.service.scheduler.weights.CoreRamWeigher
+import org.opendc.compute.workload.*
+import org.opendc.compute.workload.topology.Topology
+import org.opendc.compute.workload.topology.apply
+import org.opendc.experiments.capelin.topology.clusterTopology
+import org.opendc.simulator.core.runBlockingSimulation
+import org.openjdk.jmh.annotations.*
+import java.io.File
+import java.util.*
+import java.util.concurrent.TimeUnit
+
+/**
+ * Benchmark suite for the Capelin experiments.
+ */
+@State(Scope.Thread)
+@Fork(1)
+@Warmup(iterations = 2, time = 5, timeUnit = TimeUnit.SECONDS)
+@Measurement(iterations = 5, time = 5, timeUnit = TimeUnit.SECONDS)
+@OptIn(ExperimentalCoroutinesApi::class)
+class CapelinBenchmarks {
+    private lateinit var vms: List<VirtualMachine>
+    private lateinit var topology: Topology
+
+    @Param("true", "false")
+    private var isOptimized: Boolean = false
+
+    @Setup
+    fun setUp() {
+        val loader = ComputeWorkloadLoader(File("src/test/resources/trace"))
+        val source = trace("bitbrains-small")
+        vms = source.resolve(loader, Random(1L))
+        topology = checkNotNull(object {}.javaClass.getResourceAsStream("/env/topology.txt")).use { clusterTopology(it) }
+    }
+
+    @Benchmark
+    fun benchmarkCapelin() = runBlockingSimulation {
+        val computeScheduler = FilterScheduler(
+            filters = listOf(ComputeFilter(), VCpuFilter(16.0), RamFilter(1.0)),
+            weighers = listOf(CoreRamWeigher(multiplier = 1.0))
+        )
+        val runner = ComputeWorkloadRunner(
+            coroutineContext,
+            clock,
+            computeScheduler
+        )
+
+        try {
+            runner.apply(topology, isOptimized)
+            runner.run(vms, 0)
+        } finally {
+            runner.close()
+        }
+    }
+}
diff --git a/opendc-experiments/opendc-experiments-capelin/src/jmh/resources/log4j2.xml b/opendc-experiments/opendc-experiments-capelin/src/jmh/resources/log4j2.xml
new file mode 100644
index 00000000..c496dd75
--- /dev/null
+++ b/opendc-experiments/opendc-experiments-capelin/src/jmh/resources/log4j2.xml
@@ -0,0 +1,37 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ~ MIT License
+  ~
+  ~ Copyright (c) 2020 atlarge-research
+  ~
+  ~ Permission is hereby granted, free of charge, to any person obtaining a copy
+  ~ of this software and associated documentation files (the "Software"), to deal
+  ~ in the Software without restriction, including without limitation the rights
+  ~ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+  ~ copies of the Software, and to permit persons to whom the Software is
+  ~ furnished to do so, subject to the following conditions:
+  ~
+  ~ The above copyright notice and this permission notice shall be included in all
+  ~ copies or substantial portions of the Software.
+  ~
+  ~ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+  ~ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+  ~ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+  ~ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+  ~ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+  ~ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+  ~ SOFTWARE.
+  -->
+
+<Configuration status="WARN">
+    <Appenders>
+        <Console name="Console" target="SYSTEM_OUT">
+            <PatternLayout pattern="%d{HH:mm:ss.SSS} [%highlight{%-5level}] %logger{36} - %msg%n" disableAnsi="false"/>
+        </Console>
+    </Appenders>
+    <Loggers>
+        <Root level="warn">
+            <AppenderRef ref="Console"/>
+        </Root>
+    </Loggers>
+</Configuration>
diff --git a/opendc-simulator/opendc-simulator-compute/src/jmh/kotlin/org/opendc/simulator/compute/SimMachineBenchmarks.kt b/opendc-simulator/opendc-simulator-compute/src/jmh/kotlin/org/opendc/simulator/compute/SimMachineBenchmarks.kt
index d654d58a..b8e0227a 100644
--- a/opendc-simulator/opendc-simulator-compute/src/jmh/kotlin/org/opendc/simulator/compute/SimMachineBenchmarks.kt
+++ b/opendc-simulator/opendc-simulator-compute/src/jmh/kotlin/org/opendc/simulator/compute/SimMachineBenchmarks.kt
@@ -22,6 +22,7 @@
 
 package org.opendc.simulator.compute
 
+import javafx.application.Application.launch
 import kotlinx.coroutines.ExperimentalCoroutinesApi
 import kotlinx.coroutines.coroutineScope
 import kotlinx.coroutines.launch
@@ -34,7 +35,6 @@ import org.opendc.simulator.compute.model.ProcessingUnit
 import org.opendc.simulator.compute.power.ConstantPowerModel
 import org.opendc.simulator.compute.power.SimplePowerDriver
 import org.opendc.simulator.compute.workload.SimTraceWorkload
-import org.opendc.simulator.core.SimulationCoroutineScope
 import org.opendc.simulator.core.runBlockingSimulation
 import org.opendc.simulator.flow.FlowEngine
 import org.openjdk.jmh.annotations.*
@@ -47,48 +47,38 @@ import java.util.concurrent.TimeUnit
 @Measurement(iterations = 5, time = 3, timeUnit = TimeUnit.SECONDS)
 @OptIn(ExperimentalCoroutinesApi::class)
 class SimMachineBenchmarks {
-    private lateinit var scope: SimulationCoroutineScope
-    private lateinit var engine: FlowEngine
     private lateinit var machineModel: MachineModel
+    private lateinit var trace: Sequence<SimTraceWorkload.Fragment>
 
     @Setup
     fun setUp() {
-        scope = SimulationCoroutineScope()
-        engine = FlowEngine(scope.coroutineContext, scope.clock)
-
         val cpuNode = ProcessingNode("Intel", "Xeon", "amd64", 2)
 
         machineModel = MachineModel(
             cpus = List(cpuNode.coreCount) { ProcessingUnit(cpuNode, it, 1000.0) },
             memory = List(4) { MemoryUnit("Crucial", "MTA18ASF4G72AZ-3G2B1", 3200.0, 32_000) }
         )
-    }
 
-    @State(Scope.Thread)
-    class Workload {
-        lateinit var trace: Sequence<SimTraceWorkload.Fragment>
-
-        @Setup
-        fun setUp() {
-            val random = ThreadLocalRandom.current()
-            val entries = List(10000) { SimTraceWorkload.Fragment(it * 1000L, 1000, random.nextDouble(0.0, 4500.0), 1) }
-            trace = entries.asSequence()
-        }
+        val random = ThreadLocalRandom.current()
+        val entries = List(10000) { SimTraceWorkload.Fragment(it * 1000L, 1000, random.nextDouble(0.0, 4500.0), 1) }
+        trace = entries.asSequence()
     }
 
     @Benchmark
-    fun benchmarkBareMetal(state: Workload) {
-        return scope.runBlockingSimulation {
+    fun benchmarkBareMetal() {
+        return runBlockingSimulation {
+            val engine = FlowEngine(coroutineContext, clock)
             val machine = SimBareMetalMachine(
                 engine, machineModel, SimplePowerDriver(ConstantPowerModel(0.0))
             )
-            return@runBlockingSimulation machine.run(SimTraceWorkload(state.trace))
+            return@runBlockingSimulation machine.run(SimTraceWorkload(trace))
         }
     }
 
     @Benchmark
-    fun benchmarkSpaceSharedHypervisor(state: Workload) {
-        return scope.runBlockingSimulation {
+    fun benchmarkSpaceSharedHypervisor() {
+        return runBlockingSimulation {
+            val engine = FlowEngine(coroutineContext, clock)
             val machine = SimBareMetalMachine(
                 engine, machineModel, SimplePowerDriver(ConstantPowerModel(0.0))
             )
@@ -99,7 +89,7 @@ class SimMachineBenchmarks {
             val vm = hypervisor.createMachine(machineModel)
 
             try {
-                return@runBlockingSimulation vm.run(SimTraceWorkload(state.trace))
+                return@runBlockingSimulation vm.run(SimTraceWorkload(trace))
             } finally {
                 vm.close()
                 machine.close()
@@ -108,8 +98,9 @@ class SimMachineBenchmarks {
     }
 
     @Benchmark
-    fun benchmarkFairShareHypervisorSingle(state: Workload) {
-        return scope.runBlockingSimulation {
+    fun benchmarkFairShareHypervisorSingle() {
+        return runBlockingSimulation {
+            val engine = FlowEngine(coroutineContext, clock)
             val machine = SimBareMetalMachine(
                 engine, machineModel, SimplePowerDriver(ConstantPowerModel(0.0))
             )
@@ -120,7 +111,7 @@ class SimMachineBenchmarks {
             val vm = hypervisor.createMachine(machineModel)
 
             try {
-                return@runBlockingSimulation vm.run(SimTraceWorkload(state.trace))
+                return@runBlockingSimulation vm.run(SimTraceWorkload(trace))
             } finally {
                 vm.close()
                 machine.close()
@@ -129,8 +120,9 @@ class SimMachineBenchmarks {
     }
 
     @Benchmark
-    fun benchmarkFairShareHypervisorDouble(state: Workload) {
-        return scope.runBlockingSimulation {
+    fun benchmarkFairShareHypervisorDouble() {
+        return runBlockingSimulation {
+            val engine = FlowEngine(coroutineContext, clock)
             val machine = SimBareMetalMachine(
                 engine, machineModel, SimplePowerDriver(ConstantPowerModel(0.0))
             )
@@ -144,7 +136,7 @@ class SimMachineBenchmarks {
 
                     launch {
                         try {
-                            vm.run(SimTraceWorkload(state.trace))
+                            vm.run(SimTraceWorkload(trace))
                         } finally {
                             machine.close()
                         }
diff --git a/opendc-simulator/opendc-simulator-flow/src/jmh/kotlin/org/opendc/simulator/flow/FlowBenchmarks.kt b/opendc-simulator/opendc-simulator-flow/src/jmh/kotlin/org/opendc/simulator/flow/FlowBenchmarks.kt
index e927f81d..aabd2220 100644
--- a/opendc-simulator/opendc-simulator-flow/src/jmh/kotlin/org/opendc/simulator/flow/FlowBenchmarks.kt
+++ b/opendc-simulator/opendc-simulator-flow/src/jmh/kotlin/org/opendc/simulator/flow/FlowBenchmarks.kt
@@ -24,7 +24,6 @@ package org.opendc.simulator.flow
 
 import kotlinx.coroutines.ExperimentalCoroutinesApi
 import kotlinx.coroutines.launch
-import org.opendc.simulator.core.SimulationCoroutineScope
 import org.opendc.simulator.core.runBlockingSimulation
 import org.opendc.simulator.flow.mux.ForwardingFlowMultiplexer
 import org.opendc.simulator.flow.mux.MaxMinFlowMultiplexer
@@ -39,61 +38,53 @@ import java.util.concurrent.TimeUnit
 @Measurement(iterations = 5, time = 3, timeUnit = TimeUnit.SECONDS)
 @OptIn(ExperimentalCoroutinesApi::class)
 class FlowBenchmarks {
-    private lateinit var scope: SimulationCoroutineScope
-    private lateinit var engine: FlowEngine
+    private lateinit var trace: Sequence<TraceFlowSource.Fragment>
 
     @Setup
     fun setUp() {
-        scope = SimulationCoroutineScope()
-        engine = FlowEngine(scope.coroutineContext, scope.clock)
-    }
-
-    @State(Scope.Thread)
-    class Workload {
-        lateinit var trace: Sequence<TraceFlowSource.Fragment>
-
-        @Setup
-        fun setUp() {
-            val random = ThreadLocalRandom.current()
-            val entries = List(10000) { TraceFlowSource.Fragment(1000, random.nextDouble(0.0, 4500.0)) }
-            trace = entries.asSequence()
-        }
+        val random = ThreadLocalRandom.current()
+        val entries = List(10000) { TraceFlowSource.Fragment(1000, random.nextDouble(0.0, 4500.0)) }
+        trace = entries.asSequence()
     }
 
     @Benchmark
-    fun benchmarkSink(state: Workload) {
-        return scope.runBlockingSimulation {
+    fun benchmarkSink() {
+        return runBlockingSimulation {
+            val engine = FlowEngine(coroutineContext, clock)
             val provider = FlowSink(engine, 4200.0)
-            return@runBlockingSimulation provider.consume(TraceFlowSource(state.trace))
+            return@runBlockingSimulation provider.consume(TraceFlowSource(trace))
         }
     }
 
     @Benchmark
-    fun benchmarkForward(state: Workload) {
-        return scope.runBlockingSimulation {
+    fun benchmarkForward() {
+        return runBlockingSimulation {
+            val engine = FlowEngine(coroutineContext, clock)
             val provider = FlowSink(engine, 4200.0)
             val forwarder = FlowForwarder(engine)
             provider.startConsumer(forwarder)
-            return@runBlockingSimulation forwarder.consume(TraceFlowSource(state.trace))
+            return@runBlockingSimulation forwarder.consume(TraceFlowSource(trace))
         }
     }
 
     @Benchmark
-    fun benchmarkMuxMaxMinSingleSource(state: Workload) {
-        return scope.runBlockingSimulation {
+    fun benchmarkMuxMaxMinSingleSource() {
+        return runBlockingSimulation {
+            val engine = FlowEngine(coroutineContext, clock)
             val switch = MaxMinFlowMultiplexer(engine)
 
             FlowSink(engine, 3000.0).startConsumer(switch.newOutput())
             FlowSink(engine, 3000.0).startConsumer(switch.newOutput())
 
             val provider = switch.newInput()
-            return@runBlockingSimulation provider.consume(TraceFlowSource(state.trace))
+            return@runBlockingSimulation provider.consume(TraceFlowSource(trace))
         }
     }
 
     @Benchmark
-    fun benchmarkMuxMaxMinTripleSource(state: Workload) {
-        return scope.runBlockingSimulation {
+    fun benchmarkMuxMaxMinTripleSource() {
+        return runBlockingSimulation {
+            val engine = FlowEngine(coroutineContext, clock)
             val switch = MaxMinFlowMultiplexer(engine)
 
             FlowSink(engine, 3000.0).startConsumer(switch.newOutput())
@@ -102,28 +93,30 @@ class FlowBenchmarks {
             repeat(3) {
                 launch {
                     val provider = switch.newInput()
-                    provider.consume(TraceFlowSource(state.trace))
+                    provider.consume(TraceFlowSource(trace))
                 }
             }
         }
     }
 
     @Benchmark
-    fun benchmarkMuxExclusiveSingleSource(state: Workload) {
-        return scope.runBlockingSimulation {
+    fun benchmarkMuxExclusiveSingleSource() {
+        return runBlockingSimulation {
+            val engine = FlowEngine(coroutineContext, clock)
             val switch = ForwardingFlowMultiplexer(engine)
 
             FlowSink(engine, 3000.0).startConsumer(switch.newOutput())
             FlowSink(engine, 3000.0).startConsumer(switch.newOutput())
 
             val provider = switch.newInput()
-            return@runBlockingSimulation provider.consume(TraceFlowSource(state.trace))
+            return@runBlockingSimulation provider.consume(TraceFlowSource(trace))
         }
     }
 
     @Benchmark
-    fun benchmarkMuxExclusiveTripleSource(state: Workload) {
-        return scope.runBlockingSimulation {
+    fun benchmarkMuxExclusiveTripleSource() {
+        return runBlockingSimulation {
+            val engine = FlowEngine(coroutineContext, clock)
             val switch = ForwardingFlowMultiplexer(engine)
 
             FlowSink(engine, 3000.0).startConsumer(switch.newOutput())
@@ -132,7 +125,7 @@ class FlowBenchmarks {
             repeat(2) {
                 launch {
                     val provider = switch.newInput()
-                    provider.consume(TraceFlowSource(state.trace))
+                    provider.consume(TraceFlowSource(trace))
                 }
             }
         }