From ab0ae4779a674dd07d85ded4a812332d93888bc1 Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Fri, 30 Sep 2022 20:33:37 +0200 Subject: refactor(exp/capelin): Use experiment base for Capelin experiments This change updates the Capelin experiments to use the new `opendc-experiments-base` module for setting up the experimental environment and simulate the workloads. --- .../opendc/experiments/capelin/CapelinRunner.kt | 78 +++++++++------------- 1 file changed, 33 insertions(+), 45 deletions(-) (limited to 'opendc-experiments/opendc-experiments-capelin/src/main') diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/CapelinRunner.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/CapelinRunner.kt index dbb5ced3..6bd470f3 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/CapelinRunner.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/CapelinRunner.kt @@ -22,15 +22,16 @@ package org.opendc.experiments.capelin -import org.opendc.compute.workload.ComputeServiceHelper +import org.opendc.compute.service.ComputeService import org.opendc.compute.workload.ComputeWorkloadLoader import org.opendc.compute.workload.createComputeScheduler import org.opendc.compute.workload.export.parquet.ParquetComputeMonitor import org.opendc.compute.workload.grid5000 -import org.opendc.compute.workload.telemetry.ComputeMetricReader -import org.opendc.compute.workload.topology.apply +import org.opendc.compute.workload.replay import org.opendc.experiments.capelin.model.Scenario import org.opendc.experiments.capelin.topology.clusterTopology +import org.opendc.experiments.compute.* +import org.opendc.experiments.provisioner.Provisioner import org.opendc.simulator.core.runBlockingSimulation import java.io.File import java.time.Duration @@ -58,54 +59,41 @@ public class CapelinRunner( * Run a single [scenario] with the specified seed. */ fun runScenario(scenario: Scenario, seed: Long) = runBlockingSimulation { - val seeder = Random(seed) - - val operationalPhenomena = scenario.operationalPhenomena - val computeScheduler = createComputeScheduler(scenario.allocationPolicy, seeder) - val failureModel = - if (operationalPhenomena.failureFrequency > 0) - grid5000(Duration.ofSeconds((operationalPhenomena.failureFrequency * 60).roundToLong())) - else - null - val vms = scenario.workload.source.resolve(workloadLoader, seeder) - val runner = ComputeServiceHelper( - coroutineContext, - clock, - computeScheduler, - seed, - ) - + val serviceDomain = "compute.opendc.org" val topology = clusterTopology(File(envPath, "${scenario.topology.name}.txt")) - val partitions = scenario.partitions + ("seed" to seed.toString()) - val partition = partitions.map { (k, v) -> "$k=$v" }.joinToString("/") - val exporter = if (outputPath != null) { - ComputeMetricReader( - this, - clock, - runner.service, - ParquetComputeMonitor( - outputPath, - partition, - bufferSize = 4096 - ), - exportInterval = Duration.ofMinutes(5) + Provisioner(coroutineContext, clock, seed).use { provisioner -> + provisioner.runSteps( + setupComputeService(serviceDomain, { createComputeScheduler(scenario.allocationPolicy, Random(it.seeder.nextLong())) }), + setupHosts(serviceDomain, topology.resolve(), optimize = true) ) - } else { - null - } - try { - // Instantiate the desired topology - runner.apply(topology, optimize = true) + if (outputPath != null) { + val partitions = scenario.partitions + ("seed" to seed.toString()) + val partition = partitions.map { (k, v) -> "$k=$v" }.joinToString("/") + + provisioner.runStep( + registerComputeMonitor( + serviceDomain, + ParquetComputeMonitor( + outputPath, + partition, + bufferSize = 4096 + ) + ) + ) + } - // Run the workload trace - runner.run(vms, failureModel = failureModel, interference = operationalPhenomena.hasInterference) + val service = provisioner.registry.resolve(serviceDomain, ComputeService::class.java)!! + val vms = scenario.workload.source.resolve(workloadLoader, Random(seed)) + val operationalPhenomena = scenario.operationalPhenomena + val failureModel = + if (operationalPhenomena.failureFrequency > 0) + grid5000(Duration.ofSeconds((operationalPhenomena.failureFrequency * 60).roundToLong())) + else + null - // Stop the metric collection - exporter?.close() - } finally { - runner.close() + service.replay(clock, vms, seed, failureModel = failureModel, interference = operationalPhenomena.hasInterference) } } } -- cgit v1.2.3 From 448b4cafe3c757812138a8ca7580975191ac2f9c Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Fri, 30 Sep 2022 20:57:16 +0200 Subject: refactor(exp/compute): Integrate compute workload classes This change integrates the classes from the old `opendc-compute-workload` module into the `opendc-experiments-compute` module. This new module contains helper classes for setting up experiments with the OpenDC compute service. --- .../src/main/kotlin/org/opendc/experiments/capelin/CapelinRunner.kt | 6 +----- .../main/kotlin/org/opendc/experiments/capelin/model/Workload.kt | 2 +- .../experiments/capelin/portfolio/CompositeWorkloadPortfolio.kt | 4 ++-- .../org/opendc/experiments/capelin/portfolio/HorVerPortfolio.kt | 4 ++-- .../org/opendc/experiments/capelin/portfolio/MoreHpcPortfolio.kt | 6 +++--- .../opendc/experiments/capelin/portfolio/MoreVelocityPortfolio.kt | 4 ++-- .../experiments/capelin/portfolio/OperationalPhenomenaPortfolio.kt | 4 ++-- .../org/opendc/experiments/capelin/portfolio/TestPortfolio.kt | 2 +- .../org/opendc/experiments/capelin/topology/TopologyFactories.kt | 4 ++-- 9 files changed, 16 insertions(+), 20 deletions(-) (limited to 'opendc-experiments/opendc-experiments-capelin/src/main') diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/CapelinRunner.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/CapelinRunner.kt index 6bd470f3..e019af34 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/CapelinRunner.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/CapelinRunner.kt @@ -23,14 +23,10 @@ package org.opendc.experiments.capelin import org.opendc.compute.service.ComputeService -import org.opendc.compute.workload.ComputeWorkloadLoader -import org.opendc.compute.workload.createComputeScheduler -import org.opendc.compute.workload.export.parquet.ParquetComputeMonitor -import org.opendc.compute.workload.grid5000 -import org.opendc.compute.workload.replay import org.opendc.experiments.capelin.model.Scenario import org.opendc.experiments.capelin.topology.clusterTopology import org.opendc.experiments.compute.* +import org.opendc.experiments.compute.export.parquet.ParquetComputeMonitor import org.opendc.experiments.provisioner.Provisioner import org.opendc.simulator.core.runBlockingSimulation import java.io.File diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/model/Workload.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/model/Workload.kt index a2e71243..ed2588f0 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/model/Workload.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/model/Workload.kt @@ -22,7 +22,7 @@ package org.opendc.experiments.capelin.model -import org.opendc.compute.workload.ComputeWorkload +import org.opendc.experiments.compute.ComputeWorkload /** * A single workload originating from a trace. diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/portfolio/CompositeWorkloadPortfolio.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/portfolio/CompositeWorkloadPortfolio.kt index 68eb15b3..80b8859c 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/portfolio/CompositeWorkloadPortfolio.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/portfolio/CompositeWorkloadPortfolio.kt @@ -22,12 +22,12 @@ package org.opendc.experiments.capelin.portfolio -import org.opendc.compute.workload.composite -import org.opendc.compute.workload.trace import org.opendc.experiments.capelin.model.OperationalPhenomena import org.opendc.experiments.capelin.model.Scenario import org.opendc.experiments.capelin.model.Topology import org.opendc.experiments.capelin.model.Workload +import org.opendc.experiments.compute.composite +import org.opendc.experiments.compute.trace /** * A [Portfolio] that explores the effect of a composite workload. diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/portfolio/HorVerPortfolio.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/portfolio/HorVerPortfolio.kt index 0d7f3072..f3c002ac 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/portfolio/HorVerPortfolio.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/portfolio/HorVerPortfolio.kt @@ -22,12 +22,12 @@ package org.opendc.experiments.capelin.portfolio -import org.opendc.compute.workload.sampleByLoad -import org.opendc.compute.workload.trace import org.opendc.experiments.capelin.model.OperationalPhenomena import org.opendc.experiments.capelin.model.Scenario import org.opendc.experiments.capelin.model.Topology import org.opendc.experiments.capelin.model.Workload +import org.opendc.experiments.compute.sampleByLoad +import org.opendc.experiments.compute.trace /** * A [Portfolio] that explores the difference between horizontal and vertical scaling. diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/portfolio/MoreHpcPortfolio.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/portfolio/MoreHpcPortfolio.kt index 6afffc09..22f9f3ac 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/portfolio/MoreHpcPortfolio.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/portfolio/MoreHpcPortfolio.kt @@ -22,13 +22,13 @@ package org.opendc.experiments.capelin.portfolio -import org.opendc.compute.workload.sampleByHpc -import org.opendc.compute.workload.sampleByHpcLoad -import org.opendc.compute.workload.trace import org.opendc.experiments.capelin.model.OperationalPhenomena import org.opendc.experiments.capelin.model.Scenario import org.opendc.experiments.capelin.model.Topology import org.opendc.experiments.capelin.model.Workload +import org.opendc.experiments.compute.sampleByHpc +import org.opendc.experiments.compute.sampleByHpcLoad +import org.opendc.experiments.compute.trace /** * A [Portfolio] to explore the effect of HPC workloads. diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/portfolio/MoreVelocityPortfolio.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/portfolio/MoreVelocityPortfolio.kt index 92bf80b3..e63a5807 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/portfolio/MoreVelocityPortfolio.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/portfolio/MoreVelocityPortfolio.kt @@ -22,12 +22,12 @@ package org.opendc.experiments.capelin.portfolio -import org.opendc.compute.workload.sampleByLoad -import org.opendc.compute.workload.trace import org.opendc.experiments.capelin.model.OperationalPhenomena import org.opendc.experiments.capelin.model.Scenario import org.opendc.experiments.capelin.model.Topology import org.opendc.experiments.capelin.model.Workload +import org.opendc.experiments.compute.sampleByLoad +import org.opendc.experiments.compute.trace /** * A [Portfolio] that explores the effect of adding more velocity to a cluster (e.g., faster machines). diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/portfolio/OperationalPhenomenaPortfolio.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/portfolio/OperationalPhenomenaPortfolio.kt index f9a9d681..12570108 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/portfolio/OperationalPhenomenaPortfolio.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/portfolio/OperationalPhenomenaPortfolio.kt @@ -22,12 +22,12 @@ package org.opendc.experiments.capelin.portfolio -import org.opendc.compute.workload.sampleByLoad -import org.opendc.compute.workload.trace import org.opendc.experiments.capelin.model.OperationalPhenomena import org.opendc.experiments.capelin.model.Scenario import org.opendc.experiments.capelin.model.Topology import org.opendc.experiments.capelin.model.Workload +import org.opendc.experiments.compute.sampleByLoad +import org.opendc.experiments.compute.trace /** * A [Portfolio] that explores the effect of operational phenomena on metrics. diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/portfolio/TestPortfolio.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/portfolio/TestPortfolio.kt index 944e9f43..6f126b87 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/portfolio/TestPortfolio.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/portfolio/TestPortfolio.kt @@ -22,11 +22,11 @@ package org.opendc.experiments.capelin.portfolio -import org.opendc.compute.workload.trace import org.opendc.experiments.capelin.model.OperationalPhenomena import org.opendc.experiments.capelin.model.Scenario import org.opendc.experiments.capelin.model.Topology import org.opendc.experiments.capelin.model.Workload +import org.opendc.experiments.compute.trace /** * A [Portfolio] to perform a simple test run. diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/topology/TopologyFactories.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/topology/TopologyFactories.kt index 5ab4261a..d152a6db 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/topology/TopologyFactories.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/topology/TopologyFactories.kt @@ -23,8 +23,8 @@ @file:JvmName("TopologyFactories") package org.opendc.experiments.capelin.topology -import org.opendc.compute.workload.topology.HostSpec -import org.opendc.compute.workload.topology.Topology +import org.opendc.experiments.compute.topology.HostSpec +import org.opendc.experiments.compute.topology.Topology import org.opendc.simulator.compute.model.MachineModel import org.opendc.simulator.compute.model.MemoryUnit import org.opendc.simulator.compute.model.ProcessingNode -- cgit v1.2.3 From bd476d11ab24fe745bb54e97a11133706bb96cb1 Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Mon, 3 Oct 2022 17:20:34 +0200 Subject: refactor(exp/compute): Remove Topology interface This change removes the Topology interface from the `opendc-experiments-compute` module, which was meant for provisioning the experimental topology. Howerver, with the stateless `HostSpec` class, it is not needed to resolve the topology everytime. --- .../opendc/experiments/capelin/CapelinRunner.kt | 2 +- .../opendc/experiments/capelin/model/Topology.kt | 2 +- .../capelin/topology/TopologyFactories.kt | 71 ++++++++++------------ 3 files changed, 34 insertions(+), 41 deletions(-) (limited to 'opendc-experiments/opendc-experiments-capelin/src/main') diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/CapelinRunner.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/CapelinRunner.kt index e019af34..f1214b08 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/CapelinRunner.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/CapelinRunner.kt @@ -61,7 +61,7 @@ public class CapelinRunner( Provisioner(coroutineContext, clock, seed).use { provisioner -> provisioner.runSteps( setupComputeService(serviceDomain, { createComputeScheduler(scenario.allocationPolicy, Random(it.seeder.nextLong())) }), - setupHosts(serviceDomain, topology.resolve(), optimize = true) + setupHosts(serviceDomain, topology, optimize = true) ) if (outputPath != null) { diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/model/Topology.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/model/Topology.kt index fe16a294..c90194ce 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/model/Topology.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/model/Topology.kt @@ -23,6 +23,6 @@ package org.opendc.experiments.capelin.model /** - * The topology topology on which we test the workload. + * The topology on which we simulate the workload. */ public data class Topology(val name: String) diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/topology/TopologyFactories.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/topology/TopologyFactories.kt index d152a6db..054adfcd 100644 --- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/topology/TopologyFactories.kt +++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/topology/TopologyFactories.kt @@ -24,7 +24,6 @@ package org.opendc.experiments.capelin.topology import org.opendc.experiments.compute.topology.HostSpec -import org.opendc.experiments.compute.topology.Topology import org.opendc.simulator.compute.model.MachineModel import org.opendc.simulator.compute.model.MemoryUnit import org.opendc.simulator.compute.model.ProcessingNode @@ -43,61 +42,55 @@ import kotlin.math.roundToLong private val reader = ClusterSpecReader() /** - * Construct a [Topology] from the specified [file]. + * Construct a topology from the specified [file]. */ fun clusterTopology( file: File, powerModel: PowerModel = LinearPowerModel(350.0, idlePower = 200.0), random: Random = Random(0) -): Topology = clusterTopology(reader.read(file), powerModel, random) +): List { + return clusterTopology(reader.read(file), powerModel, random) +} /** - * Construct a [Topology] from the specified [input]. + * Construct a topology from the specified [input]. */ fun clusterTopology( input: InputStream, powerModel: PowerModel = LinearPowerModel(350.0, idlePower = 200.0), random: Random = Random(0) -): Topology = clusterTopology(reader.read(input), powerModel, random) +): List { + return clusterTopology(reader.read(input), powerModel, random) +} /** - * Construct a [Topology] from the given list of [clusters]. + * Construct a topology from the given list of [clusters]. */ -fun clusterTopology( - clusters: List, - powerModel: PowerModel, - random: Random = Random(0) -): Topology { - return object : Topology { - override fun resolve(): List { - val hosts = mutableListOf() - for (cluster in clusters) { - val cpuSpeed = cluster.cpuSpeed - val memoryPerHost = cluster.memCapacityPerHost.roundToLong() - - val unknownProcessingNode = ProcessingNode("unknown", "unknown", "unknown", cluster.cpuCountPerHost) - val unknownMemoryUnit = MemoryUnit("unknown", "unknown", -1.0, memoryPerHost) - val machineModel = MachineModel( - List(cluster.cpuCountPerHost) { coreId -> ProcessingUnit(unknownProcessingNode, coreId, cpuSpeed) }, - listOf(unknownMemoryUnit) - ) - - repeat(cluster.hostCount) { - val spec = HostSpec( - UUID(random.nextLong(), it.toLong()), - "node-${cluster.name}-$it", - mapOf("cluster" to cluster.id), - machineModel, - SimplePowerDriver(powerModel) - ) +fun clusterTopology(clusters: List, powerModel: PowerModel, random: Random = Random(0)): List { + return clusters.flatMap { it.toHostSpecs(random, powerModel) } +} - hosts += spec - } - } +/** + * Helper method to convert a [ClusterSpec] into a list of [HostSpec]s. + */ +private fun ClusterSpec.toHostSpecs(random: Random, powerModel: PowerModel): List { + val cpuSpeed = cpuSpeed + val memoryPerHost = memCapacityPerHost.roundToLong() - return hosts - } + val unknownProcessingNode = ProcessingNode("unknown", "unknown", "unknown", cpuCountPerHost) + val unknownMemoryUnit = MemoryUnit("unknown", "unknown", -1.0, memoryPerHost) + val machineModel = MachineModel( + List(cpuCountPerHost) { coreId -> ProcessingUnit(unknownProcessingNode, coreId, cpuSpeed) }, + listOf(unknownMemoryUnit) + ) - override fun toString(): String = "ClusterSpecTopology" + return List(hostCount) { + HostSpec( + UUID(random.nextLong(), it.toLong()), + "node-$name-$it", + mapOf("cluster" to id), + machineModel, + SimplePowerDriver(powerModel) + ) } } -- cgit v1.2.3