From 731ac9a0f57e67a1e741aef7a7ff3ea580588c46 Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Tue, 21 Jul 2020 16:56:55 +0200 Subject: Add support for HPC portfolio --- .../com/atlarge/opendc/experiments/sc20/Main.kt | 10 +--- .../experiments/sc20/experiment/Portfolios.kt | 31 +++++++++-- .../experiments/sc20/experiment/model/Workload.kt | 8 ++- .../experiments/sc20/trace/WorkloadSampler.kt | 65 +++++++++++++++------- 4 files changed, 82 insertions(+), 32 deletions(-) (limited to 'simulator') diff --git a/simulator/opendc/opendc-experiments-sc20/src/main/kotlin/com/atlarge/opendc/experiments/sc20/Main.kt b/simulator/opendc/opendc-experiments-sc20/src/main/kotlin/com/atlarge/opendc/experiments/sc20/Main.kt index faa68e34..4781f335 100644 --- a/simulator/opendc/opendc-experiments-sc20/src/main/kotlin/com/atlarge/opendc/experiments/sc20/Main.kt +++ b/simulator/opendc/opendc-experiments-sc20/src/main/kotlin/com/atlarge/opendc/experiments/sc20/Main.kt @@ -24,14 +24,7 @@ package com.atlarge.opendc.experiments.sc20 -import com.atlarge.opendc.experiments.sc20.experiment.CompositeWorkloadPortfolio -import com.atlarge.opendc.experiments.sc20.experiment.Experiment -import com.atlarge.opendc.experiments.sc20.experiment.HorVerPortfolio -import com.atlarge.opendc.experiments.sc20.experiment.MoreVelocityPortfolio -import com.atlarge.opendc.experiments.sc20.experiment.OperationalPhenomenaPortfolio -import com.atlarge.opendc.experiments.sc20.experiment.Portfolio -import com.atlarge.opendc.experiments.sc20.experiment.ReplayPortfolio -import com.atlarge.opendc.experiments.sc20.experiment.TestPortfolio +import com.atlarge.opendc.experiments.sc20.experiment.* import com.atlarge.opendc.experiments.sc20.reporter.ConsoleExperimentReporter import com.atlarge.opendc.experiments.sc20.runner.ExperimentDescriptor import com.atlarge.opendc.experiments.sc20.runner.execution.ThreadPoolExperimentScheduler @@ -104,6 +97,7 @@ class ExperimentCli : CliktCommand(name = "sc20-experiment") { "operational-phenomena" to { experiment, i -> OperationalPhenomenaPortfolio(experiment, i) }, "replay" to { experiment, i -> ReplayPortfolio(experiment, i) }, "test" to { experiment, i -> TestPortfolio(experiment, i) }, + "more-hpc" to { experiment, i -> MoreHpcPortfolio(experiment, i) }, ignoreCase = true ) .multiple() diff --git a/simulator/opendc/opendc-experiments-sc20/src/main/kotlin/com/atlarge/opendc/experiments/sc20/experiment/Portfolios.kt b/simulator/opendc/opendc-experiments-sc20/src/main/kotlin/com/atlarge/opendc/experiments/sc20/experiment/Portfolios.kt index 4d8e2f1d..b8dfb1be 100644 --- a/simulator/opendc/opendc-experiments-sc20/src/main/kotlin/com/atlarge/opendc/experiments/sc20/experiment/Portfolios.kt +++ b/simulator/opendc/opendc-experiments-sc20/src/main/kotlin/com/atlarge/opendc/experiments/sc20/experiment/Portfolios.kt @@ -24,10 +24,7 @@ package com.atlarge.opendc.experiments.sc20.experiment -import com.atlarge.opendc.experiments.sc20.experiment.model.CompositeWorkload -import com.atlarge.opendc.experiments.sc20.experiment.model.OperationalPhenomena -import com.atlarge.opendc.experiments.sc20.experiment.model.Topology -import com.atlarge.opendc.experiments.sc20.experiment.model.Workload +import com.atlarge.opendc.experiments.sc20.experiment.model.* public class HorVerPortfolio(parent: Experiment, id: Int) : Portfolio(parent, id, "horizontal_vs_vertical") { override val topologies = listOf( @@ -196,3 +193,29 @@ public class TestPortfolio(parent: Experiment, id: Int) : Portfolio(parent, id, override val allocationPolicies: List = listOf("active-servers") } + +public class MoreHpcPortfolio(parent: Experiment, id: Int) : Portfolio(parent, id, "more_hpc") { + override val topologies = listOf( + Topology("base"), + Topology("exp-vol-hor-hom"), + Topology("exp-vol-ver-hom"), + Topology("exp-vel-ver-hom") + ) + + override val workloads = listOf( + Workload("solvinity", 0.25, samplingStrategy = SamplingStrategy.HPC), + Workload("solvinity", 0.5, samplingStrategy = SamplingStrategy.HPC), + Workload("solvinity", 1.0, samplingStrategy = SamplingStrategy.HPC), + Workload("solvinity", 0.25, samplingStrategy = SamplingStrategy.HPC_LOAD), + Workload("solvinity", 0.5, samplingStrategy = SamplingStrategy.HPC_LOAD), + Workload("solvinity", 1.0, samplingStrategy = SamplingStrategy.HPC_LOAD) + ) + + override val operationalPhenomenas = listOf( + OperationalPhenomena(failureFrequency = 24.0 * 7, hasInterference = true) + ) + + override val allocationPolicies = listOf( + "active-servers" + ) +} diff --git a/simulator/opendc/opendc-experiments-sc20/src/main/kotlin/com/atlarge/opendc/experiments/sc20/experiment/model/Workload.kt b/simulator/opendc/opendc-experiments-sc20/src/main/kotlin/com/atlarge/opendc/experiments/sc20/experiment/model/Workload.kt index cc3c448a..d75ca6f9 100644 --- a/simulator/opendc/opendc-experiments-sc20/src/main/kotlin/com/atlarge/opendc/experiments/sc20/experiment/model/Workload.kt +++ b/simulator/opendc/opendc-experiments-sc20/src/main/kotlin/com/atlarge/opendc/experiments/sc20/experiment/model/Workload.kt @@ -24,10 +24,16 @@ package com.atlarge.opendc.experiments.sc20.experiment.model +enum class SamplingStrategy { + REGULAR, + HPC, + HPC_LOAD +} + /** * A workload that is considered for a scenario. */ -public open class Workload(open val name: String, val fraction: Double) +public open class Workload(open val name: String, val fraction: Double, val samplingStrategy: SamplingStrategy = SamplingStrategy.REGULAR) /** * A workload that is composed of multiple workloads. diff --git a/simulator/opendc/opendc-experiments-sc20/src/main/kotlin/com/atlarge/opendc/experiments/sc20/trace/WorkloadSampler.kt b/simulator/opendc/opendc-experiments-sc20/src/main/kotlin/com/atlarge/opendc/experiments/sc20/trace/WorkloadSampler.kt index bb4f60c7..b24d6de1 100644 --- a/simulator/opendc/opendc-experiments-sc20/src/main/kotlin/com/atlarge/opendc/experiments/sc20/trace/WorkloadSampler.kt +++ b/simulator/opendc/opendc-experiments-sc20/src/main/kotlin/com/atlarge/opendc/experiments/sc20/trace/WorkloadSampler.kt @@ -24,10 +24,13 @@ package com.atlarge.opendc.experiments.sc20.trace +import com.atlarge.opendc.compute.core.image.VmImage import com.atlarge.opendc.compute.core.workload.VmWorkload import com.atlarge.opendc.experiments.sc20.experiment.model.CompositeWorkload +import com.atlarge.opendc.experiments.sc20.experiment.model.SamplingStrategy import com.atlarge.opendc.experiments.sc20.experiment.model.Workload import com.atlarge.opendc.format.trace.TraceEntry +import java.util.* import kotlin.random.Random import mu.KotlinLogging @@ -42,10 +45,14 @@ fun sampleWorkload( subWorkload: Workload, seed: Int ): List> { - return if (workload is CompositeWorkload) { - sampleRegularWorkload(trace, workload, subWorkload, seed) - } else { - sampleRegularWorkload(trace, workload, workload, seed) + return when { + workload is CompositeWorkload -> sampleRegularWorkload(trace, workload, subWorkload, seed) + workload.samplingStrategy == SamplingStrategy.HPC -> + sampleHpcWorkload(trace, workload, seed, sampleOnLoad = false) + workload.samplingStrategy == SamplingStrategy.HPC_LOAD -> + sampleHpcWorkload(trace, workload, seed, sampleOnLoad = true) + else -> + sampleRegularWorkload(trace, workload, workload, seed) } } @@ -87,21 +94,23 @@ fun sampleRegularWorkload( /** * Sample a HPC workload. */ -fun sampleHPCWorkload( +fun sampleHpcWorkload( trace: List>, workload: Workload, seed: Int, - fraction: Double, sampleOnLoad: Boolean ): List> { - val pattern = Regex("^(ComputeNode|cn)") + val pattern = Regex("^vm__workload__(ComputeNode|cn).*") val random = Random(seed) + val fraction = workload.fraction val (hpc, nonHpc) = trace.partition { entry -> val name = entry.workload.image.name name.matches(pattern) } + logger.debug { "Found ${hpc.size} HPC workloads and ${nonHpc.size} non-HPC workloads" } + val totalLoad = if (workload is CompositeWorkload) { workload.totalLoad } else { @@ -111,16 +120,12 @@ fun sampleHPCWorkload( val res = mutableListOf>() if (sampleOnLoad) { - val hpcPool = mutableListOf>() - - // We repeat the HPC VMs 1000 times to have a large enough pool to pick VMs from. - repeat(1000) { - hpcPool.addAll(hpc) - } - hpcPool.shuffle(random) - var currentLoad = 0.0 - for (entry in hpcPool) { + var i = 0 + while (true) { + // Sample random HPC entry with replacement + val entry = sample(hpc.random(random), i++) + val entryLoad = entry.workload.image.tags.getValue("total-load") as Double if ((currentLoad + entryLoad) / totalLoad > fraction || res.size > trace.size) { break @@ -141,10 +146,13 @@ fun sampleHPCWorkload( res += entry } } else { - (hpc as MutableList>).shuffle(random) - (nonHpc as MutableList>).shuffle(random) + repeat((fraction * trace.size).toInt()) { i -> + // Sample random HPC entry with replacement + val entry = sample(hpc.random(random), i) + res.add(entry) + } - res.addAll(hpc.subList(0, (fraction * trace.size).toInt())) + (nonHpc as MutableList>).shuffle(random) res.addAll(nonHpc.subList(0, ((1 - fraction) * trace.size).toInt())) } @@ -152,3 +160,22 @@ fun sampleHPCWorkload( return res } + +/** + * Sample a random trace entry. + */ +private fun sample(entry: TraceEntry, i: Int): TraceEntry { + val id = UUID.nameUUIDFromBytes("${entry.workload.image.uid}-$i".toByteArray()) + val image = VmImage( + id, + entry.workload.image.name + "-$i", + entry.workload.image.tags, + entry.workload.image.flopsHistory, + entry.workload.image.maxCores, + entry.workload.image.requiredMemory + ) + val vmWorkload = entry.workload.copy(uid = id, image = image, name = entry.workload.name + "-$i") + return VmTraceEntry(vmWorkload, entry.submissionTime) +} + +private class VmTraceEntry(override val workload: VmWorkload, override val submissionTime: Long) : TraceEntry -- cgit v1.2.3