diff options
Diffstat (limited to 'opendc')
7 files changed, 92 insertions, 26 deletions
diff --git a/opendc/opendc-experiments-sc20/src/main/kotlin/com/atlarge/opendc/experiments/sc20/Main.kt b/opendc/opendc-experiments-sc20/src/main/kotlin/com/atlarge/opendc/experiments/sc20/Main.kt index 14de52b8..21cd2eca 100644 --- a/opendc/opendc-experiments-sc20/src/main/kotlin/com/atlarge/opendc/experiments/sc20/Main.kt +++ b/opendc/opendc-experiments-sc20/src/main/kotlin/com/atlarge/opendc/experiments/sc20/Main.kt @@ -24,9 +24,9 @@ package com.atlarge.opendc.experiments.sc20 +import com.atlarge.opendc.experiments.sc20.experiment.CompositeWorkloadPortfolio import com.atlarge.opendc.experiments.sc20.experiment.Experiment import com.atlarge.opendc.experiments.sc20.experiment.HorVerPortfolio -import com.atlarge.opendc.experiments.sc20.experiment.MoreHpcPortfolio import com.atlarge.opendc.experiments.sc20.experiment.MoreVelocityPortfolio import com.atlarge.opendc.experiments.sc20.experiment.OperationalPhenomenaPortfolio import com.atlarge.opendc.experiments.sc20.experiment.Portfolio @@ -96,9 +96,14 @@ class ExperimentCli : CliktCommand(name = "sc20-experiment") { */ private val portfolios by option("--portfolio") .choice( - "hor-ver" to { experiment: Experiment, i: Int -> HorVerPortfolio(experiment, i) } as (Experiment, Int) -> Portfolio, + "hor-ver" to { experiment: Experiment, i: Int -> + HorVerPortfolio( + experiment, + i + ) + } as (Experiment, Int) -> Portfolio, "more-velocity" to { experiment, i -> MoreVelocityPortfolio(experiment, i) }, - "more-hpc" to { experiment, i -> MoreHpcPortfolio(experiment, i) }, + "composite-workload" to { experiment, i -> CompositeWorkloadPortfolio(experiment, i) }, "operational-phenomena" to { experiment, i -> OperationalPhenomenaPortfolio(experiment, i) }, "test" to { experiment, i -> TestPortfolio(experiment, i) }, ignoreCase = true diff --git a/opendc/opendc-experiments-sc20/src/main/kotlin/com/atlarge/opendc/experiments/sc20/experiment/Portfolios.kt b/opendc/opendc-experiments-sc20/src/main/kotlin/com/atlarge/opendc/experiments/sc20/experiment/Portfolios.kt index 362144ae..77c0194b 100644 --- a/opendc/opendc-experiments-sc20/src/main/kotlin/com/atlarge/opendc/experiments/sc20/experiment/Portfolios.kt +++ b/opendc/opendc-experiments-sc20/src/main/kotlin/com/atlarge/opendc/experiments/sc20/experiment/Portfolios.kt @@ -24,6 +24,7 @@ package com.atlarge.opendc.experiments.sc20.experiment +import com.atlarge.opendc.experiments.sc20.experiment.model.CompositeWorkload import com.atlarge.opendc.experiments.sc20.experiment.model.OperationalPhenomena import com.atlarge.opendc.experiments.sc20.experiment.model.Topology import com.atlarge.opendc.experiments.sc20.experiment.model.Workload @@ -82,7 +83,9 @@ public class MoreVelocityPortfolio(parent: Experiment, id: Int) : Portfolio(pare ) } -public class MoreHpcPortfolio(parent: Experiment, id: Int) : Portfolio(parent, id, "more_hpc") { +public class CompositeWorkloadPortfolio(parent: Experiment, id: Int) : Portfolio(parent, id, "composite-workload") { + private val totalSampleLoad = 3425709788935.9976 + override val topologies = listOf( Topology("base"), Topology("exp-vol-hor-hom"), @@ -91,14 +94,35 @@ public class MoreHpcPortfolio(parent: Experiment, id: Int) : Portfolio(parent, i ) override val workloads = listOf( - Workload("solvinity", 0.1), - Workload("solvinity", 0.25), - Workload("solvinity", 0.5), - Workload("solvinity", 1.0) + CompositeWorkload( + "all-solvinity", + listOf(Workload("solvinity", 0.0), Workload("azure", 1.0)), + totalSampleLoad + ), + CompositeWorkload( + "solvinity-25-azure-75", + listOf(Workload("solvinity", 0.25), Workload("azure", 0.75)), + totalSampleLoad + ), + CompositeWorkload( + "solvinity-50-azure-50", + listOf(Workload("solvinity", 0.5), Workload("azure", 0.5)), + totalSampleLoad + ), + CompositeWorkload( + "solvinity-75-azure-25", + listOf(Workload("solvinity", 0.75), Workload("azure", 0.25)), + totalSampleLoad + ), + CompositeWorkload( + "all-azure", + listOf(Workload("solvinity", 1.0), Workload("azure", 0.0)), + totalSampleLoad + ) ) override val operationalPhenomenas = listOf( - OperationalPhenomena(failureFrequency = 24.0 * 7, hasInterference = true) + OperationalPhenomena(failureFrequency = 24.0 * 7, hasInterference = false) ) override val allocationPolicies = listOf( diff --git a/opendc/opendc-experiments-sc20/src/main/kotlin/com/atlarge/opendc/experiments/sc20/experiment/Run.kt b/opendc/opendc-experiments-sc20/src/main/kotlin/com/atlarge/opendc/experiments/sc20/experiment/Run.kt index fd3e29c8..49941cf4 100644 --- a/opendc/opendc-experiments-sc20/src/main/kotlin/com/atlarge/opendc/experiments/sc20/experiment/Run.kt +++ b/opendc/opendc-experiments-sc20/src/main/kotlin/com/atlarge/opendc/experiments/sc20/experiment/Run.kt @@ -31,6 +31,7 @@ import com.atlarge.opendc.compute.virt.service.allocation.NumberOfActiveServersA import com.atlarge.opendc.compute.virt.service.allocation.ProvisionedCoresAllocationPolicy import com.atlarge.opendc.compute.virt.service.allocation.RandomAllocationPolicy import com.atlarge.opendc.compute.virt.service.allocation.ReplayAllocationPolicy +import com.atlarge.opendc.experiments.sc20.experiment.model.CompositeWorkload import com.atlarge.opendc.experiments.sc20.experiment.monitor.ParquetExperimentMonitor import com.atlarge.opendc.experiments.sc20.runner.TrialExperimentDescriptor import com.atlarge.opendc.experiments.sc20.runner.execution.ExperimentExecutionContext @@ -83,18 +84,26 @@ public data class Run(override val parent: Scenario, val id: Int, val seed: Int) } @Suppress("UNCHECKED_CAST") - val rawTraceReaders = context.cache.computeIfAbsent("raw-trace-readers") { mutableMapOf<String, Sc20RawParquetTraceReader>() } as MutableMap<String, Sc20RawParquetTraceReader> - val raw = synchronized(rawTraceReaders) { - val name = parent.workload.name - rawTraceReaders.computeIfAbsent(name) { - logger.info { "Loading trace $name" } - Sc20RawParquetTraceReader(File(experiment.traces, name)) + val rawTraceReaders = + context.cache.computeIfAbsent("raw-trace-readers") { mutableMapOf<String, Sc20RawParquetTraceReader>() } as MutableMap<String, Sc20RawParquetTraceReader> + val rawReaders = synchronized(rawTraceReaders) { + val workloadNames = if (parent.workload is CompositeWorkload) { + parent.workload.workloads.map { it.name } + } else { + listOf(parent.workload.name) + } + + workloadNames.map { workloadName -> + rawTraceReaders.computeIfAbsent(workloadName) { + logger.info { "Loading trace $workloadName" } + Sc20RawParquetTraceReader(File(experiment.traces, workloadName)) + } } } val performanceInterferenceModel = experiment.performanceInterferenceModel ?.takeIf { parent.operationalPhenomena.hasInterference } ?.construct(seeder) ?: emptyMap() - val trace = Sc20ParquetTraceReader(raw, performanceInterferenceModel, parent.workload, seed) + val trace = Sc20ParquetTraceReader(rawReaders, performanceInterferenceModel, parent.workload, seed) val monitor = ParquetExperimentMonitor(this) diff --git a/opendc/opendc-experiments-sc20/src/main/kotlin/com/atlarge/opendc/experiments/sc20/experiment/model/Workload.kt b/opendc/opendc-experiments-sc20/src/main/kotlin/com/atlarge/opendc/experiments/sc20/experiment/model/Workload.kt index 2dbdf570..cc3c448a 100644 --- a/opendc/opendc-experiments-sc20/src/main/kotlin/com/atlarge/opendc/experiments/sc20/experiment/model/Workload.kt +++ b/opendc/opendc-experiments-sc20/src/main/kotlin/com/atlarge/opendc/experiments/sc20/experiment/model/Workload.kt @@ -27,4 +27,10 @@ package com.atlarge.opendc.experiments.sc20.experiment.model /** * A workload that is considered for a scenario. */ -public class Workload(val name: String, val fraction: Double) +public open class Workload(open val name: String, val fraction: Double) + +/** + * A workload that is composed of multiple workloads. + */ +public class CompositeWorkload(override val name: String, val workloads: List<Workload>, val totalLoad: Double) : + Workload(name, -1.0) diff --git a/opendc/opendc-experiments-sc20/src/main/kotlin/com/atlarge/opendc/experiments/sc20/trace/Sc20ParquetTraceReader.kt b/opendc/opendc-experiments-sc20/src/main/kotlin/com/atlarge/opendc/experiments/sc20/trace/Sc20ParquetTraceReader.kt index ad50bf18..06bececf 100644 --- a/opendc/opendc-experiments-sc20/src/main/kotlin/com/atlarge/opendc/experiments/sc20/trace/Sc20ParquetTraceReader.kt +++ b/opendc/opendc-experiments-sc20/src/main/kotlin/com/atlarge/opendc/experiments/sc20/trace/Sc20ParquetTraceReader.kt @@ -28,6 +28,7 @@ import com.atlarge.opendc.compute.core.image.VmImage import com.atlarge.opendc.compute.core.workload.IMAGE_PERF_INTERFERENCE_MODEL import com.atlarge.opendc.compute.core.workload.PerformanceInterferenceModel import com.atlarge.opendc.compute.core.workload.VmWorkload +import com.atlarge.opendc.experiments.sc20.experiment.model.CompositeWorkload import com.atlarge.opendc.experiments.sc20.experiment.model.Workload import com.atlarge.opendc.format.trace.TraceEntry import com.atlarge.opendc.format.trace.TraceReader @@ -42,7 +43,7 @@ import java.util.TreeSet */ @OptIn(ExperimentalStdlibApi::class) class Sc20ParquetTraceReader( - raw: Sc20RawParquetTraceReader, + rawReaders: List<Sc20RawParquetTraceReader>, performanceInterferenceModel: Map<String, PerformanceInterferenceModel>, workload: Workload, seed: Int @@ -51,8 +52,17 @@ class Sc20ParquetTraceReader( * The iterator over the actual trace. */ private val iterator: Iterator<TraceEntry<VmWorkload>> = - raw.read() - .run { sampleWorkload(this, workload, seed) } + rawReaders + .map { it.read() } + .run { + if (workload is CompositeWorkload) { + this.zip(workload.workloads) + } else { + this.zip(listOf(workload)) + } + } + .map { sampleWorkload(it.first, workload, it.second, seed) } + .flatten() .run { // Apply performance interference model if (performanceInterferenceModel.isEmpty()) @@ -62,7 +72,7 @@ class Sc20ParquetTraceReader( val image = entry.workload.image val id = image.name val relevantPerformanceInterferenceModelItems = - performanceInterferenceModel[id] ?: PerformanceInterferenceModel(TreeSet()) + performanceInterferenceModel[id] ?: PerformanceInterferenceModel(TreeSet()) val newImage = VmImage( diff --git a/opendc/opendc-experiments-sc20/src/main/kotlin/com/atlarge/opendc/experiments/sc20/trace/Sc20RawParquetTraceReader.kt b/opendc/opendc-experiments-sc20/src/main/kotlin/com/atlarge/opendc/experiments/sc20/trace/Sc20RawParquetTraceReader.kt index 3b480d33..b390a753 100644 --- a/opendc/opendc-experiments-sc20/src/main/kotlin/com/atlarge/opendc/experiments/sc20/trace/Sc20RawParquetTraceReader.kt +++ b/opendc/opendc-experiments-sc20/src/main/kotlin/com/atlarge/opendc/experiments/sc20/trace/Sc20RawParquetTraceReader.kt @@ -105,8 +105,6 @@ class Sc20RawParquetTraceReader(private val path: File) { val requiredMemory = record["requiredMemory"] as Long val uid = UUID.nameUUIDFromBytes("$id-${counter++}".toByteArray()) - logger.info { "VM $id" } - val vmFragments = fragments.getValue(id).asSequence() val totalLoad = vmFragments.sumByDouble { it.usage } * 5 * 60 // avg MHz * duration = MFLOPs val vmWorkload = VmWorkload( diff --git a/opendc/opendc-experiments-sc20/src/main/kotlin/com/atlarge/opendc/experiments/sc20/trace/WorkloadSampler.kt b/opendc/opendc-experiments-sc20/src/main/kotlin/com/atlarge/opendc/experiments/sc20/trace/WorkloadSampler.kt index a8580686..dcb7190d 100644 --- a/opendc/opendc-experiments-sc20/src/main/kotlin/com/atlarge/opendc/experiments/sc20/trace/WorkloadSampler.kt +++ b/opendc/opendc-experiments-sc20/src/main/kotlin/com/atlarge/opendc/experiments/sc20/trace/WorkloadSampler.kt @@ -25,6 +25,7 @@ package com.atlarge.opendc.experiments.sc20.trace import com.atlarge.opendc.compute.core.workload.VmWorkload +import com.atlarge.opendc.experiments.sc20.experiment.model.CompositeWorkload import com.atlarge.opendc.experiments.sc20.experiment.model.Workload import com.atlarge.opendc.format.trace.TraceEntry import mu.KotlinLogging @@ -35,8 +36,17 @@ private val logger = KotlinLogging.logger {} /** * Sample the workload for the specified [run]. */ -fun sampleWorkload(trace: List<TraceEntry<VmWorkload>>, workload: Workload, seed: Int): List<TraceEntry<VmWorkload>> { - return sampleRegularWorkload(trace, workload, seed) +fun sampleWorkload( + trace: List<TraceEntry<VmWorkload>>, + workload: Workload, + subWorkload: Workload, + seed: Int +): List<TraceEntry<VmWorkload>> { + return if (workload is CompositeWorkload) { + sampleRegularWorkload(trace, subWorkload, seed) + } else { + sampleRegularWorkload(trace, workload, seed) + } } /** @@ -50,7 +60,11 @@ fun sampleRegularWorkload(trace: List<TraceEntry<VmWorkload>>, workload: Workloa val shuffled = trace.shuffled(Random(seed)) val res = mutableListOf<TraceEntry<VmWorkload>>() - val totalLoad = shuffled.sumByDouble { it.workload.image.tags.getValue("total-load") as Double } + val totalLoad = if (workload is CompositeWorkload) { + workload.totalLoad + } else { + shuffled.sumByDouble { it.workload.image.tags.getValue("total-load") as Double } + } var currentLoad = 0.0 for (entry in shuffled) { |