summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorFabian Mastenbroek <mail.fabianm@gmail.com>2021-01-07 23:36:34 +0100
committerGitHub <noreply@github.com>2021-01-07 23:36:34 +0100
commit3441586e4a10fcc6b2f458d16301ae2770d4886a (patch)
tree82ddeaced2a42dd49a633a0669fbd8fedd1c167d
parent74a4bff83bfb6366cc193d1fc9c4a07e49649649 (diff)
parent885730e31a2bb987f2dcc8a90921e2f06a8c857a (diff)
Merge pull request #70 from atlarge-research/feat/harness
Add universal experiment harness
-rw-r--r--simulator/buildSrc/src/main/kotlin/kotlin-library-convention.gradle.kts1
-rw-r--r--simulator/opendc-experiments/opendc-experiments-sc20/build.gradle.kts11
-rw-r--r--simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/Main.kt159
-rw-r--r--simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/experiment/CompositeWorkloadPortfolio.kt79
-rw-r--r--simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/experiment/Experiment.kt76
-rw-r--r--simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/experiment/HorVerPortfolio.kt60
-rw-r--r--simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/experiment/MoreHpcPortfolio.kt56
-rw-r--r--simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/experiment/MoreVelocityPortfolio.kt56
-rw-r--r--simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/experiment/OperationalPhenomenaPortfolio.kt61
-rw-r--r--simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/experiment/Portfolio.kt199
-rw-r--r--simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/experiment/Portfolios.kt221
-rw-r--r--simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/experiment/ReplayPortfolio.kt (renamed from simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/experiment/Scenario.kt)36
-rw-r--r--simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/experiment/Run.kt157
-rw-r--r--simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/experiment/TestPortfolio.kt47
-rw-r--r--simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/runner/ContainerExperimentDescriptor.kt66
-rw-r--r--simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/runner/ExperimentDescriptor.kt79
-rw-r--r--simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/runner/execution/ThreadPoolExperimentScheduler.kt83
-rw-r--r--simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/runner/internal/DefaultExperimentRunner.kt60
-rw-r--r--simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/telemetry/RunEvent.kt5
-rw-r--r--simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/telemetry/parquet/ParquetRunEventWriter.kt28
-rw-r--r--simulator/opendc-harness/build.gradle.kts47
-rw-r--r--simulator/opendc-harness/src/main/kotlin/org/opendc/harness/api/ExperimentDefinition.kt39
-rw-r--r--simulator/opendc-harness/src/main/kotlin/org/opendc/harness/api/Parameter.kt (renamed from simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/runner/execution/ExperimentExecutionResult.kt)20
-rw-r--r--simulator/opendc-harness/src/main/kotlin/org/opendc/harness/api/Scenario.kt (renamed from simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/runner/ExperimentRunner.kt)27
-rw-r--r--simulator/opendc-harness/src/main/kotlin/org/opendc/harness/api/Trial.kt (renamed from simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/runner/TrialExperimentDescriptor.kt)12
-rw-r--r--simulator/opendc-harness/src/main/kotlin/org/opendc/harness/dsl/Experiment.kt99
-rw-r--r--simulator/opendc-harness/src/main/kotlin/org/opendc/harness/dsl/ParameterProvider.kt39
-rw-r--r--simulator/opendc-harness/src/main/kotlin/org/opendc/harness/dsl/Parameters.kt44
-rw-r--r--simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/ExperimentEngine.kt107
-rw-r--r--simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/ExperimentEngineLauncher.kt121
-rw-r--r--simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/ExperimentExecutionListener.kt77
-rw-r--r--simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/discovery/Discovery.kt39
-rw-r--r--simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/discovery/DiscoveryFilter.kt51
-rw-r--r--simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/discovery/DiscoveryProvider.kt65
-rw-r--r--simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/discovery/DiscoveryRequest.kt34
-rw-r--r--simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/discovery/DiscoverySelector.kt (renamed from simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/runner/execution/ExperimentExecutionListener.kt)27
-rw-r--r--simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/scheduler/ExperimentScheduler.kt (renamed from simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/runner/execution/ExperimentScheduler.kt)28
-rw-r--r--simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/scheduler/ExperimentSchedulerProvider.kt57
-rw-r--r--simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/scheduler/ThreadPoolExperimentScheduler.kt58
-rw-r--r--simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/scheduler/ThreadPoolExperimentSchedulerProvider.kt33
-rw-r--r--simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/strategy/CartesianExperimentStrategy.kt55
-rw-r--r--simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/strategy/CartesianExperimentStrategyProvider.kt32
-rw-r--r--simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/strategy/ExperimentStrategy.kt40
-rw-r--r--simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/strategy/ExperimentStrategyProvider.kt57
-rw-r--r--simulator/opendc-harness/src/main/kotlin/org/opendc/harness/internal/CompositeDiscovery.kt47
-rw-r--r--simulator/opendc-harness/src/main/kotlin/org/opendc/harness/internal/CompositeExperimentExecutionListener.kt57
-rw-r--r--simulator/opendc-harness/src/main/kotlin/org/opendc/harness/internal/DslDiscovery.kt101
-rw-r--r--simulator/opendc-harness/src/main/kotlin/org/opendc/harness/internal/DslDiscoveryProvider.kt (renamed from simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/runner/execution/ExperimentExecutionContext.kt)29
-rw-r--r--simulator/opendc-harness/src/main/kotlin/org/opendc/harness/internal/ParameterDelegate.kt43
-rw-r--r--simulator/opendc-harness/src/main/kotlin/org/opendc/harness/internal/ScenarioImpl.kt49
-rw-r--r--simulator/opendc-harness/src/main/kotlin/org/opendc/harness/runner/console/ConsoleExperimentReporter.kt (renamed from simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/reporter/ConsoleExperimentReporter.kt)42
-rw-r--r--simulator/opendc-harness/src/main/kotlin/org/opendc/harness/runner/console/ConsoleRunner.kt99
-rw-r--r--simulator/opendc-harness/src/main/kotlin/org/opendc/harness/runner/junit5/JUnitExperimentExecutionListener.kt152
-rw-r--r--simulator/opendc-harness/src/main/kotlin/org/opendc/harness/runner/junit5/OpenDCTestEngine.kt94
-rw-r--r--simulator/opendc-harness/src/main/resources/META-INF/services/org.junit.platform.engine.TestEngine1
-rw-r--r--simulator/opendc-harness/src/main/resources/META-INF/services/org.opendc.harness.engine.discovery.DiscoveryProvider1
-rw-r--r--simulator/opendc-harness/src/main/resources/META-INF/services/org.opendc.harness.engine.scheduler.ExperimentSchedulerProvider1
-rw-r--r--simulator/opendc-harness/src/main/resources/META-INF/services/org.opendc.harness.engine.strategy.ExperimentStrategyProvider1
-rw-r--r--simulator/opendc-harness/src/main/resources/log4j2.xml40
-rw-r--r--simulator/opendc-harness/src/test/kotlin/org/opendc/harness/EngineTest.kt61
-rw-r--r--simulator/opendc-harness/src/test/kotlin/org/opendc/harness/TestExperiment.kt54
-rw-r--r--simulator/settings.gradle.kts1
62 files changed, 2537 insertions, 1084 deletions
diff --git a/simulator/buildSrc/src/main/kotlin/kotlin-library-convention.gradle.kts b/simulator/buildSrc/src/main/kotlin/kotlin-library-convention.gradle.kts
index bbecf346..f9d8a966 100644
--- a/simulator/buildSrc/src/main/kotlin/kotlin-library-convention.gradle.kts
+++ b/simulator/buildSrc/src/main/kotlin/kotlin-library-convention.gradle.kts
@@ -33,6 +33,7 @@ plugins {
/* Project configuration */
repositories {
+ mavenCentral()
jcenter()
}
diff --git a/simulator/opendc-experiments/opendc-experiments-sc20/build.gradle.kts b/simulator/opendc-experiments/opendc-experiments-sc20/build.gradle.kts
index 3b682668..b94207ba 100644
--- a/simulator/opendc-experiments/opendc-experiments-sc20/build.gradle.kts
+++ b/simulator/opendc-experiments/opendc-experiments-sc20/build.gradle.kts
@@ -29,21 +29,22 @@ plugins {
}
application {
- mainClassName = "org.opendc.experiments.sc20.MainKt"
+ mainClass.set("org.opendc.harness.runner.console.ConsoleRunnerKt")
applicationDefaultJvmArgs = listOf("-Xms2500M")
}
dependencies {
api(project(":opendc-core"))
+ api(project(":opendc-harness"))
implementation(project(":opendc-format"))
implementation(project(":opendc-simulator:opendc-simulator-core"))
implementation(project(":opendc-simulator:opendc-simulator-compute"))
implementation(project(":opendc-simulator:opendc-simulator-failures"))
implementation(project(":opendc-compute:opendc-compute-simulator"))
- implementation("com.github.ajalt:clikt:2.6.0")
- implementation("me.tongfei:progressbar:0.8.1")
- implementation("io.github.microutils:kotlin-logging:1.7.9")
+ implementation("io.github.microutils:kotlin-logging:2.0.4")
+ implementation("me.tongfei:progressbar:0.9.0")
+ implementation("com.github.ajalt.clikt:clikt:3.1.0")
implementation("org.apache.parquet:parquet-avro:1.11.0")
implementation("org.apache.hadoop:hadoop-client:3.2.1") {
@@ -51,8 +52,6 @@ dependencies {
exclude(group = "log4j")
}
- runtimeOnly("org.apache.logging.log4j:log4j-slf4j-impl:2.13.1")
-
testImplementation("org.junit.jupiter:junit-jupiter-api:${Library.JUNIT_JUPITER}")
testRuntimeOnly("org.junit.jupiter:junit-jupiter-engine:${Library.JUNIT_JUPITER}")
testImplementation("org.junit.platform:junit-platform-launcher:${Library.JUNIT_PLATFORM}")
diff --git a/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/Main.kt b/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/Main.kt
deleted file mode 100644
index 8916261b..00000000
--- a/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/Main.kt
+++ /dev/null
@@ -1,159 +0,0 @@
-/*
- * Copyright (c) 2020 AtLarge Research
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to deal
- * in the Software without restriction, including without limitation the rights
- * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- * copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in all
- * copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-
-package org.opendc.experiments.sc20
-
-import com.github.ajalt.clikt.core.CliktCommand
-import com.github.ajalt.clikt.parameters.options.convert
-import com.github.ajalt.clikt.parameters.options.default
-import com.github.ajalt.clikt.parameters.options.defaultLazy
-import com.github.ajalt.clikt.parameters.options.multiple
-import com.github.ajalt.clikt.parameters.options.option
-import com.github.ajalt.clikt.parameters.options.required
-import com.github.ajalt.clikt.parameters.types.choice
-import com.github.ajalt.clikt.parameters.types.file
-import com.github.ajalt.clikt.parameters.types.int
-import mu.KotlinLogging
-import org.opendc.experiments.sc20.experiment.*
-import org.opendc.experiments.sc20.reporter.ConsoleExperimentReporter
-import org.opendc.experiments.sc20.runner.ExperimentDescriptor
-import org.opendc.experiments.sc20.runner.execution.ThreadPoolExperimentScheduler
-import org.opendc.experiments.sc20.runner.internal.DefaultExperimentRunner
-import org.opendc.format.trace.sc20.Sc20PerformanceInterferenceReader
-import org.opendc.format.trace.sc20.Sc20VmPlacementReader
-import java.io.File
-
-/**
- * The logger for this experiment.
- */
-private val logger = KotlinLogging.logger {}
-
-/**
- * Represents the command for running the experiment.
- */
-public class ExperimentCli : CliktCommand(name = "sc20-experiment") {
- /**
- * The path to the directory where the topology descriptions are located.
- */
- private val environmentPath by option("--environment-path", help = "path to the environment directory")
- .file(canBeFile = false)
- .required()
-
- /**
- * The path to the directory where the traces are located.
- */
- private val tracePath by option("--trace-path", help = "path to the traces directory")
- .file(canBeFile = false)
- .required()
-
- /**
- * The path to the performance interference model.
- */
- private val performanceInterferenceStream by option(
- "--performance-interference-model",
- help = "path to the performance interference file"
- )
- .file(canBeDir = false)
- .convert { it.inputStream() }
-
- /**
- * The path to the original VM placements file.
- */
- private val vmPlacements by option("--vm-placements-file", help = "path to the VM placement file")
- .file(canBeDir = false)
- .convert {
- Sc20VmPlacementReader(it.inputStream().buffered()).construct()
- }
- .default(emptyMap())
-
- /**
- * The selected portfolios to run.
- */
- private val portfolios by option("--portfolio", help = "portfolio of scenarios to explore")
- .choice(
- "hor-ver" to { experiment: Experiment, i: Int -> HorVerPortfolio(experiment, i) }
- as (Experiment, Int) -> Portfolio,
- "more-velocity" to { experiment, i -> MoreVelocityPortfolio(experiment, i) },
- "composite-workload" to { experiment, i -> CompositeWorkloadPortfolio(experiment, i) },
- "operational-phenomena" to { experiment, i -> OperationalPhenomenaPortfolio(experiment, i) },
- "replay" to { experiment, i -> ReplayPortfolio(experiment, i) },
- "test" to { experiment, i -> TestPortfolio(experiment, i) },
- "more-hpc" to { experiment, i -> MoreHpcPortfolio(experiment, i) },
- ignoreCase = true
- )
- .multiple(required = true)
-
- /**
- * The maximum number of worker threads to use.
- */
- private val parallelism by option("--parallelism", help = "maximum number of concurrent simulation runs")
- .int()
- .default(Runtime.getRuntime().availableProcessors())
-
- /**
- * The buffer size for writing results.
- */
- private val bufferSize by option("--buffer-size")
- .int()
- .default(4096)
-
- /**
- * The path to the output directory.
- */
- private val output by option("-O", "--output", help = "path to the output directory")
- .file(canBeFile = false)
- .defaultLazy { File("data") }
-
- override fun run() {
- logger.info { "Constructing performance interference model" }
-
- val performanceInterferenceModel =
- performanceInterferenceStream?.let { Sc20PerformanceInterferenceReader(it) }
-
- logger.info { "Creating experiment descriptor" }
- val descriptor = object :
- Experiment(environmentPath, tracePath, output, performanceInterferenceModel, vmPlacements, bufferSize) {
- private val descriptor = this
- override val children: Sequence<ExperimentDescriptor> = sequence {
- for ((i, producer) in portfolios.withIndex()) {
- yield(producer(descriptor, i))
- }
- }
- }
-
- logger.info { "Starting experiment runner [parallelism=$parallelism]" }
- val scheduler = ThreadPoolExperimentScheduler(parallelism)
- val runner = DefaultExperimentRunner(scheduler)
- val reporter = ConsoleExperimentReporter()
- try {
- runner.execute(descriptor, reporter)
- } finally {
- scheduler.close()
- reporter.close()
- }
- }
-}
-
-/**
- * Main entry point of the experiment.
- */
-public fun main(args: Array<String>): Unit = ExperimentCli().main(args)
diff --git a/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/experiment/CompositeWorkloadPortfolio.kt b/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/experiment/CompositeWorkloadPortfolio.kt
new file mode 100644
index 00000000..f4242456
--- /dev/null
+++ b/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/experiment/CompositeWorkloadPortfolio.kt
@@ -0,0 +1,79 @@
+/*
+ * Copyright (c) 2021 AtLarge Research
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+package org.opendc.experiments.sc20.experiment
+
+import org.opendc.experiments.sc20.experiment.model.CompositeWorkload
+import org.opendc.experiments.sc20.experiment.model.OperationalPhenomena
+import org.opendc.experiments.sc20.experiment.model.Topology
+import org.opendc.experiments.sc20.experiment.model.Workload
+import org.opendc.harness.dsl.anyOf
+
+/**
+ * A [Portfolio] that explores the effect of a composite workload.
+ */
+public class CompositeWorkloadPortfolio : Portfolio("composite-workload") {
+ private val totalSampleLoad = 1.3301733005049648E12
+
+ override val topology: Topology by anyOf(
+ Topology("base"),
+ Topology("exp-vol-hor-hom"),
+ Topology("exp-vol-ver-hom"),
+ Topology("exp-vel-ver-hom")
+ )
+
+ override val workload: Workload by anyOf(
+ CompositeWorkload(
+ "all-azure",
+ listOf(Workload("solvinity-short", 0.0), Workload("azure", 1.0)),
+ totalSampleLoad
+ ),
+ CompositeWorkload(
+ "solvinity-25-azure-75",
+ listOf(Workload("solvinity-short", 0.25), Workload("azure", 0.75)),
+ totalSampleLoad
+ ),
+ CompositeWorkload(
+ "solvinity-50-azure-50",
+ listOf(Workload("solvinity-short", 0.5), Workload("azure", 0.5)),
+ totalSampleLoad
+ ),
+ CompositeWorkload(
+ "solvinity-75-azure-25",
+ listOf(Workload("solvinity-short", 0.75), Workload("azure", 0.25)),
+ totalSampleLoad
+ ),
+ CompositeWorkload(
+ "all-solvinity",
+ listOf(Workload("solvinity-short", 1.0), Workload("azure", 0.0)),
+ totalSampleLoad
+ )
+ )
+
+ override val operationalPhenomena: OperationalPhenomena by anyOf(
+ OperationalPhenomena(failureFrequency = 24.0 * 7, hasInterference = false)
+ )
+
+ override val allocationPolicy: String by anyOf(
+ "active-servers"
+ )
+}
diff --git a/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/experiment/Experiment.kt b/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/experiment/Experiment.kt
deleted file mode 100644
index 34d7301b..00000000
--- a/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/experiment/Experiment.kt
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * Copyright (c) 2020 AtLarge Research
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to deal
- * in the Software without restriction, including without limitation the rights
- * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- * copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in all
- * copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-
-package org.opendc.experiments.sc20.experiment
-
-import org.opendc.experiments.sc20.runner.ContainerExperimentDescriptor
-import org.opendc.experiments.sc20.runner.ExperimentDescriptor
-import org.opendc.experiments.sc20.runner.execution.ExperimentExecutionContext
-import org.opendc.experiments.sc20.runner.execution.ExperimentExecutionListener
-import org.opendc.experiments.sc20.telemetry.RunEvent
-import org.opendc.experiments.sc20.telemetry.parquet.ParquetRunEventWriter
-import org.opendc.format.trace.PerformanceInterferenceModelReader
-import java.io.File
-
-/**
- * The global configuration of the experiment.
- *
- * @param environments The path to the topologies directory.
- * @param traces The path to the traces directory.
- * @param output The output directory.
- * @param performanceInterferenceModel The optional performance interference model that has been specified.
- * @param vmPlacements Original VM placement in the trace.
- * @param bufferSize The buffer size of the event reporters.
- */
-public abstract class Experiment(
- public val environments: File,
- public val traces: File,
- public val output: File,
- public val performanceInterferenceModel: PerformanceInterferenceModelReader?,
- public val vmPlacements: Map<String, String>,
- public val bufferSize: Int
-) : ContainerExperimentDescriptor() {
- override val parent: ExperimentDescriptor? = null
-
- override suspend fun invoke(context: ExperimentExecutionContext) {
- val writer = ParquetRunEventWriter(File(output, "experiments.parquet"), bufferSize)
- try {
- val listener = object : ExperimentExecutionListener by context.listener {
- override fun descriptorRegistered(descriptor: ExperimentDescriptor) {
- if (descriptor is Run) {
- writer.write(RunEvent(descriptor, System.currentTimeMillis()))
- }
-
- context.listener.descriptorRegistered(descriptor)
- }
- }
-
- val newContext = object : ExperimentExecutionContext by context {
- override val listener: ExperimentExecutionListener = listener
- }
-
- super.invoke(newContext)
- } finally {
- writer.close()
- }
- }
-}
diff --git a/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/experiment/HorVerPortfolio.kt b/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/experiment/HorVerPortfolio.kt
new file mode 100644
index 00000000..aa97b808
--- /dev/null
+++ b/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/experiment/HorVerPortfolio.kt
@@ -0,0 +1,60 @@
+/*
+ * Copyright (c) 2021 AtLarge Research
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+package org.opendc.experiments.sc20.experiment
+
+import org.opendc.experiments.sc20.experiment.model.OperationalPhenomena
+import org.opendc.experiments.sc20.experiment.model.Topology
+import org.opendc.experiments.sc20.experiment.model.Workload
+import org.opendc.harness.dsl.anyOf
+
+/**
+ * A [Portfolio] that explores the difference between horizontal and vertical scaling.
+ */
+public class HorVerPortfolio : Portfolio("horizontal_vs_vertical") {
+ override val topology: Topology by anyOf(
+ Topology("base"),
+ Topology("rep-vol-hor-hom"),
+ Topology("rep-vol-hor-het"),
+ Topology("rep-vol-ver-hom"),
+ Topology("rep-vol-ver-het"),
+ Topology("exp-vol-hor-hom"),
+ Topology("exp-vol-hor-het"),
+ Topology("exp-vol-ver-hom"),
+ Topology("exp-vol-ver-het")
+ )
+
+ override val workload: Workload by anyOf(
+ Workload("solvinity", 0.1),
+ Workload("solvinity", 0.25),
+ Workload("solvinity", 0.5),
+ Workload("solvinity", 1.0)
+ )
+
+ override val operationalPhenomena: OperationalPhenomena by anyOf(
+ OperationalPhenomena(failureFrequency = 24.0 * 7, hasInterference = true)
+ )
+
+ override val allocationPolicy: String by anyOf(
+ "active-servers"
+ )
+}
diff --git a/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/experiment/MoreHpcPortfolio.kt b/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/experiment/MoreHpcPortfolio.kt
new file mode 100644
index 00000000..bdb33f59
--- /dev/null
+++ b/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/experiment/MoreHpcPortfolio.kt
@@ -0,0 +1,56 @@
+/*
+ * Copyright (c) 2020 AtLarge Research
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+package org.opendc.experiments.sc20.experiment
+
+import org.opendc.experiments.sc20.experiment.model.*
+import org.opendc.harness.dsl.anyOf
+
+/**
+ * A [Portfolio] to explore the effect of HPC workloads.
+ */
+public class MoreHpcPortfolio : Portfolio("more_hpc") {
+ override val topology: Topology by anyOf(
+ Topology("base"),
+ Topology("exp-vol-hor-hom"),
+ Topology("exp-vol-ver-hom"),
+ Topology("exp-vel-ver-hom")
+ )
+
+ override val workload: Workload by anyOf(
+ Workload("solvinity", 0.0, samplingStrategy = SamplingStrategy.HPC),
+ Workload("solvinity", 0.25, samplingStrategy = SamplingStrategy.HPC),
+ Workload("solvinity", 0.5, samplingStrategy = SamplingStrategy.HPC),
+ Workload("solvinity", 1.0, samplingStrategy = SamplingStrategy.HPC),
+ Workload("solvinity", 0.25, samplingStrategy = SamplingStrategy.HPC_LOAD),
+ Workload("solvinity", 0.5, samplingStrategy = SamplingStrategy.HPC_LOAD),
+ Workload("solvinity", 1.0, samplingStrategy = SamplingStrategy.HPC_LOAD)
+ )
+
+ override val operationalPhenomena: OperationalPhenomena by anyOf(
+ OperationalPhenomena(failureFrequency = 24.0 * 7, hasInterference = true)
+ )
+
+ override val allocationPolicy: String by anyOf(
+ "active-servers"
+ )
+}
diff --git a/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/experiment/MoreVelocityPortfolio.kt b/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/experiment/MoreVelocityPortfolio.kt
new file mode 100644
index 00000000..733dabf6
--- /dev/null
+++ b/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/experiment/MoreVelocityPortfolio.kt
@@ -0,0 +1,56 @@
+/*
+ * Copyright (c) 2021 AtLarge Research
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+package org.opendc.experiments.sc20.experiment
+
+import org.opendc.experiments.sc20.experiment.model.OperationalPhenomena
+import org.opendc.experiments.sc20.experiment.model.Topology
+import org.opendc.experiments.sc20.experiment.model.Workload
+import org.opendc.harness.dsl.anyOf
+
+/**
+ * A [Portfolio] that explores the effect of adding more velocity to a cluster (e.g., faster machines).
+ */
+public class MoreVelocityPortfolio : Portfolio("more_velocity") {
+ override val topology: Topology by anyOf(
+ Topology("base"),
+ Topology("rep-vel-ver-hom"),
+ Topology("rep-vel-ver-het"),
+ Topology("exp-vel-ver-hom"),
+ Topology("exp-vel-ver-het")
+ )
+
+ override val workload: Workload by anyOf(
+ Workload("solvinity", 0.1),
+ Workload("solvinity", 0.25),
+ Workload("solvinity", 0.5),
+ Workload("solvinity", 1.0)
+ )
+
+ override val operationalPhenomena: OperationalPhenomena by anyOf(
+ OperationalPhenomena(failureFrequency = 24.0 * 7, hasInterference = true)
+ )
+
+ override val allocationPolicy: String by anyOf(
+ "active-servers"
+ )
+}
diff --git a/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/experiment/OperationalPhenomenaPortfolio.kt b/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/experiment/OperationalPhenomenaPortfolio.kt
new file mode 100644
index 00000000..66b94faf
--- /dev/null
+++ b/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/experiment/OperationalPhenomenaPortfolio.kt
@@ -0,0 +1,61 @@
+/*
+ * Copyright (c) 2021 AtLarge Research
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+package org.opendc.experiments.sc20.experiment
+
+import org.opendc.experiments.sc20.experiment.model.OperationalPhenomena
+import org.opendc.experiments.sc20.experiment.model.Topology
+import org.opendc.experiments.sc20.experiment.model.Workload
+import org.opendc.harness.dsl.anyOf
+
+/**
+ * A [Portfolio] that explores the effect of operational phenomena on metrics.
+ */
+public class OperationalPhenomenaPortfolio : Portfolio("operational_phenomena") {
+ override val topology: Topology by anyOf(
+ Topology("base")
+ )
+
+ override val workload: Workload by anyOf(
+ Workload("solvinity", 0.1),
+ Workload("solvinity", 0.25),
+ Workload("solvinity", 0.5),
+ Workload("solvinity", 1.0)
+ )
+
+ override val operationalPhenomena: OperationalPhenomena by anyOf(
+ OperationalPhenomena(failureFrequency = 24.0 * 7, hasInterference = true),
+ OperationalPhenomena(failureFrequency = 0.0, hasInterference = true),
+ OperationalPhenomena(failureFrequency = 24.0 * 7, hasInterference = false),
+ OperationalPhenomena(failureFrequency = 0.0, hasInterference = false)
+ )
+
+ override val allocationPolicy: String by anyOf(
+ "mem",
+ "mem-inv",
+ "core-mem",
+ "core-mem-inv",
+ "active-servers",
+ "active-servers-inv",
+ "random"
+ )
+}
diff --git a/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/experiment/Portfolio.kt b/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/experiment/Portfolio.kt
index 37cf2880..4a82ad56 100644
--- a/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/experiment/Portfolio.kt
+++ b/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/experiment/Portfolio.kt
@@ -22,67 +22,196 @@
package org.opendc.experiments.sc20.experiment
+import kotlinx.coroutines.ExperimentalCoroutinesApi
+import kotlinx.coroutines.cancel
+import kotlinx.coroutines.channels.Channel
+import kotlinx.coroutines.launch
+import kotlinx.coroutines.test.TestCoroutineScope
+import mu.KotlinLogging
+import org.opendc.compute.simulator.allocation.*
+import org.opendc.experiments.sc20.experiment.model.CompositeWorkload
import org.opendc.experiments.sc20.experiment.model.OperationalPhenomena
import org.opendc.experiments.sc20.experiment.model.Topology
import org.opendc.experiments.sc20.experiment.model.Workload
-import org.opendc.experiments.sc20.runner.ContainerExperimentDescriptor
+import org.opendc.experiments.sc20.experiment.monitor.ParquetExperimentMonitor
+import org.opendc.experiments.sc20.trace.Sc20ParquetTraceReader
+import org.opendc.experiments.sc20.trace.Sc20RawParquetTraceReader
+import org.opendc.format.environment.sc20.Sc20ClusterEnvironmentReader
+import org.opendc.format.trace.PerformanceInterferenceModelReader
+import org.opendc.harness.dsl.Experiment
+import org.opendc.harness.dsl.anyOf
+import org.opendc.simulator.utils.DelayControllerClockAdapter
+import org.opendc.trace.core.EventTracer
+import java.io.File
+import java.util.concurrent.ConcurrentHashMap
+import kotlin.random.Random
/**
- * A portfolio represents a collection of scenarios are tested.
+ * A portfolio represents a collection of scenarios are tested for the work.
+ *
+ * @param name The name of the portfolio.
*/
-public abstract class Portfolio(
- override val parent: Experiment,
- public val id: Int,
- public val name: String
-) : ContainerExperimentDescriptor() {
+public abstract class Portfolio(name: String) : Experiment(name) {
+ /**
+ * The logger for this portfolio instance.
+ */
+ private val logger = KotlinLogging.logger {}
+
+ /**
+ * The path to where the environments are located.
+ */
+ private val environmentPath by anyOf(File("environments/"))
+
+ /**
+ * The path to where the traces are located.
+ */
+ private val tracePath by anyOf(File("traces/"))
+
/**
- * The topologies to consider.
+ * The path to where the output results should be written.
*/
- protected abstract val topologies: List<Topology>
+ private val outputPath by anyOf(File("results/"))
/**
- * The workloads to consider.
+ * The path to the original VM placements file.
*/
- protected abstract val workloads: List<Workload>
+ private val vmPlacements by anyOf(emptyMap<String, String>())
+
+ /**
+ * The path to the performance interference model.
+ */
+ private val performanceInterferenceModel by anyOf<PerformanceInterferenceModelReader?>(null)
+
+ /**
+ * The topology to test.
+ */
+ public abstract val topology: Topology
+
+ /**
+ * The workload to test.
+ */
+ public abstract val workload: Workload
/**
* The operational phenomenas to consider.
*/
- protected abstract val operationalPhenomenas: List<OperationalPhenomena>
+ public abstract val operationalPhenomena: OperationalPhenomena
/**
* The allocation policies to consider.
*/
- protected abstract val allocationPolicies: List<String>
+ public abstract val allocationPolicy: String
/**
- * The number of repetitions to perform.
+ * A map of trace readers.
*/
- public open val repetitions: Int = 32
+ private val traceReaders = ConcurrentHashMap<String, Sc20RawParquetTraceReader>()
/**
- * Resolve the children of this container.
+ * Perform a single trial for this portfolio.
*/
- override val children: Sequence<Scenario> = sequence {
- var id = 0
- for (topology in topologies) {
- for (workload in workloads) {
- for (operationalPhenomena in operationalPhenomenas) {
- for (allocationPolicy in allocationPolicies) {
- yield(
- Scenario(
- this@Portfolio,
- id++,
- repetitions,
- topology,
- workload,
- allocationPolicy,
- operationalPhenomena
- )
- )
- }
- }
+ @OptIn(ExperimentalCoroutinesApi::class)
+ override fun doRun(repeat: Int) {
+ val testScope = TestCoroutineScope()
+ val clock = DelayControllerClockAdapter(testScope)
+ val tracer = EventTracer(clock)
+ val seeder = Random(repeat)
+ val environment = Sc20ClusterEnvironmentReader(File(environmentPath, "${topology.name}.txt"))
+
+ val chan = Channel<Unit>(Channel.CONFLATED)
+ val allocationPolicy = createAllocationPolicy(seeder)
+
+ val workload = workload
+ val workloadNames = if (workload is CompositeWorkload) {
+ workload.workloads.map { it.name }
+ } else {
+ listOf(workload.name)
+ }
+
+ val rawReaders = workloadNames.map { workloadName ->
+ traceReaders.computeIfAbsent(workloadName) {
+ logger.info { "Loading trace $workloadName" }
+ Sc20RawParquetTraceReader(File(tracePath, workloadName))
+ }
+ }
+
+ val performanceInterferenceModel = performanceInterferenceModel
+ ?.takeIf { operationalPhenomena.hasInterference }
+ ?.construct(seeder) ?: emptyMap()
+ val trace = Sc20ParquetTraceReader(rawReaders, performanceInterferenceModel, workload, seeder.nextInt())
+
+ val monitor = ParquetExperimentMonitor(
+ outputPath,
+ "portfolio_id=$name/scenario_id=$id/run_id=$repeat",
+ 4096
+ )
+
+ testScope.launch {
+ val (bareMetalProvisioner, scheduler) = createProvisioner(
+ this,
+ clock,
+ environment,
+ allocationPolicy,
+ tracer
+ )
+
+ val failureDomain = if (operationalPhenomena.failureFrequency > 0) {
+ logger.debug("ENABLING failures")
+ createFailureDomain(
+ this,
+ clock,
+ seeder.nextInt(),
+ operationalPhenomena.failureFrequency,
+ bareMetalProvisioner,
+ chan
+ )
+ } else {
+ null
}
+
+ attachMonitor(this, clock, scheduler, monitor)
+ processTrace(
+ this,
+ clock,
+ trace,
+ scheduler,
+ chan,
+ monitor
+ )
+
+ logger.debug("SUBMIT=${scheduler.submittedVms}")
+ logger.debug("FAIL=${scheduler.unscheduledVms}")
+ logger.debug("QUEUED=${scheduler.queuedVms}")
+ logger.debug("RUNNING=${scheduler.runningVms}")
+ logger.debug("FINISHED=${scheduler.finishedVms}")
+
+ failureDomain?.cancel()
+ scheduler.terminate()
+ }
+
+ try {
+ testScope.advanceUntilIdle()
+ } finally {
+ monitor.close()
+ }
+ }
+
+ /**
+ * Create the [AllocationPolicy] instance to use for the trial.
+ */
+ private fun createAllocationPolicy(seeder: Random): AllocationPolicy {
+ return when (allocationPolicy) {
+ "mem" -> AvailableMemoryAllocationPolicy()
+ "mem-inv" -> AvailableMemoryAllocationPolicy(true)
+ "core-mem" -> AvailableCoreMemoryAllocationPolicy()
+ "core-mem-inv" -> AvailableCoreMemoryAllocationPolicy(true)
+ "active-servers" -> NumberOfActiveServersAllocationPolicy()
+ "active-servers-inv" -> NumberOfActiveServersAllocationPolicy(true)
+ "provisioned-cores" -> ProvisionedCoresAllocationPolicy()
+ "provisioned-cores-inv" -> ProvisionedCoresAllocationPolicy(true)
+ "random" -> RandomAllocationPolicy(Random(seeder.nextInt()))
+ "replay" -> ReplayAllocationPolicy(vmPlacements)
+ else -> throw IllegalArgumentException("Unknown policy $allocationPolicy")
}
}
}
diff --git a/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/experiment/Portfolios.kt b/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/experiment/Portfolios.kt
deleted file mode 100644
index 249a63b9..00000000
--- a/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/experiment/Portfolios.kt
+++ /dev/null
@@ -1,221 +0,0 @@
-/*
- * Copyright (c) 2020 AtLarge Research
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to deal
- * in the Software without restriction, including without limitation the rights
- * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- * copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in all
- * copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-
-package org.opendc.experiments.sc20.experiment
-
-import org.opendc.experiments.sc20.experiment.model.*
-
-public class HorVerPortfolio(parent: Experiment, id: Int) : Portfolio(parent, id, "horizontal_vs_vertical") {
- override val topologies: List<Topology> = listOf(
- Topology("base"),
- Topology("rep-vol-hor-hom"),
- Topology("rep-vol-hor-het"),
- Topology("rep-vol-ver-hom"),
- Topology("rep-vol-ver-het"),
- Topology("exp-vol-hor-hom"),
- Topology("exp-vol-hor-het"),
- Topology("exp-vol-ver-hom"),
- Topology("exp-vol-ver-het")
- )
-
- override val workloads: List<Workload> = listOf(
- Workload("solvinity", 0.1),
- Workload("solvinity", 0.25),
- Workload("solvinity", 0.5),
- Workload("solvinity", 1.0)
- )
-
- override val operationalPhenomenas: List<OperationalPhenomena> = listOf(
- OperationalPhenomena(failureFrequency = 24.0 * 7, hasInterference = true)
- )
-
- override val allocationPolicies: List<String> = listOf(
- "active-servers"
- )
-}
-
-public class MoreVelocityPortfolio(parent: Experiment, id: Int) : Portfolio(parent, id, "more_velocity") {
- override val topologies: List<Topology> = listOf(
- Topology("base"),
- Topology("rep-vel-ver-hom"),
- Topology("rep-vel-ver-het"),
- Topology("exp-vel-ver-hom"),
- Topology("exp-vel-ver-het")
- )
-
- override val workloads: List<Workload> = listOf(
- Workload("solvinity", 0.1),
- Workload("solvinity", 0.25),
- Workload("solvinity", 0.5),
- Workload("solvinity", 1.0)
- )
-
- override val operationalPhenomenas: List<OperationalPhenomena> = listOf(
- OperationalPhenomena(failureFrequency = 24.0 * 7, hasInterference = true)
- )
-
- override val allocationPolicies: List<String> = listOf(
- "active-servers"
- )
-}
-
-public class CompositeWorkloadPortfolio(parent: Experiment, id: Int) : Portfolio(parent, id, "composite-workload") {
- private val totalSampleLoad = 1.3301733005049648E12
-
- override val topologies: List<Topology> = listOf(
- Topology("base"),
- Topology("exp-vol-hor-hom"),
- Topology("exp-vol-ver-hom"),
- Topology("exp-vel-ver-hom")
- )
-
- override val workloads: List<Workload> = listOf(
- CompositeWorkload(
- "all-azure",
- listOf(Workload("solvinity-short", 0.0), Workload("azure", 1.0)),
- totalSampleLoad
- ),
- CompositeWorkload(
- "solvinity-25-azure-75",
- listOf(Workload("solvinity-short", 0.25), Workload("azure", 0.75)),
- totalSampleLoad
- ),
- CompositeWorkload(
- "solvinity-50-azure-50",
- listOf(Workload("solvinity-short", 0.5), Workload("azure", 0.5)),
- totalSampleLoad
- ),
- CompositeWorkload(
- "solvinity-75-azure-25",
- listOf(Workload("solvinity-short", 0.75), Workload("azure", 0.25)),
- totalSampleLoad
- ),
- CompositeWorkload(
- "all-solvinity",
- listOf(Workload("solvinity-short", 1.0), Workload("azure", 0.0)),
- totalSampleLoad
- )
- )
-
- override val operationalPhenomenas: List<OperationalPhenomena> = listOf(
- OperationalPhenomena(failureFrequency = 24.0 * 7, hasInterference = false)
- )
-
- override val allocationPolicies: List<String> = listOf(
- "active-servers"
- )
-}
-
-public class OperationalPhenomenaPortfolio(parent: Experiment, id: Int) :
- Portfolio(parent, id, "operational_phenomena") {
- override val topologies: List<Topology> = listOf(
- Topology("base")
- )
-
- override val workloads: List<Workload> = listOf(
- Workload("solvinity", 0.1),
- Workload("solvinity", 0.25),
- Workload("solvinity", 0.5),
- Workload("solvinity", 1.0)
- )
-
- override val operationalPhenomenas: List<OperationalPhenomena> = listOf(
- OperationalPhenomena(failureFrequency = 24.0 * 7, hasInterference = true),
- OperationalPhenomena(failureFrequency = 0.0, hasInterference = true),
- OperationalPhenomena(failureFrequency = 24.0 * 7, hasInterference = false),
- OperationalPhenomena(failureFrequency = 0.0, hasInterference = false)
- )
-
- override val allocationPolicies: List<String> = listOf(
- "mem",
- "mem-inv",
- "core-mem",
- "core-mem-inv",
- "active-servers",
- "active-servers-inv",
- "random"
- )
-}
-
-public class ReplayPortfolio(parent: Experiment, id: Int) : Portfolio(parent, id, "replay") {
- override val topologies: List<Topology> = listOf(
- Topology("base")
- )
-
- override val workloads: List<Workload> = listOf(
- Workload("solvinity", 1.0)
- )
-
- override val operationalPhenomenas: List<OperationalPhenomena> = listOf(
- OperationalPhenomena(failureFrequency = 0.0, hasInterference = false)
- )
-
- override val allocationPolicies: List<String> = listOf(
- "replay",
- "active-servers"
- )
-}
-
-public class TestPortfolio(parent: Experiment, id: Int) : Portfolio(parent, id, "test") {
- override val repetitions: Int = 1
-
- override val topologies: List<Topology> = listOf(
- Topology("base")
- )
-
- override val workloads: List<Workload> = listOf(
- Workload("solvinity", 1.0)
- )
-
- override val operationalPhenomenas: List<OperationalPhenomena> = listOf(
- OperationalPhenomena(failureFrequency = 24.0 * 7, hasInterference = true)
- )
-
- override val allocationPolicies: List<String> = listOf("active-servers")
-}
-
-public class MoreHpcPortfolio(parent: Experiment, id: Int) : Portfolio(parent, id, "more_hpc") {
- override val topologies: List<Topology> = listOf(
- Topology("base"),
- Topology("exp-vol-hor-hom"),
- Topology("exp-vol-ver-hom"),
- Topology("exp-vel-ver-hom")
- )
-
- override val workloads: List<Workload> = listOf(
- Workload("solvinity", 0.0, samplingStrategy = SamplingStrategy.HPC),
- Workload("solvinity", 0.25, samplingStrategy = SamplingStrategy.HPC),
- Workload("solvinity", 0.5, samplingStrategy = SamplingStrategy.HPC),
- Workload("solvinity", 1.0, samplingStrategy = SamplingStrategy.HPC),
- Workload("solvinity", 0.25, samplingStrategy = SamplingStrategy.HPC_LOAD),
- Workload("solvinity", 0.5, samplingStrategy = SamplingStrategy.HPC_LOAD),
- Workload("solvinity", 1.0, samplingStrategy = SamplingStrategy.HPC_LOAD)
- )
-
- override val operationalPhenomenas: List<OperationalPhenomena> = listOf(
- OperationalPhenomena(failureFrequency = 24.0 * 7, hasInterference = true)
- )
-
- override val allocationPolicies: List<String> = listOf(
- "active-servers"
- )
-}
diff --git a/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/experiment/Scenario.kt b/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/experiment/ReplayPortfolio.kt
index d092ddd5..8a42a3b4 100644
--- a/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/experiment/Scenario.kt
+++ b/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/experiment/ReplayPortfolio.kt
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2020 AtLarge Research
+ * Copyright (c) 2021 AtLarge Research
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
@@ -25,22 +25,26 @@ package org.opendc.experiments.sc20.experiment
import org.opendc.experiments.sc20.experiment.model.OperationalPhenomena
import org.opendc.experiments.sc20.experiment.model.Topology
import org.opendc.experiments.sc20.experiment.model.Workload
-import org.opendc.experiments.sc20.runner.ContainerExperimentDescriptor
-import org.opendc.experiments.sc20.runner.ExperimentDescriptor
+import org.opendc.harness.dsl.anyOf
/**
- * A scenario represents a single point in the design space (a unique combination of parameters).
+ * A [Portfolio] that compares the original VM placements against our policies.
*/
-public class Scenario(
- override val parent: Portfolio,
- public val id: Int,
- public val repetitions: Int,
- public val topology: Topology,
- public val workload: Workload,
- public val allocationPolicy: String,
- public val operationalPhenomena: OperationalPhenomena
-) : ContainerExperimentDescriptor() {
- override val children: Sequence<ExperimentDescriptor> = sequence {
- repeat(repetitions) { i -> yield(Run(this@Scenario, i, i)) }
- }
+public class ReplayPortfolio : Portfolio("replay") {
+ override val topology: Topology by anyOf(
+ Topology("base")
+ )
+
+ override val workload: Workload by anyOf(
+ Workload("solvinity", 1.0)
+ )
+
+ override val operationalPhenomena: OperationalPhenomena by anyOf(
+ OperationalPhenomena(failureFrequency = 0.0, hasInterference = false)
+ )
+
+ override val allocationPolicy: String by anyOf(
+ "replay",
+ "active-servers"
+ )
}
diff --git a/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/experiment/Run.kt b/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/experiment/Run.kt
deleted file mode 100644
index 8d8d608d..00000000
--- a/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/experiment/Run.kt
+++ /dev/null
@@ -1,157 +0,0 @@
-/*
- * Copyright (c) 2020 AtLarge Research
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to deal
- * in the Software without restriction, including without limitation the rights
- * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- * copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in all
- * copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-
-package org.opendc.experiments.sc20.experiment
-
-import kotlinx.coroutines.ExperimentalCoroutinesApi
-import kotlinx.coroutines.cancel
-import kotlinx.coroutines.channels.Channel
-import kotlinx.coroutines.launch
-import kotlinx.coroutines.test.TestCoroutineScope
-import mu.KotlinLogging
-import org.opendc.compute.simulator.allocation.*
-import org.opendc.experiments.sc20.experiment.model.CompositeWorkload
-import org.opendc.experiments.sc20.experiment.monitor.ParquetExperimentMonitor
-import org.opendc.experiments.sc20.runner.TrialExperimentDescriptor
-import org.opendc.experiments.sc20.runner.execution.ExperimentExecutionContext
-import org.opendc.experiments.sc20.trace.Sc20ParquetTraceReader
-import org.opendc.experiments.sc20.trace.Sc20RawParquetTraceReader
-import org.opendc.format.environment.sc20.Sc20ClusterEnvironmentReader
-import org.opendc.simulator.utils.DelayControllerClockAdapter
-import org.opendc.trace.core.EventTracer
-import java.io.File
-import kotlin.random.Random
-
-/**
- * The logger for the experiment scenario.
- */
-private val logger = KotlinLogging.logger {}
-
-/**
- * An experiment run represent a single invocation of a trial and is used to distinguish between repetitions of the
- * same set of parameters.
- */
-@OptIn(ExperimentalCoroutinesApi::class)
-public data class Run(override val parent: Scenario, val id: Int, val seed: Int) : TrialExperimentDescriptor() {
- override suspend fun invoke(context: ExperimentExecutionContext) {
- val experiment = parent.parent.parent
- val testScope = TestCoroutineScope()
- val clock = DelayControllerClockAdapter(testScope)
- val seeder = Random(seed)
- val environment = Sc20ClusterEnvironmentReader(File(experiment.environments, "${parent.topology.name}.txt"))
-
- val chan = Channel<Unit>(Channel.CONFLATED)
- val allocationPolicy = when (parent.allocationPolicy) {
- "mem" -> AvailableMemoryAllocationPolicy()
- "mem-inv" -> AvailableMemoryAllocationPolicy(true)
- "core-mem" -> AvailableCoreMemoryAllocationPolicy()
- "core-mem-inv" -> AvailableCoreMemoryAllocationPolicy(true)
- "active-servers" -> NumberOfActiveServersAllocationPolicy()
- "active-servers-inv" -> NumberOfActiveServersAllocationPolicy(true)
- "provisioned-cores" -> ProvisionedCoresAllocationPolicy()
- "provisioned-cores-inv" -> ProvisionedCoresAllocationPolicy(true)
- "random" -> RandomAllocationPolicy(Random(seeder.nextInt()))
- "replay" -> ReplayAllocationPolicy(experiment.vmPlacements)
- else -> throw IllegalArgumentException("Unknown policy ${parent.allocationPolicy}")
- }
-
- @Suppress("UNCHECKED_CAST")
- val rawTraceReaders =
- context.cache.computeIfAbsent("raw-trace-readers") { mutableMapOf<String, Sc20RawParquetTraceReader>() } as MutableMap<String, Sc20RawParquetTraceReader>
- val rawReaders = synchronized(rawTraceReaders) {
- val workloadNames = if (parent.workload is CompositeWorkload) {
- parent.workload.workloads.map { it.name }
- } else {
- listOf(parent.workload.name)
- }
-
- workloadNames.map { workloadName ->
- rawTraceReaders.computeIfAbsent(workloadName) {
- logger.info { "Loading trace $workloadName" }
- Sc20RawParquetTraceReader(File(experiment.traces, workloadName))
- }
- }
- }
-
- val performanceInterferenceModel = experiment.performanceInterferenceModel
- ?.takeIf { parent.operationalPhenomena.hasInterference }
- ?.construct(seeder) ?: emptyMap()
- val trace = Sc20ParquetTraceReader(rawReaders, performanceInterferenceModel, parent.workload, seed)
-
- val monitor = ParquetExperimentMonitor(
- parent.parent.parent.output,
- "portfolio_id=${parent.parent.id}/scenario_id=${parent.id}/run_id=$id",
- parent.parent.parent.bufferSize
- )
-
- val tracer = EventTracer(clock)
-
- testScope.launch {
- val (bareMetalProvisioner, scheduler) = createProvisioner(
- this,
- clock,
- environment,
- allocationPolicy,
- tracer
- )
-
- val failureDomain = if (parent.operationalPhenomena.failureFrequency > 0) {
- logger.debug("ENABLING failures")
- createFailureDomain(
- this,
- clock,
- seeder.nextInt(),
- parent.operationalPhenomena.failureFrequency,
- bareMetalProvisioner,
- chan
- )
- } else {
- null
- }
-
- attachMonitor(this, clock, scheduler, monitor)
- processTrace(
- this,
- clock,
- trace,
- scheduler,
- chan,
- monitor
- )
-
- logger.debug("SUBMIT=${scheduler.submittedVms}")
- logger.debug("FAIL=${scheduler.unscheduledVms}")
- logger.debug("QUEUED=${scheduler.queuedVms}")
- logger.debug("RUNNING=${scheduler.runningVms}")
- logger.debug("FINISHED=${scheduler.finishedVms}")
-
- failureDomain?.cancel()
- scheduler.terminate()
- }
-
- try {
- testScope.advanceUntilIdle()
- } finally {
- monitor.close()
- }
- }
-}
diff --git a/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/experiment/TestPortfolio.kt b/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/experiment/TestPortfolio.kt
new file mode 100644
index 00000000..2210fc97
--- /dev/null
+++ b/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/experiment/TestPortfolio.kt
@@ -0,0 +1,47 @@
+/*
+ * Copyright (c) 2021 AtLarge Research
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+package org.opendc.experiments.sc20.experiment
+
+import org.opendc.experiments.sc20.experiment.model.OperationalPhenomena
+import org.opendc.experiments.sc20.experiment.model.Topology
+import org.opendc.experiments.sc20.experiment.model.Workload
+import org.opendc.harness.dsl.anyOf
+
+/**
+ * A [Portfolio] to perform a simple test run.
+ */
+public class TestPortfolio : Portfolio("test") {
+ override val topology: Topology by anyOf(
+ Topology("base")
+ )
+
+ override val workload: Workload by anyOf(
+ Workload("solvinity", 1.0)
+ )
+
+ override val operationalPhenomena: OperationalPhenomena by anyOf(
+ OperationalPhenomena(failureFrequency = 24.0 * 7, hasInterference = true)
+ )
+
+ override val allocationPolicy: String by anyOf("active-servers")
+}
diff --git a/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/runner/ContainerExperimentDescriptor.kt b/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/runner/ContainerExperimentDescriptor.kt
deleted file mode 100644
index d70e8c9a..00000000
--- a/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/runner/ContainerExperimentDescriptor.kt
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Copyright (c) 2020 AtLarge Research
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to deal
- * in the Software without restriction, including without limitation the rights
- * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- * copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in all
- * copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-
-package org.opendc.experiments.sc20.runner
-
-import kotlinx.coroutines.launch
-import kotlinx.coroutines.supervisorScope
-import org.opendc.experiments.sc20.runner.execution.ExperimentExecutionContext
-import org.opendc.experiments.sc20.runner.execution.ExperimentExecutionResult
-
-/**
- * An abstract [ExperimentDescriptor] specifically for containers.
- */
-public abstract class ContainerExperimentDescriptor : ExperimentDescriptor() {
- /**
- * The child descriptors of this container.
- */
- public abstract val children: Sequence<ExperimentDescriptor>
-
- override val type: Type = Type.CONTAINER
-
- override suspend fun invoke(context: ExperimentExecutionContext) {
- val materializedChildren = children.toList()
- for (child in materializedChildren) {
- context.listener.descriptorRegistered(child)
- }
-
- supervisorScope {
- for (child in materializedChildren) {
- if (child.isTrial) {
- launch {
- val worker = context.scheduler.allocate()
- context.listener.executionStarted(child)
- try {
- worker(child, context)
- context.listener.executionFinished(child, ExperimentExecutionResult.Success)
- } catch (e: Throwable) {
- context.listener.executionFinished(child, ExperimentExecutionResult.Failed(e))
- }
- }
- } else {
- launch { child(context) }
- }
- }
- }
- }
-}
diff --git a/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/runner/ExperimentDescriptor.kt b/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/runner/ExperimentDescriptor.kt
deleted file mode 100644
index 1e67c086..00000000
--- a/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/runner/ExperimentDescriptor.kt
+++ /dev/null
@@ -1,79 +0,0 @@
-/*
- * Copyright (c) 2020 AtLarge Research
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to deal
- * in the Software without restriction, including without limitation the rights
- * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- * copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in all
- * copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-
-package org.opendc.experiments.sc20.runner
-
-import org.opendc.experiments.sc20.runner.execution.ExperimentExecutionContext
-import java.io.Serializable
-
-/**
- * An immutable description of an experiment in the **odcsim* simulation framework, which may be a single atomic trial
- * or a composition of multiple trials.
- *
- * This class represents a dynamic tree-like structure where the children of the nodes are not known at instantiation
- * since they might be generated dynamically.
- */
-public abstract class ExperimentDescriptor : Serializable {
- /**
- * The parent of this descriptor, or `null` if it has no parent.
- */
- public abstract val parent: ExperimentDescriptor?
-
- /**
- * The type of descriptor.
- */
- public abstract val type: Type
-
- /**
- * A flag to indicate that this descriptor is a root descriptor.
- */
- public open val isRoot: Boolean
- get() = parent == null
-
- /**
- * A flag to indicate that this descriptor describes an experiment trial.
- */
- public val isTrial: Boolean
- get() = type == Type.TRIAL
-
- /**
- * Execute this [ExperimentDescriptor].
- *
- * @param context The context to execute the descriptor in.
- */
- public abstract suspend operator fun invoke(context: ExperimentExecutionContext)
-
- /**
- * The types of experiment descriptors.
- */
- public enum class Type {
- /**
- * A composition of multiple experiment descriptions whose invocation happens on a single thread.
- */
- CONTAINER,
-
- /**
- * An invocation of a single scenario of an experiment whose invocation may happen on different threads.
- */
- TRIAL
- }
-}
diff --git a/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/runner/execution/ThreadPoolExperimentScheduler.kt b/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/runner/execution/ThreadPoolExperimentScheduler.kt
deleted file mode 100644
index 942faed1..00000000
--- a/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/runner/execution/ThreadPoolExperimentScheduler.kt
+++ /dev/null
@@ -1,83 +0,0 @@
-/*
- * Copyright (c) 2020 AtLarge Research
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to deal
- * in the Software without restriction, including without limitation the rights
- * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- * copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in all
- * copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-
-package org.opendc.experiments.sc20.runner.execution
-
-import kotlinx.coroutines.asCoroutineDispatcher
-import kotlinx.coroutines.launch
-import kotlinx.coroutines.supervisorScope
-import kotlinx.coroutines.sync.Semaphore
-import kotlinx.coroutines.withContext
-import org.opendc.experiments.sc20.runner.ExperimentDescriptor
-import java.util.concurrent.Executors
-
-/**
- * An [ExperimentScheduler] that runs experiments using a local thread pool.
- *
- * @param parallelism The maximum amount of parallel workers (default is the number of available processors).
- */
-public class ThreadPoolExperimentScheduler(parallelism: Int = Runtime.getRuntime().availableProcessors() + 1) : ExperimentScheduler {
- private val dispatcher = Executors.newCachedThreadPool().asCoroutineDispatcher()
- private val tickets = Semaphore(parallelism)
-
- override suspend fun allocate(): ExperimentScheduler.Worker {
- tickets.acquire()
- return object : ExperimentScheduler.Worker {
- override suspend fun invoke(
- descriptor: ExperimentDescriptor,
- context: ExperimentExecutionContext
- ) = supervisorScope {
- val listener =
- object : ExperimentExecutionListener {
- override fun descriptorRegistered(descriptor: ExperimentDescriptor) {
- launch { context.listener.descriptorRegistered(descriptor) }
- }
-
- override fun executionFinished(
- descriptor: ExperimentDescriptor,
- result: ExperimentExecutionResult
- ) {
- launch { context.listener.executionFinished(descriptor, result) }
- }
-
- override fun executionStarted(descriptor: ExperimentDescriptor) {
- launch { context.listener.executionStarted(descriptor) }
- }
- }
-
- val newContext = object : ExperimentExecutionContext by context {
- override val listener: ExperimentExecutionListener = listener
- }
-
- try {
- withContext(dispatcher) {
- descriptor(newContext)
- }
- } finally {
- tickets.release()
- }
- }
- }
- }
-
- override fun close(): Unit = dispatcher.close()
-}
diff --git a/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/runner/internal/DefaultExperimentRunner.kt b/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/runner/internal/DefaultExperimentRunner.kt
deleted file mode 100644
index 26e4df89..00000000
--- a/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/runner/internal/DefaultExperimentRunner.kt
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- * Copyright (c) 2020 AtLarge Research
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to deal
- * in the Software without restriction, including without limitation the rights
- * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- * copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in all
- * copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-
-package org.opendc.experiments.sc20.runner.internal
-
-import kotlinx.coroutines.runBlocking
-import org.opendc.experiments.sc20.runner.ExperimentDescriptor
-import org.opendc.experiments.sc20.runner.ExperimentRunner
-import org.opendc.experiments.sc20.runner.execution.ExperimentExecutionContext
-import org.opendc.experiments.sc20.runner.execution.ExperimentExecutionListener
-import org.opendc.experiments.sc20.runner.execution.ExperimentExecutionResult
-import org.opendc.experiments.sc20.runner.execution.ExperimentScheduler
-import java.util.concurrent.ConcurrentHashMap
-
-/**
- * The default implementation of the [ExperimentRunner] interface.
- *
- * @param scheduler The scheduler to use.
- */
-public class DefaultExperimentRunner(private val scheduler: ExperimentScheduler) : ExperimentRunner {
- override val id: String = "default"
-
- override val version: String? = "1.0"
-
- override fun execute(root: ExperimentDescriptor, listener: ExperimentExecutionListener): Unit = runBlocking {
- val context = object : ExperimentExecutionContext {
- override val listener: ExperimentExecutionListener = listener
- override val scheduler: ExperimentScheduler = this@DefaultExperimentRunner.scheduler
- override val cache: MutableMap<Any?, Any?> = ConcurrentHashMap()
- }
-
- listener.descriptorRegistered(root)
- context.listener.executionStarted(root)
- try {
- root(context)
- context.listener.executionFinished(root, ExperimentExecutionResult.Success)
- } catch (e: Throwable) {
- context.listener.executionFinished(root, ExperimentExecutionResult.Failed(e))
- }
- }
-}
diff --git a/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/telemetry/RunEvent.kt b/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/telemetry/RunEvent.kt
index 3bcd10a1..4f4706f0 100644
--- a/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/telemetry/RunEvent.kt
+++ b/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/telemetry/RunEvent.kt
@@ -22,12 +22,13 @@
package org.opendc.experiments.sc20.telemetry
-import org.opendc.experiments.sc20.experiment.Run
+import org.opendc.experiments.sc20.experiment.Portfolio
/**
* A periodic report of the host machine metrics.
*/
public data class RunEvent(
- public val run: Run,
+ val portfolio: Portfolio,
+ val repeat: Int,
override val timestamp: Long
) : Event("run")
diff --git a/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/telemetry/parquet/ParquetRunEventWriter.kt b/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/telemetry/parquet/ParquetRunEventWriter.kt
index 74efb660..b50a698c 100644
--- a/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/telemetry/parquet/ParquetRunEventWriter.kt
+++ b/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/telemetry/parquet/ParquetRunEventWriter.kt
@@ -38,33 +38,27 @@ public class ParquetRunEventWriter(path: File, bufferSize: Int) :
public companion object {
private val convert: (RunEvent, GenericData.Record) -> Unit = { event, record ->
- val run = event.run
- val scenario = run.parent
- val portfolio = scenario.parent
- record.put("portfolio_id", portfolio.id)
+ val portfolio = event.portfolio
record.put("portfolio_name", portfolio.name)
- record.put("scenario_id", scenario.id)
- record.put("run_id", run.id)
- record.put("repetitions", scenario.repetitions)
- record.put("topology", scenario.topology.name)
- record.put("workload_name", scenario.workload.name)
- record.put("workload_fraction", scenario.workload.fraction)
- record.put("workload_sampler", scenario.workload.samplingStrategy)
- record.put("allocation_policy", scenario.allocationPolicy)
- record.put("failure_frequency", scenario.operationalPhenomena.failureFrequency)
- record.put("interference", scenario.operationalPhenomena.hasInterference)
- record.put("seed", run.seed)
+ record.put("scenario_id", portfolio.id)
+ record.put("run_id", event.repeat)
+ record.put("topology", portfolio.topology.name)
+ record.put("workload_name", portfolio.workload.name)
+ record.put("workload_fraction", portfolio.workload.fraction)
+ record.put("workload_sampler", portfolio.workload.samplingStrategy)
+ record.put("allocation_policy", portfolio.allocationPolicy)
+ record.put("failure_frequency", portfolio.operationalPhenomena.failureFrequency)
+ record.put("interference", portfolio.operationalPhenomena.hasInterference)
+ record.put("seed", event.repeat)
}
private val schema: Schema = SchemaBuilder
.record("runs")
.namespace("org.opendc.experiments.sc20")
.fields()
- .name("portfolio_id").type().intType().noDefault()
.name("portfolio_name").type().stringType().noDefault()
.name("scenario_id").type().intType().noDefault()
.name("run_id").type().intType().noDefault()
- .name("repetitions").type().intType().noDefault()
.name("topology").type().stringType().noDefault()
.name("workload_name").type().stringType().noDefault()
.name("workload_fraction").type().doubleType().noDefault()
diff --git a/simulator/opendc-harness/build.gradle.kts b/simulator/opendc-harness/build.gradle.kts
new file mode 100644
index 00000000..359d2384
--- /dev/null
+++ b/simulator/opendc-harness/build.gradle.kts
@@ -0,0 +1,47 @@
+/*
+ * Copyright (c) 2020 AtLarge Research
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+description = "Harness for defining repeatable experiments using OpenDC"
+
+/* Build configuration */
+plugins {
+ `kotlin-library-convention`
+}
+
+dependencies {
+ api("org.jetbrains.kotlinx:kotlinx-coroutines-core:${Library.KOTLINX_COROUTINES}")
+ api("org.junit.platform:junit-platform-commons:${Library.JUNIT_PLATFORM}")
+
+ implementation("io.github.classgraph:classgraph:4.8.98")
+ implementation("me.tongfei:progressbar:0.9.0")
+ implementation("io.github.microutils:kotlin-logging:2.0.4")
+ implementation("com.github.ajalt.clikt:clikt:3.1.0")
+
+ api("org.junit.platform:junit-platform-engine:${Library.JUNIT_PLATFORM}")
+ api("org.junit.platform:junit-platform-suite-api:${Library.JUNIT_PLATFORM}")
+ api("org.junit.platform:junit-platform-launcher:${Library.JUNIT_PLATFORM}")
+
+ runtimeOnly("org.apache.logging.log4j:log4j-slf4j-impl:2.14.0")
+
+ testImplementation("org.junit.jupiter:junit-jupiter-api:${Library.JUNIT_JUPITER}")
+ testRuntimeOnly("org.junit.jupiter:junit-jupiter-engine:${Library.JUNIT_JUPITER}")
+}
diff --git a/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/api/ExperimentDefinition.kt b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/api/ExperimentDefinition.kt
new file mode 100644
index 00000000..88b26ee1
--- /dev/null
+++ b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/api/ExperimentDefinition.kt
@@ -0,0 +1,39 @@
+/*
+ * Copyright (c) 2020 AtLarge Research
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+package org.opendc.harness.api
+
+/**
+ * A definition for a repeatable experiment which consists of multiple scenarios derived from pre-defined experiment
+ * parameters.
+ *
+ * @property name The name of the experiment.
+ * @property parameters The parameters of the experiments.
+ * @property evaluator The function to evaluate a single experiment trial.
+ * @property meta The metadata for the experiment.
+ */
+public data class ExperimentDefinition(
+ val name: String,
+ val parameters: Set<Parameter<*>>,
+ val evaluator: (Trial) -> Unit,
+ val meta: Map<String, Any> = emptyMap()
+)
diff --git a/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/runner/execution/ExperimentExecutionResult.kt b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/api/Parameter.kt
index a765c264..bb5c8c2b 100644
--- a/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/runner/execution/ExperimentExecutionResult.kt
+++ b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/api/Parameter.kt
@@ -1,7 +1,5 @@
/*
- * MIT License
- *
- * Copyright (c) 2020 atlarge-research
+ * Copyright (c) 2020 AtLarge Research
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
@@ -22,21 +20,19 @@
* SOFTWARE.
*/
-package org.opendc.experiments.sc20.runner.execution
-
-import java.io.Serializable
+package org.opendc.harness.api
/**
- * The result of executing an experiment.
+ * A [Parameter] defines a single dimension of exploration in the design space of an experiment.
*/
-public sealed class ExperimentExecutionResult : Serializable {
+public sealed class Parameter<T> {
/**
- * The experiment executed successfully
+ * The name of the parameter.
*/
- public object Success : ExperimentExecutionResult()
+ public abstract val name: String
/**
- * The experiment failed during execution.
+ * A generic dimension of the experiment design space that is defined fully by a collection of [values].
*/
- public data class Failed(val throwable: Throwable) : ExperimentExecutionResult()
+ public data class Generic<T>(override val name: String, val values: Collection<T>) : Parameter<T>()
}
diff --git a/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/runner/ExperimentRunner.kt b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/api/Scenario.kt
index a59481c0..a8dbf01e 100644
--- a/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/runner/ExperimentRunner.kt
+++ b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/api/Scenario.kt
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2020 AtLarge Research
+ * Copyright (c) 2021 AtLarge Research
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
@@ -20,30 +20,27 @@
* SOFTWARE.
*/
-package org.opendc.experiments.sc20.runner
-
-import org.opendc.experiments.sc20.runner.execution.ExperimentExecutionListener
+package org.opendc.harness.api
/**
- * An [ExperimentRunner] facilitates discovery and execution of experiments.
+ * A [Scenario] represents a single point in the design space of an experiment.
*/
-public interface ExperimentRunner {
+public interface Scenario {
/**
- * The unique identifier of this runner.
+ * A unique identifier that identifies a single scenario.
*/
- public val id: String
+ public val id: Int
/**
- * The version of this runner.
+ * The [ExperimentDefinition] describing the experiment this scenario is part of.
*/
- public val version: String?
- get() = null
+ public val experiment: ExperimentDefinition
/**
- * Execute the specified experiment represented as [ExperimentDescriptor].
+ * Obtain the instantiated value for a [parameter][param] of the experiment.
*
- * @param root The experiment to execute.
- * @param listener The listener to report events to.
+ * @param param The parameter to obtain the value of.
+ * @throws IllegalArgumentException if [param] is not defined for the experiment.
*/
- public fun execute(root: ExperimentDescriptor, listener: ExperimentExecutionListener)
+ public operator fun <T> get(param: Parameter<T>): T
}
diff --git a/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/runner/TrialExperimentDescriptor.kt b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/api/Trial.kt
index abc52997..2d6ecd19 100644
--- a/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/runner/TrialExperimentDescriptor.kt
+++ b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/api/Trial.kt
@@ -1,7 +1,5 @@
/*
- * MIT License
- *
- * Copyright (c) 2020 atlarge-research
+ * Copyright (c) 2021 AtLarge Research
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
@@ -22,11 +20,9 @@
* SOFTWARE.
*/
-package org.opendc.experiments.sc20.runner
+package org.opendc.harness.api
/**
- * An abstract [ExperimentDescriptor] specifically for trials.
+ * A [Trial] represents a single trial (run) of an experiment.
*/
-public abstract class TrialExperimentDescriptor : ExperimentDescriptor() {
- override val type: Type = Type.TRIAL
-}
+public data class Trial(val scenario: Scenario, val repeat: Int)
diff --git a/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/dsl/Experiment.kt b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/dsl/Experiment.kt
new file mode 100644
index 00000000..41d4207a
--- /dev/null
+++ b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/dsl/Experiment.kt
@@ -0,0 +1,99 @@
+/*
+ * Copyright (c) 2021 AtLarge Research
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+package org.opendc.harness.dsl
+
+import org.junit.platform.commons.annotation.Testable
+import org.opendc.harness.api.ExperimentDefinition
+import org.opendc.harness.api.Scenario
+import org.opendc.harness.api.Trial
+import org.opendc.harness.internal.ParameterDelegate
+
+/**
+ * An [Experiment] defines a blueprint for a repeatable experiment consisting of multiple scenarios based on pre-defined
+ * experiment parameters.
+ *
+ * @param name The name of the experiment or `null` to select the class name.
+ */
+@Testable
+public abstract class Experiment(name: String? = null) : Cloneable {
+ /**
+ * The name of the experiment.
+ */
+ public val name: String = name ?: javaClass.simpleName
+
+ /**
+ * An identifier that uniquely identifies a single point in the design space of this [Experiment].
+ */
+ public val id: Int
+ get() {
+ val scenario = scenario ?: throw IllegalStateException("Cannot use id before activation")
+ return scenario.id
+ }
+
+ /**
+ * Convert this experiment to an [ExperimentDefinition].
+ */
+ public fun toDefinition(): ExperimentDefinition =
+ ExperimentDefinition(name, HashSet(delegates.map { it.parameter }), this::run, mapOf("class.name" to javaClass.name))
+
+ /**
+ * Perform a single execution of the experiment based on the experiment parameters.
+ *
+ * @param repeat A number representing the repeat index of an identical scenario.
+ */
+ protected abstract fun doRun(repeat: Int)
+
+ /**
+ * A map to track the parameter delegates registered with this experiment.
+ */
+ private val delegates: MutableSet<ParameterDelegate<*>> = mutableSetOf()
+
+ /**
+ * The current active scenario.
+ */
+ internal var scenario: Scenario? = null
+
+ /**
+ * Perform a single execution of the experiment based on the experiment parameters.
+ *
+ * This operation will cause the [ParameterProvider]s of this group to be instantiated to some value in its design
+ * space.
+ *
+ * @param trial The experiment trial to run.
+ */
+ private fun run(trial: Trial) {
+ val scenario = trial.scenario
+
+ // XXX We clone the current class to prevent concurrency issues.
+ val res = clone() as Experiment
+ res.scenario = scenario
+ res.doRun(trial.repeat)
+ }
+
+ /**
+ * Register a delegate for this experiment.
+ */
+ internal fun register(delegate: ParameterDelegate<*>) {
+ delegates += delegate
+ }
+}
diff --git a/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/dsl/ParameterProvider.kt b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/dsl/ParameterProvider.kt
new file mode 100644
index 00000000..e4bb9c64
--- /dev/null
+++ b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/dsl/ParameterProvider.kt
@@ -0,0 +1,39 @@
+/*
+ * Copyright (c) 2021 AtLarge Research
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+package org.opendc.harness.dsl
+
+import kotlin.properties.ReadOnlyProperty
+import kotlin.reflect.KProperty
+
+/**
+ * A [ParameterProvider] defines a single dimension of exploration in the design space of an [Experiment].
+ */
+public interface ParameterProvider<T> {
+ /**
+ * Provide a delegate defining a parameter for the specified [Experiment][experiment].
+ *
+ * @param experiment The experiment for which the parameter is defined.
+ * @param prop The property to create the delegate for.
+ */
+ public operator fun provideDelegate(experiment: Experiment, prop: KProperty<*>): ReadOnlyProperty<Experiment, T>
+}
diff --git a/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/dsl/Parameters.kt b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/dsl/Parameters.kt
new file mode 100644
index 00000000..7d269ba1
--- /dev/null
+++ b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/dsl/Parameters.kt
@@ -0,0 +1,44 @@
+/*
+ * Copyright (c) 2021 AtLarge Research
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+package org.opendc.harness.dsl
+
+import org.opendc.harness.api.Parameter
+import org.opendc.harness.internal.ParameterDelegate
+import kotlin.properties.ReadOnlyProperty
+import kotlin.reflect.KProperty
+
+/**
+ * Define a dimension in the design space of an [Experiment] of type [T] consisting of the specified collection of
+ * [values].
+ *
+ * @param values The values in the dimension to define..
+ */
+public fun <T> anyOf(vararg values: T): ParameterProvider<T> = object : ParameterProvider<T> {
+ override fun provideDelegate(experiment: Experiment, prop: KProperty<*>): ReadOnlyProperty<Experiment, T> {
+ val delegate = ParameterDelegate(Parameter.Generic(prop.name, listOf(*values)))
+ experiment.register(delegate)
+ return delegate
+ }
+
+ override fun toString(): String = "GenericParameter"
+}
diff --git a/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/ExperimentEngine.kt b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/ExperimentEngine.kt
new file mode 100644
index 00000000..db2cd191
--- /dev/null
+++ b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/ExperimentEngine.kt
@@ -0,0 +1,107 @@
+/*
+ * Copyright (c) 2021 AtLarge Research
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+package org.opendc.harness.engine
+
+import kotlinx.coroutines.CancellationException
+import kotlinx.coroutines.flow.asFlow
+import kotlinx.coroutines.flow.buffer
+import kotlinx.coroutines.flow.collect
+import kotlinx.coroutines.flow.map
+import kotlinx.coroutines.launch
+import kotlinx.coroutines.supervisorScope
+import org.opendc.harness.api.ExperimentDefinition
+import org.opendc.harness.api.Trial
+import org.opendc.harness.engine.scheduler.ExperimentScheduler
+import org.opendc.harness.engine.strategy.ExperimentStrategy
+
+/**
+ * The [ExperimentEngine] orchestrates the execution of experiments.
+ *
+ * @property strategy The [ExperimentStrategy] used to explore the experiment design space.
+ * @property scheduler The [ExperimentScheduler] to schedule the trials over compute resources.
+ * @property listener The [ExperimentExecutionListener] to observe the progress.
+ * @property repeats The number of repeats to perform.
+ */
+public class ExperimentEngine(
+ private val strategy: ExperimentStrategy,
+ private val scheduler: ExperimentScheduler,
+ private val listener: ExperimentExecutionListener,
+ private val repeats: Int
+) {
+ /**
+ * Execute the specified [experiment][root].
+ *
+ * @param root The experiment to execute.
+ */
+ public suspend fun execute(root: ExperimentDefinition): Unit = supervisorScope {
+ listener.experimentStarted(root)
+
+ try {
+ strategy.generate(root)
+ .asFlow()
+ .map { scenario ->
+ listener.scenarioStarted(scenario)
+ scenario
+ }
+ .buffer(100)
+ .collect { scenario ->
+ val jobs = (0 until repeats).map { repeat ->
+ val worker = scheduler.allocate()
+ launch {
+ val trial = Trial(scenario, repeat)
+ try {
+ listener.trialStarted(trial)
+ worker.dispatch(trial)
+ listener.trialFinished(trial, null)
+ } catch (e: Throwable) {
+ listener.trialFinished(trial, e)
+ throw e
+ }
+ }
+ }
+
+ launch {
+ var error: Throwable? = null
+ for (job in jobs) {
+ try {
+ job.join()
+ } catch (e: CancellationException) {
+ // Propagate cancellation
+ throw e
+ } catch (e: Throwable) {
+ error = e
+ }
+ }
+
+ listener.scenarioFinished(scenario, error)
+ }
+ }
+ listener.experimentFinished(root, null)
+ } catch (e: Throwable) {
+ listener.experimentFinished(root, e)
+ throw e
+ }
+ }
+
+ override fun toString(): String = "ExperimentEngine"
+}
diff --git a/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/ExperimentEngineLauncher.kt b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/ExperimentEngineLauncher.kt
new file mode 100644
index 00000000..ddd30483
--- /dev/null
+++ b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/ExperimentEngineLauncher.kt
@@ -0,0 +1,121 @@
+/*
+ * Copyright (c) 2021 AtLarge Research
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+package org.opendc.harness.engine
+
+import kotlinx.coroutines.flow.Flow
+import kotlinx.coroutines.flow.collect
+import kotlinx.coroutines.runBlocking
+import org.opendc.harness.api.ExperimentDefinition
+import org.opendc.harness.engine.scheduler.ExperimentScheduler
+import org.opendc.harness.engine.scheduler.ThreadPoolExperimentScheduler
+import org.opendc.harness.engine.strategy.CartesianExperimentStrategy
+import org.opendc.harness.engine.strategy.ExperimentStrategy
+import org.opendc.harness.internal.CompositeExperimentExecutionListener
+
+/**
+ * A builder class for conducting experiments via the [ExperimentEngine].
+ */
+public class ExperimentEngineLauncher private constructor(
+ private val strategy: ExperimentStrategy?,
+ private val scheduler: ExperimentScheduler?,
+ private val listeners: List<ExperimentExecutionListener>,
+ private val repeats: Int
+) {
+ /**
+ * Construct an [ExperimentEngineLauncher] instance.
+ */
+ public constructor() : this(null, null, emptyList(), 1)
+
+ /**
+ * Create an [ExperimentEngineLauncher] with the specified [strategy].
+ */
+ public fun withScheduler(strategy: ExperimentStrategy): ExperimentEngineLauncher {
+ return ExperimentEngineLauncher(strategy, scheduler, listeners, repeats)
+ }
+
+ /**
+ * Create an [ExperimentEngineLauncher] with the specified [scheduler].
+ */
+ public fun withScheduler(scheduler: ExperimentScheduler): ExperimentEngineLauncher {
+ return ExperimentEngineLauncher(strategy, scheduler, listeners, repeats)
+ }
+
+ /**
+ * Create an [ExperimentEngineLauncher] with the specified [listener] added.
+ */
+ public fun withListener(listener: ExperimentExecutionListener): ExperimentEngineLauncher {
+ return ExperimentEngineLauncher(strategy, scheduler, listeners + listener, repeats)
+ }
+
+ /**
+ * Create an [ExperimentEngineLauncher] with the specified number of repeats.
+ */
+ public fun withRepeats(repeats: Int): ExperimentEngineLauncher {
+ require(repeats > 0) { "Invalid number of repeats; must be greater than zero. " }
+ return ExperimentEngineLauncher(strategy, scheduler, listeners, repeats)
+ }
+
+ /**
+ * Launch the specified experiments via the [ExperimentEngine] and block execution until finished.
+ */
+ public suspend fun run(experiments: Flow<ExperimentDefinition>) {
+ val engine = ExperimentEngine(createStrategy(), createScheduler(), createListener(), repeats)
+ experiments.collect { experiment -> engine.execute(experiment) }
+ }
+
+ /**
+ * Launch the specified experiments via the [ExperimentEngine] and block the current thread until finished.
+ */
+ public fun runBlocking(experiments: Flow<ExperimentDefinition>) {
+ runBlocking {
+ run(experiments)
+ }
+ }
+
+ /**
+ * Return a string representation of this instance.
+ */
+ public override fun toString(): String = "ExperimentEngineLauncher"
+
+ /**
+ * Create the [ExperimentStrategy] that explores the experiment design space.
+ */
+ private fun createStrategy(): ExperimentStrategy {
+ return strategy ?: CartesianExperimentStrategy
+ }
+
+ /**
+ * Create the [ExperimentScheduler] that schedules the trials over the compute resources.
+ */
+ private fun createScheduler(): ExperimentScheduler {
+ return scheduler ?: ThreadPoolExperimentScheduler(Runtime.getRuntime().availableProcessors())
+ }
+
+ /**
+ * Create the [ExperimentExecutionListener] that listens the to the execution of the experiments.
+ */
+ private fun createListener(): ExperimentExecutionListener {
+ require(listeners.isNotEmpty()) { "No listeners registered." }
+ return CompositeExperimentExecutionListener(listeners)
+ }
+}
diff --git a/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/ExperimentExecutionListener.kt b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/ExperimentExecutionListener.kt
new file mode 100644
index 00000000..9ef71863
--- /dev/null
+++ b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/ExperimentExecutionListener.kt
@@ -0,0 +1,77 @@
+/*
+ * Copyright (c) 2021 AtLarge Research
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+package org.opendc.harness.engine
+
+import org.opendc.harness.api.ExperimentDefinition
+import org.opendc.harness.api.Scenario
+import org.opendc.harness.api.Trial
+
+/**
+ * Listener to be notified of experiment execution events by experiment runners.
+ */
+public interface ExperimentExecutionListener {
+ /**
+ * A method that is invoked when an experiment is started.
+ *
+ * @param experiment The [ExperimentDefinition] that started.
+ */
+ public fun experimentStarted(experiment: ExperimentDefinition) {}
+
+ /**
+ * A method that is invoked when an experiment is finished, regardless of the outcome.
+ *
+ * @param experiment The [ExperimentDefinition] that finished.
+ * @param throwable The exception that was thrown during execution or `null` if the execution completed successfully.
+ */
+ public fun experimentFinished(experiment: ExperimentDefinition, throwable: Throwable?) {}
+
+ /**
+ * A method that is invoked when a scenario is started.
+ *
+ * @param scenario The scenario that is started.
+ */
+ public fun scenarioStarted(scenario: Scenario) {}
+
+ /**
+ * A method that is invoked when a scenario is finished, regardless of the outcome.
+ *
+ * @param scenario The [Scenario] that has finished.
+ * @param throwable The exception that was thrown during execution or `null` if the execution completed successfully.
+ */
+ public fun scenarioFinished(scenario: Scenario, throwable: Throwable?) {}
+
+ /**
+ * A method that is invoked when a trial is started.
+ *
+ * @param trial The trial that is started.
+ */
+ public fun trialStarted(trial: Trial) {}
+
+ /**
+ * A method that is invoked when a scenario is finished, regardless of the outcome.
+ *
+ * @param trial The [Trial] that has finished.
+ * @param throwable The exception that was thrown during execution or `null` if the execution completed successfully.
+ */
+ public fun trialFinished(trial: Trial, throwable: Throwable?) {}
+}
diff --git a/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/discovery/Discovery.kt b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/discovery/Discovery.kt
new file mode 100644
index 00000000..f7f73b38
--- /dev/null
+++ b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/discovery/Discovery.kt
@@ -0,0 +1,39 @@
+/*
+ * Copyright (c) 2021 AtLarge Research
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+package org.opendc.harness.engine.discovery
+
+import kotlinx.coroutines.flow.Flow
+import org.opendc.harness.api.ExperimentDefinition
+
+/**
+ * Component responsible for scanning for [ExperimentDefinition]s.
+ */
+public interface Discovery {
+ /**
+ * Start discovery of experiments.
+ *
+ * @param request The [DiscoveryRequest] to determine the experiments to discover.
+ * @return A flow of [ExperimentDefinition]s that have been discovered by the implementation.
+ */
+ public fun discover(request: DiscoveryRequest): Flow<ExperimentDefinition>
+}
diff --git a/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/discovery/DiscoveryFilter.kt b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/discovery/DiscoveryFilter.kt
new file mode 100644
index 00000000..219d09cd
--- /dev/null
+++ b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/discovery/DiscoveryFilter.kt
@@ -0,0 +1,51 @@
+/*
+ * Copyright (c) 2021 AtLarge Research
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+package org.opendc.harness.engine.discovery
+
+import org.opendc.harness.api.ExperimentDefinition
+import java.util.function.Predicate
+
+/**
+ * A [DiscoveryFilter] decides how the selected experiments are filtered.
+ */
+public sealed class DiscoveryFilter {
+ /**
+ * Test whether the specified [ExperimentDefinition] should be selected.
+ */
+ public abstract fun test(definition: ExperimentDefinition): Boolean
+
+ /**
+ * Filter an experiment based on its name.
+ */
+ public data class Name(val predicate: Predicate<String>) : DiscoveryFilter() {
+ override fun test(definition: ExperimentDefinition): Boolean = predicate.test(definition.name)
+ }
+
+ /**
+ * Filter an experiment based on its metadata.
+ */
+ public data class Meta(val key: String, val predicate: Predicate<Any>) : DiscoveryFilter() {
+ override fun test(definition: ExperimentDefinition): Boolean =
+ definition.meta[key]?.let { predicate.test(it) } ?: false
+ }
+}
diff --git a/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/discovery/DiscoveryProvider.kt b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/discovery/DiscoveryProvider.kt
new file mode 100644
index 00000000..fad255de
--- /dev/null
+++ b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/discovery/DiscoveryProvider.kt
@@ -0,0 +1,65 @@
+/*
+ * Copyright (c) 2021 AtLarge Research
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+package org.opendc.harness.engine.discovery
+
+import org.opendc.harness.internal.CompositeDiscovery
+import java.util.*
+
+/**
+ * A provider interface for the [Discovery] component.
+ */
+public interface DiscoveryProvider {
+ /**
+ * A unique identifier for this discovery implementation.
+ *
+ * Each discovery implementation must provide a unique ID, so that they can be selected by the user.
+ * When in doubt, you may use the fully qualified name of your custom [Discovery] implementation class.
+ */
+ public val id: String
+
+ /**
+ * Factory method for creating a new [Discovery] instance.
+ */
+ public fun create(): Discovery
+
+ public companion object {
+ /**
+ * The available [DiscoveryProvider]s.
+ */
+ private val providers by lazy { ServiceLoader.load(DiscoveryProvider::class.java) }
+
+ /**
+ * Obtain the [DiscoveryProvider] with the specified [id] or return `null`.
+ */
+ public fun findById(id: String): DiscoveryProvider? {
+ return providers.find { it.id == id }
+ }
+
+ /**
+ * Obtain a composite [Discovery] that combines the results of all available providers.
+ */
+ public fun createComposite(): Discovery {
+ return CompositeDiscovery(providers)
+ }
+ }
+}
diff --git a/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/discovery/DiscoveryRequest.kt b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/discovery/DiscoveryRequest.kt
new file mode 100644
index 00000000..5bc08dac
--- /dev/null
+++ b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/discovery/DiscoveryRequest.kt
@@ -0,0 +1,34 @@
+/*
+ * Copyright (c) 2021 AtLarge Research
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+package org.opendc.harness.engine.discovery
+
+/**
+ * A request for discovering experiments according to the specified information.
+ *
+ * @param selectors The selectors for this discovery request.
+ * @param filters The filters for this discovery request.
+ */
+public data class DiscoveryRequest(
+ val selectors: List<DiscoverySelector> = emptyList(),
+ val filters: List<DiscoveryFilter> = emptyList(),
+)
diff --git a/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/runner/execution/ExperimentExecutionListener.kt b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/discovery/DiscoverySelector.kt
index 42fef164..67681303 100644
--- a/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/runner/execution/ExperimentExecutionListener.kt
+++ b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/discovery/DiscoverySelector.kt
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2020 AtLarge Research
+ * Copyright (c) 2021 AtLarge Research
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
@@ -20,27 +20,30 @@
* SOFTWARE.
*/
-package org.opendc.experiments.sc20.runner.execution
+package org.opendc.harness.engine.discovery
-import org.opendc.experiments.sc20.runner.ExperimentDescriptor
+import org.opendc.harness.api.ExperimentDefinition
/**
- * Listener to be notified of experiment execution events by experiment runners.
+ * A [DiscoverySelector] defines the properties used to discover experiments.
*/
-public interface ExperimentExecutionListener {
+public sealed class DiscoverySelector {
/**
- * A method that is invoked when a new [ExperimentDescriptor] is registered.
+ * Test whether the specified [ExperimentDefinition] should be selected.
*/
- public fun descriptorRegistered(descriptor: ExperimentDescriptor)
+ public abstract fun test(definition: ExperimentDefinition): Boolean
/**
- * A method that is invoked when when the execution of a leaf or subtree of the experiment tree has finished,
- * regardless of the outcome.
+ * Select an experiment based on its name.
*/
- public fun executionFinished(descriptor: ExperimentDescriptor, result: ExperimentExecutionResult)
+ public data class Name(val name: String) : DiscoverySelector() {
+ override fun test(definition: ExperimentDefinition): Boolean = definition.name == name
+ }
/**
- * A method that is invoked when the execution of a leaf or subtree of the experiment tree is about to be started.
+ * Select an experiment based on its metadata.
*/
- public fun executionStarted(descriptor: ExperimentDescriptor)
+ public data class Meta(val key: String, val value: Any) : DiscoverySelector() {
+ override fun test(definition: ExperimentDefinition): Boolean = definition.meta[key] == value
+ }
}
diff --git a/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/runner/execution/ExperimentScheduler.kt b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/scheduler/ExperimentScheduler.kt
index 70095ccd..0265554a 100644
--- a/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/runner/execution/ExperimentScheduler.kt
+++ b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/scheduler/ExperimentScheduler.kt
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2020 AtLarge Research
+ * Copyright (c) 2021 AtLarge Research
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
@@ -20,37 +20,33 @@
* SOFTWARE.
*/
-package org.opendc.experiments.sc20.runner.execution
+package org.opendc.harness.engine.scheduler
-import org.opendc.experiments.sc20.runner.ExperimentDescriptor
-import java.io.Closeable
+import org.opendc.harness.api.Trial
/**
- * A interface for scheduling the execution of experiment trials over compute resources (threads/containers/vms)
+ * The [ExperimentScheduler] is responsible for scheduling the execution of experiment runs over some set of compute
+ * resources (e.g., threads or even multiple machines).
*/
-public interface ExperimentScheduler : Closeable {
+public interface ExperimentScheduler : AutoCloseable {
/**
* Allocate a [Worker] for executing an experiment trial. This method may suspend in case no resources are directly
* available at the moment.
*
* @return The available worker.
*/
- public suspend fun allocate(): ExperimentScheduler.Worker
+ public suspend fun allocate(): Worker
/**
- * An isolated worker of an [ExperimentScheduler] that is responsible for executing a single experiment trial.
+ * An isolated worker of an [ExperimentScheduler] that is responsible for conducting a single experiment trial.
*/
public interface Worker {
/**
- * Dispatch the specified [ExperimentDescriptor] to execute some time in the future and return the results of
- * the trial.
+ * Dispatch an experiment trial immediately to one of the available compute resources and block execution until
+ * the trial has finished.
*
- * @param descriptor The descriptor to execute.
- * @param context The context to execute the descriptor in.
+ * @param trial The trial to dispatch.
*/
- public suspend operator fun invoke(
- descriptor: ExperimentDescriptor,
- context: ExperimentExecutionContext
- )
+ public suspend fun dispatch(trial: Trial)
}
}
diff --git a/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/scheduler/ExperimentSchedulerProvider.kt b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/scheduler/ExperimentSchedulerProvider.kt
new file mode 100644
index 00000000..a93d4bf6
--- /dev/null
+++ b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/scheduler/ExperimentSchedulerProvider.kt
@@ -0,0 +1,57 @@
+/*
+ * Copyright (c) 2021 AtLarge Research
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+package org.opendc.harness.engine.scheduler
+
+import java.util.*
+
+/**
+ * A factory for constructing an [ExperimentScheduler].
+ */
+public interface ExperimentSchedulerProvider {
+ /**
+ * A unique identifier for this scheduler implementation.
+ *
+ * Each experiment scheduler must provide a unique ID, so that they can be selected by the user.
+ * When in doubt, you may use the fully qualified name of your custom [ExperimentScheduler] implementation class.
+ */
+ public val id: String
+
+ /**
+ * Factory method for creating a new [ExperimentScheduler] instance.
+ */
+ public fun create(): ExperimentScheduler
+
+ public companion object {
+ /**
+ * The available [ExperimentSchedulerProvider]s.
+ */
+ private val providers by lazy { ServiceLoader.load(ExperimentSchedulerProvider::class.java) }
+
+ /**
+ * Obtain the [ExperimentScheduler] with the specified [id] or return `null`.
+ */
+ public fun findById(id: String): ExperimentSchedulerProvider? {
+ return providers.find { it.id == id }
+ }
+ }
+}
diff --git a/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/scheduler/ThreadPoolExperimentScheduler.kt b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/scheduler/ThreadPoolExperimentScheduler.kt
new file mode 100644
index 00000000..1ae533cf
--- /dev/null
+++ b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/scheduler/ThreadPoolExperimentScheduler.kt
@@ -0,0 +1,58 @@
+/*
+ * Copyright (c) 2021 AtLarge Research
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+package org.opendc.harness.engine.scheduler
+
+import kotlinx.coroutines.asCoroutineDispatcher
+import kotlinx.coroutines.sync.Semaphore
+import kotlinx.coroutines.withContext
+import org.opendc.harness.api.Trial
+import java.util.concurrent.Executors
+
+/**
+ * An [ExperimentScheduler] that runs experiment trials using a local thread pool.
+ *
+ * @param parallelism The maximum amount of concurrent workers.
+ */
+public class ThreadPoolExperimentScheduler(parallelism: Int) : ExperimentScheduler {
+ private val dispatcher = Executors.newCachedThreadPool().asCoroutineDispatcher()
+ private val tickets = Semaphore(parallelism)
+
+ override suspend fun allocate(): ExperimentScheduler.Worker {
+ tickets.acquire()
+ return object : ExperimentScheduler.Worker {
+ override suspend fun dispatch(trial: Trial) {
+ try {
+ withContext(dispatcher) {
+ trial.scenario.experiment.evaluator(trial)
+ }
+ } finally {
+ tickets.release()
+ }
+ }
+ }
+ }
+
+ override fun close(): Unit = dispatcher.close()
+
+ override fun toString(): String = "ThreadPoolScheduler"
+}
diff --git a/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/scheduler/ThreadPoolExperimentSchedulerProvider.kt b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/scheduler/ThreadPoolExperimentSchedulerProvider.kt
new file mode 100644
index 00000000..cf9a132f
--- /dev/null
+++ b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/scheduler/ThreadPoolExperimentSchedulerProvider.kt
@@ -0,0 +1,33 @@
+/*
+ * Copyright (c) 2021 AtLarge Research
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+package org.opendc.harness.engine.scheduler
+
+/**
+ * An [ExperimentSchedulerProvider] for constructing a [ThreadPoolExperimentScheduler].
+ */
+public class ThreadPoolExperimentSchedulerProvider : ExperimentSchedulerProvider {
+ override val id: String = "thread-pool"
+
+ override fun create(): ExperimentScheduler =
+ ThreadPoolExperimentScheduler(Runtime.getRuntime().availableProcessors())
+}
diff --git a/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/strategy/CartesianExperimentStrategy.kt b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/strategy/CartesianExperimentStrategy.kt
new file mode 100644
index 00000000..e5e08003
--- /dev/null
+++ b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/strategy/CartesianExperimentStrategy.kt
@@ -0,0 +1,55 @@
+/*
+ * Copyright (c) 2021 AtLarge Research
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+package org.opendc.harness.engine.strategy
+
+import org.opendc.harness.api.ExperimentDefinition
+import org.opendc.harness.api.Parameter
+import org.opendc.harness.api.Scenario
+import org.opendc.harness.internal.ScenarioImpl
+
+/**
+ * An [ExperimentStrategy] that takes the cartesian product of the parameters and evaluates every combination.
+ */
+public object CartesianExperimentStrategy : ExperimentStrategy {
+ /**
+ * Build the trials of an experiment.
+ */
+ override fun generate(experiment: ExperimentDefinition): Sequence<Scenario> {
+ return experiment.parameters
+ .asSequence()
+ .map { param -> mapParameter(param).map { value -> listOf(param to value) } }
+ .reduce { acc, param ->
+ acc.flatMap { x -> param.map { y -> x + y } }
+ }
+ .mapIndexed { id, values -> ScenarioImpl(id, experiment, values.toMap()) }
+ }
+
+ /**
+ * Instantiate a parameter and return a sequence of possible values.
+ */
+ private fun <T> mapParameter(param: Parameter<T>): Sequence<T> {
+ return when (param) {
+ is Parameter.Generic<T> -> param.values.asSequence()
+ }
+ }
+}
diff --git a/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/strategy/CartesianExperimentStrategyProvider.kt b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/strategy/CartesianExperimentStrategyProvider.kt
new file mode 100644
index 00000000..f18795a3
--- /dev/null
+++ b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/strategy/CartesianExperimentStrategyProvider.kt
@@ -0,0 +1,32 @@
+/*
+ * Copyright (c) 2021 AtLarge Research
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+package org.opendc.harness.engine.strategy
+
+/**
+ * An [ExperimentStrategyProvider] for constructing a [CartesianExperimentStrategy].
+ */
+public class CartesianExperimentStrategyProvider : ExperimentStrategyProvider {
+ override val id: String = "cartesian"
+
+ override fun create(): ExperimentStrategy = CartesianExperimentStrategy
+}
diff --git a/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/strategy/ExperimentStrategy.kt b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/strategy/ExperimentStrategy.kt
new file mode 100644
index 00000000..3a0148ad
--- /dev/null
+++ b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/strategy/ExperimentStrategy.kt
@@ -0,0 +1,40 @@
+/*
+ * Copyright (c) 2021 AtLarge Research
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+package org.opendc.harness.engine.strategy
+
+import org.opendc.harness.api.ExperimentDefinition
+import org.opendc.harness.api.Scenario
+
+/**
+ * The [ExperimentStrategy] is responsible for traversing the design space of an [ExperimentDefinition] based on its
+ * parameters, generating concrete points in the space represented as [Scenario]s.
+ */
+public interface ExperimentStrategy {
+ /**
+ * Generate the points in the design space of the specified [experiment] to explore.
+ *
+ * @param experiment The experiment design space to explore.
+ * @return A sequence of [Scenario]s which may be explored by the [ExperimentEngine].
+ */
+ public fun generate(experiment: ExperimentDefinition): Sequence<Scenario>
+}
diff --git a/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/strategy/ExperimentStrategyProvider.kt b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/strategy/ExperimentStrategyProvider.kt
new file mode 100644
index 00000000..7fa05f34
--- /dev/null
+++ b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/strategy/ExperimentStrategyProvider.kt
@@ -0,0 +1,57 @@
+/*
+ * Copyright (c) 2021 AtLarge Research
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+package org.opendc.harness.engine.strategy
+
+import java.util.*
+
+/**
+ * A factory for constructing an [ExperimentStrategy].
+ */
+public interface ExperimentStrategyProvider {
+ /**
+ * A unique identifier for this strategy implementation.
+ *
+ * Each experiment strategy must provide a unique ID, so that they can be selected by the user.
+ * When in doubt, you may use the fully qualified name of your custom [ExperimentStrategy] implementation class.
+ */
+ public val id: String
+
+ /**
+ * Factory method for creating a new [ExperimentStrategy] instance.
+ */
+ public fun create(): ExperimentStrategy
+
+ public companion object {
+ /**
+ * The available [ExperimentStrategyProvider]s.
+ */
+ private val providers by lazy { ServiceLoader.load(ExperimentStrategyProvider::class.java) }
+
+ /**
+ * Obtain the [ExperimentStrategy] with the specified [id] or return `null`.
+ */
+ public fun findById(id: String): ExperimentStrategyProvider? {
+ return providers.find { it.id == id }
+ }
+ }
+}
diff --git a/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/internal/CompositeDiscovery.kt b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/internal/CompositeDiscovery.kt
new file mode 100644
index 00000000..67a895e4
--- /dev/null
+++ b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/internal/CompositeDiscovery.kt
@@ -0,0 +1,47 @@
+/*
+ * Copyright (c) 2021 AtLarge Research
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+package org.opendc.harness.internal
+
+import kotlinx.coroutines.FlowPreview
+import kotlinx.coroutines.flow.*
+import org.opendc.harness.api.ExperimentDefinition
+import org.opendc.harness.engine.discovery.Discovery
+import org.opendc.harness.engine.discovery.DiscoveryProvider
+import org.opendc.harness.engine.discovery.DiscoveryRequest
+
+/**
+ * A composite [Discovery] instance that combines the results of multiple delegate instances.
+ */
+internal class CompositeDiscovery(providers: Iterable<DiscoveryProvider>) : Discovery {
+ /**
+ * The [Discovery] instances to delegate to.
+ */
+ private val delegates = providers.map { it.create() }
+
+ @OptIn(FlowPreview::class)
+ override fun discover(request: DiscoveryRequest): Flow<ExperimentDefinition> {
+ return delegates.asFlow()
+ .map { it.discover(request) }
+ .flattenMerge(delegates.size)
+ }
+}
diff --git a/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/internal/CompositeExperimentExecutionListener.kt b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/internal/CompositeExperimentExecutionListener.kt
new file mode 100644
index 00000000..a3cd6bd2
--- /dev/null
+++ b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/internal/CompositeExperimentExecutionListener.kt
@@ -0,0 +1,57 @@
+/*
+ * Copyright (c) 2021 AtLarge Research
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+package org.opendc.harness.internal
+
+import org.opendc.harness.api.ExperimentDefinition
+import org.opendc.harness.api.Scenario
+import org.opendc.harness.api.Trial
+import org.opendc.harness.engine.ExperimentExecutionListener
+
+/**
+ * An [ExperimentExecutionListener] that composes multiple other listeners.
+ */
+public class CompositeExperimentExecutionListener(private val listeners: List<ExperimentExecutionListener>) : ExperimentExecutionListener {
+ override fun experimentStarted(experiment: ExperimentDefinition) {
+ listeners.forEach { it.experimentStarted(experiment) }
+ }
+
+ override fun experimentFinished(experiment: ExperimentDefinition, throwable: Throwable?) {
+ listeners.forEach { it.experimentFinished(experiment, throwable) }
+ }
+
+ override fun scenarioStarted(scenario: Scenario) {
+ listeners.forEach { it.scenarioStarted(scenario) }
+ }
+
+ override fun scenarioFinished(scenario: Scenario, throwable: Throwable?) {
+ listeners.forEach { it.scenarioFinished(scenario, throwable) }
+ }
+
+ override fun trialStarted(trial: Trial) {
+ listeners.forEach { it.trialStarted(trial) }
+ }
+
+ override fun trialFinished(trial: Trial, throwable: Throwable?) {
+ listeners.forEach { it.trialFinished(trial, throwable) }
+ }
+}
diff --git a/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/internal/DslDiscovery.kt b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/internal/DslDiscovery.kt
new file mode 100644
index 00000000..eb6303d6
--- /dev/null
+++ b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/internal/DslDiscovery.kt
@@ -0,0 +1,101 @@
+/*
+ * Copyright (c) 2021 AtLarge Research
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+package org.opendc.harness.internal
+
+import io.github.classgraph.ClassGraph
+import io.github.classgraph.ScanResult
+import kotlinx.coroutines.flow.Flow
+import kotlinx.coroutines.flow.asFlow
+import org.opendc.harness.api.ExperimentDefinition
+import org.opendc.harness.dsl.Experiment
+import org.opendc.harness.engine.discovery.Discovery
+import org.opendc.harness.engine.discovery.DiscoveryFilter
+import org.opendc.harness.engine.discovery.DiscoveryRequest
+import org.opendc.harness.engine.discovery.DiscoverySelector
+
+/**
+ * A [Discovery] implementation that discovers [Experiment] instances on the classpath.
+ */
+internal class DslDiscovery : Discovery {
+ /*
+ * Lazily memoize the results of the classpath scan.
+ */
+ private val scanResult by lazy { scan() }
+
+ override fun discover(request: DiscoveryRequest): Flow<ExperimentDefinition> {
+ return findExperiments()
+ .map { cls ->
+ val exp = cls.constructors[0].newInstance() as Experiment
+ exp.toDefinition()
+ }
+ .filter(select(request.selectors))
+ .filter(filter(request.filters))
+ .asFlow()
+ }
+
+ /**
+ * Find the classes on the classpath implementing the [Experiment] class.
+ */
+ private fun findExperiments(): Sequence<Class<out Experiment>> {
+ return scanResult
+ .getSubclasses(Experiment::class.java.name)
+ .filter { !(it.isAbstract || it.isInterface) }
+ .map { it.loadClass() }
+ .filterIsInstance<Class<out Experiment>>()
+ .asSequence()
+ }
+
+ /**
+ * Create a predicate for filtering the experiments based on the specified [filters].
+ */
+ private fun filter(filters: List<DiscoveryFilter>): (ExperimentDefinition) -> Boolean = { def ->
+ filters.isEmpty() || filters.all { it.test(def) }
+ }
+
+ /**
+ * Create a predicate for selecting the experiments based on the specified [selectors].
+ */
+ private fun select(selectors: List<DiscoverySelector>): (ExperimentDefinition) -> Boolean = { def ->
+ selectors.isEmpty() || selectors.any { it.test(def) }
+ }
+
+ /**
+ * Scan the classpath using [ClassGraph].
+ */
+ private fun scan(): ScanResult {
+ return ClassGraph()
+ .enableClassInfo()
+ .enableExternalClasses()
+ .ignoreClassVisibility()
+ .rejectPackages(
+ "java.*",
+ "javax.*",
+ "sun.*",
+ "com.sun.*",
+ "kotlin.*",
+ "androidx.*",
+ "org.jetbrains.kotlin.*",
+ "org.junit.*"
+ ).scan()
+ }
+}
diff --git a/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/runner/execution/ExperimentExecutionContext.kt b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/internal/DslDiscoveryProvider.kt
index 942eb891..752ba4bb 100644
--- a/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/runner/execution/ExperimentExecutionContext.kt
+++ b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/internal/DslDiscoveryProvider.kt
@@ -1,7 +1,5 @@
/*
- * MIT License
- *
- * Copyright (c) 2020 atlarge-research
+ * Copyright (c) 2021 AtLarge Research
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
@@ -22,24 +20,17 @@
* SOFTWARE.
*/
-package org.opendc.experiments.sc20.runner.execution
+package org.opendc.harness.internal
+
+import org.opendc.harness.dsl.Experiment
+import org.opendc.harness.engine.discovery.Discovery
+import org.opendc.harness.engine.discovery.DiscoveryProvider
/**
- * The execution context of an experiment.
+ * A [DiscoveryProvider] for the [Experiment]s on the classpath.
*/
-public interface ExperimentExecutionContext {
- /**
- * The execution listener to use.
- */
- public val listener: ExperimentExecutionListener
-
- /**
- * The experiment scheduler to use.
- */
- public val scheduler: ExperimentScheduler
+public class DslDiscoveryProvider : DiscoveryProvider {
+ override val id: String = "dsl"
- /**
- * A cache for objects within a single runner.
- */
- public val cache: MutableMap<Any?, Any?>
+ override fun create(): Discovery = DslDiscovery()
}
diff --git a/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/internal/ParameterDelegate.kt b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/internal/ParameterDelegate.kt
new file mode 100644
index 00000000..aaf90b99
--- /dev/null
+++ b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/internal/ParameterDelegate.kt
@@ -0,0 +1,43 @@
+/*
+ * Copyright (c) 2021 AtLarge Research
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+package org.opendc.harness.internal
+
+import org.opendc.harness.api.Parameter
+import org.opendc.harness.dsl.Experiment
+import kotlin.properties.ReadOnlyProperty
+import kotlin.reflect.KProperty
+
+/**
+ * A delegate for an experiment parameter.
+ *
+ * @property parameter The parameter descriptor of this delegate.
+ */
+internal class ParameterDelegate<T>(val parameter: Parameter<T>) : ReadOnlyProperty<Experiment, T> {
+ /**
+ * Obtain the value for the parameter.
+ */
+ override fun getValue(thisRef: Experiment, property: KProperty<*>): T {
+ val scenario = thisRef.scenario ?: throw IllegalStateException("Cannot use parameters before activation")
+ return scenario[parameter]
+ }
+}
diff --git a/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/internal/ScenarioImpl.kt b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/internal/ScenarioImpl.kt
new file mode 100644
index 00000000..d255004d
--- /dev/null
+++ b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/internal/ScenarioImpl.kt
@@ -0,0 +1,49 @@
+/*
+ * Copyright (c) 2021 AtLarge Research
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+package org.opendc.harness.internal
+
+import org.opendc.harness.api.ExperimentDefinition
+import org.opendc.harness.api.Parameter
+import org.opendc.harness.api.Scenario
+
+/**
+ * Internal implementation of a [Scenario].
+ */
+internal data class ScenarioImpl(
+ override val id: Int,
+ override val experiment: ExperimentDefinition,
+ val parameters: Map<Parameter<*>, Any?>
+) : Scenario {
+
+ override fun <T> get(param: Parameter<T>): T {
+ if (!parameters.containsKey(param)) {
+ throw IllegalArgumentException("Unknown parameter for this scenario.")
+ }
+
+ // This cast should always succeed
+ @Suppress("UNCHECKED_CAST")
+ return parameters[param] as T
+ }
+
+ override fun toString(): String = "Scenario"
+}
diff --git a/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/reporter/ConsoleExperimentReporter.kt b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/runner/console/ConsoleExperimentReporter.kt
index af61622a..2db74ef4 100644
--- a/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/reporter/ConsoleExperimentReporter.kt
+++ b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/runner/console/ConsoleExperimentReporter.kt
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2020 AtLarge Research
+ * Copyright (c) 2021 AtLarge Research
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
@@ -20,24 +20,22 @@
* SOFTWARE.
*/
-package org.opendc.experiments.sc20.reporter
+package org.opendc.harness.runner.console
import me.tongfei.progressbar.ProgressBar
import me.tongfei.progressbar.ProgressBarBuilder
import mu.KotlinLogging
-import org.opendc.experiments.sc20.experiment.Run
-import org.opendc.experiments.sc20.runner.ExperimentDescriptor
-import org.opendc.experiments.sc20.runner.execution.ExperimentExecutionListener
-import org.opendc.experiments.sc20.runner.execution.ExperimentExecutionResult
+import org.opendc.harness.api.Trial
+import org.opendc.harness.engine.ExperimentExecutionListener
/**
* A reporter that reports the experiment progress to the console.
*/
public class ConsoleExperimentReporter : ExperimentExecutionListener, AutoCloseable {
/**
- * The active [Run]s.
+ * The active [Trial]s.
*/
- private val runs: MutableSet<Run> = mutableSetOf()
+ private val trials: MutableSet<Trial> = mutableSetOf()
/**
* The total number of runs.
@@ -57,29 +55,23 @@ public class ConsoleExperimentReporter : ExperimentExecutionListener, AutoClosea
.setInitialMax(1)
.build()
- override fun descriptorRegistered(descriptor: ExperimentDescriptor) {
- if (descriptor is Run) {
- runs += descriptor
- pb.maxHint((++total).toLong())
- }
- }
+ override fun trialFinished(trial: Trial, throwable: Throwable?) {
+ trials -= trial
- override fun executionFinished(descriptor: ExperimentDescriptor, result: ExperimentExecutionResult) {
- if (descriptor is Run) {
- runs -= descriptor
-
- pb.stepTo(total - runs.size.toLong())
- if (runs.isEmpty()) {
- pb.close()
- }
+ pb.stepTo(total - trials.size.toLong())
+ if (trials.isEmpty()) {
+ pb.close()
}
- if (result is ExperimentExecutionResult.Failed) {
- logger.warn(result.throwable) { "Descriptor $descriptor failed" }
+ if (throwable != null) {
+ logger.warn(throwable) { "Trial $trial failed" }
}
}
- override fun executionStarted(descriptor: ExperimentDescriptor) {}
+ override fun trialStarted(trial: Trial) {
+ trials += trial
+ pb.maxHint((++total).toLong())
+ }
override fun close() {
pb.close()
diff --git a/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/runner/console/ConsoleRunner.kt b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/runner/console/ConsoleRunner.kt
new file mode 100644
index 00000000..ae221c7f
--- /dev/null
+++ b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/runner/console/ConsoleRunner.kt
@@ -0,0 +1,99 @@
+/*
+ * Copyright (c) 2021 AtLarge Research
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+package org.opendc.harness.runner.console
+
+import com.github.ajalt.clikt.core.CliktCommand
+import com.github.ajalt.clikt.parameters.options.default
+import com.github.ajalt.clikt.parameters.options.multiple
+import com.github.ajalt.clikt.parameters.options.option
+import com.github.ajalt.clikt.parameters.types.int
+import mu.KotlinLogging
+import org.opendc.harness.engine.ExperimentEngineLauncher
+import org.opendc.harness.engine.discovery.DiscoveryProvider
+import org.opendc.harness.engine.discovery.DiscoveryRequest
+import org.opendc.harness.engine.discovery.DiscoverySelector
+import org.opendc.harness.engine.scheduler.ThreadPoolExperimentScheduler
+
+/**
+ * The logger for this experiment runner.
+ */
+private val logger = KotlinLogging.logger {}
+
+/**
+ * The command line interface for the console experiment runner.
+ */
+public class ConsoleRunner : CliktCommand(name = "opendc-harness") {
+ /**
+ * The number of repeats per scenario.
+ */
+ private val repeats by option("-r", "--repeats", help = "Number of repeats per scenario")
+ .int()
+ .default(1)
+
+ /**
+ * The selected experiments to run by name.
+ */
+ private val experiments by option("-e", "--experiments", help = "Names of experiments to explore")
+ .multiple(emptyList())
+
+ /**
+ * The maximum number of worker threads to use.
+ */
+ private val parallelism by option("-p", "--parallelism", help = "Maximum number of concurrent simulation runs")
+ .int()
+ .default(Runtime.getRuntime().availableProcessors())
+
+ override fun run() {
+ logger.info { "Starting OpenDC Console Experiment Runner" }
+
+ val discovery = DiscoveryProvider.createComposite()
+ val experiments = discovery.discover(
+ DiscoveryRequest(
+ selectors = experiments.map { DiscoverySelector.Name(it) }
+ )
+ )
+
+ val reporter = ConsoleExperimentReporter()
+ val scheduler = ThreadPoolExperimentScheduler(parallelism)
+
+ try {
+ ExperimentEngineLauncher()
+ .withListener(reporter)
+ .withRepeats(repeats)
+ .withScheduler(scheduler)
+ .runBlocking(experiments)
+ } catch (e: Throwable) {
+ logger.error(e) { "Failed to finish experiments" }
+ } finally {
+ reporter.close()
+ scheduler.close()
+ }
+
+ logger.info { "Finished all experiments. Exiting." }
+ }
+}
+
+/**
+ * Main entry point of the experiment runner.
+ */
+public fun main(args: Array<String>): Unit = ConsoleRunner().main(args)
diff --git a/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/runner/junit5/JUnitExperimentExecutionListener.kt b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/runner/junit5/JUnitExperimentExecutionListener.kt
new file mode 100644
index 00000000..58791549
--- /dev/null
+++ b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/runner/junit5/JUnitExperimentExecutionListener.kt
@@ -0,0 +1,152 @@
+/*
+ * Copyright (c) 2021 AtLarge Research
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+package org.opendc.harness.runner.junit5
+
+import org.junit.platform.engine.*
+import org.junit.platform.engine.support.descriptor.AbstractTestDescriptor
+import org.junit.platform.engine.support.descriptor.EngineDescriptor
+import org.opendc.harness.api.ExperimentDefinition
+import org.opendc.harness.api.Scenario
+import org.opendc.harness.api.Trial
+import org.opendc.harness.engine.ExperimentExecutionListener
+
+/**
+ * An [ExperimentExecutionListener] that notifies JUnit platform of the progress of the experiment trials.
+ */
+public class JUnitExperimentExecutionListener(
+ private val listener: EngineExecutionListener,
+ private val root: EngineDescriptor
+) : ExperimentExecutionListener {
+ /**
+ * The current active experiments.
+ */
+ private val experiments = mutableMapOf<ExperimentDefinition, TestDescriptor>()
+
+ /**
+ * The current active scenarios.
+ */
+ private val scenarios = mutableMapOf<Scenario, TestDescriptor>()
+
+ /**
+ * The current active trials.
+ */
+ private val trials = mutableMapOf<Trial, TestDescriptor>()
+
+ override fun experimentStarted(experiment: ExperimentDefinition) {
+ val descriptor = experiment.toDescriptor(root)
+ root.addChild(descriptor)
+ experiments[experiment] = descriptor
+
+ listener.dynamicTestRegistered(descriptor)
+ listener.executionStarted(descriptor)
+ }
+
+ override fun experimentFinished(experiment: ExperimentDefinition, throwable: Throwable?) {
+ val descriptor = experiments.remove(experiment)
+
+ if (throwable != null) {
+ listener.executionFinished(descriptor, TestExecutionResult.failed(throwable))
+ } else {
+ listener.executionFinished(descriptor, TestExecutionResult.successful())
+ }
+ }
+
+ override fun scenarioStarted(scenario: Scenario) {
+ val parent = experiments[scenario.experiment] ?: return
+ val descriptor = scenario.toDescriptor(parent)
+ parent.addChild(descriptor)
+ scenarios[scenario] = descriptor
+
+ listener.dynamicTestRegistered(descriptor)
+ listener.executionStarted(descriptor)
+ }
+
+ override fun scenarioFinished(scenario: Scenario, throwable: Throwable?) {
+ val descriptor = scenarios.remove(scenario)
+
+ if (throwable != null) {
+ listener.executionFinished(descriptor, TestExecutionResult.failed(throwable))
+ } else {
+ listener.executionFinished(descriptor, TestExecutionResult.successful())
+ }
+ }
+
+ override fun trialStarted(trial: Trial) {
+ val parent = scenarios[trial.scenario] ?: return
+ val descriptor = trial.toDescriptor(parent)
+ parent.addChild(descriptor)
+ trials[trial] = descriptor
+
+ listener.dynamicTestRegistered(descriptor)
+ listener.executionStarted(descriptor)
+ }
+
+ override fun trialFinished(trial: Trial, throwable: Throwable?) {
+ val descriptor = trials.remove(trial)
+
+ if (throwable != null) {
+ listener.executionFinished(descriptor, TestExecutionResult.failed(throwable))
+ } else {
+ listener.executionFinished(descriptor, TestExecutionResult.successful())
+ }
+ }
+
+ /**
+ * Create a [TestDescriptor] for an [ExperimentDefinition].
+ */
+ private fun ExperimentDefinition.toDescriptor(parent: TestDescriptor): TestDescriptor {
+ return object : AbstractTestDescriptor(parent.uniqueId.append("experiment", name), name) {
+ override fun getType(): TestDescriptor.Type = TestDescriptor.Type.CONTAINER
+
+ override fun mayRegisterTests(): Boolean = true
+
+ override fun toString(): String = "ExperimentDescriptor"
+ }
+ }
+
+ /**
+ * Create a [TestDescriptor] for a [Scenario].
+ */
+ private fun Scenario.toDescriptor(parent: TestDescriptor): TestDescriptor {
+ return object : AbstractTestDescriptor(parent.uniqueId.append("scenario", id.toString()), "Scenario $id") {
+ override fun getType(): TestDescriptor.Type = TestDescriptor.Type.CONTAINER_AND_TEST
+
+ override fun mayRegisterTests(): Boolean = true
+
+ override fun toString(): String = "ScenarioDescriptor"
+ }
+ }
+
+ /**
+ * Create a [TestDescriptor] for a [Trial].
+ */
+ private fun Trial.toDescriptor(parent: TestDescriptor): TestDescriptor {
+ return object : AbstractTestDescriptor(parent.uniqueId.append("repeat", repeat.toString()), "Repeat $repeat") {
+ override fun getType(): TestDescriptor.Type = TestDescriptor.Type.TEST
+
+ override fun mayRegisterTests(): Boolean = false
+
+ override fun toString(): String = "TrialDescriptor"
+ }
+ }
+}
diff --git a/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/runner/junit5/OpenDCTestEngine.kt b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/runner/junit5/OpenDCTestEngine.kt
new file mode 100644
index 00000000..685cd41a
--- /dev/null
+++ b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/runner/junit5/OpenDCTestEngine.kt
@@ -0,0 +1,94 @@
+/*
+ * Copyright (c) 2021 AtLarge Research
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+package org.opendc.harness.runner.junit5
+
+import kotlinx.coroutines.flow.Flow
+import kotlinx.coroutines.flow.emptyFlow
+import mu.KotlinLogging
+import org.junit.platform.engine.*
+import org.junit.platform.engine.discovery.ClassNameFilter
+import org.junit.platform.engine.discovery.ClassSelector
+import org.junit.platform.engine.discovery.MethodSelector
+import org.junit.platform.engine.support.descriptor.EngineDescriptor
+import org.junit.platform.launcher.LauncherDiscoveryRequest
+import org.opendc.harness.api.ExperimentDefinition
+import org.opendc.harness.engine.ExperimentEngineLauncher
+import org.opendc.harness.engine.discovery.DiscoveryFilter
+import org.opendc.harness.engine.discovery.DiscoveryProvider
+import org.opendc.harness.engine.discovery.DiscoveryRequest
+import org.opendc.harness.engine.discovery.DiscoverySelector
+
+/**
+ * A [TestEngine] implementation that is able to run experiments defined using the harness.
+ */
+public class OpenDCTestEngine : TestEngine {
+ /**
+ * The logging instance for this engine.
+ */
+ private val logger = KotlinLogging.logger {}
+
+ override fun getId(): String = "opendc"
+
+ override fun discover(request: EngineDiscoveryRequest, uniqueId: UniqueId): TestDescriptor {
+ // Test whether are excluded from the engines
+ val isEnabled = (request as? LauncherDiscoveryRequest)?.engineFilters?.all { it.toPredicate().test(this) } ?: true
+ if (!isEnabled) {
+ return ExperimentEngineDescriptor(uniqueId, emptyFlow())
+ }
+
+ // IntelliJ will pass a [MethodSelector] to run just a single method inside a file. In that
+ // case, no experiments should be discovered, since we support only experiments by class.
+ if (request.getSelectorsByType(MethodSelector::class.java).isNotEmpty()) {
+ return ExperimentEngineDescriptor(uniqueId, emptyFlow())
+ }
+
+ val classNames = request.getSelectorsByType(ClassSelector::class.java).map { DiscoverySelector.Meta("class.name", it.className) }
+ val classNameFilters = request.getFiltersByType(ClassNameFilter::class.java).map { DiscoveryFilter.Name(it.toPredicate()) }
+
+ val discovery = DiscoveryProvider.createComposite()
+ val definitions = discovery.discover(DiscoveryRequest(classNames, classNameFilters))
+
+ return ExperimentEngineDescriptor(uniqueId, definitions)
+ }
+
+ override fun execute(request: ExecutionRequest) {
+ logger.debug { "JUnit ExecutionRequest[${request::class.java.name}] [configurationParameters=${request.configurationParameters}; rootTestDescriptor=${request.rootTestDescriptor}]" }
+ val root = request.rootTestDescriptor as ExperimentEngineDescriptor
+ val listener = request.engineExecutionListener
+
+ listener.executionStarted(root)
+
+ try {
+ ExperimentEngineLauncher()
+ .withListener(JUnitExperimentExecutionListener(listener, root))
+ .runBlocking(root.experiments)
+ listener.executionFinished(root, TestExecutionResult.successful())
+ } catch (e: Throwable) {
+ listener.executionFinished(root, TestExecutionResult.failed(e))
+ }
+ }
+
+ private class ExperimentEngineDescriptor(id: UniqueId, val experiments: Flow<ExperimentDefinition>) : EngineDescriptor(id, "opendc") {
+ override fun mayRegisterTests(): Boolean = true
+ }
+}
diff --git a/simulator/opendc-harness/src/main/resources/META-INF/services/org.junit.platform.engine.TestEngine b/simulator/opendc-harness/src/main/resources/META-INF/services/org.junit.platform.engine.TestEngine
new file mode 100644
index 00000000..b83eec0c
--- /dev/null
+++ b/simulator/opendc-harness/src/main/resources/META-INF/services/org.junit.platform.engine.TestEngine
@@ -0,0 +1 @@
+org.opendc.harness.runner.junit5.OpenDCTestEngine
diff --git a/simulator/opendc-harness/src/main/resources/META-INF/services/org.opendc.harness.engine.discovery.DiscoveryProvider b/simulator/opendc-harness/src/main/resources/META-INF/services/org.opendc.harness.engine.discovery.DiscoveryProvider
new file mode 100644
index 00000000..d6a73ded
--- /dev/null
+++ b/simulator/opendc-harness/src/main/resources/META-INF/services/org.opendc.harness.engine.discovery.DiscoveryProvider
@@ -0,0 +1 @@
+org.opendc.harness.internal.DslDiscoveryProvider
diff --git a/simulator/opendc-harness/src/main/resources/META-INF/services/org.opendc.harness.engine.scheduler.ExperimentSchedulerProvider b/simulator/opendc-harness/src/main/resources/META-INF/services/org.opendc.harness.engine.scheduler.ExperimentSchedulerProvider
new file mode 100644
index 00000000..2ba3a7cb
--- /dev/null
+++ b/simulator/opendc-harness/src/main/resources/META-INF/services/org.opendc.harness.engine.scheduler.ExperimentSchedulerProvider
@@ -0,0 +1 @@
+org.opendc.harness.engine.scheduler.ThreadPoolExperimentSchedulerProvider
diff --git a/simulator/opendc-harness/src/main/resources/META-INF/services/org.opendc.harness.engine.strategy.ExperimentStrategyProvider b/simulator/opendc-harness/src/main/resources/META-INF/services/org.opendc.harness.engine.strategy.ExperimentStrategyProvider
new file mode 100644
index 00000000..cb1c70ac
--- /dev/null
+++ b/simulator/opendc-harness/src/main/resources/META-INF/services/org.opendc.harness.engine.strategy.ExperimentStrategyProvider
@@ -0,0 +1 @@
+org.opendc.harness.engine.strategy.CartesianExperimentStrategyProvider
diff --git a/simulator/opendc-harness/src/main/resources/log4j2.xml b/simulator/opendc-harness/src/main/resources/log4j2.xml
new file mode 100644
index 00000000..9553d964
--- /dev/null
+++ b/simulator/opendc-harness/src/main/resources/log4j2.xml
@@ -0,0 +1,40 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ MIT License
+ ~
+ ~ Copyright (c) 2020 atlarge-research
+ ~
+ ~ Permission is hereby granted, free of charge, to any person obtaining a copy
+ ~ of this software and associated documentation files (the "Software"), to deal
+ ~ in the Software without restriction, including without limitation the rights
+ ~ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ ~ copies of the Software, and to permit persons to whom the Software is
+ ~ furnished to do so, subject to the following conditions:
+ ~
+ ~ The above copyright notice and this permission notice shall be included in all
+ ~ copies or substantial portions of the Software.
+ ~
+ ~ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ ~ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ ~ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ ~ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ ~ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ ~ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ ~ SOFTWARE.
+ -->
+
+<Configuration status="WARN">
+ <Appenders>
+ <Console name="Console" target="SYSTEM_OUT">
+ <PatternLayout pattern="%d{HH:mm:ss.SSS} [%highlight{%-5level}] %logger{36} - %msg%n" disableAnsi="false"/>
+ </Console>
+ </Appenders>
+ <Loggers>
+ <Logger name="org.opendc" level="info" additivity="false">
+ <AppenderRef ref="Console"/>
+ </Logger>
+ <Root level="error">
+ <AppenderRef ref="Console"/>
+ </Root>
+ </Loggers>
+</Configuration>
diff --git a/simulator/opendc-harness/src/test/kotlin/org/opendc/harness/EngineTest.kt b/simulator/opendc-harness/src/test/kotlin/org/opendc/harness/EngineTest.kt
new file mode 100644
index 00000000..6f2989db
--- /dev/null
+++ b/simulator/opendc-harness/src/test/kotlin/org/opendc/harness/EngineTest.kt
@@ -0,0 +1,61 @@
+/*
+ * Copyright (c) 2021 AtLarge Research
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+package org.opendc.harness
+
+import kotlinx.coroutines.flow.flowOf
+import kotlinx.coroutines.flow.toList
+import kotlinx.coroutines.runBlocking
+import org.junit.jupiter.api.Assertions.assertEquals
+import org.junit.jupiter.api.Assertions.assertNotNull
+import org.junit.jupiter.api.Test
+import org.opendc.harness.api.ExperimentDefinition
+import org.opendc.harness.engine.ExperimentEngine
+import org.opendc.harness.engine.ExperimentEngineLauncher
+import org.opendc.harness.engine.ExperimentExecutionListener
+import org.opendc.harness.engine.discovery.DiscoveryProvider
+import org.opendc.harness.engine.discovery.DiscoveryRequest
+
+/**
+ * A test suite for the [ExperimentEngine].
+ */
+internal class EngineTest {
+ @Test
+ fun test() {
+ val listener = object : ExperimentExecutionListener {}
+ ExperimentEngineLauncher()
+ .withListener(listener)
+ .runBlocking(flowOf(TestExperiment().toDefinition()))
+ }
+
+ @Test
+ fun discovery() {
+ runBlocking {
+ val discovery = DiscoveryProvider.findById("dsl")?.create()
+ assertNotNull(discovery)
+ val res = mutableListOf<ExperimentDefinition>()
+ discovery?.discover(DiscoveryRequest())?.toList(res)
+ println(res)
+ assertEquals(1, res.size)
+ }
+ }
+}
diff --git a/simulator/opendc-harness/src/test/kotlin/org/opendc/harness/TestExperiment.kt b/simulator/opendc-harness/src/test/kotlin/org/opendc/harness/TestExperiment.kt
new file mode 100644
index 00000000..bedd1c76
--- /dev/null
+++ b/simulator/opendc-harness/src/test/kotlin/org/opendc/harness/TestExperiment.kt
@@ -0,0 +1,54 @@
+/*
+ * Copyright (c) 2021 AtLarge Research
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+package org.opendc.harness
+
+import org.opendc.harness.dsl.Experiment
+import org.opendc.harness.dsl.anyOf
+
+/**
+ * An experiment to test the harness' functionality.
+ */
+class TestExperiment : Experiment("Design Space Exploration") {
+ /**
+ * The cloud environment to use.
+ */
+ private val environment: String by anyOf(
+ "../../traces/setup-small.json",
+ "../../traces/setup.json",
+ "../../traces/setup-large.json"
+ )
+
+ /**
+ * The trace to use.
+ */
+ private val trace: String by anyOf(
+ "../../traces/gwf/askalon_workload_olde.gwf",
+ "../../traces/gwf/askalon_workload_ee.gwf",
+ "../../traces/gwf/chronos_exp_noscaler_ca.gwf"
+ )
+
+ override fun doRun(repeat: Int) {
+ println("Id $id, Run $repeat, Environment $environment, Trace $trace")
+ Thread.sleep(500 * id.toLong())
+ }
+}
diff --git a/simulator/settings.gradle.kts b/simulator/settings.gradle.kts
index e6f42574..470303f4 100644
--- a/simulator/settings.gradle.kts
+++ b/simulator/settings.gradle.kts
@@ -33,4 +33,5 @@ include(":opendc-simulator:opendc-simulator-core")
include(":opendc-simulator:opendc-simulator-compute")
include(":opendc-simulator:opendc-simulator-failures")
include(":opendc-trace:opendc-trace-core")
+include(":opendc-harness")
include(":opendc-utils")