diff options
| author | Fabian Mastenbroek <mail.fabianm@gmail.com> | 2021-01-07 23:36:34 +0100 |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2021-01-07 23:36:34 +0100 |
| commit | 3441586e4a10fcc6b2f458d16301ae2770d4886a (patch) | |
| tree | 82ddeaced2a42dd49a633a0669fbd8fedd1c167d | |
| parent | 74a4bff83bfb6366cc193d1fc9c4a07e49649649 (diff) | |
| parent | 885730e31a2bb987f2dcc8a90921e2f06a8c857a (diff) | |
Merge pull request #70 from atlarge-research/feat/harness
Add universal experiment harness
62 files changed, 2537 insertions, 1084 deletions
diff --git a/simulator/buildSrc/src/main/kotlin/kotlin-library-convention.gradle.kts b/simulator/buildSrc/src/main/kotlin/kotlin-library-convention.gradle.kts index bbecf346..f9d8a966 100644 --- a/simulator/buildSrc/src/main/kotlin/kotlin-library-convention.gradle.kts +++ b/simulator/buildSrc/src/main/kotlin/kotlin-library-convention.gradle.kts @@ -33,6 +33,7 @@ plugins { /* Project configuration */ repositories { + mavenCentral() jcenter() } diff --git a/simulator/opendc-experiments/opendc-experiments-sc20/build.gradle.kts b/simulator/opendc-experiments/opendc-experiments-sc20/build.gradle.kts index 3b682668..b94207ba 100644 --- a/simulator/opendc-experiments/opendc-experiments-sc20/build.gradle.kts +++ b/simulator/opendc-experiments/opendc-experiments-sc20/build.gradle.kts @@ -29,21 +29,22 @@ plugins { } application { - mainClassName = "org.opendc.experiments.sc20.MainKt" + mainClass.set("org.opendc.harness.runner.console.ConsoleRunnerKt") applicationDefaultJvmArgs = listOf("-Xms2500M") } dependencies { api(project(":opendc-core")) + api(project(":opendc-harness")) implementation(project(":opendc-format")) implementation(project(":opendc-simulator:opendc-simulator-core")) implementation(project(":opendc-simulator:opendc-simulator-compute")) implementation(project(":opendc-simulator:opendc-simulator-failures")) implementation(project(":opendc-compute:opendc-compute-simulator")) - implementation("com.github.ajalt:clikt:2.6.0") - implementation("me.tongfei:progressbar:0.8.1") - implementation("io.github.microutils:kotlin-logging:1.7.9") + implementation("io.github.microutils:kotlin-logging:2.0.4") + implementation("me.tongfei:progressbar:0.9.0") + implementation("com.github.ajalt.clikt:clikt:3.1.0") implementation("org.apache.parquet:parquet-avro:1.11.0") implementation("org.apache.hadoop:hadoop-client:3.2.1") { @@ -51,8 +52,6 @@ dependencies { exclude(group = "log4j") } - runtimeOnly("org.apache.logging.log4j:log4j-slf4j-impl:2.13.1") - testImplementation("org.junit.jupiter:junit-jupiter-api:${Library.JUNIT_JUPITER}") testRuntimeOnly("org.junit.jupiter:junit-jupiter-engine:${Library.JUNIT_JUPITER}") testImplementation("org.junit.platform:junit-platform-launcher:${Library.JUNIT_PLATFORM}") diff --git a/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/Main.kt b/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/Main.kt deleted file mode 100644 index 8916261b..00000000 --- a/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/Main.kt +++ /dev/null @@ -1,159 +0,0 @@ -/* - * Copyright (c) 2020 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.experiments.sc20 - -import com.github.ajalt.clikt.core.CliktCommand -import com.github.ajalt.clikt.parameters.options.convert -import com.github.ajalt.clikt.parameters.options.default -import com.github.ajalt.clikt.parameters.options.defaultLazy -import com.github.ajalt.clikt.parameters.options.multiple -import com.github.ajalt.clikt.parameters.options.option -import com.github.ajalt.clikt.parameters.options.required -import com.github.ajalt.clikt.parameters.types.choice -import com.github.ajalt.clikt.parameters.types.file -import com.github.ajalt.clikt.parameters.types.int -import mu.KotlinLogging -import org.opendc.experiments.sc20.experiment.* -import org.opendc.experiments.sc20.reporter.ConsoleExperimentReporter -import org.opendc.experiments.sc20.runner.ExperimentDescriptor -import org.opendc.experiments.sc20.runner.execution.ThreadPoolExperimentScheduler -import org.opendc.experiments.sc20.runner.internal.DefaultExperimentRunner -import org.opendc.format.trace.sc20.Sc20PerformanceInterferenceReader -import org.opendc.format.trace.sc20.Sc20VmPlacementReader -import java.io.File - -/** - * The logger for this experiment. - */ -private val logger = KotlinLogging.logger {} - -/** - * Represents the command for running the experiment. - */ -public class ExperimentCli : CliktCommand(name = "sc20-experiment") { - /** - * The path to the directory where the topology descriptions are located. - */ - private val environmentPath by option("--environment-path", help = "path to the environment directory") - .file(canBeFile = false) - .required() - - /** - * The path to the directory where the traces are located. - */ - private val tracePath by option("--trace-path", help = "path to the traces directory") - .file(canBeFile = false) - .required() - - /** - * The path to the performance interference model. - */ - private val performanceInterferenceStream by option( - "--performance-interference-model", - help = "path to the performance interference file" - ) - .file(canBeDir = false) - .convert { it.inputStream() } - - /** - * The path to the original VM placements file. - */ - private val vmPlacements by option("--vm-placements-file", help = "path to the VM placement file") - .file(canBeDir = false) - .convert { - Sc20VmPlacementReader(it.inputStream().buffered()).construct() - } - .default(emptyMap()) - - /** - * The selected portfolios to run. - */ - private val portfolios by option("--portfolio", help = "portfolio of scenarios to explore") - .choice( - "hor-ver" to { experiment: Experiment, i: Int -> HorVerPortfolio(experiment, i) } - as (Experiment, Int) -> Portfolio, - "more-velocity" to { experiment, i -> MoreVelocityPortfolio(experiment, i) }, - "composite-workload" to { experiment, i -> CompositeWorkloadPortfolio(experiment, i) }, - "operational-phenomena" to { experiment, i -> OperationalPhenomenaPortfolio(experiment, i) }, - "replay" to { experiment, i -> ReplayPortfolio(experiment, i) }, - "test" to { experiment, i -> TestPortfolio(experiment, i) }, - "more-hpc" to { experiment, i -> MoreHpcPortfolio(experiment, i) }, - ignoreCase = true - ) - .multiple(required = true) - - /** - * The maximum number of worker threads to use. - */ - private val parallelism by option("--parallelism", help = "maximum number of concurrent simulation runs") - .int() - .default(Runtime.getRuntime().availableProcessors()) - - /** - * The buffer size for writing results. - */ - private val bufferSize by option("--buffer-size") - .int() - .default(4096) - - /** - * The path to the output directory. - */ - private val output by option("-O", "--output", help = "path to the output directory") - .file(canBeFile = false) - .defaultLazy { File("data") } - - override fun run() { - logger.info { "Constructing performance interference model" } - - val performanceInterferenceModel = - performanceInterferenceStream?.let { Sc20PerformanceInterferenceReader(it) } - - logger.info { "Creating experiment descriptor" } - val descriptor = object : - Experiment(environmentPath, tracePath, output, performanceInterferenceModel, vmPlacements, bufferSize) { - private val descriptor = this - override val children: Sequence<ExperimentDescriptor> = sequence { - for ((i, producer) in portfolios.withIndex()) { - yield(producer(descriptor, i)) - } - } - } - - logger.info { "Starting experiment runner [parallelism=$parallelism]" } - val scheduler = ThreadPoolExperimentScheduler(parallelism) - val runner = DefaultExperimentRunner(scheduler) - val reporter = ConsoleExperimentReporter() - try { - runner.execute(descriptor, reporter) - } finally { - scheduler.close() - reporter.close() - } - } -} - -/** - * Main entry point of the experiment. - */ -public fun main(args: Array<String>): Unit = ExperimentCli().main(args) diff --git a/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/experiment/CompositeWorkloadPortfolio.kt b/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/experiment/CompositeWorkloadPortfolio.kt new file mode 100644 index 00000000..f4242456 --- /dev/null +++ b/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/experiment/CompositeWorkloadPortfolio.kt @@ -0,0 +1,79 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.experiments.sc20.experiment + +import org.opendc.experiments.sc20.experiment.model.CompositeWorkload +import org.opendc.experiments.sc20.experiment.model.OperationalPhenomena +import org.opendc.experiments.sc20.experiment.model.Topology +import org.opendc.experiments.sc20.experiment.model.Workload +import org.opendc.harness.dsl.anyOf + +/** + * A [Portfolio] that explores the effect of a composite workload. + */ +public class CompositeWorkloadPortfolio : Portfolio("composite-workload") { + private val totalSampleLoad = 1.3301733005049648E12 + + override val topology: Topology by anyOf( + Topology("base"), + Topology("exp-vol-hor-hom"), + Topology("exp-vol-ver-hom"), + Topology("exp-vel-ver-hom") + ) + + override val workload: Workload by anyOf( + CompositeWorkload( + "all-azure", + listOf(Workload("solvinity-short", 0.0), Workload("azure", 1.0)), + totalSampleLoad + ), + CompositeWorkload( + "solvinity-25-azure-75", + listOf(Workload("solvinity-short", 0.25), Workload("azure", 0.75)), + totalSampleLoad + ), + CompositeWorkload( + "solvinity-50-azure-50", + listOf(Workload("solvinity-short", 0.5), Workload("azure", 0.5)), + totalSampleLoad + ), + CompositeWorkload( + "solvinity-75-azure-25", + listOf(Workload("solvinity-short", 0.75), Workload("azure", 0.25)), + totalSampleLoad + ), + CompositeWorkload( + "all-solvinity", + listOf(Workload("solvinity-short", 1.0), Workload("azure", 0.0)), + totalSampleLoad + ) + ) + + override val operationalPhenomena: OperationalPhenomena by anyOf( + OperationalPhenomena(failureFrequency = 24.0 * 7, hasInterference = false) + ) + + override val allocationPolicy: String by anyOf( + "active-servers" + ) +} diff --git a/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/experiment/Experiment.kt b/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/experiment/Experiment.kt deleted file mode 100644 index 34d7301b..00000000 --- a/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/experiment/Experiment.kt +++ /dev/null @@ -1,76 +0,0 @@ -/* - * Copyright (c) 2020 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.experiments.sc20.experiment - -import org.opendc.experiments.sc20.runner.ContainerExperimentDescriptor -import org.opendc.experiments.sc20.runner.ExperimentDescriptor -import org.opendc.experiments.sc20.runner.execution.ExperimentExecutionContext -import org.opendc.experiments.sc20.runner.execution.ExperimentExecutionListener -import org.opendc.experiments.sc20.telemetry.RunEvent -import org.opendc.experiments.sc20.telemetry.parquet.ParquetRunEventWriter -import org.opendc.format.trace.PerformanceInterferenceModelReader -import java.io.File - -/** - * The global configuration of the experiment. - * - * @param environments The path to the topologies directory. - * @param traces The path to the traces directory. - * @param output The output directory. - * @param performanceInterferenceModel The optional performance interference model that has been specified. - * @param vmPlacements Original VM placement in the trace. - * @param bufferSize The buffer size of the event reporters. - */ -public abstract class Experiment( - public val environments: File, - public val traces: File, - public val output: File, - public val performanceInterferenceModel: PerformanceInterferenceModelReader?, - public val vmPlacements: Map<String, String>, - public val bufferSize: Int -) : ContainerExperimentDescriptor() { - override val parent: ExperimentDescriptor? = null - - override suspend fun invoke(context: ExperimentExecutionContext) { - val writer = ParquetRunEventWriter(File(output, "experiments.parquet"), bufferSize) - try { - val listener = object : ExperimentExecutionListener by context.listener { - override fun descriptorRegistered(descriptor: ExperimentDescriptor) { - if (descriptor is Run) { - writer.write(RunEvent(descriptor, System.currentTimeMillis())) - } - - context.listener.descriptorRegistered(descriptor) - } - } - - val newContext = object : ExperimentExecutionContext by context { - override val listener: ExperimentExecutionListener = listener - } - - super.invoke(newContext) - } finally { - writer.close() - } - } -} diff --git a/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/experiment/HorVerPortfolio.kt b/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/experiment/HorVerPortfolio.kt new file mode 100644 index 00000000..aa97b808 --- /dev/null +++ b/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/experiment/HorVerPortfolio.kt @@ -0,0 +1,60 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.experiments.sc20.experiment + +import org.opendc.experiments.sc20.experiment.model.OperationalPhenomena +import org.opendc.experiments.sc20.experiment.model.Topology +import org.opendc.experiments.sc20.experiment.model.Workload +import org.opendc.harness.dsl.anyOf + +/** + * A [Portfolio] that explores the difference between horizontal and vertical scaling. + */ +public class HorVerPortfolio : Portfolio("horizontal_vs_vertical") { + override val topology: Topology by anyOf( + Topology("base"), + Topology("rep-vol-hor-hom"), + Topology("rep-vol-hor-het"), + Topology("rep-vol-ver-hom"), + Topology("rep-vol-ver-het"), + Topology("exp-vol-hor-hom"), + Topology("exp-vol-hor-het"), + Topology("exp-vol-ver-hom"), + Topology("exp-vol-ver-het") + ) + + override val workload: Workload by anyOf( + Workload("solvinity", 0.1), + Workload("solvinity", 0.25), + Workload("solvinity", 0.5), + Workload("solvinity", 1.0) + ) + + override val operationalPhenomena: OperationalPhenomena by anyOf( + OperationalPhenomena(failureFrequency = 24.0 * 7, hasInterference = true) + ) + + override val allocationPolicy: String by anyOf( + "active-servers" + ) +} diff --git a/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/experiment/MoreHpcPortfolio.kt b/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/experiment/MoreHpcPortfolio.kt new file mode 100644 index 00000000..bdb33f59 --- /dev/null +++ b/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/experiment/MoreHpcPortfolio.kt @@ -0,0 +1,56 @@ +/* + * Copyright (c) 2020 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.experiments.sc20.experiment + +import org.opendc.experiments.sc20.experiment.model.* +import org.opendc.harness.dsl.anyOf + +/** + * A [Portfolio] to explore the effect of HPC workloads. + */ +public class MoreHpcPortfolio : Portfolio("more_hpc") { + override val topology: Topology by anyOf( + Topology("base"), + Topology("exp-vol-hor-hom"), + Topology("exp-vol-ver-hom"), + Topology("exp-vel-ver-hom") + ) + + override val workload: Workload by anyOf( + Workload("solvinity", 0.0, samplingStrategy = SamplingStrategy.HPC), + Workload("solvinity", 0.25, samplingStrategy = SamplingStrategy.HPC), + Workload("solvinity", 0.5, samplingStrategy = SamplingStrategy.HPC), + Workload("solvinity", 1.0, samplingStrategy = SamplingStrategy.HPC), + Workload("solvinity", 0.25, samplingStrategy = SamplingStrategy.HPC_LOAD), + Workload("solvinity", 0.5, samplingStrategy = SamplingStrategy.HPC_LOAD), + Workload("solvinity", 1.0, samplingStrategy = SamplingStrategy.HPC_LOAD) + ) + + override val operationalPhenomena: OperationalPhenomena by anyOf( + OperationalPhenomena(failureFrequency = 24.0 * 7, hasInterference = true) + ) + + override val allocationPolicy: String by anyOf( + "active-servers" + ) +} diff --git a/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/experiment/MoreVelocityPortfolio.kt b/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/experiment/MoreVelocityPortfolio.kt new file mode 100644 index 00000000..733dabf6 --- /dev/null +++ b/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/experiment/MoreVelocityPortfolio.kt @@ -0,0 +1,56 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.experiments.sc20.experiment + +import org.opendc.experiments.sc20.experiment.model.OperationalPhenomena +import org.opendc.experiments.sc20.experiment.model.Topology +import org.opendc.experiments.sc20.experiment.model.Workload +import org.opendc.harness.dsl.anyOf + +/** + * A [Portfolio] that explores the effect of adding more velocity to a cluster (e.g., faster machines). + */ +public class MoreVelocityPortfolio : Portfolio("more_velocity") { + override val topology: Topology by anyOf( + Topology("base"), + Topology("rep-vel-ver-hom"), + Topology("rep-vel-ver-het"), + Topology("exp-vel-ver-hom"), + Topology("exp-vel-ver-het") + ) + + override val workload: Workload by anyOf( + Workload("solvinity", 0.1), + Workload("solvinity", 0.25), + Workload("solvinity", 0.5), + Workload("solvinity", 1.0) + ) + + override val operationalPhenomena: OperationalPhenomena by anyOf( + OperationalPhenomena(failureFrequency = 24.0 * 7, hasInterference = true) + ) + + override val allocationPolicy: String by anyOf( + "active-servers" + ) +} diff --git a/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/experiment/OperationalPhenomenaPortfolio.kt b/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/experiment/OperationalPhenomenaPortfolio.kt new file mode 100644 index 00000000..66b94faf --- /dev/null +++ b/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/experiment/OperationalPhenomenaPortfolio.kt @@ -0,0 +1,61 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.experiments.sc20.experiment + +import org.opendc.experiments.sc20.experiment.model.OperationalPhenomena +import org.opendc.experiments.sc20.experiment.model.Topology +import org.opendc.experiments.sc20.experiment.model.Workload +import org.opendc.harness.dsl.anyOf + +/** + * A [Portfolio] that explores the effect of operational phenomena on metrics. + */ +public class OperationalPhenomenaPortfolio : Portfolio("operational_phenomena") { + override val topology: Topology by anyOf( + Topology("base") + ) + + override val workload: Workload by anyOf( + Workload("solvinity", 0.1), + Workload("solvinity", 0.25), + Workload("solvinity", 0.5), + Workload("solvinity", 1.0) + ) + + override val operationalPhenomena: OperationalPhenomena by anyOf( + OperationalPhenomena(failureFrequency = 24.0 * 7, hasInterference = true), + OperationalPhenomena(failureFrequency = 0.0, hasInterference = true), + OperationalPhenomena(failureFrequency = 24.0 * 7, hasInterference = false), + OperationalPhenomena(failureFrequency = 0.0, hasInterference = false) + ) + + override val allocationPolicy: String by anyOf( + "mem", + "mem-inv", + "core-mem", + "core-mem-inv", + "active-servers", + "active-servers-inv", + "random" + ) +} diff --git a/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/experiment/Portfolio.kt b/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/experiment/Portfolio.kt index 37cf2880..4a82ad56 100644 --- a/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/experiment/Portfolio.kt +++ b/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/experiment/Portfolio.kt @@ -22,67 +22,196 @@ package org.opendc.experiments.sc20.experiment +import kotlinx.coroutines.ExperimentalCoroutinesApi +import kotlinx.coroutines.cancel +import kotlinx.coroutines.channels.Channel +import kotlinx.coroutines.launch +import kotlinx.coroutines.test.TestCoroutineScope +import mu.KotlinLogging +import org.opendc.compute.simulator.allocation.* +import org.opendc.experiments.sc20.experiment.model.CompositeWorkload import org.opendc.experiments.sc20.experiment.model.OperationalPhenomena import org.opendc.experiments.sc20.experiment.model.Topology import org.opendc.experiments.sc20.experiment.model.Workload -import org.opendc.experiments.sc20.runner.ContainerExperimentDescriptor +import org.opendc.experiments.sc20.experiment.monitor.ParquetExperimentMonitor +import org.opendc.experiments.sc20.trace.Sc20ParquetTraceReader +import org.opendc.experiments.sc20.trace.Sc20RawParquetTraceReader +import org.opendc.format.environment.sc20.Sc20ClusterEnvironmentReader +import org.opendc.format.trace.PerformanceInterferenceModelReader +import org.opendc.harness.dsl.Experiment +import org.opendc.harness.dsl.anyOf +import org.opendc.simulator.utils.DelayControllerClockAdapter +import org.opendc.trace.core.EventTracer +import java.io.File +import java.util.concurrent.ConcurrentHashMap +import kotlin.random.Random /** - * A portfolio represents a collection of scenarios are tested. + * A portfolio represents a collection of scenarios are tested for the work. + * + * @param name The name of the portfolio. */ -public abstract class Portfolio( - override val parent: Experiment, - public val id: Int, - public val name: String -) : ContainerExperimentDescriptor() { +public abstract class Portfolio(name: String) : Experiment(name) { + /** + * The logger for this portfolio instance. + */ + private val logger = KotlinLogging.logger {} + + /** + * The path to where the environments are located. + */ + private val environmentPath by anyOf(File("environments/")) + + /** + * The path to where the traces are located. + */ + private val tracePath by anyOf(File("traces/")) + /** - * The topologies to consider. + * The path to where the output results should be written. */ - protected abstract val topologies: List<Topology> + private val outputPath by anyOf(File("results/")) /** - * The workloads to consider. + * The path to the original VM placements file. */ - protected abstract val workloads: List<Workload> + private val vmPlacements by anyOf(emptyMap<String, String>()) + + /** + * The path to the performance interference model. + */ + private val performanceInterferenceModel by anyOf<PerformanceInterferenceModelReader?>(null) + + /** + * The topology to test. + */ + public abstract val topology: Topology + + /** + * The workload to test. + */ + public abstract val workload: Workload /** * The operational phenomenas to consider. */ - protected abstract val operationalPhenomenas: List<OperationalPhenomena> + public abstract val operationalPhenomena: OperationalPhenomena /** * The allocation policies to consider. */ - protected abstract val allocationPolicies: List<String> + public abstract val allocationPolicy: String /** - * The number of repetitions to perform. + * A map of trace readers. */ - public open val repetitions: Int = 32 + private val traceReaders = ConcurrentHashMap<String, Sc20RawParquetTraceReader>() /** - * Resolve the children of this container. + * Perform a single trial for this portfolio. */ - override val children: Sequence<Scenario> = sequence { - var id = 0 - for (topology in topologies) { - for (workload in workloads) { - for (operationalPhenomena in operationalPhenomenas) { - for (allocationPolicy in allocationPolicies) { - yield( - Scenario( - this@Portfolio, - id++, - repetitions, - topology, - workload, - allocationPolicy, - operationalPhenomena - ) - ) - } - } + @OptIn(ExperimentalCoroutinesApi::class) + override fun doRun(repeat: Int) { + val testScope = TestCoroutineScope() + val clock = DelayControllerClockAdapter(testScope) + val tracer = EventTracer(clock) + val seeder = Random(repeat) + val environment = Sc20ClusterEnvironmentReader(File(environmentPath, "${topology.name}.txt")) + + val chan = Channel<Unit>(Channel.CONFLATED) + val allocationPolicy = createAllocationPolicy(seeder) + + val workload = workload + val workloadNames = if (workload is CompositeWorkload) { + workload.workloads.map { it.name } + } else { + listOf(workload.name) + } + + val rawReaders = workloadNames.map { workloadName -> + traceReaders.computeIfAbsent(workloadName) { + logger.info { "Loading trace $workloadName" } + Sc20RawParquetTraceReader(File(tracePath, workloadName)) + } + } + + val performanceInterferenceModel = performanceInterferenceModel + ?.takeIf { operationalPhenomena.hasInterference } + ?.construct(seeder) ?: emptyMap() + val trace = Sc20ParquetTraceReader(rawReaders, performanceInterferenceModel, workload, seeder.nextInt()) + + val monitor = ParquetExperimentMonitor( + outputPath, + "portfolio_id=$name/scenario_id=$id/run_id=$repeat", + 4096 + ) + + testScope.launch { + val (bareMetalProvisioner, scheduler) = createProvisioner( + this, + clock, + environment, + allocationPolicy, + tracer + ) + + val failureDomain = if (operationalPhenomena.failureFrequency > 0) { + logger.debug("ENABLING failures") + createFailureDomain( + this, + clock, + seeder.nextInt(), + operationalPhenomena.failureFrequency, + bareMetalProvisioner, + chan + ) + } else { + null } + + attachMonitor(this, clock, scheduler, monitor) + processTrace( + this, + clock, + trace, + scheduler, + chan, + monitor + ) + + logger.debug("SUBMIT=${scheduler.submittedVms}") + logger.debug("FAIL=${scheduler.unscheduledVms}") + logger.debug("QUEUED=${scheduler.queuedVms}") + logger.debug("RUNNING=${scheduler.runningVms}") + logger.debug("FINISHED=${scheduler.finishedVms}") + + failureDomain?.cancel() + scheduler.terminate() + } + + try { + testScope.advanceUntilIdle() + } finally { + monitor.close() + } + } + + /** + * Create the [AllocationPolicy] instance to use for the trial. + */ + private fun createAllocationPolicy(seeder: Random): AllocationPolicy { + return when (allocationPolicy) { + "mem" -> AvailableMemoryAllocationPolicy() + "mem-inv" -> AvailableMemoryAllocationPolicy(true) + "core-mem" -> AvailableCoreMemoryAllocationPolicy() + "core-mem-inv" -> AvailableCoreMemoryAllocationPolicy(true) + "active-servers" -> NumberOfActiveServersAllocationPolicy() + "active-servers-inv" -> NumberOfActiveServersAllocationPolicy(true) + "provisioned-cores" -> ProvisionedCoresAllocationPolicy() + "provisioned-cores-inv" -> ProvisionedCoresAllocationPolicy(true) + "random" -> RandomAllocationPolicy(Random(seeder.nextInt())) + "replay" -> ReplayAllocationPolicy(vmPlacements) + else -> throw IllegalArgumentException("Unknown policy $allocationPolicy") } } } diff --git a/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/experiment/Portfolios.kt b/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/experiment/Portfolios.kt deleted file mode 100644 index 249a63b9..00000000 --- a/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/experiment/Portfolios.kt +++ /dev/null @@ -1,221 +0,0 @@ -/* - * Copyright (c) 2020 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.experiments.sc20.experiment - -import org.opendc.experiments.sc20.experiment.model.* - -public class HorVerPortfolio(parent: Experiment, id: Int) : Portfolio(parent, id, "horizontal_vs_vertical") { - override val topologies: List<Topology> = listOf( - Topology("base"), - Topology("rep-vol-hor-hom"), - Topology("rep-vol-hor-het"), - Topology("rep-vol-ver-hom"), - Topology("rep-vol-ver-het"), - Topology("exp-vol-hor-hom"), - Topology("exp-vol-hor-het"), - Topology("exp-vol-ver-hom"), - Topology("exp-vol-ver-het") - ) - - override val workloads: List<Workload> = listOf( - Workload("solvinity", 0.1), - Workload("solvinity", 0.25), - Workload("solvinity", 0.5), - Workload("solvinity", 1.0) - ) - - override val operationalPhenomenas: List<OperationalPhenomena> = listOf( - OperationalPhenomena(failureFrequency = 24.0 * 7, hasInterference = true) - ) - - override val allocationPolicies: List<String> = listOf( - "active-servers" - ) -} - -public class MoreVelocityPortfolio(parent: Experiment, id: Int) : Portfolio(parent, id, "more_velocity") { - override val topologies: List<Topology> = listOf( - Topology("base"), - Topology("rep-vel-ver-hom"), - Topology("rep-vel-ver-het"), - Topology("exp-vel-ver-hom"), - Topology("exp-vel-ver-het") - ) - - override val workloads: List<Workload> = listOf( - Workload("solvinity", 0.1), - Workload("solvinity", 0.25), - Workload("solvinity", 0.5), - Workload("solvinity", 1.0) - ) - - override val operationalPhenomenas: List<OperationalPhenomena> = listOf( - OperationalPhenomena(failureFrequency = 24.0 * 7, hasInterference = true) - ) - - override val allocationPolicies: List<String> = listOf( - "active-servers" - ) -} - -public class CompositeWorkloadPortfolio(parent: Experiment, id: Int) : Portfolio(parent, id, "composite-workload") { - private val totalSampleLoad = 1.3301733005049648E12 - - override val topologies: List<Topology> = listOf( - Topology("base"), - Topology("exp-vol-hor-hom"), - Topology("exp-vol-ver-hom"), - Topology("exp-vel-ver-hom") - ) - - override val workloads: List<Workload> = listOf( - CompositeWorkload( - "all-azure", - listOf(Workload("solvinity-short", 0.0), Workload("azure", 1.0)), - totalSampleLoad - ), - CompositeWorkload( - "solvinity-25-azure-75", - listOf(Workload("solvinity-short", 0.25), Workload("azure", 0.75)), - totalSampleLoad - ), - CompositeWorkload( - "solvinity-50-azure-50", - listOf(Workload("solvinity-short", 0.5), Workload("azure", 0.5)), - totalSampleLoad - ), - CompositeWorkload( - "solvinity-75-azure-25", - listOf(Workload("solvinity-short", 0.75), Workload("azure", 0.25)), - totalSampleLoad - ), - CompositeWorkload( - "all-solvinity", - listOf(Workload("solvinity-short", 1.0), Workload("azure", 0.0)), - totalSampleLoad - ) - ) - - override val operationalPhenomenas: List<OperationalPhenomena> = listOf( - OperationalPhenomena(failureFrequency = 24.0 * 7, hasInterference = false) - ) - - override val allocationPolicies: List<String> = listOf( - "active-servers" - ) -} - -public class OperationalPhenomenaPortfolio(parent: Experiment, id: Int) : - Portfolio(parent, id, "operational_phenomena") { - override val topologies: List<Topology> = listOf( - Topology("base") - ) - - override val workloads: List<Workload> = listOf( - Workload("solvinity", 0.1), - Workload("solvinity", 0.25), - Workload("solvinity", 0.5), - Workload("solvinity", 1.0) - ) - - override val operationalPhenomenas: List<OperationalPhenomena> = listOf( - OperationalPhenomena(failureFrequency = 24.0 * 7, hasInterference = true), - OperationalPhenomena(failureFrequency = 0.0, hasInterference = true), - OperationalPhenomena(failureFrequency = 24.0 * 7, hasInterference = false), - OperationalPhenomena(failureFrequency = 0.0, hasInterference = false) - ) - - override val allocationPolicies: List<String> = listOf( - "mem", - "mem-inv", - "core-mem", - "core-mem-inv", - "active-servers", - "active-servers-inv", - "random" - ) -} - -public class ReplayPortfolio(parent: Experiment, id: Int) : Portfolio(parent, id, "replay") { - override val topologies: List<Topology> = listOf( - Topology("base") - ) - - override val workloads: List<Workload> = listOf( - Workload("solvinity", 1.0) - ) - - override val operationalPhenomenas: List<OperationalPhenomena> = listOf( - OperationalPhenomena(failureFrequency = 0.0, hasInterference = false) - ) - - override val allocationPolicies: List<String> = listOf( - "replay", - "active-servers" - ) -} - -public class TestPortfolio(parent: Experiment, id: Int) : Portfolio(parent, id, "test") { - override val repetitions: Int = 1 - - override val topologies: List<Topology> = listOf( - Topology("base") - ) - - override val workloads: List<Workload> = listOf( - Workload("solvinity", 1.0) - ) - - override val operationalPhenomenas: List<OperationalPhenomena> = listOf( - OperationalPhenomena(failureFrequency = 24.0 * 7, hasInterference = true) - ) - - override val allocationPolicies: List<String> = listOf("active-servers") -} - -public class MoreHpcPortfolio(parent: Experiment, id: Int) : Portfolio(parent, id, "more_hpc") { - override val topologies: List<Topology> = listOf( - Topology("base"), - Topology("exp-vol-hor-hom"), - Topology("exp-vol-ver-hom"), - Topology("exp-vel-ver-hom") - ) - - override val workloads: List<Workload> = listOf( - Workload("solvinity", 0.0, samplingStrategy = SamplingStrategy.HPC), - Workload("solvinity", 0.25, samplingStrategy = SamplingStrategy.HPC), - Workload("solvinity", 0.5, samplingStrategy = SamplingStrategy.HPC), - Workload("solvinity", 1.0, samplingStrategy = SamplingStrategy.HPC), - Workload("solvinity", 0.25, samplingStrategy = SamplingStrategy.HPC_LOAD), - Workload("solvinity", 0.5, samplingStrategy = SamplingStrategy.HPC_LOAD), - Workload("solvinity", 1.0, samplingStrategy = SamplingStrategy.HPC_LOAD) - ) - - override val operationalPhenomenas: List<OperationalPhenomena> = listOf( - OperationalPhenomena(failureFrequency = 24.0 * 7, hasInterference = true) - ) - - override val allocationPolicies: List<String> = listOf( - "active-servers" - ) -} diff --git a/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/experiment/Scenario.kt b/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/experiment/ReplayPortfolio.kt index d092ddd5..8a42a3b4 100644 --- a/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/experiment/Scenario.kt +++ b/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/experiment/ReplayPortfolio.kt @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020 AtLarge Research + * Copyright (c) 2021 AtLarge Research * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -25,22 +25,26 @@ package org.opendc.experiments.sc20.experiment import org.opendc.experiments.sc20.experiment.model.OperationalPhenomena import org.opendc.experiments.sc20.experiment.model.Topology import org.opendc.experiments.sc20.experiment.model.Workload -import org.opendc.experiments.sc20.runner.ContainerExperimentDescriptor -import org.opendc.experiments.sc20.runner.ExperimentDescriptor +import org.opendc.harness.dsl.anyOf /** - * A scenario represents a single point in the design space (a unique combination of parameters). + * A [Portfolio] that compares the original VM placements against our policies. */ -public class Scenario( - override val parent: Portfolio, - public val id: Int, - public val repetitions: Int, - public val topology: Topology, - public val workload: Workload, - public val allocationPolicy: String, - public val operationalPhenomena: OperationalPhenomena -) : ContainerExperimentDescriptor() { - override val children: Sequence<ExperimentDescriptor> = sequence { - repeat(repetitions) { i -> yield(Run(this@Scenario, i, i)) } - } +public class ReplayPortfolio : Portfolio("replay") { + override val topology: Topology by anyOf( + Topology("base") + ) + + override val workload: Workload by anyOf( + Workload("solvinity", 1.0) + ) + + override val operationalPhenomena: OperationalPhenomena by anyOf( + OperationalPhenomena(failureFrequency = 0.0, hasInterference = false) + ) + + override val allocationPolicy: String by anyOf( + "replay", + "active-servers" + ) } diff --git a/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/experiment/Run.kt b/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/experiment/Run.kt deleted file mode 100644 index 8d8d608d..00000000 --- a/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/experiment/Run.kt +++ /dev/null @@ -1,157 +0,0 @@ -/* - * Copyright (c) 2020 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.experiments.sc20.experiment - -import kotlinx.coroutines.ExperimentalCoroutinesApi -import kotlinx.coroutines.cancel -import kotlinx.coroutines.channels.Channel -import kotlinx.coroutines.launch -import kotlinx.coroutines.test.TestCoroutineScope -import mu.KotlinLogging -import org.opendc.compute.simulator.allocation.* -import org.opendc.experiments.sc20.experiment.model.CompositeWorkload -import org.opendc.experiments.sc20.experiment.monitor.ParquetExperimentMonitor -import org.opendc.experiments.sc20.runner.TrialExperimentDescriptor -import org.opendc.experiments.sc20.runner.execution.ExperimentExecutionContext -import org.opendc.experiments.sc20.trace.Sc20ParquetTraceReader -import org.opendc.experiments.sc20.trace.Sc20RawParquetTraceReader -import org.opendc.format.environment.sc20.Sc20ClusterEnvironmentReader -import org.opendc.simulator.utils.DelayControllerClockAdapter -import org.opendc.trace.core.EventTracer -import java.io.File -import kotlin.random.Random - -/** - * The logger for the experiment scenario. - */ -private val logger = KotlinLogging.logger {} - -/** - * An experiment run represent a single invocation of a trial and is used to distinguish between repetitions of the - * same set of parameters. - */ -@OptIn(ExperimentalCoroutinesApi::class) -public data class Run(override val parent: Scenario, val id: Int, val seed: Int) : TrialExperimentDescriptor() { - override suspend fun invoke(context: ExperimentExecutionContext) { - val experiment = parent.parent.parent - val testScope = TestCoroutineScope() - val clock = DelayControllerClockAdapter(testScope) - val seeder = Random(seed) - val environment = Sc20ClusterEnvironmentReader(File(experiment.environments, "${parent.topology.name}.txt")) - - val chan = Channel<Unit>(Channel.CONFLATED) - val allocationPolicy = when (parent.allocationPolicy) { - "mem" -> AvailableMemoryAllocationPolicy() - "mem-inv" -> AvailableMemoryAllocationPolicy(true) - "core-mem" -> AvailableCoreMemoryAllocationPolicy() - "core-mem-inv" -> AvailableCoreMemoryAllocationPolicy(true) - "active-servers" -> NumberOfActiveServersAllocationPolicy() - "active-servers-inv" -> NumberOfActiveServersAllocationPolicy(true) - "provisioned-cores" -> ProvisionedCoresAllocationPolicy() - "provisioned-cores-inv" -> ProvisionedCoresAllocationPolicy(true) - "random" -> RandomAllocationPolicy(Random(seeder.nextInt())) - "replay" -> ReplayAllocationPolicy(experiment.vmPlacements) - else -> throw IllegalArgumentException("Unknown policy ${parent.allocationPolicy}") - } - - @Suppress("UNCHECKED_CAST") - val rawTraceReaders = - context.cache.computeIfAbsent("raw-trace-readers") { mutableMapOf<String, Sc20RawParquetTraceReader>() } as MutableMap<String, Sc20RawParquetTraceReader> - val rawReaders = synchronized(rawTraceReaders) { - val workloadNames = if (parent.workload is CompositeWorkload) { - parent.workload.workloads.map { it.name } - } else { - listOf(parent.workload.name) - } - - workloadNames.map { workloadName -> - rawTraceReaders.computeIfAbsent(workloadName) { - logger.info { "Loading trace $workloadName" } - Sc20RawParquetTraceReader(File(experiment.traces, workloadName)) - } - } - } - - val performanceInterferenceModel = experiment.performanceInterferenceModel - ?.takeIf { parent.operationalPhenomena.hasInterference } - ?.construct(seeder) ?: emptyMap() - val trace = Sc20ParquetTraceReader(rawReaders, performanceInterferenceModel, parent.workload, seed) - - val monitor = ParquetExperimentMonitor( - parent.parent.parent.output, - "portfolio_id=${parent.parent.id}/scenario_id=${parent.id}/run_id=$id", - parent.parent.parent.bufferSize - ) - - val tracer = EventTracer(clock) - - testScope.launch { - val (bareMetalProvisioner, scheduler) = createProvisioner( - this, - clock, - environment, - allocationPolicy, - tracer - ) - - val failureDomain = if (parent.operationalPhenomena.failureFrequency > 0) { - logger.debug("ENABLING failures") - createFailureDomain( - this, - clock, - seeder.nextInt(), - parent.operationalPhenomena.failureFrequency, - bareMetalProvisioner, - chan - ) - } else { - null - } - - attachMonitor(this, clock, scheduler, monitor) - processTrace( - this, - clock, - trace, - scheduler, - chan, - monitor - ) - - logger.debug("SUBMIT=${scheduler.submittedVms}") - logger.debug("FAIL=${scheduler.unscheduledVms}") - logger.debug("QUEUED=${scheduler.queuedVms}") - logger.debug("RUNNING=${scheduler.runningVms}") - logger.debug("FINISHED=${scheduler.finishedVms}") - - failureDomain?.cancel() - scheduler.terminate() - } - - try { - testScope.advanceUntilIdle() - } finally { - monitor.close() - } - } -} diff --git a/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/experiment/TestPortfolio.kt b/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/experiment/TestPortfolio.kt new file mode 100644 index 00000000..2210fc97 --- /dev/null +++ b/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/experiment/TestPortfolio.kt @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.experiments.sc20.experiment + +import org.opendc.experiments.sc20.experiment.model.OperationalPhenomena +import org.opendc.experiments.sc20.experiment.model.Topology +import org.opendc.experiments.sc20.experiment.model.Workload +import org.opendc.harness.dsl.anyOf + +/** + * A [Portfolio] to perform a simple test run. + */ +public class TestPortfolio : Portfolio("test") { + override val topology: Topology by anyOf( + Topology("base") + ) + + override val workload: Workload by anyOf( + Workload("solvinity", 1.0) + ) + + override val operationalPhenomena: OperationalPhenomena by anyOf( + OperationalPhenomena(failureFrequency = 24.0 * 7, hasInterference = true) + ) + + override val allocationPolicy: String by anyOf("active-servers") +} diff --git a/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/runner/ContainerExperimentDescriptor.kt b/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/runner/ContainerExperimentDescriptor.kt deleted file mode 100644 index d70e8c9a..00000000 --- a/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/runner/ContainerExperimentDescriptor.kt +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Copyright (c) 2020 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.experiments.sc20.runner - -import kotlinx.coroutines.launch -import kotlinx.coroutines.supervisorScope -import org.opendc.experiments.sc20.runner.execution.ExperimentExecutionContext -import org.opendc.experiments.sc20.runner.execution.ExperimentExecutionResult - -/** - * An abstract [ExperimentDescriptor] specifically for containers. - */ -public abstract class ContainerExperimentDescriptor : ExperimentDescriptor() { - /** - * The child descriptors of this container. - */ - public abstract val children: Sequence<ExperimentDescriptor> - - override val type: Type = Type.CONTAINER - - override suspend fun invoke(context: ExperimentExecutionContext) { - val materializedChildren = children.toList() - for (child in materializedChildren) { - context.listener.descriptorRegistered(child) - } - - supervisorScope { - for (child in materializedChildren) { - if (child.isTrial) { - launch { - val worker = context.scheduler.allocate() - context.listener.executionStarted(child) - try { - worker(child, context) - context.listener.executionFinished(child, ExperimentExecutionResult.Success) - } catch (e: Throwable) { - context.listener.executionFinished(child, ExperimentExecutionResult.Failed(e)) - } - } - } else { - launch { child(context) } - } - } - } - } -} diff --git a/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/runner/ExperimentDescriptor.kt b/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/runner/ExperimentDescriptor.kt deleted file mode 100644 index 1e67c086..00000000 --- a/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/runner/ExperimentDescriptor.kt +++ /dev/null @@ -1,79 +0,0 @@ -/* - * Copyright (c) 2020 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.experiments.sc20.runner - -import org.opendc.experiments.sc20.runner.execution.ExperimentExecutionContext -import java.io.Serializable - -/** - * An immutable description of an experiment in the **odcsim* simulation framework, which may be a single atomic trial - * or a composition of multiple trials. - * - * This class represents a dynamic tree-like structure where the children of the nodes are not known at instantiation - * since they might be generated dynamically. - */ -public abstract class ExperimentDescriptor : Serializable { - /** - * The parent of this descriptor, or `null` if it has no parent. - */ - public abstract val parent: ExperimentDescriptor? - - /** - * The type of descriptor. - */ - public abstract val type: Type - - /** - * A flag to indicate that this descriptor is a root descriptor. - */ - public open val isRoot: Boolean - get() = parent == null - - /** - * A flag to indicate that this descriptor describes an experiment trial. - */ - public val isTrial: Boolean - get() = type == Type.TRIAL - - /** - * Execute this [ExperimentDescriptor]. - * - * @param context The context to execute the descriptor in. - */ - public abstract suspend operator fun invoke(context: ExperimentExecutionContext) - - /** - * The types of experiment descriptors. - */ - public enum class Type { - /** - * A composition of multiple experiment descriptions whose invocation happens on a single thread. - */ - CONTAINER, - - /** - * An invocation of a single scenario of an experiment whose invocation may happen on different threads. - */ - TRIAL - } -} diff --git a/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/runner/execution/ThreadPoolExperimentScheduler.kt b/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/runner/execution/ThreadPoolExperimentScheduler.kt deleted file mode 100644 index 942faed1..00000000 --- a/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/runner/execution/ThreadPoolExperimentScheduler.kt +++ /dev/null @@ -1,83 +0,0 @@ -/* - * Copyright (c) 2020 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.experiments.sc20.runner.execution - -import kotlinx.coroutines.asCoroutineDispatcher -import kotlinx.coroutines.launch -import kotlinx.coroutines.supervisorScope -import kotlinx.coroutines.sync.Semaphore -import kotlinx.coroutines.withContext -import org.opendc.experiments.sc20.runner.ExperimentDescriptor -import java.util.concurrent.Executors - -/** - * An [ExperimentScheduler] that runs experiments using a local thread pool. - * - * @param parallelism The maximum amount of parallel workers (default is the number of available processors). - */ -public class ThreadPoolExperimentScheduler(parallelism: Int = Runtime.getRuntime().availableProcessors() + 1) : ExperimentScheduler { - private val dispatcher = Executors.newCachedThreadPool().asCoroutineDispatcher() - private val tickets = Semaphore(parallelism) - - override suspend fun allocate(): ExperimentScheduler.Worker { - tickets.acquire() - return object : ExperimentScheduler.Worker { - override suspend fun invoke( - descriptor: ExperimentDescriptor, - context: ExperimentExecutionContext - ) = supervisorScope { - val listener = - object : ExperimentExecutionListener { - override fun descriptorRegistered(descriptor: ExperimentDescriptor) { - launch { context.listener.descriptorRegistered(descriptor) } - } - - override fun executionFinished( - descriptor: ExperimentDescriptor, - result: ExperimentExecutionResult - ) { - launch { context.listener.executionFinished(descriptor, result) } - } - - override fun executionStarted(descriptor: ExperimentDescriptor) { - launch { context.listener.executionStarted(descriptor) } - } - } - - val newContext = object : ExperimentExecutionContext by context { - override val listener: ExperimentExecutionListener = listener - } - - try { - withContext(dispatcher) { - descriptor(newContext) - } - } finally { - tickets.release() - } - } - } - } - - override fun close(): Unit = dispatcher.close() -} diff --git a/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/runner/internal/DefaultExperimentRunner.kt b/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/runner/internal/DefaultExperimentRunner.kt deleted file mode 100644 index 26e4df89..00000000 --- a/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/runner/internal/DefaultExperimentRunner.kt +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Copyright (c) 2020 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.experiments.sc20.runner.internal - -import kotlinx.coroutines.runBlocking -import org.opendc.experiments.sc20.runner.ExperimentDescriptor -import org.opendc.experiments.sc20.runner.ExperimentRunner -import org.opendc.experiments.sc20.runner.execution.ExperimentExecutionContext -import org.opendc.experiments.sc20.runner.execution.ExperimentExecutionListener -import org.opendc.experiments.sc20.runner.execution.ExperimentExecutionResult -import org.opendc.experiments.sc20.runner.execution.ExperimentScheduler -import java.util.concurrent.ConcurrentHashMap - -/** - * The default implementation of the [ExperimentRunner] interface. - * - * @param scheduler The scheduler to use. - */ -public class DefaultExperimentRunner(private val scheduler: ExperimentScheduler) : ExperimentRunner { - override val id: String = "default" - - override val version: String? = "1.0" - - override fun execute(root: ExperimentDescriptor, listener: ExperimentExecutionListener): Unit = runBlocking { - val context = object : ExperimentExecutionContext { - override val listener: ExperimentExecutionListener = listener - override val scheduler: ExperimentScheduler = this@DefaultExperimentRunner.scheduler - override val cache: MutableMap<Any?, Any?> = ConcurrentHashMap() - } - - listener.descriptorRegistered(root) - context.listener.executionStarted(root) - try { - root(context) - context.listener.executionFinished(root, ExperimentExecutionResult.Success) - } catch (e: Throwable) { - context.listener.executionFinished(root, ExperimentExecutionResult.Failed(e)) - } - } -} diff --git a/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/telemetry/RunEvent.kt b/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/telemetry/RunEvent.kt index 3bcd10a1..4f4706f0 100644 --- a/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/telemetry/RunEvent.kt +++ b/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/telemetry/RunEvent.kt @@ -22,12 +22,13 @@ package org.opendc.experiments.sc20.telemetry -import org.opendc.experiments.sc20.experiment.Run +import org.opendc.experiments.sc20.experiment.Portfolio /** * A periodic report of the host machine metrics. */ public data class RunEvent( - public val run: Run, + val portfolio: Portfolio, + val repeat: Int, override val timestamp: Long ) : Event("run") diff --git a/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/telemetry/parquet/ParquetRunEventWriter.kt b/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/telemetry/parquet/ParquetRunEventWriter.kt index 74efb660..b50a698c 100644 --- a/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/telemetry/parquet/ParquetRunEventWriter.kt +++ b/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/telemetry/parquet/ParquetRunEventWriter.kt @@ -38,33 +38,27 @@ public class ParquetRunEventWriter(path: File, bufferSize: Int) : public companion object { private val convert: (RunEvent, GenericData.Record) -> Unit = { event, record -> - val run = event.run - val scenario = run.parent - val portfolio = scenario.parent - record.put("portfolio_id", portfolio.id) + val portfolio = event.portfolio record.put("portfolio_name", portfolio.name) - record.put("scenario_id", scenario.id) - record.put("run_id", run.id) - record.put("repetitions", scenario.repetitions) - record.put("topology", scenario.topology.name) - record.put("workload_name", scenario.workload.name) - record.put("workload_fraction", scenario.workload.fraction) - record.put("workload_sampler", scenario.workload.samplingStrategy) - record.put("allocation_policy", scenario.allocationPolicy) - record.put("failure_frequency", scenario.operationalPhenomena.failureFrequency) - record.put("interference", scenario.operationalPhenomena.hasInterference) - record.put("seed", run.seed) + record.put("scenario_id", portfolio.id) + record.put("run_id", event.repeat) + record.put("topology", portfolio.topology.name) + record.put("workload_name", portfolio.workload.name) + record.put("workload_fraction", portfolio.workload.fraction) + record.put("workload_sampler", portfolio.workload.samplingStrategy) + record.put("allocation_policy", portfolio.allocationPolicy) + record.put("failure_frequency", portfolio.operationalPhenomena.failureFrequency) + record.put("interference", portfolio.operationalPhenomena.hasInterference) + record.put("seed", event.repeat) } private val schema: Schema = SchemaBuilder .record("runs") .namespace("org.opendc.experiments.sc20") .fields() - .name("portfolio_id").type().intType().noDefault() .name("portfolio_name").type().stringType().noDefault() .name("scenario_id").type().intType().noDefault() .name("run_id").type().intType().noDefault() - .name("repetitions").type().intType().noDefault() .name("topology").type().stringType().noDefault() .name("workload_name").type().stringType().noDefault() .name("workload_fraction").type().doubleType().noDefault() diff --git a/simulator/opendc-harness/build.gradle.kts b/simulator/opendc-harness/build.gradle.kts new file mode 100644 index 00000000..359d2384 --- /dev/null +++ b/simulator/opendc-harness/build.gradle.kts @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2020 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +description = "Harness for defining repeatable experiments using OpenDC" + +/* Build configuration */ +plugins { + `kotlin-library-convention` +} + +dependencies { + api("org.jetbrains.kotlinx:kotlinx-coroutines-core:${Library.KOTLINX_COROUTINES}") + api("org.junit.platform:junit-platform-commons:${Library.JUNIT_PLATFORM}") + + implementation("io.github.classgraph:classgraph:4.8.98") + implementation("me.tongfei:progressbar:0.9.0") + implementation("io.github.microutils:kotlin-logging:2.0.4") + implementation("com.github.ajalt.clikt:clikt:3.1.0") + + api("org.junit.platform:junit-platform-engine:${Library.JUNIT_PLATFORM}") + api("org.junit.platform:junit-platform-suite-api:${Library.JUNIT_PLATFORM}") + api("org.junit.platform:junit-platform-launcher:${Library.JUNIT_PLATFORM}") + + runtimeOnly("org.apache.logging.log4j:log4j-slf4j-impl:2.14.0") + + testImplementation("org.junit.jupiter:junit-jupiter-api:${Library.JUNIT_JUPITER}") + testRuntimeOnly("org.junit.jupiter:junit-jupiter-engine:${Library.JUNIT_JUPITER}") +} diff --git a/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/api/ExperimentDefinition.kt b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/api/ExperimentDefinition.kt new file mode 100644 index 00000000..88b26ee1 --- /dev/null +++ b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/api/ExperimentDefinition.kt @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2020 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.harness.api + +/** + * A definition for a repeatable experiment which consists of multiple scenarios derived from pre-defined experiment + * parameters. + * + * @property name The name of the experiment. + * @property parameters The parameters of the experiments. + * @property evaluator The function to evaluate a single experiment trial. + * @property meta The metadata for the experiment. + */ +public data class ExperimentDefinition( + val name: String, + val parameters: Set<Parameter<*>>, + val evaluator: (Trial) -> Unit, + val meta: Map<String, Any> = emptyMap() +) diff --git a/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/runner/execution/ExperimentExecutionResult.kt b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/api/Parameter.kt index a765c264..bb5c8c2b 100644 --- a/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/runner/execution/ExperimentExecutionResult.kt +++ b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/api/Parameter.kt @@ -1,7 +1,5 @@ /* - * MIT License - * - * Copyright (c) 2020 atlarge-research + * Copyright (c) 2020 AtLarge Research * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -22,21 +20,19 @@ * SOFTWARE. */ -package org.opendc.experiments.sc20.runner.execution - -import java.io.Serializable +package org.opendc.harness.api /** - * The result of executing an experiment. + * A [Parameter] defines a single dimension of exploration in the design space of an experiment. */ -public sealed class ExperimentExecutionResult : Serializable { +public sealed class Parameter<T> { /** - * The experiment executed successfully + * The name of the parameter. */ - public object Success : ExperimentExecutionResult() + public abstract val name: String /** - * The experiment failed during execution. + * A generic dimension of the experiment design space that is defined fully by a collection of [values]. */ - public data class Failed(val throwable: Throwable) : ExperimentExecutionResult() + public data class Generic<T>(override val name: String, val values: Collection<T>) : Parameter<T>() } diff --git a/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/runner/ExperimentRunner.kt b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/api/Scenario.kt index a59481c0..a8dbf01e 100644 --- a/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/runner/ExperimentRunner.kt +++ b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/api/Scenario.kt @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020 AtLarge Research + * Copyright (c) 2021 AtLarge Research * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -20,30 +20,27 @@ * SOFTWARE. */ -package org.opendc.experiments.sc20.runner - -import org.opendc.experiments.sc20.runner.execution.ExperimentExecutionListener +package org.opendc.harness.api /** - * An [ExperimentRunner] facilitates discovery and execution of experiments. + * A [Scenario] represents a single point in the design space of an experiment. */ -public interface ExperimentRunner { +public interface Scenario { /** - * The unique identifier of this runner. + * A unique identifier that identifies a single scenario. */ - public val id: String + public val id: Int /** - * The version of this runner. + * The [ExperimentDefinition] describing the experiment this scenario is part of. */ - public val version: String? - get() = null + public val experiment: ExperimentDefinition /** - * Execute the specified experiment represented as [ExperimentDescriptor]. + * Obtain the instantiated value for a [parameter][param] of the experiment. * - * @param root The experiment to execute. - * @param listener The listener to report events to. + * @param param The parameter to obtain the value of. + * @throws IllegalArgumentException if [param] is not defined for the experiment. */ - public fun execute(root: ExperimentDescriptor, listener: ExperimentExecutionListener) + public operator fun <T> get(param: Parameter<T>): T } diff --git a/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/runner/TrialExperimentDescriptor.kt b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/api/Trial.kt index abc52997..2d6ecd19 100644 --- a/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/runner/TrialExperimentDescriptor.kt +++ b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/api/Trial.kt @@ -1,7 +1,5 @@ /* - * MIT License - * - * Copyright (c) 2020 atlarge-research + * Copyright (c) 2021 AtLarge Research * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -22,11 +20,9 @@ * SOFTWARE. */ -package org.opendc.experiments.sc20.runner +package org.opendc.harness.api /** - * An abstract [ExperimentDescriptor] specifically for trials. + * A [Trial] represents a single trial (run) of an experiment. */ -public abstract class TrialExperimentDescriptor : ExperimentDescriptor() { - override val type: Type = Type.TRIAL -} +public data class Trial(val scenario: Scenario, val repeat: Int) diff --git a/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/dsl/Experiment.kt b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/dsl/Experiment.kt new file mode 100644 index 00000000..41d4207a --- /dev/null +++ b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/dsl/Experiment.kt @@ -0,0 +1,99 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.harness.dsl + +import org.junit.platform.commons.annotation.Testable +import org.opendc.harness.api.ExperimentDefinition +import org.opendc.harness.api.Scenario +import org.opendc.harness.api.Trial +import org.opendc.harness.internal.ParameterDelegate + +/** + * An [Experiment] defines a blueprint for a repeatable experiment consisting of multiple scenarios based on pre-defined + * experiment parameters. + * + * @param name The name of the experiment or `null` to select the class name. + */ +@Testable +public abstract class Experiment(name: String? = null) : Cloneable { + /** + * The name of the experiment. + */ + public val name: String = name ?: javaClass.simpleName + + /** + * An identifier that uniquely identifies a single point in the design space of this [Experiment]. + */ + public val id: Int + get() { + val scenario = scenario ?: throw IllegalStateException("Cannot use id before activation") + return scenario.id + } + + /** + * Convert this experiment to an [ExperimentDefinition]. + */ + public fun toDefinition(): ExperimentDefinition = + ExperimentDefinition(name, HashSet(delegates.map { it.parameter }), this::run, mapOf("class.name" to javaClass.name)) + + /** + * Perform a single execution of the experiment based on the experiment parameters. + * + * @param repeat A number representing the repeat index of an identical scenario. + */ + protected abstract fun doRun(repeat: Int) + + /** + * A map to track the parameter delegates registered with this experiment. + */ + private val delegates: MutableSet<ParameterDelegate<*>> = mutableSetOf() + + /** + * The current active scenario. + */ + internal var scenario: Scenario? = null + + /** + * Perform a single execution of the experiment based on the experiment parameters. + * + * This operation will cause the [ParameterProvider]s of this group to be instantiated to some value in its design + * space. + * + * @param trial The experiment trial to run. + */ + private fun run(trial: Trial) { + val scenario = trial.scenario + + // XXX We clone the current class to prevent concurrency issues. + val res = clone() as Experiment + res.scenario = scenario + res.doRun(trial.repeat) + } + + /** + * Register a delegate for this experiment. + */ + internal fun register(delegate: ParameterDelegate<*>) { + delegates += delegate + } +} diff --git a/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/dsl/ParameterProvider.kt b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/dsl/ParameterProvider.kt new file mode 100644 index 00000000..e4bb9c64 --- /dev/null +++ b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/dsl/ParameterProvider.kt @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.harness.dsl + +import kotlin.properties.ReadOnlyProperty +import kotlin.reflect.KProperty + +/** + * A [ParameterProvider] defines a single dimension of exploration in the design space of an [Experiment]. + */ +public interface ParameterProvider<T> { + /** + * Provide a delegate defining a parameter for the specified [Experiment][experiment]. + * + * @param experiment The experiment for which the parameter is defined. + * @param prop The property to create the delegate for. + */ + public operator fun provideDelegate(experiment: Experiment, prop: KProperty<*>): ReadOnlyProperty<Experiment, T> +} diff --git a/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/dsl/Parameters.kt b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/dsl/Parameters.kt new file mode 100644 index 00000000..7d269ba1 --- /dev/null +++ b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/dsl/Parameters.kt @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.harness.dsl + +import org.opendc.harness.api.Parameter +import org.opendc.harness.internal.ParameterDelegate +import kotlin.properties.ReadOnlyProperty +import kotlin.reflect.KProperty + +/** + * Define a dimension in the design space of an [Experiment] of type [T] consisting of the specified collection of + * [values]. + * + * @param values The values in the dimension to define.. + */ +public fun <T> anyOf(vararg values: T): ParameterProvider<T> = object : ParameterProvider<T> { + override fun provideDelegate(experiment: Experiment, prop: KProperty<*>): ReadOnlyProperty<Experiment, T> { + val delegate = ParameterDelegate(Parameter.Generic(prop.name, listOf(*values))) + experiment.register(delegate) + return delegate + } + + override fun toString(): String = "GenericParameter" +} diff --git a/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/ExperimentEngine.kt b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/ExperimentEngine.kt new file mode 100644 index 00000000..db2cd191 --- /dev/null +++ b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/ExperimentEngine.kt @@ -0,0 +1,107 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.harness.engine + +import kotlinx.coroutines.CancellationException +import kotlinx.coroutines.flow.asFlow +import kotlinx.coroutines.flow.buffer +import kotlinx.coroutines.flow.collect +import kotlinx.coroutines.flow.map +import kotlinx.coroutines.launch +import kotlinx.coroutines.supervisorScope +import org.opendc.harness.api.ExperimentDefinition +import org.opendc.harness.api.Trial +import org.opendc.harness.engine.scheduler.ExperimentScheduler +import org.opendc.harness.engine.strategy.ExperimentStrategy + +/** + * The [ExperimentEngine] orchestrates the execution of experiments. + * + * @property strategy The [ExperimentStrategy] used to explore the experiment design space. + * @property scheduler The [ExperimentScheduler] to schedule the trials over compute resources. + * @property listener The [ExperimentExecutionListener] to observe the progress. + * @property repeats The number of repeats to perform. + */ +public class ExperimentEngine( + private val strategy: ExperimentStrategy, + private val scheduler: ExperimentScheduler, + private val listener: ExperimentExecutionListener, + private val repeats: Int +) { + /** + * Execute the specified [experiment][root]. + * + * @param root The experiment to execute. + */ + public suspend fun execute(root: ExperimentDefinition): Unit = supervisorScope { + listener.experimentStarted(root) + + try { + strategy.generate(root) + .asFlow() + .map { scenario -> + listener.scenarioStarted(scenario) + scenario + } + .buffer(100) + .collect { scenario -> + val jobs = (0 until repeats).map { repeat -> + val worker = scheduler.allocate() + launch { + val trial = Trial(scenario, repeat) + try { + listener.trialStarted(trial) + worker.dispatch(trial) + listener.trialFinished(trial, null) + } catch (e: Throwable) { + listener.trialFinished(trial, e) + throw e + } + } + } + + launch { + var error: Throwable? = null + for (job in jobs) { + try { + job.join() + } catch (e: CancellationException) { + // Propagate cancellation + throw e + } catch (e: Throwable) { + error = e + } + } + + listener.scenarioFinished(scenario, error) + } + } + listener.experimentFinished(root, null) + } catch (e: Throwable) { + listener.experimentFinished(root, e) + throw e + } + } + + override fun toString(): String = "ExperimentEngine" +} diff --git a/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/ExperimentEngineLauncher.kt b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/ExperimentEngineLauncher.kt new file mode 100644 index 00000000..ddd30483 --- /dev/null +++ b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/ExperimentEngineLauncher.kt @@ -0,0 +1,121 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.harness.engine + +import kotlinx.coroutines.flow.Flow +import kotlinx.coroutines.flow.collect +import kotlinx.coroutines.runBlocking +import org.opendc.harness.api.ExperimentDefinition +import org.opendc.harness.engine.scheduler.ExperimentScheduler +import org.opendc.harness.engine.scheduler.ThreadPoolExperimentScheduler +import org.opendc.harness.engine.strategy.CartesianExperimentStrategy +import org.opendc.harness.engine.strategy.ExperimentStrategy +import org.opendc.harness.internal.CompositeExperimentExecutionListener + +/** + * A builder class for conducting experiments via the [ExperimentEngine]. + */ +public class ExperimentEngineLauncher private constructor( + private val strategy: ExperimentStrategy?, + private val scheduler: ExperimentScheduler?, + private val listeners: List<ExperimentExecutionListener>, + private val repeats: Int +) { + /** + * Construct an [ExperimentEngineLauncher] instance. + */ + public constructor() : this(null, null, emptyList(), 1) + + /** + * Create an [ExperimentEngineLauncher] with the specified [strategy]. + */ + public fun withScheduler(strategy: ExperimentStrategy): ExperimentEngineLauncher { + return ExperimentEngineLauncher(strategy, scheduler, listeners, repeats) + } + + /** + * Create an [ExperimentEngineLauncher] with the specified [scheduler]. + */ + public fun withScheduler(scheduler: ExperimentScheduler): ExperimentEngineLauncher { + return ExperimentEngineLauncher(strategy, scheduler, listeners, repeats) + } + + /** + * Create an [ExperimentEngineLauncher] with the specified [listener] added. + */ + public fun withListener(listener: ExperimentExecutionListener): ExperimentEngineLauncher { + return ExperimentEngineLauncher(strategy, scheduler, listeners + listener, repeats) + } + + /** + * Create an [ExperimentEngineLauncher] with the specified number of repeats. + */ + public fun withRepeats(repeats: Int): ExperimentEngineLauncher { + require(repeats > 0) { "Invalid number of repeats; must be greater than zero. " } + return ExperimentEngineLauncher(strategy, scheduler, listeners, repeats) + } + + /** + * Launch the specified experiments via the [ExperimentEngine] and block execution until finished. + */ + public suspend fun run(experiments: Flow<ExperimentDefinition>) { + val engine = ExperimentEngine(createStrategy(), createScheduler(), createListener(), repeats) + experiments.collect { experiment -> engine.execute(experiment) } + } + + /** + * Launch the specified experiments via the [ExperimentEngine] and block the current thread until finished. + */ + public fun runBlocking(experiments: Flow<ExperimentDefinition>) { + runBlocking { + run(experiments) + } + } + + /** + * Return a string representation of this instance. + */ + public override fun toString(): String = "ExperimentEngineLauncher" + + /** + * Create the [ExperimentStrategy] that explores the experiment design space. + */ + private fun createStrategy(): ExperimentStrategy { + return strategy ?: CartesianExperimentStrategy + } + + /** + * Create the [ExperimentScheduler] that schedules the trials over the compute resources. + */ + private fun createScheduler(): ExperimentScheduler { + return scheduler ?: ThreadPoolExperimentScheduler(Runtime.getRuntime().availableProcessors()) + } + + /** + * Create the [ExperimentExecutionListener] that listens the to the execution of the experiments. + */ + private fun createListener(): ExperimentExecutionListener { + require(listeners.isNotEmpty()) { "No listeners registered." } + return CompositeExperimentExecutionListener(listeners) + } +} diff --git a/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/ExperimentExecutionListener.kt b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/ExperimentExecutionListener.kt new file mode 100644 index 00000000..9ef71863 --- /dev/null +++ b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/ExperimentExecutionListener.kt @@ -0,0 +1,77 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.harness.engine + +import org.opendc.harness.api.ExperimentDefinition +import org.opendc.harness.api.Scenario +import org.opendc.harness.api.Trial + +/** + * Listener to be notified of experiment execution events by experiment runners. + */ +public interface ExperimentExecutionListener { + /** + * A method that is invoked when an experiment is started. + * + * @param experiment The [ExperimentDefinition] that started. + */ + public fun experimentStarted(experiment: ExperimentDefinition) {} + + /** + * A method that is invoked when an experiment is finished, regardless of the outcome. + * + * @param experiment The [ExperimentDefinition] that finished. + * @param throwable The exception that was thrown during execution or `null` if the execution completed successfully. + */ + public fun experimentFinished(experiment: ExperimentDefinition, throwable: Throwable?) {} + + /** + * A method that is invoked when a scenario is started. + * + * @param scenario The scenario that is started. + */ + public fun scenarioStarted(scenario: Scenario) {} + + /** + * A method that is invoked when a scenario is finished, regardless of the outcome. + * + * @param scenario The [Scenario] that has finished. + * @param throwable The exception that was thrown during execution or `null` if the execution completed successfully. + */ + public fun scenarioFinished(scenario: Scenario, throwable: Throwable?) {} + + /** + * A method that is invoked when a trial is started. + * + * @param trial The trial that is started. + */ + public fun trialStarted(trial: Trial) {} + + /** + * A method that is invoked when a scenario is finished, regardless of the outcome. + * + * @param trial The [Trial] that has finished. + * @param throwable The exception that was thrown during execution or `null` if the execution completed successfully. + */ + public fun trialFinished(trial: Trial, throwable: Throwable?) {} +} diff --git a/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/discovery/Discovery.kt b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/discovery/Discovery.kt new file mode 100644 index 00000000..f7f73b38 --- /dev/null +++ b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/discovery/Discovery.kt @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.harness.engine.discovery + +import kotlinx.coroutines.flow.Flow +import org.opendc.harness.api.ExperimentDefinition + +/** + * Component responsible for scanning for [ExperimentDefinition]s. + */ +public interface Discovery { + /** + * Start discovery of experiments. + * + * @param request The [DiscoveryRequest] to determine the experiments to discover. + * @return A flow of [ExperimentDefinition]s that have been discovered by the implementation. + */ + public fun discover(request: DiscoveryRequest): Flow<ExperimentDefinition> +} diff --git a/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/discovery/DiscoveryFilter.kt b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/discovery/DiscoveryFilter.kt new file mode 100644 index 00000000..219d09cd --- /dev/null +++ b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/discovery/DiscoveryFilter.kt @@ -0,0 +1,51 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.harness.engine.discovery + +import org.opendc.harness.api.ExperimentDefinition +import java.util.function.Predicate + +/** + * A [DiscoveryFilter] decides how the selected experiments are filtered. + */ +public sealed class DiscoveryFilter { + /** + * Test whether the specified [ExperimentDefinition] should be selected. + */ + public abstract fun test(definition: ExperimentDefinition): Boolean + + /** + * Filter an experiment based on its name. + */ + public data class Name(val predicate: Predicate<String>) : DiscoveryFilter() { + override fun test(definition: ExperimentDefinition): Boolean = predicate.test(definition.name) + } + + /** + * Filter an experiment based on its metadata. + */ + public data class Meta(val key: String, val predicate: Predicate<Any>) : DiscoveryFilter() { + override fun test(definition: ExperimentDefinition): Boolean = + definition.meta[key]?.let { predicate.test(it) } ?: false + } +} diff --git a/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/discovery/DiscoveryProvider.kt b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/discovery/DiscoveryProvider.kt new file mode 100644 index 00000000..fad255de --- /dev/null +++ b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/discovery/DiscoveryProvider.kt @@ -0,0 +1,65 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.harness.engine.discovery + +import org.opendc.harness.internal.CompositeDiscovery +import java.util.* + +/** + * A provider interface for the [Discovery] component. + */ +public interface DiscoveryProvider { + /** + * A unique identifier for this discovery implementation. + * + * Each discovery implementation must provide a unique ID, so that they can be selected by the user. + * When in doubt, you may use the fully qualified name of your custom [Discovery] implementation class. + */ + public val id: String + + /** + * Factory method for creating a new [Discovery] instance. + */ + public fun create(): Discovery + + public companion object { + /** + * The available [DiscoveryProvider]s. + */ + private val providers by lazy { ServiceLoader.load(DiscoveryProvider::class.java) } + + /** + * Obtain the [DiscoveryProvider] with the specified [id] or return `null`. + */ + public fun findById(id: String): DiscoveryProvider? { + return providers.find { it.id == id } + } + + /** + * Obtain a composite [Discovery] that combines the results of all available providers. + */ + public fun createComposite(): Discovery { + return CompositeDiscovery(providers) + } + } +} diff --git a/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/discovery/DiscoveryRequest.kt b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/discovery/DiscoveryRequest.kt new file mode 100644 index 00000000..5bc08dac --- /dev/null +++ b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/discovery/DiscoveryRequest.kt @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.harness.engine.discovery + +/** + * A request for discovering experiments according to the specified information. + * + * @param selectors The selectors for this discovery request. + * @param filters The filters for this discovery request. + */ +public data class DiscoveryRequest( + val selectors: List<DiscoverySelector> = emptyList(), + val filters: List<DiscoveryFilter> = emptyList(), +) diff --git a/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/runner/execution/ExperimentExecutionListener.kt b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/discovery/DiscoverySelector.kt index 42fef164..67681303 100644 --- a/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/runner/execution/ExperimentExecutionListener.kt +++ b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/discovery/DiscoverySelector.kt @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020 AtLarge Research + * Copyright (c) 2021 AtLarge Research * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -20,27 +20,30 @@ * SOFTWARE. */ -package org.opendc.experiments.sc20.runner.execution +package org.opendc.harness.engine.discovery -import org.opendc.experiments.sc20.runner.ExperimentDescriptor +import org.opendc.harness.api.ExperimentDefinition /** - * Listener to be notified of experiment execution events by experiment runners. + * A [DiscoverySelector] defines the properties used to discover experiments. */ -public interface ExperimentExecutionListener { +public sealed class DiscoverySelector { /** - * A method that is invoked when a new [ExperimentDescriptor] is registered. + * Test whether the specified [ExperimentDefinition] should be selected. */ - public fun descriptorRegistered(descriptor: ExperimentDescriptor) + public abstract fun test(definition: ExperimentDefinition): Boolean /** - * A method that is invoked when when the execution of a leaf or subtree of the experiment tree has finished, - * regardless of the outcome. + * Select an experiment based on its name. */ - public fun executionFinished(descriptor: ExperimentDescriptor, result: ExperimentExecutionResult) + public data class Name(val name: String) : DiscoverySelector() { + override fun test(definition: ExperimentDefinition): Boolean = definition.name == name + } /** - * A method that is invoked when the execution of a leaf or subtree of the experiment tree is about to be started. + * Select an experiment based on its metadata. */ - public fun executionStarted(descriptor: ExperimentDescriptor) + public data class Meta(val key: String, val value: Any) : DiscoverySelector() { + override fun test(definition: ExperimentDefinition): Boolean = definition.meta[key] == value + } } diff --git a/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/runner/execution/ExperimentScheduler.kt b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/scheduler/ExperimentScheduler.kt index 70095ccd..0265554a 100644 --- a/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/runner/execution/ExperimentScheduler.kt +++ b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/scheduler/ExperimentScheduler.kt @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020 AtLarge Research + * Copyright (c) 2021 AtLarge Research * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -20,37 +20,33 @@ * SOFTWARE. */ -package org.opendc.experiments.sc20.runner.execution +package org.opendc.harness.engine.scheduler -import org.opendc.experiments.sc20.runner.ExperimentDescriptor -import java.io.Closeable +import org.opendc.harness.api.Trial /** - * A interface for scheduling the execution of experiment trials over compute resources (threads/containers/vms) + * The [ExperimentScheduler] is responsible for scheduling the execution of experiment runs over some set of compute + * resources (e.g., threads or even multiple machines). */ -public interface ExperimentScheduler : Closeable { +public interface ExperimentScheduler : AutoCloseable { /** * Allocate a [Worker] for executing an experiment trial. This method may suspend in case no resources are directly * available at the moment. * * @return The available worker. */ - public suspend fun allocate(): ExperimentScheduler.Worker + public suspend fun allocate(): Worker /** - * An isolated worker of an [ExperimentScheduler] that is responsible for executing a single experiment trial. + * An isolated worker of an [ExperimentScheduler] that is responsible for conducting a single experiment trial. */ public interface Worker { /** - * Dispatch the specified [ExperimentDescriptor] to execute some time in the future and return the results of - * the trial. + * Dispatch an experiment trial immediately to one of the available compute resources and block execution until + * the trial has finished. * - * @param descriptor The descriptor to execute. - * @param context The context to execute the descriptor in. + * @param trial The trial to dispatch. */ - public suspend operator fun invoke( - descriptor: ExperimentDescriptor, - context: ExperimentExecutionContext - ) + public suspend fun dispatch(trial: Trial) } } diff --git a/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/scheduler/ExperimentSchedulerProvider.kt b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/scheduler/ExperimentSchedulerProvider.kt new file mode 100644 index 00000000..a93d4bf6 --- /dev/null +++ b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/scheduler/ExperimentSchedulerProvider.kt @@ -0,0 +1,57 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.harness.engine.scheduler + +import java.util.* + +/** + * A factory for constructing an [ExperimentScheduler]. + */ +public interface ExperimentSchedulerProvider { + /** + * A unique identifier for this scheduler implementation. + * + * Each experiment scheduler must provide a unique ID, so that they can be selected by the user. + * When in doubt, you may use the fully qualified name of your custom [ExperimentScheduler] implementation class. + */ + public val id: String + + /** + * Factory method for creating a new [ExperimentScheduler] instance. + */ + public fun create(): ExperimentScheduler + + public companion object { + /** + * The available [ExperimentSchedulerProvider]s. + */ + private val providers by lazy { ServiceLoader.load(ExperimentSchedulerProvider::class.java) } + + /** + * Obtain the [ExperimentScheduler] with the specified [id] or return `null`. + */ + public fun findById(id: String): ExperimentSchedulerProvider? { + return providers.find { it.id == id } + } + } +} diff --git a/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/scheduler/ThreadPoolExperimentScheduler.kt b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/scheduler/ThreadPoolExperimentScheduler.kt new file mode 100644 index 00000000..1ae533cf --- /dev/null +++ b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/scheduler/ThreadPoolExperimentScheduler.kt @@ -0,0 +1,58 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.harness.engine.scheduler + +import kotlinx.coroutines.asCoroutineDispatcher +import kotlinx.coroutines.sync.Semaphore +import kotlinx.coroutines.withContext +import org.opendc.harness.api.Trial +import java.util.concurrent.Executors + +/** + * An [ExperimentScheduler] that runs experiment trials using a local thread pool. + * + * @param parallelism The maximum amount of concurrent workers. + */ +public class ThreadPoolExperimentScheduler(parallelism: Int) : ExperimentScheduler { + private val dispatcher = Executors.newCachedThreadPool().asCoroutineDispatcher() + private val tickets = Semaphore(parallelism) + + override suspend fun allocate(): ExperimentScheduler.Worker { + tickets.acquire() + return object : ExperimentScheduler.Worker { + override suspend fun dispatch(trial: Trial) { + try { + withContext(dispatcher) { + trial.scenario.experiment.evaluator(trial) + } + } finally { + tickets.release() + } + } + } + } + + override fun close(): Unit = dispatcher.close() + + override fun toString(): String = "ThreadPoolScheduler" +} diff --git a/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/scheduler/ThreadPoolExperimentSchedulerProvider.kt b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/scheduler/ThreadPoolExperimentSchedulerProvider.kt new file mode 100644 index 00000000..cf9a132f --- /dev/null +++ b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/scheduler/ThreadPoolExperimentSchedulerProvider.kt @@ -0,0 +1,33 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.harness.engine.scheduler + +/** + * An [ExperimentSchedulerProvider] for constructing a [ThreadPoolExperimentScheduler]. + */ +public class ThreadPoolExperimentSchedulerProvider : ExperimentSchedulerProvider { + override val id: String = "thread-pool" + + override fun create(): ExperimentScheduler = + ThreadPoolExperimentScheduler(Runtime.getRuntime().availableProcessors()) +} diff --git a/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/strategy/CartesianExperimentStrategy.kt b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/strategy/CartesianExperimentStrategy.kt new file mode 100644 index 00000000..e5e08003 --- /dev/null +++ b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/strategy/CartesianExperimentStrategy.kt @@ -0,0 +1,55 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.harness.engine.strategy + +import org.opendc.harness.api.ExperimentDefinition +import org.opendc.harness.api.Parameter +import org.opendc.harness.api.Scenario +import org.opendc.harness.internal.ScenarioImpl + +/** + * An [ExperimentStrategy] that takes the cartesian product of the parameters and evaluates every combination. + */ +public object CartesianExperimentStrategy : ExperimentStrategy { + /** + * Build the trials of an experiment. + */ + override fun generate(experiment: ExperimentDefinition): Sequence<Scenario> { + return experiment.parameters + .asSequence() + .map { param -> mapParameter(param).map { value -> listOf(param to value) } } + .reduce { acc, param -> + acc.flatMap { x -> param.map { y -> x + y } } + } + .mapIndexed { id, values -> ScenarioImpl(id, experiment, values.toMap()) } + } + + /** + * Instantiate a parameter and return a sequence of possible values. + */ + private fun <T> mapParameter(param: Parameter<T>): Sequence<T> { + return when (param) { + is Parameter.Generic<T> -> param.values.asSequence() + } + } +} diff --git a/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/strategy/CartesianExperimentStrategyProvider.kt b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/strategy/CartesianExperimentStrategyProvider.kt new file mode 100644 index 00000000..f18795a3 --- /dev/null +++ b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/strategy/CartesianExperimentStrategyProvider.kt @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.harness.engine.strategy + +/** + * An [ExperimentStrategyProvider] for constructing a [CartesianExperimentStrategy]. + */ +public class CartesianExperimentStrategyProvider : ExperimentStrategyProvider { + override val id: String = "cartesian" + + override fun create(): ExperimentStrategy = CartesianExperimentStrategy +} diff --git a/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/strategy/ExperimentStrategy.kt b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/strategy/ExperimentStrategy.kt new file mode 100644 index 00000000..3a0148ad --- /dev/null +++ b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/strategy/ExperimentStrategy.kt @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.harness.engine.strategy + +import org.opendc.harness.api.ExperimentDefinition +import org.opendc.harness.api.Scenario + +/** + * The [ExperimentStrategy] is responsible for traversing the design space of an [ExperimentDefinition] based on its + * parameters, generating concrete points in the space represented as [Scenario]s. + */ +public interface ExperimentStrategy { + /** + * Generate the points in the design space of the specified [experiment] to explore. + * + * @param experiment The experiment design space to explore. + * @return A sequence of [Scenario]s which may be explored by the [ExperimentEngine]. + */ + public fun generate(experiment: ExperimentDefinition): Sequence<Scenario> +} diff --git a/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/strategy/ExperimentStrategyProvider.kt b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/strategy/ExperimentStrategyProvider.kt new file mode 100644 index 00000000..7fa05f34 --- /dev/null +++ b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/strategy/ExperimentStrategyProvider.kt @@ -0,0 +1,57 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.harness.engine.strategy + +import java.util.* + +/** + * A factory for constructing an [ExperimentStrategy]. + */ +public interface ExperimentStrategyProvider { + /** + * A unique identifier for this strategy implementation. + * + * Each experiment strategy must provide a unique ID, so that they can be selected by the user. + * When in doubt, you may use the fully qualified name of your custom [ExperimentStrategy] implementation class. + */ + public val id: String + + /** + * Factory method for creating a new [ExperimentStrategy] instance. + */ + public fun create(): ExperimentStrategy + + public companion object { + /** + * The available [ExperimentStrategyProvider]s. + */ + private val providers by lazy { ServiceLoader.load(ExperimentStrategyProvider::class.java) } + + /** + * Obtain the [ExperimentStrategy] with the specified [id] or return `null`. + */ + public fun findById(id: String): ExperimentStrategyProvider? { + return providers.find { it.id == id } + } + } +} diff --git a/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/internal/CompositeDiscovery.kt b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/internal/CompositeDiscovery.kt new file mode 100644 index 00000000..67a895e4 --- /dev/null +++ b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/internal/CompositeDiscovery.kt @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.harness.internal + +import kotlinx.coroutines.FlowPreview +import kotlinx.coroutines.flow.* +import org.opendc.harness.api.ExperimentDefinition +import org.opendc.harness.engine.discovery.Discovery +import org.opendc.harness.engine.discovery.DiscoveryProvider +import org.opendc.harness.engine.discovery.DiscoveryRequest + +/** + * A composite [Discovery] instance that combines the results of multiple delegate instances. + */ +internal class CompositeDiscovery(providers: Iterable<DiscoveryProvider>) : Discovery { + /** + * The [Discovery] instances to delegate to. + */ + private val delegates = providers.map { it.create() } + + @OptIn(FlowPreview::class) + override fun discover(request: DiscoveryRequest): Flow<ExperimentDefinition> { + return delegates.asFlow() + .map { it.discover(request) } + .flattenMerge(delegates.size) + } +} diff --git a/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/internal/CompositeExperimentExecutionListener.kt b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/internal/CompositeExperimentExecutionListener.kt new file mode 100644 index 00000000..a3cd6bd2 --- /dev/null +++ b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/internal/CompositeExperimentExecutionListener.kt @@ -0,0 +1,57 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.harness.internal + +import org.opendc.harness.api.ExperimentDefinition +import org.opendc.harness.api.Scenario +import org.opendc.harness.api.Trial +import org.opendc.harness.engine.ExperimentExecutionListener + +/** + * An [ExperimentExecutionListener] that composes multiple other listeners. + */ +public class CompositeExperimentExecutionListener(private val listeners: List<ExperimentExecutionListener>) : ExperimentExecutionListener { + override fun experimentStarted(experiment: ExperimentDefinition) { + listeners.forEach { it.experimentStarted(experiment) } + } + + override fun experimentFinished(experiment: ExperimentDefinition, throwable: Throwable?) { + listeners.forEach { it.experimentFinished(experiment, throwable) } + } + + override fun scenarioStarted(scenario: Scenario) { + listeners.forEach { it.scenarioStarted(scenario) } + } + + override fun scenarioFinished(scenario: Scenario, throwable: Throwable?) { + listeners.forEach { it.scenarioFinished(scenario, throwable) } + } + + override fun trialStarted(trial: Trial) { + listeners.forEach { it.trialStarted(trial) } + } + + override fun trialFinished(trial: Trial, throwable: Throwable?) { + listeners.forEach { it.trialFinished(trial, throwable) } + } +} diff --git a/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/internal/DslDiscovery.kt b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/internal/DslDiscovery.kt new file mode 100644 index 00000000..eb6303d6 --- /dev/null +++ b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/internal/DslDiscovery.kt @@ -0,0 +1,101 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.harness.internal + +import io.github.classgraph.ClassGraph +import io.github.classgraph.ScanResult +import kotlinx.coroutines.flow.Flow +import kotlinx.coroutines.flow.asFlow +import org.opendc.harness.api.ExperimentDefinition +import org.opendc.harness.dsl.Experiment +import org.opendc.harness.engine.discovery.Discovery +import org.opendc.harness.engine.discovery.DiscoveryFilter +import org.opendc.harness.engine.discovery.DiscoveryRequest +import org.opendc.harness.engine.discovery.DiscoverySelector + +/** + * A [Discovery] implementation that discovers [Experiment] instances on the classpath. + */ +internal class DslDiscovery : Discovery { + /* + * Lazily memoize the results of the classpath scan. + */ + private val scanResult by lazy { scan() } + + override fun discover(request: DiscoveryRequest): Flow<ExperimentDefinition> { + return findExperiments() + .map { cls -> + val exp = cls.constructors[0].newInstance() as Experiment + exp.toDefinition() + } + .filter(select(request.selectors)) + .filter(filter(request.filters)) + .asFlow() + } + + /** + * Find the classes on the classpath implementing the [Experiment] class. + */ + private fun findExperiments(): Sequence<Class<out Experiment>> { + return scanResult + .getSubclasses(Experiment::class.java.name) + .filter { !(it.isAbstract || it.isInterface) } + .map { it.loadClass() } + .filterIsInstance<Class<out Experiment>>() + .asSequence() + } + + /** + * Create a predicate for filtering the experiments based on the specified [filters]. + */ + private fun filter(filters: List<DiscoveryFilter>): (ExperimentDefinition) -> Boolean = { def -> + filters.isEmpty() || filters.all { it.test(def) } + } + + /** + * Create a predicate for selecting the experiments based on the specified [selectors]. + */ + private fun select(selectors: List<DiscoverySelector>): (ExperimentDefinition) -> Boolean = { def -> + selectors.isEmpty() || selectors.any { it.test(def) } + } + + /** + * Scan the classpath using [ClassGraph]. + */ + private fun scan(): ScanResult { + return ClassGraph() + .enableClassInfo() + .enableExternalClasses() + .ignoreClassVisibility() + .rejectPackages( + "java.*", + "javax.*", + "sun.*", + "com.sun.*", + "kotlin.*", + "androidx.*", + "org.jetbrains.kotlin.*", + "org.junit.*" + ).scan() + } +} diff --git a/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/runner/execution/ExperimentExecutionContext.kt b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/internal/DslDiscoveryProvider.kt index 942eb891..752ba4bb 100644 --- a/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/runner/execution/ExperimentExecutionContext.kt +++ b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/internal/DslDiscoveryProvider.kt @@ -1,7 +1,5 @@ /* - * MIT License - * - * Copyright (c) 2020 atlarge-research + * Copyright (c) 2021 AtLarge Research * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -22,24 +20,17 @@ * SOFTWARE. */ -package org.opendc.experiments.sc20.runner.execution +package org.opendc.harness.internal + +import org.opendc.harness.dsl.Experiment +import org.opendc.harness.engine.discovery.Discovery +import org.opendc.harness.engine.discovery.DiscoveryProvider /** - * The execution context of an experiment. + * A [DiscoveryProvider] for the [Experiment]s on the classpath. */ -public interface ExperimentExecutionContext { - /** - * The execution listener to use. - */ - public val listener: ExperimentExecutionListener - - /** - * The experiment scheduler to use. - */ - public val scheduler: ExperimentScheduler +public class DslDiscoveryProvider : DiscoveryProvider { + override val id: String = "dsl" - /** - * A cache for objects within a single runner. - */ - public val cache: MutableMap<Any?, Any?> + override fun create(): Discovery = DslDiscovery() } diff --git a/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/internal/ParameterDelegate.kt b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/internal/ParameterDelegate.kt new file mode 100644 index 00000000..aaf90b99 --- /dev/null +++ b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/internal/ParameterDelegate.kt @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.harness.internal + +import org.opendc.harness.api.Parameter +import org.opendc.harness.dsl.Experiment +import kotlin.properties.ReadOnlyProperty +import kotlin.reflect.KProperty + +/** + * A delegate for an experiment parameter. + * + * @property parameter The parameter descriptor of this delegate. + */ +internal class ParameterDelegate<T>(val parameter: Parameter<T>) : ReadOnlyProperty<Experiment, T> { + /** + * Obtain the value for the parameter. + */ + override fun getValue(thisRef: Experiment, property: KProperty<*>): T { + val scenario = thisRef.scenario ?: throw IllegalStateException("Cannot use parameters before activation") + return scenario[parameter] + } +} diff --git a/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/internal/ScenarioImpl.kt b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/internal/ScenarioImpl.kt new file mode 100644 index 00000000..d255004d --- /dev/null +++ b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/internal/ScenarioImpl.kt @@ -0,0 +1,49 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.harness.internal + +import org.opendc.harness.api.ExperimentDefinition +import org.opendc.harness.api.Parameter +import org.opendc.harness.api.Scenario + +/** + * Internal implementation of a [Scenario]. + */ +internal data class ScenarioImpl( + override val id: Int, + override val experiment: ExperimentDefinition, + val parameters: Map<Parameter<*>, Any?> +) : Scenario { + + override fun <T> get(param: Parameter<T>): T { + if (!parameters.containsKey(param)) { + throw IllegalArgumentException("Unknown parameter for this scenario.") + } + + // This cast should always succeed + @Suppress("UNCHECKED_CAST") + return parameters[param] as T + } + + override fun toString(): String = "Scenario" +} diff --git a/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/reporter/ConsoleExperimentReporter.kt b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/runner/console/ConsoleExperimentReporter.kt index af61622a..2db74ef4 100644 --- a/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/reporter/ConsoleExperimentReporter.kt +++ b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/runner/console/ConsoleExperimentReporter.kt @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020 AtLarge Research + * Copyright (c) 2021 AtLarge Research * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -20,24 +20,22 @@ * SOFTWARE. */ -package org.opendc.experiments.sc20.reporter +package org.opendc.harness.runner.console import me.tongfei.progressbar.ProgressBar import me.tongfei.progressbar.ProgressBarBuilder import mu.KotlinLogging -import org.opendc.experiments.sc20.experiment.Run -import org.opendc.experiments.sc20.runner.ExperimentDescriptor -import org.opendc.experiments.sc20.runner.execution.ExperimentExecutionListener -import org.opendc.experiments.sc20.runner.execution.ExperimentExecutionResult +import org.opendc.harness.api.Trial +import org.opendc.harness.engine.ExperimentExecutionListener /** * A reporter that reports the experiment progress to the console. */ public class ConsoleExperimentReporter : ExperimentExecutionListener, AutoCloseable { /** - * The active [Run]s. + * The active [Trial]s. */ - private val runs: MutableSet<Run> = mutableSetOf() + private val trials: MutableSet<Trial> = mutableSetOf() /** * The total number of runs. @@ -57,29 +55,23 @@ public class ConsoleExperimentReporter : ExperimentExecutionListener, AutoClosea .setInitialMax(1) .build() - override fun descriptorRegistered(descriptor: ExperimentDescriptor) { - if (descriptor is Run) { - runs += descriptor - pb.maxHint((++total).toLong()) - } - } + override fun trialFinished(trial: Trial, throwable: Throwable?) { + trials -= trial - override fun executionFinished(descriptor: ExperimentDescriptor, result: ExperimentExecutionResult) { - if (descriptor is Run) { - runs -= descriptor - - pb.stepTo(total - runs.size.toLong()) - if (runs.isEmpty()) { - pb.close() - } + pb.stepTo(total - trials.size.toLong()) + if (trials.isEmpty()) { + pb.close() } - if (result is ExperimentExecutionResult.Failed) { - logger.warn(result.throwable) { "Descriptor $descriptor failed" } + if (throwable != null) { + logger.warn(throwable) { "Trial $trial failed" } } } - override fun executionStarted(descriptor: ExperimentDescriptor) {} + override fun trialStarted(trial: Trial) { + trials += trial + pb.maxHint((++total).toLong()) + } override fun close() { pb.close() diff --git a/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/runner/console/ConsoleRunner.kt b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/runner/console/ConsoleRunner.kt new file mode 100644 index 00000000..ae221c7f --- /dev/null +++ b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/runner/console/ConsoleRunner.kt @@ -0,0 +1,99 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.harness.runner.console + +import com.github.ajalt.clikt.core.CliktCommand +import com.github.ajalt.clikt.parameters.options.default +import com.github.ajalt.clikt.parameters.options.multiple +import com.github.ajalt.clikt.parameters.options.option +import com.github.ajalt.clikt.parameters.types.int +import mu.KotlinLogging +import org.opendc.harness.engine.ExperimentEngineLauncher +import org.opendc.harness.engine.discovery.DiscoveryProvider +import org.opendc.harness.engine.discovery.DiscoveryRequest +import org.opendc.harness.engine.discovery.DiscoverySelector +import org.opendc.harness.engine.scheduler.ThreadPoolExperimentScheduler + +/** + * The logger for this experiment runner. + */ +private val logger = KotlinLogging.logger {} + +/** + * The command line interface for the console experiment runner. + */ +public class ConsoleRunner : CliktCommand(name = "opendc-harness") { + /** + * The number of repeats per scenario. + */ + private val repeats by option("-r", "--repeats", help = "Number of repeats per scenario") + .int() + .default(1) + + /** + * The selected experiments to run by name. + */ + private val experiments by option("-e", "--experiments", help = "Names of experiments to explore") + .multiple(emptyList()) + + /** + * The maximum number of worker threads to use. + */ + private val parallelism by option("-p", "--parallelism", help = "Maximum number of concurrent simulation runs") + .int() + .default(Runtime.getRuntime().availableProcessors()) + + override fun run() { + logger.info { "Starting OpenDC Console Experiment Runner" } + + val discovery = DiscoveryProvider.createComposite() + val experiments = discovery.discover( + DiscoveryRequest( + selectors = experiments.map { DiscoverySelector.Name(it) } + ) + ) + + val reporter = ConsoleExperimentReporter() + val scheduler = ThreadPoolExperimentScheduler(parallelism) + + try { + ExperimentEngineLauncher() + .withListener(reporter) + .withRepeats(repeats) + .withScheduler(scheduler) + .runBlocking(experiments) + } catch (e: Throwable) { + logger.error(e) { "Failed to finish experiments" } + } finally { + reporter.close() + scheduler.close() + } + + logger.info { "Finished all experiments. Exiting." } + } +} + +/** + * Main entry point of the experiment runner. + */ +public fun main(args: Array<String>): Unit = ConsoleRunner().main(args) diff --git a/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/runner/junit5/JUnitExperimentExecutionListener.kt b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/runner/junit5/JUnitExperimentExecutionListener.kt new file mode 100644 index 00000000..58791549 --- /dev/null +++ b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/runner/junit5/JUnitExperimentExecutionListener.kt @@ -0,0 +1,152 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.harness.runner.junit5 + +import org.junit.platform.engine.* +import org.junit.platform.engine.support.descriptor.AbstractTestDescriptor +import org.junit.platform.engine.support.descriptor.EngineDescriptor +import org.opendc.harness.api.ExperimentDefinition +import org.opendc.harness.api.Scenario +import org.opendc.harness.api.Trial +import org.opendc.harness.engine.ExperimentExecutionListener + +/** + * An [ExperimentExecutionListener] that notifies JUnit platform of the progress of the experiment trials. + */ +public class JUnitExperimentExecutionListener( + private val listener: EngineExecutionListener, + private val root: EngineDescriptor +) : ExperimentExecutionListener { + /** + * The current active experiments. + */ + private val experiments = mutableMapOf<ExperimentDefinition, TestDescriptor>() + + /** + * The current active scenarios. + */ + private val scenarios = mutableMapOf<Scenario, TestDescriptor>() + + /** + * The current active trials. + */ + private val trials = mutableMapOf<Trial, TestDescriptor>() + + override fun experimentStarted(experiment: ExperimentDefinition) { + val descriptor = experiment.toDescriptor(root) + root.addChild(descriptor) + experiments[experiment] = descriptor + + listener.dynamicTestRegistered(descriptor) + listener.executionStarted(descriptor) + } + + override fun experimentFinished(experiment: ExperimentDefinition, throwable: Throwable?) { + val descriptor = experiments.remove(experiment) + + if (throwable != null) { + listener.executionFinished(descriptor, TestExecutionResult.failed(throwable)) + } else { + listener.executionFinished(descriptor, TestExecutionResult.successful()) + } + } + + override fun scenarioStarted(scenario: Scenario) { + val parent = experiments[scenario.experiment] ?: return + val descriptor = scenario.toDescriptor(parent) + parent.addChild(descriptor) + scenarios[scenario] = descriptor + + listener.dynamicTestRegistered(descriptor) + listener.executionStarted(descriptor) + } + + override fun scenarioFinished(scenario: Scenario, throwable: Throwable?) { + val descriptor = scenarios.remove(scenario) + + if (throwable != null) { + listener.executionFinished(descriptor, TestExecutionResult.failed(throwable)) + } else { + listener.executionFinished(descriptor, TestExecutionResult.successful()) + } + } + + override fun trialStarted(trial: Trial) { + val parent = scenarios[trial.scenario] ?: return + val descriptor = trial.toDescriptor(parent) + parent.addChild(descriptor) + trials[trial] = descriptor + + listener.dynamicTestRegistered(descriptor) + listener.executionStarted(descriptor) + } + + override fun trialFinished(trial: Trial, throwable: Throwable?) { + val descriptor = trials.remove(trial) + + if (throwable != null) { + listener.executionFinished(descriptor, TestExecutionResult.failed(throwable)) + } else { + listener.executionFinished(descriptor, TestExecutionResult.successful()) + } + } + + /** + * Create a [TestDescriptor] for an [ExperimentDefinition]. + */ + private fun ExperimentDefinition.toDescriptor(parent: TestDescriptor): TestDescriptor { + return object : AbstractTestDescriptor(parent.uniqueId.append("experiment", name), name) { + override fun getType(): TestDescriptor.Type = TestDescriptor.Type.CONTAINER + + override fun mayRegisterTests(): Boolean = true + + override fun toString(): String = "ExperimentDescriptor" + } + } + + /** + * Create a [TestDescriptor] for a [Scenario]. + */ + private fun Scenario.toDescriptor(parent: TestDescriptor): TestDescriptor { + return object : AbstractTestDescriptor(parent.uniqueId.append("scenario", id.toString()), "Scenario $id") { + override fun getType(): TestDescriptor.Type = TestDescriptor.Type.CONTAINER_AND_TEST + + override fun mayRegisterTests(): Boolean = true + + override fun toString(): String = "ScenarioDescriptor" + } + } + + /** + * Create a [TestDescriptor] for a [Trial]. + */ + private fun Trial.toDescriptor(parent: TestDescriptor): TestDescriptor { + return object : AbstractTestDescriptor(parent.uniqueId.append("repeat", repeat.toString()), "Repeat $repeat") { + override fun getType(): TestDescriptor.Type = TestDescriptor.Type.TEST + + override fun mayRegisterTests(): Boolean = false + + override fun toString(): String = "TrialDescriptor" + } + } +} diff --git a/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/runner/junit5/OpenDCTestEngine.kt b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/runner/junit5/OpenDCTestEngine.kt new file mode 100644 index 00000000..685cd41a --- /dev/null +++ b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/runner/junit5/OpenDCTestEngine.kt @@ -0,0 +1,94 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.harness.runner.junit5 + +import kotlinx.coroutines.flow.Flow +import kotlinx.coroutines.flow.emptyFlow +import mu.KotlinLogging +import org.junit.platform.engine.* +import org.junit.platform.engine.discovery.ClassNameFilter +import org.junit.platform.engine.discovery.ClassSelector +import org.junit.platform.engine.discovery.MethodSelector +import org.junit.platform.engine.support.descriptor.EngineDescriptor +import org.junit.platform.launcher.LauncherDiscoveryRequest +import org.opendc.harness.api.ExperimentDefinition +import org.opendc.harness.engine.ExperimentEngineLauncher +import org.opendc.harness.engine.discovery.DiscoveryFilter +import org.opendc.harness.engine.discovery.DiscoveryProvider +import org.opendc.harness.engine.discovery.DiscoveryRequest +import org.opendc.harness.engine.discovery.DiscoverySelector + +/** + * A [TestEngine] implementation that is able to run experiments defined using the harness. + */ +public class OpenDCTestEngine : TestEngine { + /** + * The logging instance for this engine. + */ + private val logger = KotlinLogging.logger {} + + override fun getId(): String = "opendc" + + override fun discover(request: EngineDiscoveryRequest, uniqueId: UniqueId): TestDescriptor { + // Test whether are excluded from the engines + val isEnabled = (request as? LauncherDiscoveryRequest)?.engineFilters?.all { it.toPredicate().test(this) } ?: true + if (!isEnabled) { + return ExperimentEngineDescriptor(uniqueId, emptyFlow()) + } + + // IntelliJ will pass a [MethodSelector] to run just a single method inside a file. In that + // case, no experiments should be discovered, since we support only experiments by class. + if (request.getSelectorsByType(MethodSelector::class.java).isNotEmpty()) { + return ExperimentEngineDescriptor(uniqueId, emptyFlow()) + } + + val classNames = request.getSelectorsByType(ClassSelector::class.java).map { DiscoverySelector.Meta("class.name", it.className) } + val classNameFilters = request.getFiltersByType(ClassNameFilter::class.java).map { DiscoveryFilter.Name(it.toPredicate()) } + + val discovery = DiscoveryProvider.createComposite() + val definitions = discovery.discover(DiscoveryRequest(classNames, classNameFilters)) + + return ExperimentEngineDescriptor(uniqueId, definitions) + } + + override fun execute(request: ExecutionRequest) { + logger.debug { "JUnit ExecutionRequest[${request::class.java.name}] [configurationParameters=${request.configurationParameters}; rootTestDescriptor=${request.rootTestDescriptor}]" } + val root = request.rootTestDescriptor as ExperimentEngineDescriptor + val listener = request.engineExecutionListener + + listener.executionStarted(root) + + try { + ExperimentEngineLauncher() + .withListener(JUnitExperimentExecutionListener(listener, root)) + .runBlocking(root.experiments) + listener.executionFinished(root, TestExecutionResult.successful()) + } catch (e: Throwable) { + listener.executionFinished(root, TestExecutionResult.failed(e)) + } + } + + private class ExperimentEngineDescriptor(id: UniqueId, val experiments: Flow<ExperimentDefinition>) : EngineDescriptor(id, "opendc") { + override fun mayRegisterTests(): Boolean = true + } +} diff --git a/simulator/opendc-harness/src/main/resources/META-INF/services/org.junit.platform.engine.TestEngine b/simulator/opendc-harness/src/main/resources/META-INF/services/org.junit.platform.engine.TestEngine new file mode 100644 index 00000000..b83eec0c --- /dev/null +++ b/simulator/opendc-harness/src/main/resources/META-INF/services/org.junit.platform.engine.TestEngine @@ -0,0 +1 @@ +org.opendc.harness.runner.junit5.OpenDCTestEngine diff --git a/simulator/opendc-harness/src/main/resources/META-INF/services/org.opendc.harness.engine.discovery.DiscoveryProvider b/simulator/opendc-harness/src/main/resources/META-INF/services/org.opendc.harness.engine.discovery.DiscoveryProvider new file mode 100644 index 00000000..d6a73ded --- /dev/null +++ b/simulator/opendc-harness/src/main/resources/META-INF/services/org.opendc.harness.engine.discovery.DiscoveryProvider @@ -0,0 +1 @@ +org.opendc.harness.internal.DslDiscoveryProvider diff --git a/simulator/opendc-harness/src/main/resources/META-INF/services/org.opendc.harness.engine.scheduler.ExperimentSchedulerProvider b/simulator/opendc-harness/src/main/resources/META-INF/services/org.opendc.harness.engine.scheduler.ExperimentSchedulerProvider new file mode 100644 index 00000000..2ba3a7cb --- /dev/null +++ b/simulator/opendc-harness/src/main/resources/META-INF/services/org.opendc.harness.engine.scheduler.ExperimentSchedulerProvider @@ -0,0 +1 @@ +org.opendc.harness.engine.scheduler.ThreadPoolExperimentSchedulerProvider diff --git a/simulator/opendc-harness/src/main/resources/META-INF/services/org.opendc.harness.engine.strategy.ExperimentStrategyProvider b/simulator/opendc-harness/src/main/resources/META-INF/services/org.opendc.harness.engine.strategy.ExperimentStrategyProvider new file mode 100644 index 00000000..cb1c70ac --- /dev/null +++ b/simulator/opendc-harness/src/main/resources/META-INF/services/org.opendc.harness.engine.strategy.ExperimentStrategyProvider @@ -0,0 +1 @@ +org.opendc.harness.engine.strategy.CartesianExperimentStrategyProvider diff --git a/simulator/opendc-harness/src/main/resources/log4j2.xml b/simulator/opendc-harness/src/main/resources/log4j2.xml new file mode 100644 index 00000000..9553d964 --- /dev/null +++ b/simulator/opendc-harness/src/main/resources/log4j2.xml @@ -0,0 +1,40 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + ~ MIT License + ~ + ~ Copyright (c) 2020 atlarge-research + ~ + ~ Permission is hereby granted, free of charge, to any person obtaining a copy + ~ of this software and associated documentation files (the "Software"), to deal + ~ in the Software without restriction, including without limitation the rights + ~ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + ~ copies of the Software, and to permit persons to whom the Software is + ~ furnished to do so, subject to the following conditions: + ~ + ~ The above copyright notice and this permission notice shall be included in all + ~ copies or substantial portions of the Software. + ~ + ~ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + ~ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + ~ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + ~ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + ~ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + ~ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + ~ SOFTWARE. + --> + +<Configuration status="WARN"> + <Appenders> + <Console name="Console" target="SYSTEM_OUT"> + <PatternLayout pattern="%d{HH:mm:ss.SSS} [%highlight{%-5level}] %logger{36} - %msg%n" disableAnsi="false"/> + </Console> + </Appenders> + <Loggers> + <Logger name="org.opendc" level="info" additivity="false"> + <AppenderRef ref="Console"/> + </Logger> + <Root level="error"> + <AppenderRef ref="Console"/> + </Root> + </Loggers> +</Configuration> diff --git a/simulator/opendc-harness/src/test/kotlin/org/opendc/harness/EngineTest.kt b/simulator/opendc-harness/src/test/kotlin/org/opendc/harness/EngineTest.kt new file mode 100644 index 00000000..6f2989db --- /dev/null +++ b/simulator/opendc-harness/src/test/kotlin/org/opendc/harness/EngineTest.kt @@ -0,0 +1,61 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.harness + +import kotlinx.coroutines.flow.flowOf +import kotlinx.coroutines.flow.toList +import kotlinx.coroutines.runBlocking +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertNotNull +import org.junit.jupiter.api.Test +import org.opendc.harness.api.ExperimentDefinition +import org.opendc.harness.engine.ExperimentEngine +import org.opendc.harness.engine.ExperimentEngineLauncher +import org.opendc.harness.engine.ExperimentExecutionListener +import org.opendc.harness.engine.discovery.DiscoveryProvider +import org.opendc.harness.engine.discovery.DiscoveryRequest + +/** + * A test suite for the [ExperimentEngine]. + */ +internal class EngineTest { + @Test + fun test() { + val listener = object : ExperimentExecutionListener {} + ExperimentEngineLauncher() + .withListener(listener) + .runBlocking(flowOf(TestExperiment().toDefinition())) + } + + @Test + fun discovery() { + runBlocking { + val discovery = DiscoveryProvider.findById("dsl")?.create() + assertNotNull(discovery) + val res = mutableListOf<ExperimentDefinition>() + discovery?.discover(DiscoveryRequest())?.toList(res) + println(res) + assertEquals(1, res.size) + } + } +} diff --git a/simulator/opendc-harness/src/test/kotlin/org/opendc/harness/TestExperiment.kt b/simulator/opendc-harness/src/test/kotlin/org/opendc/harness/TestExperiment.kt new file mode 100644 index 00000000..bedd1c76 --- /dev/null +++ b/simulator/opendc-harness/src/test/kotlin/org/opendc/harness/TestExperiment.kt @@ -0,0 +1,54 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.harness + +import org.opendc.harness.dsl.Experiment +import org.opendc.harness.dsl.anyOf + +/** + * An experiment to test the harness' functionality. + */ +class TestExperiment : Experiment("Design Space Exploration") { + /** + * The cloud environment to use. + */ + private val environment: String by anyOf( + "../../traces/setup-small.json", + "../../traces/setup.json", + "../../traces/setup-large.json" + ) + + /** + * The trace to use. + */ + private val trace: String by anyOf( + "../../traces/gwf/askalon_workload_olde.gwf", + "../../traces/gwf/askalon_workload_ee.gwf", + "../../traces/gwf/chronos_exp_noscaler_ca.gwf" + ) + + override fun doRun(repeat: Int) { + println("Id $id, Run $repeat, Environment $environment, Trace $trace") + Thread.sleep(500 * id.toLong()) + } +} diff --git a/simulator/settings.gradle.kts b/simulator/settings.gradle.kts index e6f42574..470303f4 100644 --- a/simulator/settings.gradle.kts +++ b/simulator/settings.gradle.kts @@ -33,4 +33,5 @@ include(":opendc-simulator:opendc-simulator-core") include(":opendc-simulator:opendc-simulator-compute") include(":opendc-simulator:opendc-simulator-failures") include(":opendc-trace:opendc-trace-core") +include(":opendc-harness") include(":opendc-utils") |
