From cba8446a04ef77d36596774cf0bbd5d3a6f0c70f Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Wed, 6 Jan 2021 21:32:19 +0100 Subject: Add initial experiment harness to OpenDC 2.0 This change adds an experiment harness to OpenDC 2.0 based on previous work on separate branches. This harness enables users to declaratively specify and orchestrate experiment designs. --- simulator/opendc-harness/build.gradle.kts | 43 ++++++ .../org/opendc/harness/api/ExperimentDefinition.kt | 39 ++++++ .../kotlin/org/opendc/harness/api/Parameter.kt | 38 ++++++ .../main/kotlin/org/opendc/harness/api/Scenario.kt | 46 +++++++ .../main/kotlin/org/opendc/harness/api/Trial.kt | 28 ++++ .../kotlin/org/opendc/harness/dsl/Experiment.kt | 99 ++++++++++++++ .../org/opendc/harness/dsl/ParameterProvider.kt | 39 ++++++ .../kotlin/org/opendc/harness/dsl/Parameters.kt | 44 ++++++ .../org/opendc/harness/engine/ExperimentEngine.kt | 107 +++++++++++++++ .../harness/engine/ExperimentEngineLauncher.kt | 121 ++++++++++++++++ .../harness/engine/ExperimentExecutionListener.kt | 77 +++++++++++ .../opendc/harness/engine/discovery/Discovery.kt | 39 ++++++ .../harness/engine/discovery/DiscoveryFilter.kt | 51 +++++++ .../harness/engine/discovery/DiscoveryProvider.kt | 65 +++++++++ .../harness/engine/discovery/DiscoveryRequest.kt | 34 +++++ .../harness/engine/discovery/DiscoverySelector.kt | 49 +++++++ .../engine/scheduler/ExperimentScheduler.kt | 52 +++++++ .../scheduler/ExperimentSchedulerProvider.kt | 57 ++++++++ .../scheduler/ThreadPoolExperimentScheduler.kt | 58 ++++++++ .../ThreadPoolExperimentSchedulerProvider.kt | 33 +++++ .../engine/strategy/CartesianExperimentStrategy.kt | 55 ++++++++ .../CartesianExperimentStrategyProvider.kt | 32 +++++ .../harness/engine/strategy/ExperimentStrategy.kt | 40 ++++++ .../engine/strategy/ExperimentStrategyProvider.kt | 57 ++++++++ .../opendc/harness/internal/CompositeDiscovery.kt | 47 +++++++ .../CompositeExperimentExecutionListener.kt | 57 ++++++++ .../org/opendc/harness/internal/DslDiscovery.kt | 101 ++++++++++++++ .../harness/internal/DslDiscoveryProvider.kt | 36 +++++ .../opendc/harness/internal/ParameterDelegate.kt | 43 ++++++ .../org/opendc/harness/internal/ScenarioImpl.kt | 49 +++++++ .../junit5/JUnitExperimentExecutionListener.kt | 152 +++++++++++++++++++++ .../harness/runner/junit5/OpenDCTestEngine.kt | 94 +++++++++++++ .../services/org.junit.platform.engine.TestEngine | 1 + ...endc.harness.engine.discovery.DiscoveryProvider | 1 + ...ss.engine.scheduler.ExperimentSchedulerProvider | 1 + ...ness.engine.strategy.ExperimentStrategyProvider | 1 + .../test/kotlin/org/opendc/harness/EngineTest.kt | 61 +++++++++ .../kotlin/org/opendc/harness/TestExperiment.kt | 54 ++++++++ simulator/settings.gradle.kts | 1 + 39 files changed, 2002 insertions(+) create mode 100644 simulator/opendc-harness/build.gradle.kts create mode 100644 simulator/opendc-harness/src/main/kotlin/org/opendc/harness/api/ExperimentDefinition.kt create mode 100644 simulator/opendc-harness/src/main/kotlin/org/opendc/harness/api/Parameter.kt create mode 100644 simulator/opendc-harness/src/main/kotlin/org/opendc/harness/api/Scenario.kt create mode 100644 simulator/opendc-harness/src/main/kotlin/org/opendc/harness/api/Trial.kt create mode 100644 simulator/opendc-harness/src/main/kotlin/org/opendc/harness/dsl/Experiment.kt create mode 100644 simulator/opendc-harness/src/main/kotlin/org/opendc/harness/dsl/ParameterProvider.kt create mode 100644 simulator/opendc-harness/src/main/kotlin/org/opendc/harness/dsl/Parameters.kt create mode 100644 simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/ExperimentEngine.kt create mode 100644 simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/ExperimentEngineLauncher.kt create mode 100644 simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/ExperimentExecutionListener.kt create mode 100644 simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/discovery/Discovery.kt create mode 100644 simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/discovery/DiscoveryFilter.kt create mode 100644 simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/discovery/DiscoveryProvider.kt create mode 100644 simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/discovery/DiscoveryRequest.kt create mode 100644 simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/discovery/DiscoverySelector.kt create mode 100644 simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/scheduler/ExperimentScheduler.kt create mode 100644 simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/scheduler/ExperimentSchedulerProvider.kt create mode 100644 simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/scheduler/ThreadPoolExperimentScheduler.kt create mode 100644 simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/scheduler/ThreadPoolExperimentSchedulerProvider.kt create mode 100644 simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/strategy/CartesianExperimentStrategy.kt create mode 100644 simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/strategy/CartesianExperimentStrategyProvider.kt create mode 100644 simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/strategy/ExperimentStrategy.kt create mode 100644 simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/strategy/ExperimentStrategyProvider.kt create mode 100644 simulator/opendc-harness/src/main/kotlin/org/opendc/harness/internal/CompositeDiscovery.kt create mode 100644 simulator/opendc-harness/src/main/kotlin/org/opendc/harness/internal/CompositeExperimentExecutionListener.kt create mode 100644 simulator/opendc-harness/src/main/kotlin/org/opendc/harness/internal/DslDiscovery.kt create mode 100644 simulator/opendc-harness/src/main/kotlin/org/opendc/harness/internal/DslDiscoveryProvider.kt create mode 100644 simulator/opendc-harness/src/main/kotlin/org/opendc/harness/internal/ParameterDelegate.kt create mode 100644 simulator/opendc-harness/src/main/kotlin/org/opendc/harness/internal/ScenarioImpl.kt create mode 100644 simulator/opendc-harness/src/main/kotlin/org/opendc/harness/runner/junit5/JUnitExperimentExecutionListener.kt create mode 100644 simulator/opendc-harness/src/main/kotlin/org/opendc/harness/runner/junit5/OpenDCTestEngine.kt create mode 100644 simulator/opendc-harness/src/main/resources/META-INF/services/org.junit.platform.engine.TestEngine create mode 100644 simulator/opendc-harness/src/main/resources/META-INF/services/org.opendc.harness.engine.discovery.DiscoveryProvider create mode 100644 simulator/opendc-harness/src/main/resources/META-INF/services/org.opendc.harness.engine.scheduler.ExperimentSchedulerProvider create mode 100644 simulator/opendc-harness/src/main/resources/META-INF/services/org.opendc.harness.engine.strategy.ExperimentStrategyProvider create mode 100644 simulator/opendc-harness/src/test/kotlin/org/opendc/harness/EngineTest.kt create mode 100644 simulator/opendc-harness/src/test/kotlin/org/opendc/harness/TestExperiment.kt diff --git a/simulator/opendc-harness/build.gradle.kts b/simulator/opendc-harness/build.gradle.kts new file mode 100644 index 00000000..76d1f2cd --- /dev/null +++ b/simulator/opendc-harness/build.gradle.kts @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2020 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +description = "Harness for defining repeatable experiments using OpenDC" + +/* Build configuration */ +plugins { + `kotlin-library-convention` +} + +dependencies { + api("org.jetbrains.kotlinx:kotlinx-coroutines-core:${Library.KOTLINX_COROUTINES}") + api("org.junit.platform:junit-platform-commons:${Library.JUNIT_PLATFORM}") + implementation("io.github.classgraph:classgraph:4.8.98") + implementation("me.tongfei:progressbar:0.9.0") + implementation("io.github.microutils:kotlin-logging:2.0.4") + + api("org.junit.platform:junit-platform-engine:${Library.JUNIT_PLATFORM}") + api("org.junit.platform:junit-platform-suite-api:${Library.JUNIT_PLATFORM}") + api("org.junit.platform:junit-platform-launcher:${Library.JUNIT_PLATFORM}") + + testImplementation("org.junit.jupiter:junit-jupiter-api:${Library.JUNIT_JUPITER}") + testRuntimeOnly("org.junit.jupiter:junit-jupiter-engine:${Library.JUNIT_JUPITER}") +} diff --git a/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/api/ExperimentDefinition.kt b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/api/ExperimentDefinition.kt new file mode 100644 index 00000000..88b26ee1 --- /dev/null +++ b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/api/ExperimentDefinition.kt @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2020 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.harness.api + +/** + * A definition for a repeatable experiment which consists of multiple scenarios derived from pre-defined experiment + * parameters. + * + * @property name The name of the experiment. + * @property parameters The parameters of the experiments. + * @property evaluator The function to evaluate a single experiment trial. + * @property meta The metadata for the experiment. + */ +public data class ExperimentDefinition( + val name: String, + val parameters: Set>, + val evaluator: (Trial) -> Unit, + val meta: Map = emptyMap() +) diff --git a/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/api/Parameter.kt b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/api/Parameter.kt new file mode 100644 index 00000000..bb5c8c2b --- /dev/null +++ b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/api/Parameter.kt @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2020 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.harness.api + +/** + * A [Parameter] defines a single dimension of exploration in the design space of an experiment. + */ +public sealed class Parameter { + /** + * The name of the parameter. + */ + public abstract val name: String + + /** + * A generic dimension of the experiment design space that is defined fully by a collection of [values]. + */ + public data class Generic(override val name: String, val values: Collection) : Parameter() +} diff --git a/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/api/Scenario.kt b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/api/Scenario.kt new file mode 100644 index 00000000..a8dbf01e --- /dev/null +++ b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/api/Scenario.kt @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.harness.api + +/** + * A [Scenario] represents a single point in the design space of an experiment. + */ +public interface Scenario { + /** + * A unique identifier that identifies a single scenario. + */ + public val id: Int + + /** + * The [ExperimentDefinition] describing the experiment this scenario is part of. + */ + public val experiment: ExperimentDefinition + + /** + * Obtain the instantiated value for a [parameter][param] of the experiment. + * + * @param param The parameter to obtain the value of. + * @throws IllegalArgumentException if [param] is not defined for the experiment. + */ + public operator fun get(param: Parameter): T +} diff --git a/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/api/Trial.kt b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/api/Trial.kt new file mode 100644 index 00000000..2d6ecd19 --- /dev/null +++ b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/api/Trial.kt @@ -0,0 +1,28 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.harness.api + +/** + * A [Trial] represents a single trial (run) of an experiment. + */ +public data class Trial(val scenario: Scenario, val repeat: Int) diff --git a/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/dsl/Experiment.kt b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/dsl/Experiment.kt new file mode 100644 index 00000000..41d4207a --- /dev/null +++ b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/dsl/Experiment.kt @@ -0,0 +1,99 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.harness.dsl + +import org.junit.platform.commons.annotation.Testable +import org.opendc.harness.api.ExperimentDefinition +import org.opendc.harness.api.Scenario +import org.opendc.harness.api.Trial +import org.opendc.harness.internal.ParameterDelegate + +/** + * An [Experiment] defines a blueprint for a repeatable experiment consisting of multiple scenarios based on pre-defined + * experiment parameters. + * + * @param name The name of the experiment or `null` to select the class name. + */ +@Testable +public abstract class Experiment(name: String? = null) : Cloneable { + /** + * The name of the experiment. + */ + public val name: String = name ?: javaClass.simpleName + + /** + * An identifier that uniquely identifies a single point in the design space of this [Experiment]. + */ + public val id: Int + get() { + val scenario = scenario ?: throw IllegalStateException("Cannot use id before activation") + return scenario.id + } + + /** + * Convert this experiment to an [ExperimentDefinition]. + */ + public fun toDefinition(): ExperimentDefinition = + ExperimentDefinition(name, HashSet(delegates.map { it.parameter }), this::run, mapOf("class.name" to javaClass.name)) + + /** + * Perform a single execution of the experiment based on the experiment parameters. + * + * @param repeat A number representing the repeat index of an identical scenario. + */ + protected abstract fun doRun(repeat: Int) + + /** + * A map to track the parameter delegates registered with this experiment. + */ + private val delegates: MutableSet> = mutableSetOf() + + /** + * The current active scenario. + */ + internal var scenario: Scenario? = null + + /** + * Perform a single execution of the experiment based on the experiment parameters. + * + * This operation will cause the [ParameterProvider]s of this group to be instantiated to some value in its design + * space. + * + * @param trial The experiment trial to run. + */ + private fun run(trial: Trial) { + val scenario = trial.scenario + + // XXX We clone the current class to prevent concurrency issues. + val res = clone() as Experiment + res.scenario = scenario + res.doRun(trial.repeat) + } + + /** + * Register a delegate for this experiment. + */ + internal fun register(delegate: ParameterDelegate<*>) { + delegates += delegate + } +} diff --git a/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/dsl/ParameterProvider.kt b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/dsl/ParameterProvider.kt new file mode 100644 index 00000000..e4bb9c64 --- /dev/null +++ b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/dsl/ParameterProvider.kt @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.harness.dsl + +import kotlin.properties.ReadOnlyProperty +import kotlin.reflect.KProperty + +/** + * A [ParameterProvider] defines a single dimension of exploration in the design space of an [Experiment]. + */ +public interface ParameterProvider { + /** + * Provide a delegate defining a parameter for the specified [Experiment][experiment]. + * + * @param experiment The experiment for which the parameter is defined. + * @param prop The property to create the delegate for. + */ + public operator fun provideDelegate(experiment: Experiment, prop: KProperty<*>): ReadOnlyProperty +} diff --git a/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/dsl/Parameters.kt b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/dsl/Parameters.kt new file mode 100644 index 00000000..7d269ba1 --- /dev/null +++ b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/dsl/Parameters.kt @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.harness.dsl + +import org.opendc.harness.api.Parameter +import org.opendc.harness.internal.ParameterDelegate +import kotlin.properties.ReadOnlyProperty +import kotlin.reflect.KProperty + +/** + * Define a dimension in the design space of an [Experiment] of type [T] consisting of the specified collection of + * [values]. + * + * @param values The values in the dimension to define.. + */ +public fun anyOf(vararg values: T): ParameterProvider = object : ParameterProvider { + override fun provideDelegate(experiment: Experiment, prop: KProperty<*>): ReadOnlyProperty { + val delegate = ParameterDelegate(Parameter.Generic(prop.name, listOf(*values))) + experiment.register(delegate) + return delegate + } + + override fun toString(): String = "GenericParameter" +} diff --git a/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/ExperimentEngine.kt b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/ExperimentEngine.kt new file mode 100644 index 00000000..db2cd191 --- /dev/null +++ b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/ExperimentEngine.kt @@ -0,0 +1,107 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.harness.engine + +import kotlinx.coroutines.CancellationException +import kotlinx.coroutines.flow.asFlow +import kotlinx.coroutines.flow.buffer +import kotlinx.coroutines.flow.collect +import kotlinx.coroutines.flow.map +import kotlinx.coroutines.launch +import kotlinx.coroutines.supervisorScope +import org.opendc.harness.api.ExperimentDefinition +import org.opendc.harness.api.Trial +import org.opendc.harness.engine.scheduler.ExperimentScheduler +import org.opendc.harness.engine.strategy.ExperimentStrategy + +/** + * The [ExperimentEngine] orchestrates the execution of experiments. + * + * @property strategy The [ExperimentStrategy] used to explore the experiment design space. + * @property scheduler The [ExperimentScheduler] to schedule the trials over compute resources. + * @property listener The [ExperimentExecutionListener] to observe the progress. + * @property repeats The number of repeats to perform. + */ +public class ExperimentEngine( + private val strategy: ExperimentStrategy, + private val scheduler: ExperimentScheduler, + private val listener: ExperimentExecutionListener, + private val repeats: Int +) { + /** + * Execute the specified [experiment][root]. + * + * @param root The experiment to execute. + */ + public suspend fun execute(root: ExperimentDefinition): Unit = supervisorScope { + listener.experimentStarted(root) + + try { + strategy.generate(root) + .asFlow() + .map { scenario -> + listener.scenarioStarted(scenario) + scenario + } + .buffer(100) + .collect { scenario -> + val jobs = (0 until repeats).map { repeat -> + val worker = scheduler.allocate() + launch { + val trial = Trial(scenario, repeat) + try { + listener.trialStarted(trial) + worker.dispatch(trial) + listener.trialFinished(trial, null) + } catch (e: Throwable) { + listener.trialFinished(trial, e) + throw e + } + } + } + + launch { + var error: Throwable? = null + for (job in jobs) { + try { + job.join() + } catch (e: CancellationException) { + // Propagate cancellation + throw e + } catch (e: Throwable) { + error = e + } + } + + listener.scenarioFinished(scenario, error) + } + } + listener.experimentFinished(root, null) + } catch (e: Throwable) { + listener.experimentFinished(root, e) + throw e + } + } + + override fun toString(): String = "ExperimentEngine" +} diff --git a/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/ExperimentEngineLauncher.kt b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/ExperimentEngineLauncher.kt new file mode 100644 index 00000000..ddd30483 --- /dev/null +++ b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/ExperimentEngineLauncher.kt @@ -0,0 +1,121 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.harness.engine + +import kotlinx.coroutines.flow.Flow +import kotlinx.coroutines.flow.collect +import kotlinx.coroutines.runBlocking +import org.opendc.harness.api.ExperimentDefinition +import org.opendc.harness.engine.scheduler.ExperimentScheduler +import org.opendc.harness.engine.scheduler.ThreadPoolExperimentScheduler +import org.opendc.harness.engine.strategy.CartesianExperimentStrategy +import org.opendc.harness.engine.strategy.ExperimentStrategy +import org.opendc.harness.internal.CompositeExperimentExecutionListener + +/** + * A builder class for conducting experiments via the [ExperimentEngine]. + */ +public class ExperimentEngineLauncher private constructor( + private val strategy: ExperimentStrategy?, + private val scheduler: ExperimentScheduler?, + private val listeners: List, + private val repeats: Int +) { + /** + * Construct an [ExperimentEngineLauncher] instance. + */ + public constructor() : this(null, null, emptyList(), 1) + + /** + * Create an [ExperimentEngineLauncher] with the specified [strategy]. + */ + public fun withScheduler(strategy: ExperimentStrategy): ExperimentEngineLauncher { + return ExperimentEngineLauncher(strategy, scheduler, listeners, repeats) + } + + /** + * Create an [ExperimentEngineLauncher] with the specified [scheduler]. + */ + public fun withScheduler(scheduler: ExperimentScheduler): ExperimentEngineLauncher { + return ExperimentEngineLauncher(strategy, scheduler, listeners, repeats) + } + + /** + * Create an [ExperimentEngineLauncher] with the specified [listener] added. + */ + public fun withListener(listener: ExperimentExecutionListener): ExperimentEngineLauncher { + return ExperimentEngineLauncher(strategy, scheduler, listeners + listener, repeats) + } + + /** + * Create an [ExperimentEngineLauncher] with the specified number of repeats. + */ + public fun withRepeats(repeats: Int): ExperimentEngineLauncher { + require(repeats > 0) { "Invalid number of repeats; must be greater than zero. " } + return ExperimentEngineLauncher(strategy, scheduler, listeners, repeats) + } + + /** + * Launch the specified experiments via the [ExperimentEngine] and block execution until finished. + */ + public suspend fun run(experiments: Flow) { + val engine = ExperimentEngine(createStrategy(), createScheduler(), createListener(), repeats) + experiments.collect { experiment -> engine.execute(experiment) } + } + + /** + * Launch the specified experiments via the [ExperimentEngine] and block the current thread until finished. + */ + public fun runBlocking(experiments: Flow) { + runBlocking { + run(experiments) + } + } + + /** + * Return a string representation of this instance. + */ + public override fun toString(): String = "ExperimentEngineLauncher" + + /** + * Create the [ExperimentStrategy] that explores the experiment design space. + */ + private fun createStrategy(): ExperimentStrategy { + return strategy ?: CartesianExperimentStrategy + } + + /** + * Create the [ExperimentScheduler] that schedules the trials over the compute resources. + */ + private fun createScheduler(): ExperimentScheduler { + return scheduler ?: ThreadPoolExperimentScheduler(Runtime.getRuntime().availableProcessors()) + } + + /** + * Create the [ExperimentExecutionListener] that listens the to the execution of the experiments. + */ + private fun createListener(): ExperimentExecutionListener { + require(listeners.isNotEmpty()) { "No listeners registered." } + return CompositeExperimentExecutionListener(listeners) + } +} diff --git a/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/ExperimentExecutionListener.kt b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/ExperimentExecutionListener.kt new file mode 100644 index 00000000..9ef71863 --- /dev/null +++ b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/ExperimentExecutionListener.kt @@ -0,0 +1,77 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.harness.engine + +import org.opendc.harness.api.ExperimentDefinition +import org.opendc.harness.api.Scenario +import org.opendc.harness.api.Trial + +/** + * Listener to be notified of experiment execution events by experiment runners. + */ +public interface ExperimentExecutionListener { + /** + * A method that is invoked when an experiment is started. + * + * @param experiment The [ExperimentDefinition] that started. + */ + public fun experimentStarted(experiment: ExperimentDefinition) {} + + /** + * A method that is invoked when an experiment is finished, regardless of the outcome. + * + * @param experiment The [ExperimentDefinition] that finished. + * @param throwable The exception that was thrown during execution or `null` if the execution completed successfully. + */ + public fun experimentFinished(experiment: ExperimentDefinition, throwable: Throwable?) {} + + /** + * A method that is invoked when a scenario is started. + * + * @param scenario The scenario that is started. + */ + public fun scenarioStarted(scenario: Scenario) {} + + /** + * A method that is invoked when a scenario is finished, regardless of the outcome. + * + * @param scenario The [Scenario] that has finished. + * @param throwable The exception that was thrown during execution or `null` if the execution completed successfully. + */ + public fun scenarioFinished(scenario: Scenario, throwable: Throwable?) {} + + /** + * A method that is invoked when a trial is started. + * + * @param trial The trial that is started. + */ + public fun trialStarted(trial: Trial) {} + + /** + * A method that is invoked when a scenario is finished, regardless of the outcome. + * + * @param trial The [Trial] that has finished. + * @param throwable The exception that was thrown during execution or `null` if the execution completed successfully. + */ + public fun trialFinished(trial: Trial, throwable: Throwable?) {} +} diff --git a/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/discovery/Discovery.kt b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/discovery/Discovery.kt new file mode 100644 index 00000000..f7f73b38 --- /dev/null +++ b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/discovery/Discovery.kt @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.harness.engine.discovery + +import kotlinx.coroutines.flow.Flow +import org.opendc.harness.api.ExperimentDefinition + +/** + * Component responsible for scanning for [ExperimentDefinition]s. + */ +public interface Discovery { + /** + * Start discovery of experiments. + * + * @param request The [DiscoveryRequest] to determine the experiments to discover. + * @return A flow of [ExperimentDefinition]s that have been discovered by the implementation. + */ + public fun discover(request: DiscoveryRequest): Flow +} diff --git a/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/discovery/DiscoveryFilter.kt b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/discovery/DiscoveryFilter.kt new file mode 100644 index 00000000..219d09cd --- /dev/null +++ b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/discovery/DiscoveryFilter.kt @@ -0,0 +1,51 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.harness.engine.discovery + +import org.opendc.harness.api.ExperimentDefinition +import java.util.function.Predicate + +/** + * A [DiscoveryFilter] decides how the selected experiments are filtered. + */ +public sealed class DiscoveryFilter { + /** + * Test whether the specified [ExperimentDefinition] should be selected. + */ + public abstract fun test(definition: ExperimentDefinition): Boolean + + /** + * Filter an experiment based on its name. + */ + public data class Name(val predicate: Predicate) : DiscoveryFilter() { + override fun test(definition: ExperimentDefinition): Boolean = predicate.test(definition.name) + } + + /** + * Filter an experiment based on its metadata. + */ + public data class Meta(val key: String, val predicate: Predicate) : DiscoveryFilter() { + override fun test(definition: ExperimentDefinition): Boolean = + definition.meta[key]?.let { predicate.test(it) } ?: false + } +} diff --git a/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/discovery/DiscoveryProvider.kt b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/discovery/DiscoveryProvider.kt new file mode 100644 index 00000000..fad255de --- /dev/null +++ b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/discovery/DiscoveryProvider.kt @@ -0,0 +1,65 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.harness.engine.discovery + +import org.opendc.harness.internal.CompositeDiscovery +import java.util.* + +/** + * A provider interface for the [Discovery] component. + */ +public interface DiscoveryProvider { + /** + * A unique identifier for this discovery implementation. + * + * Each discovery implementation must provide a unique ID, so that they can be selected by the user. + * When in doubt, you may use the fully qualified name of your custom [Discovery] implementation class. + */ + public val id: String + + /** + * Factory method for creating a new [Discovery] instance. + */ + public fun create(): Discovery + + public companion object { + /** + * The available [DiscoveryProvider]s. + */ + private val providers by lazy { ServiceLoader.load(DiscoveryProvider::class.java) } + + /** + * Obtain the [DiscoveryProvider] with the specified [id] or return `null`. + */ + public fun findById(id: String): DiscoveryProvider? { + return providers.find { it.id == id } + } + + /** + * Obtain a composite [Discovery] that combines the results of all available providers. + */ + public fun createComposite(): Discovery { + return CompositeDiscovery(providers) + } + } +} diff --git a/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/discovery/DiscoveryRequest.kt b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/discovery/DiscoveryRequest.kt new file mode 100644 index 00000000..5bc08dac --- /dev/null +++ b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/discovery/DiscoveryRequest.kt @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.harness.engine.discovery + +/** + * A request for discovering experiments according to the specified information. + * + * @param selectors The selectors for this discovery request. + * @param filters The filters for this discovery request. + */ +public data class DiscoveryRequest( + val selectors: List = emptyList(), + val filters: List = emptyList(), +) diff --git a/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/discovery/DiscoverySelector.kt b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/discovery/DiscoverySelector.kt new file mode 100644 index 00000000..67681303 --- /dev/null +++ b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/discovery/DiscoverySelector.kt @@ -0,0 +1,49 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.harness.engine.discovery + +import org.opendc.harness.api.ExperimentDefinition + +/** + * A [DiscoverySelector] defines the properties used to discover experiments. + */ +public sealed class DiscoverySelector { + /** + * Test whether the specified [ExperimentDefinition] should be selected. + */ + public abstract fun test(definition: ExperimentDefinition): Boolean + + /** + * Select an experiment based on its name. + */ + public data class Name(val name: String) : DiscoverySelector() { + override fun test(definition: ExperimentDefinition): Boolean = definition.name == name + } + + /** + * Select an experiment based on its metadata. + */ + public data class Meta(val key: String, val value: Any) : DiscoverySelector() { + override fun test(definition: ExperimentDefinition): Boolean = definition.meta[key] == value + } +} diff --git a/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/scheduler/ExperimentScheduler.kt b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/scheduler/ExperimentScheduler.kt new file mode 100644 index 00000000..0265554a --- /dev/null +++ b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/scheduler/ExperimentScheduler.kt @@ -0,0 +1,52 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.harness.engine.scheduler + +import org.opendc.harness.api.Trial + +/** + * The [ExperimentScheduler] is responsible for scheduling the execution of experiment runs over some set of compute + * resources (e.g., threads or even multiple machines). + */ +public interface ExperimentScheduler : AutoCloseable { + /** + * Allocate a [Worker] for executing an experiment trial. This method may suspend in case no resources are directly + * available at the moment. + * + * @return The available worker. + */ + public suspend fun allocate(): Worker + + /** + * An isolated worker of an [ExperimentScheduler] that is responsible for conducting a single experiment trial. + */ + public interface Worker { + /** + * Dispatch an experiment trial immediately to one of the available compute resources and block execution until + * the trial has finished. + * + * @param trial The trial to dispatch. + */ + public suspend fun dispatch(trial: Trial) + } +} diff --git a/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/scheduler/ExperimentSchedulerProvider.kt b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/scheduler/ExperimentSchedulerProvider.kt new file mode 100644 index 00000000..a93d4bf6 --- /dev/null +++ b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/scheduler/ExperimentSchedulerProvider.kt @@ -0,0 +1,57 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.harness.engine.scheduler + +import java.util.* + +/** + * A factory for constructing an [ExperimentScheduler]. + */ +public interface ExperimentSchedulerProvider { + /** + * A unique identifier for this scheduler implementation. + * + * Each experiment scheduler must provide a unique ID, so that they can be selected by the user. + * When in doubt, you may use the fully qualified name of your custom [ExperimentScheduler] implementation class. + */ + public val id: String + + /** + * Factory method for creating a new [ExperimentScheduler] instance. + */ + public fun create(): ExperimentScheduler + + public companion object { + /** + * The available [ExperimentSchedulerProvider]s. + */ + private val providers by lazy { ServiceLoader.load(ExperimentSchedulerProvider::class.java) } + + /** + * Obtain the [ExperimentScheduler] with the specified [id] or return `null`. + */ + public fun findById(id: String): ExperimentSchedulerProvider? { + return providers.find { it.id == id } + } + } +} diff --git a/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/scheduler/ThreadPoolExperimentScheduler.kt b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/scheduler/ThreadPoolExperimentScheduler.kt new file mode 100644 index 00000000..1ae533cf --- /dev/null +++ b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/scheduler/ThreadPoolExperimentScheduler.kt @@ -0,0 +1,58 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.harness.engine.scheduler + +import kotlinx.coroutines.asCoroutineDispatcher +import kotlinx.coroutines.sync.Semaphore +import kotlinx.coroutines.withContext +import org.opendc.harness.api.Trial +import java.util.concurrent.Executors + +/** + * An [ExperimentScheduler] that runs experiment trials using a local thread pool. + * + * @param parallelism The maximum amount of concurrent workers. + */ +public class ThreadPoolExperimentScheduler(parallelism: Int) : ExperimentScheduler { + private val dispatcher = Executors.newCachedThreadPool().asCoroutineDispatcher() + private val tickets = Semaphore(parallelism) + + override suspend fun allocate(): ExperimentScheduler.Worker { + tickets.acquire() + return object : ExperimentScheduler.Worker { + override suspend fun dispatch(trial: Trial) { + try { + withContext(dispatcher) { + trial.scenario.experiment.evaluator(trial) + } + } finally { + tickets.release() + } + } + } + } + + override fun close(): Unit = dispatcher.close() + + override fun toString(): String = "ThreadPoolScheduler" +} diff --git a/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/scheduler/ThreadPoolExperimentSchedulerProvider.kt b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/scheduler/ThreadPoolExperimentSchedulerProvider.kt new file mode 100644 index 00000000..cf9a132f --- /dev/null +++ b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/scheduler/ThreadPoolExperimentSchedulerProvider.kt @@ -0,0 +1,33 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.harness.engine.scheduler + +/** + * An [ExperimentSchedulerProvider] for constructing a [ThreadPoolExperimentScheduler]. + */ +public class ThreadPoolExperimentSchedulerProvider : ExperimentSchedulerProvider { + override val id: String = "thread-pool" + + override fun create(): ExperimentScheduler = + ThreadPoolExperimentScheduler(Runtime.getRuntime().availableProcessors()) +} diff --git a/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/strategy/CartesianExperimentStrategy.kt b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/strategy/CartesianExperimentStrategy.kt new file mode 100644 index 00000000..e5e08003 --- /dev/null +++ b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/strategy/CartesianExperimentStrategy.kt @@ -0,0 +1,55 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.harness.engine.strategy + +import org.opendc.harness.api.ExperimentDefinition +import org.opendc.harness.api.Parameter +import org.opendc.harness.api.Scenario +import org.opendc.harness.internal.ScenarioImpl + +/** + * An [ExperimentStrategy] that takes the cartesian product of the parameters and evaluates every combination. + */ +public object CartesianExperimentStrategy : ExperimentStrategy { + /** + * Build the trials of an experiment. + */ + override fun generate(experiment: ExperimentDefinition): Sequence { + return experiment.parameters + .asSequence() + .map { param -> mapParameter(param).map { value -> listOf(param to value) } } + .reduce { acc, param -> + acc.flatMap { x -> param.map { y -> x + y } } + } + .mapIndexed { id, values -> ScenarioImpl(id, experiment, values.toMap()) } + } + + /** + * Instantiate a parameter and return a sequence of possible values. + */ + private fun mapParameter(param: Parameter): Sequence { + return when (param) { + is Parameter.Generic -> param.values.asSequence() + } + } +} diff --git a/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/strategy/CartesianExperimentStrategyProvider.kt b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/strategy/CartesianExperimentStrategyProvider.kt new file mode 100644 index 00000000..f18795a3 --- /dev/null +++ b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/strategy/CartesianExperimentStrategyProvider.kt @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.harness.engine.strategy + +/** + * An [ExperimentStrategyProvider] for constructing a [CartesianExperimentStrategy]. + */ +public class CartesianExperimentStrategyProvider : ExperimentStrategyProvider { + override val id: String = "cartesian" + + override fun create(): ExperimentStrategy = CartesianExperimentStrategy +} diff --git a/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/strategy/ExperimentStrategy.kt b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/strategy/ExperimentStrategy.kt new file mode 100644 index 00000000..3a0148ad --- /dev/null +++ b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/strategy/ExperimentStrategy.kt @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.harness.engine.strategy + +import org.opendc.harness.api.ExperimentDefinition +import org.opendc.harness.api.Scenario + +/** + * The [ExperimentStrategy] is responsible for traversing the design space of an [ExperimentDefinition] based on its + * parameters, generating concrete points in the space represented as [Scenario]s. + */ +public interface ExperimentStrategy { + /** + * Generate the points in the design space of the specified [experiment] to explore. + * + * @param experiment The experiment design space to explore. + * @return A sequence of [Scenario]s which may be explored by the [ExperimentEngine]. + */ + public fun generate(experiment: ExperimentDefinition): Sequence +} diff --git a/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/strategy/ExperimentStrategyProvider.kt b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/strategy/ExperimentStrategyProvider.kt new file mode 100644 index 00000000..7fa05f34 --- /dev/null +++ b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/engine/strategy/ExperimentStrategyProvider.kt @@ -0,0 +1,57 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.harness.engine.strategy + +import java.util.* + +/** + * A factory for constructing an [ExperimentStrategy]. + */ +public interface ExperimentStrategyProvider { + /** + * A unique identifier for this strategy implementation. + * + * Each experiment strategy must provide a unique ID, so that they can be selected by the user. + * When in doubt, you may use the fully qualified name of your custom [ExperimentStrategy] implementation class. + */ + public val id: String + + /** + * Factory method for creating a new [ExperimentStrategy] instance. + */ + public fun create(): ExperimentStrategy + + public companion object { + /** + * The available [ExperimentStrategyProvider]s. + */ + private val providers by lazy { ServiceLoader.load(ExperimentStrategyProvider::class.java) } + + /** + * Obtain the [ExperimentStrategy] with the specified [id] or return `null`. + */ + public fun findById(id: String): ExperimentStrategyProvider? { + return providers.find { it.id == id } + } + } +} diff --git a/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/internal/CompositeDiscovery.kt b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/internal/CompositeDiscovery.kt new file mode 100644 index 00000000..67a895e4 --- /dev/null +++ b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/internal/CompositeDiscovery.kt @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.harness.internal + +import kotlinx.coroutines.FlowPreview +import kotlinx.coroutines.flow.* +import org.opendc.harness.api.ExperimentDefinition +import org.opendc.harness.engine.discovery.Discovery +import org.opendc.harness.engine.discovery.DiscoveryProvider +import org.opendc.harness.engine.discovery.DiscoveryRequest + +/** + * A composite [Discovery] instance that combines the results of multiple delegate instances. + */ +internal class CompositeDiscovery(providers: Iterable) : Discovery { + /** + * The [Discovery] instances to delegate to. + */ + private val delegates = providers.map { it.create() } + + @OptIn(FlowPreview::class) + override fun discover(request: DiscoveryRequest): Flow { + return delegates.asFlow() + .map { it.discover(request) } + .flattenMerge(delegates.size) + } +} diff --git a/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/internal/CompositeExperimentExecutionListener.kt b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/internal/CompositeExperimentExecutionListener.kt new file mode 100644 index 00000000..a3cd6bd2 --- /dev/null +++ b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/internal/CompositeExperimentExecutionListener.kt @@ -0,0 +1,57 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.harness.internal + +import org.opendc.harness.api.ExperimentDefinition +import org.opendc.harness.api.Scenario +import org.opendc.harness.api.Trial +import org.opendc.harness.engine.ExperimentExecutionListener + +/** + * An [ExperimentExecutionListener] that composes multiple other listeners. + */ +public class CompositeExperimentExecutionListener(private val listeners: List) : ExperimentExecutionListener { + override fun experimentStarted(experiment: ExperimentDefinition) { + listeners.forEach { it.experimentStarted(experiment) } + } + + override fun experimentFinished(experiment: ExperimentDefinition, throwable: Throwable?) { + listeners.forEach { it.experimentFinished(experiment, throwable) } + } + + override fun scenarioStarted(scenario: Scenario) { + listeners.forEach { it.scenarioStarted(scenario) } + } + + override fun scenarioFinished(scenario: Scenario, throwable: Throwable?) { + listeners.forEach { it.scenarioFinished(scenario, throwable) } + } + + override fun trialStarted(trial: Trial) { + listeners.forEach { it.trialStarted(trial) } + } + + override fun trialFinished(trial: Trial, throwable: Throwable?) { + listeners.forEach { it.trialFinished(trial, throwable) } + } +} diff --git a/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/internal/DslDiscovery.kt b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/internal/DslDiscovery.kt new file mode 100644 index 00000000..eb6303d6 --- /dev/null +++ b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/internal/DslDiscovery.kt @@ -0,0 +1,101 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.harness.internal + +import io.github.classgraph.ClassGraph +import io.github.classgraph.ScanResult +import kotlinx.coroutines.flow.Flow +import kotlinx.coroutines.flow.asFlow +import org.opendc.harness.api.ExperimentDefinition +import org.opendc.harness.dsl.Experiment +import org.opendc.harness.engine.discovery.Discovery +import org.opendc.harness.engine.discovery.DiscoveryFilter +import org.opendc.harness.engine.discovery.DiscoveryRequest +import org.opendc.harness.engine.discovery.DiscoverySelector + +/** + * A [Discovery] implementation that discovers [Experiment] instances on the classpath. + */ +internal class DslDiscovery : Discovery { + /* + * Lazily memoize the results of the classpath scan. + */ + private val scanResult by lazy { scan() } + + override fun discover(request: DiscoveryRequest): Flow { + return findExperiments() + .map { cls -> + val exp = cls.constructors[0].newInstance() as Experiment + exp.toDefinition() + } + .filter(select(request.selectors)) + .filter(filter(request.filters)) + .asFlow() + } + + /** + * Find the classes on the classpath implementing the [Experiment] class. + */ + private fun findExperiments(): Sequence> { + return scanResult + .getSubclasses(Experiment::class.java.name) + .filter { !(it.isAbstract || it.isInterface) } + .map { it.loadClass() } + .filterIsInstance>() + .asSequence() + } + + /** + * Create a predicate for filtering the experiments based on the specified [filters]. + */ + private fun filter(filters: List): (ExperimentDefinition) -> Boolean = { def -> + filters.isEmpty() || filters.all { it.test(def) } + } + + /** + * Create a predicate for selecting the experiments based on the specified [selectors]. + */ + private fun select(selectors: List): (ExperimentDefinition) -> Boolean = { def -> + selectors.isEmpty() || selectors.any { it.test(def) } + } + + /** + * Scan the classpath using [ClassGraph]. + */ + private fun scan(): ScanResult { + return ClassGraph() + .enableClassInfo() + .enableExternalClasses() + .ignoreClassVisibility() + .rejectPackages( + "java.*", + "javax.*", + "sun.*", + "com.sun.*", + "kotlin.*", + "androidx.*", + "org.jetbrains.kotlin.*", + "org.junit.*" + ).scan() + } +} diff --git a/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/internal/DslDiscoveryProvider.kt b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/internal/DslDiscoveryProvider.kt new file mode 100644 index 00000000..752ba4bb --- /dev/null +++ b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/internal/DslDiscoveryProvider.kt @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.harness.internal + +import org.opendc.harness.dsl.Experiment +import org.opendc.harness.engine.discovery.Discovery +import org.opendc.harness.engine.discovery.DiscoveryProvider + +/** + * A [DiscoveryProvider] for the [Experiment]s on the classpath. + */ +public class DslDiscoveryProvider : DiscoveryProvider { + override val id: String = "dsl" + + override fun create(): Discovery = DslDiscovery() +} diff --git a/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/internal/ParameterDelegate.kt b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/internal/ParameterDelegate.kt new file mode 100644 index 00000000..aaf90b99 --- /dev/null +++ b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/internal/ParameterDelegate.kt @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.harness.internal + +import org.opendc.harness.api.Parameter +import org.opendc.harness.dsl.Experiment +import kotlin.properties.ReadOnlyProperty +import kotlin.reflect.KProperty + +/** + * A delegate for an experiment parameter. + * + * @property parameter The parameter descriptor of this delegate. + */ +internal class ParameterDelegate(val parameter: Parameter) : ReadOnlyProperty { + /** + * Obtain the value for the parameter. + */ + override fun getValue(thisRef: Experiment, property: KProperty<*>): T { + val scenario = thisRef.scenario ?: throw IllegalStateException("Cannot use parameters before activation") + return scenario[parameter] + } +} diff --git a/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/internal/ScenarioImpl.kt b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/internal/ScenarioImpl.kt new file mode 100644 index 00000000..d255004d --- /dev/null +++ b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/internal/ScenarioImpl.kt @@ -0,0 +1,49 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.harness.internal + +import org.opendc.harness.api.ExperimentDefinition +import org.opendc.harness.api.Parameter +import org.opendc.harness.api.Scenario + +/** + * Internal implementation of a [Scenario]. + */ +internal data class ScenarioImpl( + override val id: Int, + override val experiment: ExperimentDefinition, + val parameters: Map, Any?> +) : Scenario { + + override fun get(param: Parameter): T { + if (!parameters.containsKey(param)) { + throw IllegalArgumentException("Unknown parameter for this scenario.") + } + + // This cast should always succeed + @Suppress("UNCHECKED_CAST") + return parameters[param] as T + } + + override fun toString(): String = "Scenario" +} diff --git a/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/runner/junit5/JUnitExperimentExecutionListener.kt b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/runner/junit5/JUnitExperimentExecutionListener.kt new file mode 100644 index 00000000..58791549 --- /dev/null +++ b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/runner/junit5/JUnitExperimentExecutionListener.kt @@ -0,0 +1,152 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.harness.runner.junit5 + +import org.junit.platform.engine.* +import org.junit.platform.engine.support.descriptor.AbstractTestDescriptor +import org.junit.platform.engine.support.descriptor.EngineDescriptor +import org.opendc.harness.api.ExperimentDefinition +import org.opendc.harness.api.Scenario +import org.opendc.harness.api.Trial +import org.opendc.harness.engine.ExperimentExecutionListener + +/** + * An [ExperimentExecutionListener] that notifies JUnit platform of the progress of the experiment trials. + */ +public class JUnitExperimentExecutionListener( + private val listener: EngineExecutionListener, + private val root: EngineDescriptor +) : ExperimentExecutionListener { + /** + * The current active experiments. + */ + private val experiments = mutableMapOf() + + /** + * The current active scenarios. + */ + private val scenarios = mutableMapOf() + + /** + * The current active trials. + */ + private val trials = mutableMapOf() + + override fun experimentStarted(experiment: ExperimentDefinition) { + val descriptor = experiment.toDescriptor(root) + root.addChild(descriptor) + experiments[experiment] = descriptor + + listener.dynamicTestRegistered(descriptor) + listener.executionStarted(descriptor) + } + + override fun experimentFinished(experiment: ExperimentDefinition, throwable: Throwable?) { + val descriptor = experiments.remove(experiment) + + if (throwable != null) { + listener.executionFinished(descriptor, TestExecutionResult.failed(throwable)) + } else { + listener.executionFinished(descriptor, TestExecutionResult.successful()) + } + } + + override fun scenarioStarted(scenario: Scenario) { + val parent = experiments[scenario.experiment] ?: return + val descriptor = scenario.toDescriptor(parent) + parent.addChild(descriptor) + scenarios[scenario] = descriptor + + listener.dynamicTestRegistered(descriptor) + listener.executionStarted(descriptor) + } + + override fun scenarioFinished(scenario: Scenario, throwable: Throwable?) { + val descriptor = scenarios.remove(scenario) + + if (throwable != null) { + listener.executionFinished(descriptor, TestExecutionResult.failed(throwable)) + } else { + listener.executionFinished(descriptor, TestExecutionResult.successful()) + } + } + + override fun trialStarted(trial: Trial) { + val parent = scenarios[trial.scenario] ?: return + val descriptor = trial.toDescriptor(parent) + parent.addChild(descriptor) + trials[trial] = descriptor + + listener.dynamicTestRegistered(descriptor) + listener.executionStarted(descriptor) + } + + override fun trialFinished(trial: Trial, throwable: Throwable?) { + val descriptor = trials.remove(trial) + + if (throwable != null) { + listener.executionFinished(descriptor, TestExecutionResult.failed(throwable)) + } else { + listener.executionFinished(descriptor, TestExecutionResult.successful()) + } + } + + /** + * Create a [TestDescriptor] for an [ExperimentDefinition]. + */ + private fun ExperimentDefinition.toDescriptor(parent: TestDescriptor): TestDescriptor { + return object : AbstractTestDescriptor(parent.uniqueId.append("experiment", name), name) { + override fun getType(): TestDescriptor.Type = TestDescriptor.Type.CONTAINER + + override fun mayRegisterTests(): Boolean = true + + override fun toString(): String = "ExperimentDescriptor" + } + } + + /** + * Create a [TestDescriptor] for a [Scenario]. + */ + private fun Scenario.toDescriptor(parent: TestDescriptor): TestDescriptor { + return object : AbstractTestDescriptor(parent.uniqueId.append("scenario", id.toString()), "Scenario $id") { + override fun getType(): TestDescriptor.Type = TestDescriptor.Type.CONTAINER_AND_TEST + + override fun mayRegisterTests(): Boolean = true + + override fun toString(): String = "ScenarioDescriptor" + } + } + + /** + * Create a [TestDescriptor] for a [Trial]. + */ + private fun Trial.toDescriptor(parent: TestDescriptor): TestDescriptor { + return object : AbstractTestDescriptor(parent.uniqueId.append("repeat", repeat.toString()), "Repeat $repeat") { + override fun getType(): TestDescriptor.Type = TestDescriptor.Type.TEST + + override fun mayRegisterTests(): Boolean = false + + override fun toString(): String = "TrialDescriptor" + } + } +} diff --git a/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/runner/junit5/OpenDCTestEngine.kt b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/runner/junit5/OpenDCTestEngine.kt new file mode 100644 index 00000000..685cd41a --- /dev/null +++ b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/runner/junit5/OpenDCTestEngine.kt @@ -0,0 +1,94 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.harness.runner.junit5 + +import kotlinx.coroutines.flow.Flow +import kotlinx.coroutines.flow.emptyFlow +import mu.KotlinLogging +import org.junit.platform.engine.* +import org.junit.platform.engine.discovery.ClassNameFilter +import org.junit.platform.engine.discovery.ClassSelector +import org.junit.platform.engine.discovery.MethodSelector +import org.junit.platform.engine.support.descriptor.EngineDescriptor +import org.junit.platform.launcher.LauncherDiscoveryRequest +import org.opendc.harness.api.ExperimentDefinition +import org.opendc.harness.engine.ExperimentEngineLauncher +import org.opendc.harness.engine.discovery.DiscoveryFilter +import org.opendc.harness.engine.discovery.DiscoveryProvider +import org.opendc.harness.engine.discovery.DiscoveryRequest +import org.opendc.harness.engine.discovery.DiscoverySelector + +/** + * A [TestEngine] implementation that is able to run experiments defined using the harness. + */ +public class OpenDCTestEngine : TestEngine { + /** + * The logging instance for this engine. + */ + private val logger = KotlinLogging.logger {} + + override fun getId(): String = "opendc" + + override fun discover(request: EngineDiscoveryRequest, uniqueId: UniqueId): TestDescriptor { + // Test whether are excluded from the engines + val isEnabled = (request as? LauncherDiscoveryRequest)?.engineFilters?.all { it.toPredicate().test(this) } ?: true + if (!isEnabled) { + return ExperimentEngineDescriptor(uniqueId, emptyFlow()) + } + + // IntelliJ will pass a [MethodSelector] to run just a single method inside a file. In that + // case, no experiments should be discovered, since we support only experiments by class. + if (request.getSelectorsByType(MethodSelector::class.java).isNotEmpty()) { + return ExperimentEngineDescriptor(uniqueId, emptyFlow()) + } + + val classNames = request.getSelectorsByType(ClassSelector::class.java).map { DiscoverySelector.Meta("class.name", it.className) } + val classNameFilters = request.getFiltersByType(ClassNameFilter::class.java).map { DiscoveryFilter.Name(it.toPredicate()) } + + val discovery = DiscoveryProvider.createComposite() + val definitions = discovery.discover(DiscoveryRequest(classNames, classNameFilters)) + + return ExperimentEngineDescriptor(uniqueId, definitions) + } + + override fun execute(request: ExecutionRequest) { + logger.debug { "JUnit ExecutionRequest[${request::class.java.name}] [configurationParameters=${request.configurationParameters}; rootTestDescriptor=${request.rootTestDescriptor}]" } + val root = request.rootTestDescriptor as ExperimentEngineDescriptor + val listener = request.engineExecutionListener + + listener.executionStarted(root) + + try { + ExperimentEngineLauncher() + .withListener(JUnitExperimentExecutionListener(listener, root)) + .runBlocking(root.experiments) + listener.executionFinished(root, TestExecutionResult.successful()) + } catch (e: Throwable) { + listener.executionFinished(root, TestExecutionResult.failed(e)) + } + } + + private class ExperimentEngineDescriptor(id: UniqueId, val experiments: Flow) : EngineDescriptor(id, "opendc") { + override fun mayRegisterTests(): Boolean = true + } +} diff --git a/simulator/opendc-harness/src/main/resources/META-INF/services/org.junit.platform.engine.TestEngine b/simulator/opendc-harness/src/main/resources/META-INF/services/org.junit.platform.engine.TestEngine new file mode 100644 index 00000000..b83eec0c --- /dev/null +++ b/simulator/opendc-harness/src/main/resources/META-INF/services/org.junit.platform.engine.TestEngine @@ -0,0 +1 @@ +org.opendc.harness.runner.junit5.OpenDCTestEngine diff --git a/simulator/opendc-harness/src/main/resources/META-INF/services/org.opendc.harness.engine.discovery.DiscoveryProvider b/simulator/opendc-harness/src/main/resources/META-INF/services/org.opendc.harness.engine.discovery.DiscoveryProvider new file mode 100644 index 00000000..d6a73ded --- /dev/null +++ b/simulator/opendc-harness/src/main/resources/META-INF/services/org.opendc.harness.engine.discovery.DiscoveryProvider @@ -0,0 +1 @@ +org.opendc.harness.internal.DslDiscoveryProvider diff --git a/simulator/opendc-harness/src/main/resources/META-INF/services/org.opendc.harness.engine.scheduler.ExperimentSchedulerProvider b/simulator/opendc-harness/src/main/resources/META-INF/services/org.opendc.harness.engine.scheduler.ExperimentSchedulerProvider new file mode 100644 index 00000000..2ba3a7cb --- /dev/null +++ b/simulator/opendc-harness/src/main/resources/META-INF/services/org.opendc.harness.engine.scheduler.ExperimentSchedulerProvider @@ -0,0 +1 @@ +org.opendc.harness.engine.scheduler.ThreadPoolExperimentSchedulerProvider diff --git a/simulator/opendc-harness/src/main/resources/META-INF/services/org.opendc.harness.engine.strategy.ExperimentStrategyProvider b/simulator/opendc-harness/src/main/resources/META-INF/services/org.opendc.harness.engine.strategy.ExperimentStrategyProvider new file mode 100644 index 00000000..cb1c70ac --- /dev/null +++ b/simulator/opendc-harness/src/main/resources/META-INF/services/org.opendc.harness.engine.strategy.ExperimentStrategyProvider @@ -0,0 +1 @@ +org.opendc.harness.engine.strategy.CartesianExperimentStrategyProvider diff --git a/simulator/opendc-harness/src/test/kotlin/org/opendc/harness/EngineTest.kt b/simulator/opendc-harness/src/test/kotlin/org/opendc/harness/EngineTest.kt new file mode 100644 index 00000000..6f2989db --- /dev/null +++ b/simulator/opendc-harness/src/test/kotlin/org/opendc/harness/EngineTest.kt @@ -0,0 +1,61 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.harness + +import kotlinx.coroutines.flow.flowOf +import kotlinx.coroutines.flow.toList +import kotlinx.coroutines.runBlocking +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertNotNull +import org.junit.jupiter.api.Test +import org.opendc.harness.api.ExperimentDefinition +import org.opendc.harness.engine.ExperimentEngine +import org.opendc.harness.engine.ExperimentEngineLauncher +import org.opendc.harness.engine.ExperimentExecutionListener +import org.opendc.harness.engine.discovery.DiscoveryProvider +import org.opendc.harness.engine.discovery.DiscoveryRequest + +/** + * A test suite for the [ExperimentEngine]. + */ +internal class EngineTest { + @Test + fun test() { + val listener = object : ExperimentExecutionListener {} + ExperimentEngineLauncher() + .withListener(listener) + .runBlocking(flowOf(TestExperiment().toDefinition())) + } + + @Test + fun discovery() { + runBlocking { + val discovery = DiscoveryProvider.findById("dsl")?.create() + assertNotNull(discovery) + val res = mutableListOf() + discovery?.discover(DiscoveryRequest())?.toList(res) + println(res) + assertEquals(1, res.size) + } + } +} diff --git a/simulator/opendc-harness/src/test/kotlin/org/opendc/harness/TestExperiment.kt b/simulator/opendc-harness/src/test/kotlin/org/opendc/harness/TestExperiment.kt new file mode 100644 index 00000000..bedd1c76 --- /dev/null +++ b/simulator/opendc-harness/src/test/kotlin/org/opendc/harness/TestExperiment.kt @@ -0,0 +1,54 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.harness + +import org.opendc.harness.dsl.Experiment +import org.opendc.harness.dsl.anyOf + +/** + * An experiment to test the harness' functionality. + */ +class TestExperiment : Experiment("Design Space Exploration") { + /** + * The cloud environment to use. + */ + private val environment: String by anyOf( + "../../traces/setup-small.json", + "../../traces/setup.json", + "../../traces/setup-large.json" + ) + + /** + * The trace to use. + */ + private val trace: String by anyOf( + "../../traces/gwf/askalon_workload_olde.gwf", + "../../traces/gwf/askalon_workload_ee.gwf", + "../../traces/gwf/chronos_exp_noscaler_ca.gwf" + ) + + override fun doRun(repeat: Int) { + println("Id $id, Run $repeat, Environment $environment, Trace $trace") + Thread.sleep(500 * id.toLong()) + } +} diff --git a/simulator/settings.gradle.kts b/simulator/settings.gradle.kts index e6f42574..470303f4 100644 --- a/simulator/settings.gradle.kts +++ b/simulator/settings.gradle.kts @@ -33,4 +33,5 @@ include(":opendc-simulator:opendc-simulator-core") include(":opendc-simulator:opendc-simulator-compute") include(":opendc-simulator:opendc-simulator-failures") include(":opendc-trace:opendc-trace-core") +include(":opendc-harness") include(":opendc-utils") -- cgit v1.2.3 From 127686493ff44f536ee134575cd23bc96eebf075 Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Wed, 6 Jan 2021 22:52:07 +0100 Subject: Add rudimentary console runner for harness --- .../kotlin/kotlin-library-convention.gradle.kts | 1 + simulator/opendc-harness/build.gradle.kts | 4 + .../runner/console/ConsoleExperimentReporter.kt | 79 +++++++++++++++++ .../opendc/harness/runner/console/ConsoleRunner.kt | 99 ++++++++++++++++++++++ .../opendc-harness/src/main/resources/log4j2.xml | 40 +++++++++ 5 files changed, 223 insertions(+) create mode 100644 simulator/opendc-harness/src/main/kotlin/org/opendc/harness/runner/console/ConsoleExperimentReporter.kt create mode 100644 simulator/opendc-harness/src/main/kotlin/org/opendc/harness/runner/console/ConsoleRunner.kt create mode 100644 simulator/opendc-harness/src/main/resources/log4j2.xml diff --git a/simulator/buildSrc/src/main/kotlin/kotlin-library-convention.gradle.kts b/simulator/buildSrc/src/main/kotlin/kotlin-library-convention.gradle.kts index bbecf346..f9d8a966 100644 --- a/simulator/buildSrc/src/main/kotlin/kotlin-library-convention.gradle.kts +++ b/simulator/buildSrc/src/main/kotlin/kotlin-library-convention.gradle.kts @@ -33,6 +33,7 @@ plugins { /* Project configuration */ repositories { + mavenCentral() jcenter() } diff --git a/simulator/opendc-harness/build.gradle.kts b/simulator/opendc-harness/build.gradle.kts index 76d1f2cd..359d2384 100644 --- a/simulator/opendc-harness/build.gradle.kts +++ b/simulator/opendc-harness/build.gradle.kts @@ -30,14 +30,18 @@ plugins { dependencies { api("org.jetbrains.kotlinx:kotlinx-coroutines-core:${Library.KOTLINX_COROUTINES}") api("org.junit.platform:junit-platform-commons:${Library.JUNIT_PLATFORM}") + implementation("io.github.classgraph:classgraph:4.8.98") implementation("me.tongfei:progressbar:0.9.0") implementation("io.github.microutils:kotlin-logging:2.0.4") + implementation("com.github.ajalt.clikt:clikt:3.1.0") api("org.junit.platform:junit-platform-engine:${Library.JUNIT_PLATFORM}") api("org.junit.platform:junit-platform-suite-api:${Library.JUNIT_PLATFORM}") api("org.junit.platform:junit-platform-launcher:${Library.JUNIT_PLATFORM}") + runtimeOnly("org.apache.logging.log4j:log4j-slf4j-impl:2.14.0") + testImplementation("org.junit.jupiter:junit-jupiter-api:${Library.JUNIT_JUPITER}") testRuntimeOnly("org.junit.jupiter:junit-jupiter-engine:${Library.JUNIT_JUPITER}") } diff --git a/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/runner/console/ConsoleExperimentReporter.kt b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/runner/console/ConsoleExperimentReporter.kt new file mode 100644 index 00000000..2db74ef4 --- /dev/null +++ b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/runner/console/ConsoleExperimentReporter.kt @@ -0,0 +1,79 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.harness.runner.console + +import me.tongfei.progressbar.ProgressBar +import me.tongfei.progressbar.ProgressBarBuilder +import mu.KotlinLogging +import org.opendc.harness.api.Trial +import org.opendc.harness.engine.ExperimentExecutionListener + +/** + * A reporter that reports the experiment progress to the console. + */ +public class ConsoleExperimentReporter : ExperimentExecutionListener, AutoCloseable { + /** + * The active [Trial]s. + */ + private val trials: MutableSet = mutableSetOf() + + /** + * The total number of runs. + */ + private var total = 0 + + /** + * The logger for this reporter. + */ + private val logger = KotlinLogging.logger {} + + /** + * The progress bar to keep track of the progress. + */ + private val pb: ProgressBar = ProgressBarBuilder() + .setTaskName("") + .setInitialMax(1) + .build() + + override fun trialFinished(trial: Trial, throwable: Throwable?) { + trials -= trial + + pb.stepTo(total - trials.size.toLong()) + if (trials.isEmpty()) { + pb.close() + } + + if (throwable != null) { + logger.warn(throwable) { "Trial $trial failed" } + } + } + + override fun trialStarted(trial: Trial) { + trials += trial + pb.maxHint((++total).toLong()) + } + + override fun close() { + pb.close() + } +} diff --git a/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/runner/console/ConsoleRunner.kt b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/runner/console/ConsoleRunner.kt new file mode 100644 index 00000000..ae221c7f --- /dev/null +++ b/simulator/opendc-harness/src/main/kotlin/org/opendc/harness/runner/console/ConsoleRunner.kt @@ -0,0 +1,99 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.harness.runner.console + +import com.github.ajalt.clikt.core.CliktCommand +import com.github.ajalt.clikt.parameters.options.default +import com.github.ajalt.clikt.parameters.options.multiple +import com.github.ajalt.clikt.parameters.options.option +import com.github.ajalt.clikt.parameters.types.int +import mu.KotlinLogging +import org.opendc.harness.engine.ExperimentEngineLauncher +import org.opendc.harness.engine.discovery.DiscoveryProvider +import org.opendc.harness.engine.discovery.DiscoveryRequest +import org.opendc.harness.engine.discovery.DiscoverySelector +import org.opendc.harness.engine.scheduler.ThreadPoolExperimentScheduler + +/** + * The logger for this experiment runner. + */ +private val logger = KotlinLogging.logger {} + +/** + * The command line interface for the console experiment runner. + */ +public class ConsoleRunner : CliktCommand(name = "opendc-harness") { + /** + * The number of repeats per scenario. + */ + private val repeats by option("-r", "--repeats", help = "Number of repeats per scenario") + .int() + .default(1) + + /** + * The selected experiments to run by name. + */ + private val experiments by option("-e", "--experiments", help = "Names of experiments to explore") + .multiple(emptyList()) + + /** + * The maximum number of worker threads to use. + */ + private val parallelism by option("-p", "--parallelism", help = "Maximum number of concurrent simulation runs") + .int() + .default(Runtime.getRuntime().availableProcessors()) + + override fun run() { + logger.info { "Starting OpenDC Console Experiment Runner" } + + val discovery = DiscoveryProvider.createComposite() + val experiments = discovery.discover( + DiscoveryRequest( + selectors = experiments.map { DiscoverySelector.Name(it) } + ) + ) + + val reporter = ConsoleExperimentReporter() + val scheduler = ThreadPoolExperimentScheduler(parallelism) + + try { + ExperimentEngineLauncher() + .withListener(reporter) + .withRepeats(repeats) + .withScheduler(scheduler) + .runBlocking(experiments) + } catch (e: Throwable) { + logger.error(e) { "Failed to finish experiments" } + } finally { + reporter.close() + scheduler.close() + } + + logger.info { "Finished all experiments. Exiting." } + } +} + +/** + * Main entry point of the experiment runner. + */ +public fun main(args: Array): Unit = ConsoleRunner().main(args) diff --git a/simulator/opendc-harness/src/main/resources/log4j2.xml b/simulator/opendc-harness/src/main/resources/log4j2.xml new file mode 100644 index 00000000..9553d964 --- /dev/null +++ b/simulator/opendc-harness/src/main/resources/log4j2.xml @@ -0,0 +1,40 @@ + + + + + + + + + + + + + + + + + + -- cgit v1.2.3 From 885730e31a2bb987f2dcc8a90921e2f06a8c857a Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Thu, 7 Jan 2021 00:27:42 +0100 Subject: Migrate OpenDC SC20 experiments to experiment harness This change refactors the SC20 experiments in OpenDC to use the standalone experiment harness as opposed to its built-in ad-hoc experiment framework. --- .../opendc-experiments-sc20/build.gradle.kts | 11 +- .../kotlin/org/opendc/experiments/sc20/Main.kt | 159 --------------- .../sc20/experiment/CompositeWorkloadPortfolio.kt | 79 ++++++++ .../experiments/sc20/experiment/Experiment.kt | 76 ------- .../experiments/sc20/experiment/HorVerPortfolio.kt | 60 ++++++ .../sc20/experiment/MoreHpcPortfolio.kt | 56 ++++++ .../sc20/experiment/MoreVelocityPortfolio.kt | 56 ++++++ .../experiment/OperationalPhenomenaPortfolio.kt | 61 ++++++ .../experiments/sc20/experiment/Portfolio.kt | 199 +++++++++++++++---- .../experiments/sc20/experiment/Portfolios.kt | 221 --------------------- .../experiments/sc20/experiment/ReplayPortfolio.kt | 50 +++++ .../org/opendc/experiments/sc20/experiment/Run.kt | 157 --------------- .../opendc/experiments/sc20/experiment/Scenario.kt | 46 ----- .../experiments/sc20/experiment/TestPortfolio.kt | 47 +++++ .../sc20/reporter/ConsoleExperimentReporter.kt | 87 -------- .../sc20/runner/ContainerExperimentDescriptor.kt | 66 ------ .../sc20/runner/ExperimentDescriptor.kt | 79 -------- .../experiments/sc20/runner/ExperimentRunner.kt | 49 ----- .../sc20/runner/TrialExperimentDescriptor.kt | 32 --- .../runner/execution/ExperimentExecutionContext.kt | 45 ----- .../execution/ExperimentExecutionListener.kt | 46 ----- .../runner/execution/ExperimentExecutionResult.kt | 42 ---- .../sc20/runner/execution/ExperimentScheduler.kt | 56 ------ .../execution/ThreadPoolExperimentScheduler.kt | 83 -------- .../runner/internal/DefaultExperimentRunner.kt | 60 ------ .../opendc/experiments/sc20/telemetry/RunEvent.kt | 5 +- .../telemetry/parquet/ParquetRunEventWriter.kt | 28 +-- 27 files changed, 592 insertions(+), 1364 deletions(-) delete mode 100644 simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/Main.kt create mode 100644 simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/experiment/CompositeWorkloadPortfolio.kt delete mode 100644 simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/experiment/Experiment.kt create mode 100644 simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/experiment/HorVerPortfolio.kt create mode 100644 simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/experiment/MoreHpcPortfolio.kt create mode 100644 simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/experiment/MoreVelocityPortfolio.kt create mode 100644 simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/experiment/OperationalPhenomenaPortfolio.kt delete mode 100644 simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/experiment/Portfolios.kt create mode 100644 simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/experiment/ReplayPortfolio.kt delete mode 100644 simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/experiment/Run.kt delete mode 100644 simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/experiment/Scenario.kt create mode 100644 simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/experiment/TestPortfolio.kt delete mode 100644 simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/reporter/ConsoleExperimentReporter.kt delete mode 100644 simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/runner/ContainerExperimentDescriptor.kt delete mode 100644 simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/runner/ExperimentDescriptor.kt delete mode 100644 simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/runner/ExperimentRunner.kt delete mode 100644 simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/runner/TrialExperimentDescriptor.kt delete mode 100644 simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/runner/execution/ExperimentExecutionContext.kt delete mode 100644 simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/runner/execution/ExperimentExecutionListener.kt delete mode 100644 simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/runner/execution/ExperimentExecutionResult.kt delete mode 100644 simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/runner/execution/ExperimentScheduler.kt delete mode 100644 simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/runner/execution/ThreadPoolExperimentScheduler.kt delete mode 100644 simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/runner/internal/DefaultExperimentRunner.kt diff --git a/simulator/opendc-experiments/opendc-experiments-sc20/build.gradle.kts b/simulator/opendc-experiments/opendc-experiments-sc20/build.gradle.kts index 3b682668..b94207ba 100644 --- a/simulator/opendc-experiments/opendc-experiments-sc20/build.gradle.kts +++ b/simulator/opendc-experiments/opendc-experiments-sc20/build.gradle.kts @@ -29,21 +29,22 @@ plugins { } application { - mainClassName = "org.opendc.experiments.sc20.MainKt" + mainClass.set("org.opendc.harness.runner.console.ConsoleRunnerKt") applicationDefaultJvmArgs = listOf("-Xms2500M") } dependencies { api(project(":opendc-core")) + api(project(":opendc-harness")) implementation(project(":opendc-format")) implementation(project(":opendc-simulator:opendc-simulator-core")) implementation(project(":opendc-simulator:opendc-simulator-compute")) implementation(project(":opendc-simulator:opendc-simulator-failures")) implementation(project(":opendc-compute:opendc-compute-simulator")) - implementation("com.github.ajalt:clikt:2.6.0") - implementation("me.tongfei:progressbar:0.8.1") - implementation("io.github.microutils:kotlin-logging:1.7.9") + implementation("io.github.microutils:kotlin-logging:2.0.4") + implementation("me.tongfei:progressbar:0.9.0") + implementation("com.github.ajalt.clikt:clikt:3.1.0") implementation("org.apache.parquet:parquet-avro:1.11.0") implementation("org.apache.hadoop:hadoop-client:3.2.1") { @@ -51,8 +52,6 @@ dependencies { exclude(group = "log4j") } - runtimeOnly("org.apache.logging.log4j:log4j-slf4j-impl:2.13.1") - testImplementation("org.junit.jupiter:junit-jupiter-api:${Library.JUNIT_JUPITER}") testRuntimeOnly("org.junit.jupiter:junit-jupiter-engine:${Library.JUNIT_JUPITER}") testImplementation("org.junit.platform:junit-platform-launcher:${Library.JUNIT_PLATFORM}") diff --git a/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/Main.kt b/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/Main.kt deleted file mode 100644 index 8916261b..00000000 --- a/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/Main.kt +++ /dev/null @@ -1,159 +0,0 @@ -/* - * Copyright (c) 2020 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.experiments.sc20 - -import com.github.ajalt.clikt.core.CliktCommand -import com.github.ajalt.clikt.parameters.options.convert -import com.github.ajalt.clikt.parameters.options.default -import com.github.ajalt.clikt.parameters.options.defaultLazy -import com.github.ajalt.clikt.parameters.options.multiple -import com.github.ajalt.clikt.parameters.options.option -import com.github.ajalt.clikt.parameters.options.required -import com.github.ajalt.clikt.parameters.types.choice -import com.github.ajalt.clikt.parameters.types.file -import com.github.ajalt.clikt.parameters.types.int -import mu.KotlinLogging -import org.opendc.experiments.sc20.experiment.* -import org.opendc.experiments.sc20.reporter.ConsoleExperimentReporter -import org.opendc.experiments.sc20.runner.ExperimentDescriptor -import org.opendc.experiments.sc20.runner.execution.ThreadPoolExperimentScheduler -import org.opendc.experiments.sc20.runner.internal.DefaultExperimentRunner -import org.opendc.format.trace.sc20.Sc20PerformanceInterferenceReader -import org.opendc.format.trace.sc20.Sc20VmPlacementReader -import java.io.File - -/** - * The logger for this experiment. - */ -private val logger = KotlinLogging.logger {} - -/** - * Represents the command for running the experiment. - */ -public class ExperimentCli : CliktCommand(name = "sc20-experiment") { - /** - * The path to the directory where the topology descriptions are located. - */ - private val environmentPath by option("--environment-path", help = "path to the environment directory") - .file(canBeFile = false) - .required() - - /** - * The path to the directory where the traces are located. - */ - private val tracePath by option("--trace-path", help = "path to the traces directory") - .file(canBeFile = false) - .required() - - /** - * The path to the performance interference model. - */ - private val performanceInterferenceStream by option( - "--performance-interference-model", - help = "path to the performance interference file" - ) - .file(canBeDir = false) - .convert { it.inputStream() } - - /** - * The path to the original VM placements file. - */ - private val vmPlacements by option("--vm-placements-file", help = "path to the VM placement file") - .file(canBeDir = false) - .convert { - Sc20VmPlacementReader(it.inputStream().buffered()).construct() - } - .default(emptyMap()) - - /** - * The selected portfolios to run. - */ - private val portfolios by option("--portfolio", help = "portfolio of scenarios to explore") - .choice( - "hor-ver" to { experiment: Experiment, i: Int -> HorVerPortfolio(experiment, i) } - as (Experiment, Int) -> Portfolio, - "more-velocity" to { experiment, i -> MoreVelocityPortfolio(experiment, i) }, - "composite-workload" to { experiment, i -> CompositeWorkloadPortfolio(experiment, i) }, - "operational-phenomena" to { experiment, i -> OperationalPhenomenaPortfolio(experiment, i) }, - "replay" to { experiment, i -> ReplayPortfolio(experiment, i) }, - "test" to { experiment, i -> TestPortfolio(experiment, i) }, - "more-hpc" to { experiment, i -> MoreHpcPortfolio(experiment, i) }, - ignoreCase = true - ) - .multiple(required = true) - - /** - * The maximum number of worker threads to use. - */ - private val parallelism by option("--parallelism", help = "maximum number of concurrent simulation runs") - .int() - .default(Runtime.getRuntime().availableProcessors()) - - /** - * The buffer size for writing results. - */ - private val bufferSize by option("--buffer-size") - .int() - .default(4096) - - /** - * The path to the output directory. - */ - private val output by option("-O", "--output", help = "path to the output directory") - .file(canBeFile = false) - .defaultLazy { File("data") } - - override fun run() { - logger.info { "Constructing performance interference model" } - - val performanceInterferenceModel = - performanceInterferenceStream?.let { Sc20PerformanceInterferenceReader(it) } - - logger.info { "Creating experiment descriptor" } - val descriptor = object : - Experiment(environmentPath, tracePath, output, performanceInterferenceModel, vmPlacements, bufferSize) { - private val descriptor = this - override val children: Sequence = sequence { - for ((i, producer) in portfolios.withIndex()) { - yield(producer(descriptor, i)) - } - } - } - - logger.info { "Starting experiment runner [parallelism=$parallelism]" } - val scheduler = ThreadPoolExperimentScheduler(parallelism) - val runner = DefaultExperimentRunner(scheduler) - val reporter = ConsoleExperimentReporter() - try { - runner.execute(descriptor, reporter) - } finally { - scheduler.close() - reporter.close() - } - } -} - -/** - * Main entry point of the experiment. - */ -public fun main(args: Array): Unit = ExperimentCli().main(args) diff --git a/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/experiment/CompositeWorkloadPortfolio.kt b/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/experiment/CompositeWorkloadPortfolio.kt new file mode 100644 index 00000000..f4242456 --- /dev/null +++ b/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/experiment/CompositeWorkloadPortfolio.kt @@ -0,0 +1,79 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.experiments.sc20.experiment + +import org.opendc.experiments.sc20.experiment.model.CompositeWorkload +import org.opendc.experiments.sc20.experiment.model.OperationalPhenomena +import org.opendc.experiments.sc20.experiment.model.Topology +import org.opendc.experiments.sc20.experiment.model.Workload +import org.opendc.harness.dsl.anyOf + +/** + * A [Portfolio] that explores the effect of a composite workload. + */ +public class CompositeWorkloadPortfolio : Portfolio("composite-workload") { + private val totalSampleLoad = 1.3301733005049648E12 + + override val topology: Topology by anyOf( + Topology("base"), + Topology("exp-vol-hor-hom"), + Topology("exp-vol-ver-hom"), + Topology("exp-vel-ver-hom") + ) + + override val workload: Workload by anyOf( + CompositeWorkload( + "all-azure", + listOf(Workload("solvinity-short", 0.0), Workload("azure", 1.0)), + totalSampleLoad + ), + CompositeWorkload( + "solvinity-25-azure-75", + listOf(Workload("solvinity-short", 0.25), Workload("azure", 0.75)), + totalSampleLoad + ), + CompositeWorkload( + "solvinity-50-azure-50", + listOf(Workload("solvinity-short", 0.5), Workload("azure", 0.5)), + totalSampleLoad + ), + CompositeWorkload( + "solvinity-75-azure-25", + listOf(Workload("solvinity-short", 0.75), Workload("azure", 0.25)), + totalSampleLoad + ), + CompositeWorkload( + "all-solvinity", + listOf(Workload("solvinity-short", 1.0), Workload("azure", 0.0)), + totalSampleLoad + ) + ) + + override val operationalPhenomena: OperationalPhenomena by anyOf( + OperationalPhenomena(failureFrequency = 24.0 * 7, hasInterference = false) + ) + + override val allocationPolicy: String by anyOf( + "active-servers" + ) +} diff --git a/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/experiment/Experiment.kt b/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/experiment/Experiment.kt deleted file mode 100644 index 34d7301b..00000000 --- a/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/experiment/Experiment.kt +++ /dev/null @@ -1,76 +0,0 @@ -/* - * Copyright (c) 2020 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.experiments.sc20.experiment - -import org.opendc.experiments.sc20.runner.ContainerExperimentDescriptor -import org.opendc.experiments.sc20.runner.ExperimentDescriptor -import org.opendc.experiments.sc20.runner.execution.ExperimentExecutionContext -import org.opendc.experiments.sc20.runner.execution.ExperimentExecutionListener -import org.opendc.experiments.sc20.telemetry.RunEvent -import org.opendc.experiments.sc20.telemetry.parquet.ParquetRunEventWriter -import org.opendc.format.trace.PerformanceInterferenceModelReader -import java.io.File - -/** - * The global configuration of the experiment. - * - * @param environments The path to the topologies directory. - * @param traces The path to the traces directory. - * @param output The output directory. - * @param performanceInterferenceModel The optional performance interference model that has been specified. - * @param vmPlacements Original VM placement in the trace. - * @param bufferSize The buffer size of the event reporters. - */ -public abstract class Experiment( - public val environments: File, - public val traces: File, - public val output: File, - public val performanceInterferenceModel: PerformanceInterferenceModelReader?, - public val vmPlacements: Map, - public val bufferSize: Int -) : ContainerExperimentDescriptor() { - override val parent: ExperimentDescriptor? = null - - override suspend fun invoke(context: ExperimentExecutionContext) { - val writer = ParquetRunEventWriter(File(output, "experiments.parquet"), bufferSize) - try { - val listener = object : ExperimentExecutionListener by context.listener { - override fun descriptorRegistered(descriptor: ExperimentDescriptor) { - if (descriptor is Run) { - writer.write(RunEvent(descriptor, System.currentTimeMillis())) - } - - context.listener.descriptorRegistered(descriptor) - } - } - - val newContext = object : ExperimentExecutionContext by context { - override val listener: ExperimentExecutionListener = listener - } - - super.invoke(newContext) - } finally { - writer.close() - } - } -} diff --git a/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/experiment/HorVerPortfolio.kt b/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/experiment/HorVerPortfolio.kt new file mode 100644 index 00000000..aa97b808 --- /dev/null +++ b/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/experiment/HorVerPortfolio.kt @@ -0,0 +1,60 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.experiments.sc20.experiment + +import org.opendc.experiments.sc20.experiment.model.OperationalPhenomena +import org.opendc.experiments.sc20.experiment.model.Topology +import org.opendc.experiments.sc20.experiment.model.Workload +import org.opendc.harness.dsl.anyOf + +/** + * A [Portfolio] that explores the difference between horizontal and vertical scaling. + */ +public class HorVerPortfolio : Portfolio("horizontal_vs_vertical") { + override val topology: Topology by anyOf( + Topology("base"), + Topology("rep-vol-hor-hom"), + Topology("rep-vol-hor-het"), + Topology("rep-vol-ver-hom"), + Topology("rep-vol-ver-het"), + Topology("exp-vol-hor-hom"), + Topology("exp-vol-hor-het"), + Topology("exp-vol-ver-hom"), + Topology("exp-vol-ver-het") + ) + + override val workload: Workload by anyOf( + Workload("solvinity", 0.1), + Workload("solvinity", 0.25), + Workload("solvinity", 0.5), + Workload("solvinity", 1.0) + ) + + override val operationalPhenomena: OperationalPhenomena by anyOf( + OperationalPhenomena(failureFrequency = 24.0 * 7, hasInterference = true) + ) + + override val allocationPolicy: String by anyOf( + "active-servers" + ) +} diff --git a/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/experiment/MoreHpcPortfolio.kt b/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/experiment/MoreHpcPortfolio.kt new file mode 100644 index 00000000..bdb33f59 --- /dev/null +++ b/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/experiment/MoreHpcPortfolio.kt @@ -0,0 +1,56 @@ +/* + * Copyright (c) 2020 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.experiments.sc20.experiment + +import org.opendc.experiments.sc20.experiment.model.* +import org.opendc.harness.dsl.anyOf + +/** + * A [Portfolio] to explore the effect of HPC workloads. + */ +public class MoreHpcPortfolio : Portfolio("more_hpc") { + override val topology: Topology by anyOf( + Topology("base"), + Topology("exp-vol-hor-hom"), + Topology("exp-vol-ver-hom"), + Topology("exp-vel-ver-hom") + ) + + override val workload: Workload by anyOf( + Workload("solvinity", 0.0, samplingStrategy = SamplingStrategy.HPC), + Workload("solvinity", 0.25, samplingStrategy = SamplingStrategy.HPC), + Workload("solvinity", 0.5, samplingStrategy = SamplingStrategy.HPC), + Workload("solvinity", 1.0, samplingStrategy = SamplingStrategy.HPC), + Workload("solvinity", 0.25, samplingStrategy = SamplingStrategy.HPC_LOAD), + Workload("solvinity", 0.5, samplingStrategy = SamplingStrategy.HPC_LOAD), + Workload("solvinity", 1.0, samplingStrategy = SamplingStrategy.HPC_LOAD) + ) + + override val operationalPhenomena: OperationalPhenomena by anyOf( + OperationalPhenomena(failureFrequency = 24.0 * 7, hasInterference = true) + ) + + override val allocationPolicy: String by anyOf( + "active-servers" + ) +} diff --git a/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/experiment/MoreVelocityPortfolio.kt b/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/experiment/MoreVelocityPortfolio.kt new file mode 100644 index 00000000..733dabf6 --- /dev/null +++ b/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/experiment/MoreVelocityPortfolio.kt @@ -0,0 +1,56 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.experiments.sc20.experiment + +import org.opendc.experiments.sc20.experiment.model.OperationalPhenomena +import org.opendc.experiments.sc20.experiment.model.Topology +import org.opendc.experiments.sc20.experiment.model.Workload +import org.opendc.harness.dsl.anyOf + +/** + * A [Portfolio] that explores the effect of adding more velocity to a cluster (e.g., faster machines). + */ +public class MoreVelocityPortfolio : Portfolio("more_velocity") { + override val topology: Topology by anyOf( + Topology("base"), + Topology("rep-vel-ver-hom"), + Topology("rep-vel-ver-het"), + Topology("exp-vel-ver-hom"), + Topology("exp-vel-ver-het") + ) + + override val workload: Workload by anyOf( + Workload("solvinity", 0.1), + Workload("solvinity", 0.25), + Workload("solvinity", 0.5), + Workload("solvinity", 1.0) + ) + + override val operationalPhenomena: OperationalPhenomena by anyOf( + OperationalPhenomena(failureFrequency = 24.0 * 7, hasInterference = true) + ) + + override val allocationPolicy: String by anyOf( + "active-servers" + ) +} diff --git a/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/experiment/OperationalPhenomenaPortfolio.kt b/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/experiment/OperationalPhenomenaPortfolio.kt new file mode 100644 index 00000000..66b94faf --- /dev/null +++ b/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/experiment/OperationalPhenomenaPortfolio.kt @@ -0,0 +1,61 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.experiments.sc20.experiment + +import org.opendc.experiments.sc20.experiment.model.OperationalPhenomena +import org.opendc.experiments.sc20.experiment.model.Topology +import org.opendc.experiments.sc20.experiment.model.Workload +import org.opendc.harness.dsl.anyOf + +/** + * A [Portfolio] that explores the effect of operational phenomena on metrics. + */ +public class OperationalPhenomenaPortfolio : Portfolio("operational_phenomena") { + override val topology: Topology by anyOf( + Topology("base") + ) + + override val workload: Workload by anyOf( + Workload("solvinity", 0.1), + Workload("solvinity", 0.25), + Workload("solvinity", 0.5), + Workload("solvinity", 1.0) + ) + + override val operationalPhenomena: OperationalPhenomena by anyOf( + OperationalPhenomena(failureFrequency = 24.0 * 7, hasInterference = true), + OperationalPhenomena(failureFrequency = 0.0, hasInterference = true), + OperationalPhenomena(failureFrequency = 24.0 * 7, hasInterference = false), + OperationalPhenomena(failureFrequency = 0.0, hasInterference = false) + ) + + override val allocationPolicy: String by anyOf( + "mem", + "mem-inv", + "core-mem", + "core-mem-inv", + "active-servers", + "active-servers-inv", + "random" + ) +} diff --git a/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/experiment/Portfolio.kt b/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/experiment/Portfolio.kt index 37cf2880..4a82ad56 100644 --- a/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/experiment/Portfolio.kt +++ b/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/experiment/Portfolio.kt @@ -22,67 +22,196 @@ package org.opendc.experiments.sc20.experiment +import kotlinx.coroutines.ExperimentalCoroutinesApi +import kotlinx.coroutines.cancel +import kotlinx.coroutines.channels.Channel +import kotlinx.coroutines.launch +import kotlinx.coroutines.test.TestCoroutineScope +import mu.KotlinLogging +import org.opendc.compute.simulator.allocation.* +import org.opendc.experiments.sc20.experiment.model.CompositeWorkload import org.opendc.experiments.sc20.experiment.model.OperationalPhenomena import org.opendc.experiments.sc20.experiment.model.Topology import org.opendc.experiments.sc20.experiment.model.Workload -import org.opendc.experiments.sc20.runner.ContainerExperimentDescriptor +import org.opendc.experiments.sc20.experiment.monitor.ParquetExperimentMonitor +import org.opendc.experiments.sc20.trace.Sc20ParquetTraceReader +import org.opendc.experiments.sc20.trace.Sc20RawParquetTraceReader +import org.opendc.format.environment.sc20.Sc20ClusterEnvironmentReader +import org.opendc.format.trace.PerformanceInterferenceModelReader +import org.opendc.harness.dsl.Experiment +import org.opendc.harness.dsl.anyOf +import org.opendc.simulator.utils.DelayControllerClockAdapter +import org.opendc.trace.core.EventTracer +import java.io.File +import java.util.concurrent.ConcurrentHashMap +import kotlin.random.Random /** - * A portfolio represents a collection of scenarios are tested. + * A portfolio represents a collection of scenarios are tested for the work. + * + * @param name The name of the portfolio. */ -public abstract class Portfolio( - override val parent: Experiment, - public val id: Int, - public val name: String -) : ContainerExperimentDescriptor() { +public abstract class Portfolio(name: String) : Experiment(name) { + /** + * The logger for this portfolio instance. + */ + private val logger = KotlinLogging.logger {} + + /** + * The path to where the environments are located. + */ + private val environmentPath by anyOf(File("environments/")) + + /** + * The path to where the traces are located. + */ + private val tracePath by anyOf(File("traces/")) + /** - * The topologies to consider. + * The path to where the output results should be written. */ - protected abstract val topologies: List + private val outputPath by anyOf(File("results/")) /** - * The workloads to consider. + * The path to the original VM placements file. */ - protected abstract val workloads: List + private val vmPlacements by anyOf(emptyMap()) + + /** + * The path to the performance interference model. + */ + private val performanceInterferenceModel by anyOf(null) + + /** + * The topology to test. + */ + public abstract val topology: Topology + + /** + * The workload to test. + */ + public abstract val workload: Workload /** * The operational phenomenas to consider. */ - protected abstract val operationalPhenomenas: List + public abstract val operationalPhenomena: OperationalPhenomena /** * The allocation policies to consider. */ - protected abstract val allocationPolicies: List + public abstract val allocationPolicy: String /** - * The number of repetitions to perform. + * A map of trace readers. */ - public open val repetitions: Int = 32 + private val traceReaders = ConcurrentHashMap() /** - * Resolve the children of this container. + * Perform a single trial for this portfolio. */ - override val children: Sequence = sequence { - var id = 0 - for (topology in topologies) { - for (workload in workloads) { - for (operationalPhenomena in operationalPhenomenas) { - for (allocationPolicy in allocationPolicies) { - yield( - Scenario( - this@Portfolio, - id++, - repetitions, - topology, - workload, - allocationPolicy, - operationalPhenomena - ) - ) - } - } + @OptIn(ExperimentalCoroutinesApi::class) + override fun doRun(repeat: Int) { + val testScope = TestCoroutineScope() + val clock = DelayControllerClockAdapter(testScope) + val tracer = EventTracer(clock) + val seeder = Random(repeat) + val environment = Sc20ClusterEnvironmentReader(File(environmentPath, "${topology.name}.txt")) + + val chan = Channel(Channel.CONFLATED) + val allocationPolicy = createAllocationPolicy(seeder) + + val workload = workload + val workloadNames = if (workload is CompositeWorkload) { + workload.workloads.map { it.name } + } else { + listOf(workload.name) + } + + val rawReaders = workloadNames.map { workloadName -> + traceReaders.computeIfAbsent(workloadName) { + logger.info { "Loading trace $workloadName" } + Sc20RawParquetTraceReader(File(tracePath, workloadName)) + } + } + + val performanceInterferenceModel = performanceInterferenceModel + ?.takeIf { operationalPhenomena.hasInterference } + ?.construct(seeder) ?: emptyMap() + val trace = Sc20ParquetTraceReader(rawReaders, performanceInterferenceModel, workload, seeder.nextInt()) + + val monitor = ParquetExperimentMonitor( + outputPath, + "portfolio_id=$name/scenario_id=$id/run_id=$repeat", + 4096 + ) + + testScope.launch { + val (bareMetalProvisioner, scheduler) = createProvisioner( + this, + clock, + environment, + allocationPolicy, + tracer + ) + + val failureDomain = if (operationalPhenomena.failureFrequency > 0) { + logger.debug("ENABLING failures") + createFailureDomain( + this, + clock, + seeder.nextInt(), + operationalPhenomena.failureFrequency, + bareMetalProvisioner, + chan + ) + } else { + null } + + attachMonitor(this, clock, scheduler, monitor) + processTrace( + this, + clock, + trace, + scheduler, + chan, + monitor + ) + + logger.debug("SUBMIT=${scheduler.submittedVms}") + logger.debug("FAIL=${scheduler.unscheduledVms}") + logger.debug("QUEUED=${scheduler.queuedVms}") + logger.debug("RUNNING=${scheduler.runningVms}") + logger.debug("FINISHED=${scheduler.finishedVms}") + + failureDomain?.cancel() + scheduler.terminate() + } + + try { + testScope.advanceUntilIdle() + } finally { + monitor.close() + } + } + + /** + * Create the [AllocationPolicy] instance to use for the trial. + */ + private fun createAllocationPolicy(seeder: Random): AllocationPolicy { + return when (allocationPolicy) { + "mem" -> AvailableMemoryAllocationPolicy() + "mem-inv" -> AvailableMemoryAllocationPolicy(true) + "core-mem" -> AvailableCoreMemoryAllocationPolicy() + "core-mem-inv" -> AvailableCoreMemoryAllocationPolicy(true) + "active-servers" -> NumberOfActiveServersAllocationPolicy() + "active-servers-inv" -> NumberOfActiveServersAllocationPolicy(true) + "provisioned-cores" -> ProvisionedCoresAllocationPolicy() + "provisioned-cores-inv" -> ProvisionedCoresAllocationPolicy(true) + "random" -> RandomAllocationPolicy(Random(seeder.nextInt())) + "replay" -> ReplayAllocationPolicy(vmPlacements) + else -> throw IllegalArgumentException("Unknown policy $allocationPolicy") } } } diff --git a/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/experiment/Portfolios.kt b/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/experiment/Portfolios.kt deleted file mode 100644 index 249a63b9..00000000 --- a/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/experiment/Portfolios.kt +++ /dev/null @@ -1,221 +0,0 @@ -/* - * Copyright (c) 2020 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.experiments.sc20.experiment - -import org.opendc.experiments.sc20.experiment.model.* - -public class HorVerPortfolio(parent: Experiment, id: Int) : Portfolio(parent, id, "horizontal_vs_vertical") { - override val topologies: List = listOf( - Topology("base"), - Topology("rep-vol-hor-hom"), - Topology("rep-vol-hor-het"), - Topology("rep-vol-ver-hom"), - Topology("rep-vol-ver-het"), - Topology("exp-vol-hor-hom"), - Topology("exp-vol-hor-het"), - Topology("exp-vol-ver-hom"), - Topology("exp-vol-ver-het") - ) - - override val workloads: List = listOf( - Workload("solvinity", 0.1), - Workload("solvinity", 0.25), - Workload("solvinity", 0.5), - Workload("solvinity", 1.0) - ) - - override val operationalPhenomenas: List = listOf( - OperationalPhenomena(failureFrequency = 24.0 * 7, hasInterference = true) - ) - - override val allocationPolicies: List = listOf( - "active-servers" - ) -} - -public class MoreVelocityPortfolio(parent: Experiment, id: Int) : Portfolio(parent, id, "more_velocity") { - override val topologies: List = listOf( - Topology("base"), - Topology("rep-vel-ver-hom"), - Topology("rep-vel-ver-het"), - Topology("exp-vel-ver-hom"), - Topology("exp-vel-ver-het") - ) - - override val workloads: List = listOf( - Workload("solvinity", 0.1), - Workload("solvinity", 0.25), - Workload("solvinity", 0.5), - Workload("solvinity", 1.0) - ) - - override val operationalPhenomenas: List = listOf( - OperationalPhenomena(failureFrequency = 24.0 * 7, hasInterference = true) - ) - - override val allocationPolicies: List = listOf( - "active-servers" - ) -} - -public class CompositeWorkloadPortfolio(parent: Experiment, id: Int) : Portfolio(parent, id, "composite-workload") { - private val totalSampleLoad = 1.3301733005049648E12 - - override val topologies: List = listOf( - Topology("base"), - Topology("exp-vol-hor-hom"), - Topology("exp-vol-ver-hom"), - Topology("exp-vel-ver-hom") - ) - - override val workloads: List = listOf( - CompositeWorkload( - "all-azure", - listOf(Workload("solvinity-short", 0.0), Workload("azure", 1.0)), - totalSampleLoad - ), - CompositeWorkload( - "solvinity-25-azure-75", - listOf(Workload("solvinity-short", 0.25), Workload("azure", 0.75)), - totalSampleLoad - ), - CompositeWorkload( - "solvinity-50-azure-50", - listOf(Workload("solvinity-short", 0.5), Workload("azure", 0.5)), - totalSampleLoad - ), - CompositeWorkload( - "solvinity-75-azure-25", - listOf(Workload("solvinity-short", 0.75), Workload("azure", 0.25)), - totalSampleLoad - ), - CompositeWorkload( - "all-solvinity", - listOf(Workload("solvinity-short", 1.0), Workload("azure", 0.0)), - totalSampleLoad - ) - ) - - override val operationalPhenomenas: List = listOf( - OperationalPhenomena(failureFrequency = 24.0 * 7, hasInterference = false) - ) - - override val allocationPolicies: List = listOf( - "active-servers" - ) -} - -public class OperationalPhenomenaPortfolio(parent: Experiment, id: Int) : - Portfolio(parent, id, "operational_phenomena") { - override val topologies: List = listOf( - Topology("base") - ) - - override val workloads: List = listOf( - Workload("solvinity", 0.1), - Workload("solvinity", 0.25), - Workload("solvinity", 0.5), - Workload("solvinity", 1.0) - ) - - override val operationalPhenomenas: List = listOf( - OperationalPhenomena(failureFrequency = 24.0 * 7, hasInterference = true), - OperationalPhenomena(failureFrequency = 0.0, hasInterference = true), - OperationalPhenomena(failureFrequency = 24.0 * 7, hasInterference = false), - OperationalPhenomena(failureFrequency = 0.0, hasInterference = false) - ) - - override val allocationPolicies: List = listOf( - "mem", - "mem-inv", - "core-mem", - "core-mem-inv", - "active-servers", - "active-servers-inv", - "random" - ) -} - -public class ReplayPortfolio(parent: Experiment, id: Int) : Portfolio(parent, id, "replay") { - override val topologies: List = listOf( - Topology("base") - ) - - override val workloads: List = listOf( - Workload("solvinity", 1.0) - ) - - override val operationalPhenomenas: List = listOf( - OperationalPhenomena(failureFrequency = 0.0, hasInterference = false) - ) - - override val allocationPolicies: List = listOf( - "replay", - "active-servers" - ) -} - -public class TestPortfolio(parent: Experiment, id: Int) : Portfolio(parent, id, "test") { - override val repetitions: Int = 1 - - override val topologies: List = listOf( - Topology("base") - ) - - override val workloads: List = listOf( - Workload("solvinity", 1.0) - ) - - override val operationalPhenomenas: List = listOf( - OperationalPhenomena(failureFrequency = 24.0 * 7, hasInterference = true) - ) - - override val allocationPolicies: List = listOf("active-servers") -} - -public class MoreHpcPortfolio(parent: Experiment, id: Int) : Portfolio(parent, id, "more_hpc") { - override val topologies: List = listOf( - Topology("base"), - Topology("exp-vol-hor-hom"), - Topology("exp-vol-ver-hom"), - Topology("exp-vel-ver-hom") - ) - - override val workloads: List = listOf( - Workload("solvinity", 0.0, samplingStrategy = SamplingStrategy.HPC), - Workload("solvinity", 0.25, samplingStrategy = SamplingStrategy.HPC), - Workload("solvinity", 0.5, samplingStrategy = SamplingStrategy.HPC), - Workload("solvinity", 1.0, samplingStrategy = SamplingStrategy.HPC), - Workload("solvinity", 0.25, samplingStrategy = SamplingStrategy.HPC_LOAD), - Workload("solvinity", 0.5, samplingStrategy = SamplingStrategy.HPC_LOAD), - Workload("solvinity", 1.0, samplingStrategy = SamplingStrategy.HPC_LOAD) - ) - - override val operationalPhenomenas: List = listOf( - OperationalPhenomena(failureFrequency = 24.0 * 7, hasInterference = true) - ) - - override val allocationPolicies: List = listOf( - "active-servers" - ) -} diff --git a/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/experiment/ReplayPortfolio.kt b/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/experiment/ReplayPortfolio.kt new file mode 100644 index 00000000..8a42a3b4 --- /dev/null +++ b/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/experiment/ReplayPortfolio.kt @@ -0,0 +1,50 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.experiments.sc20.experiment + +import org.opendc.experiments.sc20.experiment.model.OperationalPhenomena +import org.opendc.experiments.sc20.experiment.model.Topology +import org.opendc.experiments.sc20.experiment.model.Workload +import org.opendc.harness.dsl.anyOf + +/** + * A [Portfolio] that compares the original VM placements against our policies. + */ +public class ReplayPortfolio : Portfolio("replay") { + override val topology: Topology by anyOf( + Topology("base") + ) + + override val workload: Workload by anyOf( + Workload("solvinity", 1.0) + ) + + override val operationalPhenomena: OperationalPhenomena by anyOf( + OperationalPhenomena(failureFrequency = 0.0, hasInterference = false) + ) + + override val allocationPolicy: String by anyOf( + "replay", + "active-servers" + ) +} diff --git a/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/experiment/Run.kt b/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/experiment/Run.kt deleted file mode 100644 index 8d8d608d..00000000 --- a/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/experiment/Run.kt +++ /dev/null @@ -1,157 +0,0 @@ -/* - * Copyright (c) 2020 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.experiments.sc20.experiment - -import kotlinx.coroutines.ExperimentalCoroutinesApi -import kotlinx.coroutines.cancel -import kotlinx.coroutines.channels.Channel -import kotlinx.coroutines.launch -import kotlinx.coroutines.test.TestCoroutineScope -import mu.KotlinLogging -import org.opendc.compute.simulator.allocation.* -import org.opendc.experiments.sc20.experiment.model.CompositeWorkload -import org.opendc.experiments.sc20.experiment.monitor.ParquetExperimentMonitor -import org.opendc.experiments.sc20.runner.TrialExperimentDescriptor -import org.opendc.experiments.sc20.runner.execution.ExperimentExecutionContext -import org.opendc.experiments.sc20.trace.Sc20ParquetTraceReader -import org.opendc.experiments.sc20.trace.Sc20RawParquetTraceReader -import org.opendc.format.environment.sc20.Sc20ClusterEnvironmentReader -import org.opendc.simulator.utils.DelayControllerClockAdapter -import org.opendc.trace.core.EventTracer -import java.io.File -import kotlin.random.Random - -/** - * The logger for the experiment scenario. - */ -private val logger = KotlinLogging.logger {} - -/** - * An experiment run represent a single invocation of a trial and is used to distinguish between repetitions of the - * same set of parameters. - */ -@OptIn(ExperimentalCoroutinesApi::class) -public data class Run(override val parent: Scenario, val id: Int, val seed: Int) : TrialExperimentDescriptor() { - override suspend fun invoke(context: ExperimentExecutionContext) { - val experiment = parent.parent.parent - val testScope = TestCoroutineScope() - val clock = DelayControllerClockAdapter(testScope) - val seeder = Random(seed) - val environment = Sc20ClusterEnvironmentReader(File(experiment.environments, "${parent.topology.name}.txt")) - - val chan = Channel(Channel.CONFLATED) - val allocationPolicy = when (parent.allocationPolicy) { - "mem" -> AvailableMemoryAllocationPolicy() - "mem-inv" -> AvailableMemoryAllocationPolicy(true) - "core-mem" -> AvailableCoreMemoryAllocationPolicy() - "core-mem-inv" -> AvailableCoreMemoryAllocationPolicy(true) - "active-servers" -> NumberOfActiveServersAllocationPolicy() - "active-servers-inv" -> NumberOfActiveServersAllocationPolicy(true) - "provisioned-cores" -> ProvisionedCoresAllocationPolicy() - "provisioned-cores-inv" -> ProvisionedCoresAllocationPolicy(true) - "random" -> RandomAllocationPolicy(Random(seeder.nextInt())) - "replay" -> ReplayAllocationPolicy(experiment.vmPlacements) - else -> throw IllegalArgumentException("Unknown policy ${parent.allocationPolicy}") - } - - @Suppress("UNCHECKED_CAST") - val rawTraceReaders = - context.cache.computeIfAbsent("raw-trace-readers") { mutableMapOf() } as MutableMap - val rawReaders = synchronized(rawTraceReaders) { - val workloadNames = if (parent.workload is CompositeWorkload) { - parent.workload.workloads.map { it.name } - } else { - listOf(parent.workload.name) - } - - workloadNames.map { workloadName -> - rawTraceReaders.computeIfAbsent(workloadName) { - logger.info { "Loading trace $workloadName" } - Sc20RawParquetTraceReader(File(experiment.traces, workloadName)) - } - } - } - - val performanceInterferenceModel = experiment.performanceInterferenceModel - ?.takeIf { parent.operationalPhenomena.hasInterference } - ?.construct(seeder) ?: emptyMap() - val trace = Sc20ParquetTraceReader(rawReaders, performanceInterferenceModel, parent.workload, seed) - - val monitor = ParquetExperimentMonitor( - parent.parent.parent.output, - "portfolio_id=${parent.parent.id}/scenario_id=${parent.id}/run_id=$id", - parent.parent.parent.bufferSize - ) - - val tracer = EventTracer(clock) - - testScope.launch { - val (bareMetalProvisioner, scheduler) = createProvisioner( - this, - clock, - environment, - allocationPolicy, - tracer - ) - - val failureDomain = if (parent.operationalPhenomena.failureFrequency > 0) { - logger.debug("ENABLING failures") - createFailureDomain( - this, - clock, - seeder.nextInt(), - parent.operationalPhenomena.failureFrequency, - bareMetalProvisioner, - chan - ) - } else { - null - } - - attachMonitor(this, clock, scheduler, monitor) - processTrace( - this, - clock, - trace, - scheduler, - chan, - monitor - ) - - logger.debug("SUBMIT=${scheduler.submittedVms}") - logger.debug("FAIL=${scheduler.unscheduledVms}") - logger.debug("QUEUED=${scheduler.queuedVms}") - logger.debug("RUNNING=${scheduler.runningVms}") - logger.debug("FINISHED=${scheduler.finishedVms}") - - failureDomain?.cancel() - scheduler.terminate() - } - - try { - testScope.advanceUntilIdle() - } finally { - monitor.close() - } - } -} diff --git a/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/experiment/Scenario.kt b/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/experiment/Scenario.kt deleted file mode 100644 index d092ddd5..00000000 --- a/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/experiment/Scenario.kt +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Copyright (c) 2020 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.experiments.sc20.experiment - -import org.opendc.experiments.sc20.experiment.model.OperationalPhenomena -import org.opendc.experiments.sc20.experiment.model.Topology -import org.opendc.experiments.sc20.experiment.model.Workload -import org.opendc.experiments.sc20.runner.ContainerExperimentDescriptor -import org.opendc.experiments.sc20.runner.ExperimentDescriptor - -/** - * A scenario represents a single point in the design space (a unique combination of parameters). - */ -public class Scenario( - override val parent: Portfolio, - public val id: Int, - public val repetitions: Int, - public val topology: Topology, - public val workload: Workload, - public val allocationPolicy: String, - public val operationalPhenomena: OperationalPhenomena -) : ContainerExperimentDescriptor() { - override val children: Sequence = sequence { - repeat(repetitions) { i -> yield(Run(this@Scenario, i, i)) } - } -} diff --git a/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/experiment/TestPortfolio.kt b/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/experiment/TestPortfolio.kt new file mode 100644 index 00000000..2210fc97 --- /dev/null +++ b/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/experiment/TestPortfolio.kt @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.experiments.sc20.experiment + +import org.opendc.experiments.sc20.experiment.model.OperationalPhenomena +import org.opendc.experiments.sc20.experiment.model.Topology +import org.opendc.experiments.sc20.experiment.model.Workload +import org.opendc.harness.dsl.anyOf + +/** + * A [Portfolio] to perform a simple test run. + */ +public class TestPortfolio : Portfolio("test") { + override val topology: Topology by anyOf( + Topology("base") + ) + + override val workload: Workload by anyOf( + Workload("solvinity", 1.0) + ) + + override val operationalPhenomena: OperationalPhenomena by anyOf( + OperationalPhenomena(failureFrequency = 24.0 * 7, hasInterference = true) + ) + + override val allocationPolicy: String by anyOf("active-servers") +} diff --git a/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/reporter/ConsoleExperimentReporter.kt b/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/reporter/ConsoleExperimentReporter.kt deleted file mode 100644 index af61622a..00000000 --- a/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/reporter/ConsoleExperimentReporter.kt +++ /dev/null @@ -1,87 +0,0 @@ -/* - * Copyright (c) 2020 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.experiments.sc20.reporter - -import me.tongfei.progressbar.ProgressBar -import me.tongfei.progressbar.ProgressBarBuilder -import mu.KotlinLogging -import org.opendc.experiments.sc20.experiment.Run -import org.opendc.experiments.sc20.runner.ExperimentDescriptor -import org.opendc.experiments.sc20.runner.execution.ExperimentExecutionListener -import org.opendc.experiments.sc20.runner.execution.ExperimentExecutionResult - -/** - * A reporter that reports the experiment progress to the console. - */ -public class ConsoleExperimentReporter : ExperimentExecutionListener, AutoCloseable { - /** - * The active [Run]s. - */ - private val runs: MutableSet = mutableSetOf() - - /** - * The total number of runs. - */ - private var total = 0 - - /** - * The logger for this reporter. - */ - private val logger = KotlinLogging.logger {} - - /** - * The progress bar to keep track of the progress. - */ - private val pb: ProgressBar = ProgressBarBuilder() - .setTaskName("") - .setInitialMax(1) - .build() - - override fun descriptorRegistered(descriptor: ExperimentDescriptor) { - if (descriptor is Run) { - runs += descriptor - pb.maxHint((++total).toLong()) - } - } - - override fun executionFinished(descriptor: ExperimentDescriptor, result: ExperimentExecutionResult) { - if (descriptor is Run) { - runs -= descriptor - - pb.stepTo(total - runs.size.toLong()) - if (runs.isEmpty()) { - pb.close() - } - } - - if (result is ExperimentExecutionResult.Failed) { - logger.warn(result.throwable) { "Descriptor $descriptor failed" } - } - } - - override fun executionStarted(descriptor: ExperimentDescriptor) {} - - override fun close() { - pb.close() - } -} diff --git a/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/runner/ContainerExperimentDescriptor.kt b/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/runner/ContainerExperimentDescriptor.kt deleted file mode 100644 index d70e8c9a..00000000 --- a/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/runner/ContainerExperimentDescriptor.kt +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Copyright (c) 2020 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.experiments.sc20.runner - -import kotlinx.coroutines.launch -import kotlinx.coroutines.supervisorScope -import org.opendc.experiments.sc20.runner.execution.ExperimentExecutionContext -import org.opendc.experiments.sc20.runner.execution.ExperimentExecutionResult - -/** - * An abstract [ExperimentDescriptor] specifically for containers. - */ -public abstract class ContainerExperimentDescriptor : ExperimentDescriptor() { - /** - * The child descriptors of this container. - */ - public abstract val children: Sequence - - override val type: Type = Type.CONTAINER - - override suspend fun invoke(context: ExperimentExecutionContext) { - val materializedChildren = children.toList() - for (child in materializedChildren) { - context.listener.descriptorRegistered(child) - } - - supervisorScope { - for (child in materializedChildren) { - if (child.isTrial) { - launch { - val worker = context.scheduler.allocate() - context.listener.executionStarted(child) - try { - worker(child, context) - context.listener.executionFinished(child, ExperimentExecutionResult.Success) - } catch (e: Throwable) { - context.listener.executionFinished(child, ExperimentExecutionResult.Failed(e)) - } - } - } else { - launch { child(context) } - } - } - } - } -} diff --git a/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/runner/ExperimentDescriptor.kt b/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/runner/ExperimentDescriptor.kt deleted file mode 100644 index 1e67c086..00000000 --- a/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/runner/ExperimentDescriptor.kt +++ /dev/null @@ -1,79 +0,0 @@ -/* - * Copyright (c) 2020 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.experiments.sc20.runner - -import org.opendc.experiments.sc20.runner.execution.ExperimentExecutionContext -import java.io.Serializable - -/** - * An immutable description of an experiment in the **odcsim* simulation framework, which may be a single atomic trial - * or a composition of multiple trials. - * - * This class represents a dynamic tree-like structure where the children of the nodes are not known at instantiation - * since they might be generated dynamically. - */ -public abstract class ExperimentDescriptor : Serializable { - /** - * The parent of this descriptor, or `null` if it has no parent. - */ - public abstract val parent: ExperimentDescriptor? - - /** - * The type of descriptor. - */ - public abstract val type: Type - - /** - * A flag to indicate that this descriptor is a root descriptor. - */ - public open val isRoot: Boolean - get() = parent == null - - /** - * A flag to indicate that this descriptor describes an experiment trial. - */ - public val isTrial: Boolean - get() = type == Type.TRIAL - - /** - * Execute this [ExperimentDescriptor]. - * - * @param context The context to execute the descriptor in. - */ - public abstract suspend operator fun invoke(context: ExperimentExecutionContext) - - /** - * The types of experiment descriptors. - */ - public enum class Type { - /** - * A composition of multiple experiment descriptions whose invocation happens on a single thread. - */ - CONTAINER, - - /** - * An invocation of a single scenario of an experiment whose invocation may happen on different threads. - */ - TRIAL - } -} diff --git a/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/runner/ExperimentRunner.kt b/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/runner/ExperimentRunner.kt deleted file mode 100644 index a59481c0..00000000 --- a/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/runner/ExperimentRunner.kt +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Copyright (c) 2020 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.experiments.sc20.runner - -import org.opendc.experiments.sc20.runner.execution.ExperimentExecutionListener - -/** - * An [ExperimentRunner] facilitates discovery and execution of experiments. - */ -public interface ExperimentRunner { - /** - * The unique identifier of this runner. - */ - public val id: String - - /** - * The version of this runner. - */ - public val version: String? - get() = null - - /** - * Execute the specified experiment represented as [ExperimentDescriptor]. - * - * @param root The experiment to execute. - * @param listener The listener to report events to. - */ - public fun execute(root: ExperimentDescriptor, listener: ExperimentExecutionListener) -} diff --git a/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/runner/TrialExperimentDescriptor.kt b/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/runner/TrialExperimentDescriptor.kt deleted file mode 100644 index abc52997..00000000 --- a/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/runner/TrialExperimentDescriptor.kt +++ /dev/null @@ -1,32 +0,0 @@ -/* - * MIT License - * - * Copyright (c) 2020 atlarge-research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.experiments.sc20.runner - -/** - * An abstract [ExperimentDescriptor] specifically for trials. - */ -public abstract class TrialExperimentDescriptor : ExperimentDescriptor() { - override val type: Type = Type.TRIAL -} diff --git a/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/runner/execution/ExperimentExecutionContext.kt b/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/runner/execution/ExperimentExecutionContext.kt deleted file mode 100644 index 942eb891..00000000 --- a/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/runner/execution/ExperimentExecutionContext.kt +++ /dev/null @@ -1,45 +0,0 @@ -/* - * MIT License - * - * Copyright (c) 2020 atlarge-research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.experiments.sc20.runner.execution - -/** - * The execution context of an experiment. - */ -public interface ExperimentExecutionContext { - /** - * The execution listener to use. - */ - public val listener: ExperimentExecutionListener - - /** - * The experiment scheduler to use. - */ - public val scheduler: ExperimentScheduler - - /** - * A cache for objects within a single runner. - */ - public val cache: MutableMap -} diff --git a/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/runner/execution/ExperimentExecutionListener.kt b/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/runner/execution/ExperimentExecutionListener.kt deleted file mode 100644 index 42fef164..00000000 --- a/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/runner/execution/ExperimentExecutionListener.kt +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Copyright (c) 2020 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.experiments.sc20.runner.execution - -import org.opendc.experiments.sc20.runner.ExperimentDescriptor - -/** - * Listener to be notified of experiment execution events by experiment runners. - */ -public interface ExperimentExecutionListener { - /** - * A method that is invoked when a new [ExperimentDescriptor] is registered. - */ - public fun descriptorRegistered(descriptor: ExperimentDescriptor) - - /** - * A method that is invoked when when the execution of a leaf or subtree of the experiment tree has finished, - * regardless of the outcome. - */ - public fun executionFinished(descriptor: ExperimentDescriptor, result: ExperimentExecutionResult) - - /** - * A method that is invoked when the execution of a leaf or subtree of the experiment tree is about to be started. - */ - public fun executionStarted(descriptor: ExperimentDescriptor) -} diff --git a/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/runner/execution/ExperimentExecutionResult.kt b/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/runner/execution/ExperimentExecutionResult.kt deleted file mode 100644 index a765c264..00000000 --- a/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/runner/execution/ExperimentExecutionResult.kt +++ /dev/null @@ -1,42 +0,0 @@ -/* - * MIT License - * - * Copyright (c) 2020 atlarge-research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.experiments.sc20.runner.execution - -import java.io.Serializable - -/** - * The result of executing an experiment. - */ -public sealed class ExperimentExecutionResult : Serializable { - /** - * The experiment executed successfully - */ - public object Success : ExperimentExecutionResult() - - /** - * The experiment failed during execution. - */ - public data class Failed(val throwable: Throwable) : ExperimentExecutionResult() -} diff --git a/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/runner/execution/ExperimentScheduler.kt b/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/runner/execution/ExperimentScheduler.kt deleted file mode 100644 index 70095ccd..00000000 --- a/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/runner/execution/ExperimentScheduler.kt +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Copyright (c) 2020 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.experiments.sc20.runner.execution - -import org.opendc.experiments.sc20.runner.ExperimentDescriptor -import java.io.Closeable - -/** - * A interface for scheduling the execution of experiment trials over compute resources (threads/containers/vms) - */ -public interface ExperimentScheduler : Closeable { - /** - * Allocate a [Worker] for executing an experiment trial. This method may suspend in case no resources are directly - * available at the moment. - * - * @return The available worker. - */ - public suspend fun allocate(): ExperimentScheduler.Worker - - /** - * An isolated worker of an [ExperimentScheduler] that is responsible for executing a single experiment trial. - */ - public interface Worker { - /** - * Dispatch the specified [ExperimentDescriptor] to execute some time in the future and return the results of - * the trial. - * - * @param descriptor The descriptor to execute. - * @param context The context to execute the descriptor in. - */ - public suspend operator fun invoke( - descriptor: ExperimentDescriptor, - context: ExperimentExecutionContext - ) - } -} diff --git a/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/runner/execution/ThreadPoolExperimentScheduler.kt b/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/runner/execution/ThreadPoolExperimentScheduler.kt deleted file mode 100644 index 942faed1..00000000 --- a/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/runner/execution/ThreadPoolExperimentScheduler.kt +++ /dev/null @@ -1,83 +0,0 @@ -/* - * Copyright (c) 2020 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.experiments.sc20.runner.execution - -import kotlinx.coroutines.asCoroutineDispatcher -import kotlinx.coroutines.launch -import kotlinx.coroutines.supervisorScope -import kotlinx.coroutines.sync.Semaphore -import kotlinx.coroutines.withContext -import org.opendc.experiments.sc20.runner.ExperimentDescriptor -import java.util.concurrent.Executors - -/** - * An [ExperimentScheduler] that runs experiments using a local thread pool. - * - * @param parallelism The maximum amount of parallel workers (default is the number of available processors). - */ -public class ThreadPoolExperimentScheduler(parallelism: Int = Runtime.getRuntime().availableProcessors() + 1) : ExperimentScheduler { - private val dispatcher = Executors.newCachedThreadPool().asCoroutineDispatcher() - private val tickets = Semaphore(parallelism) - - override suspend fun allocate(): ExperimentScheduler.Worker { - tickets.acquire() - return object : ExperimentScheduler.Worker { - override suspend fun invoke( - descriptor: ExperimentDescriptor, - context: ExperimentExecutionContext - ) = supervisorScope { - val listener = - object : ExperimentExecutionListener { - override fun descriptorRegistered(descriptor: ExperimentDescriptor) { - launch { context.listener.descriptorRegistered(descriptor) } - } - - override fun executionFinished( - descriptor: ExperimentDescriptor, - result: ExperimentExecutionResult - ) { - launch { context.listener.executionFinished(descriptor, result) } - } - - override fun executionStarted(descriptor: ExperimentDescriptor) { - launch { context.listener.executionStarted(descriptor) } - } - } - - val newContext = object : ExperimentExecutionContext by context { - override val listener: ExperimentExecutionListener = listener - } - - try { - withContext(dispatcher) { - descriptor(newContext) - } - } finally { - tickets.release() - } - } - } - } - - override fun close(): Unit = dispatcher.close() -} diff --git a/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/runner/internal/DefaultExperimentRunner.kt b/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/runner/internal/DefaultExperimentRunner.kt deleted file mode 100644 index 26e4df89..00000000 --- a/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/runner/internal/DefaultExperimentRunner.kt +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Copyright (c) 2020 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.experiments.sc20.runner.internal - -import kotlinx.coroutines.runBlocking -import org.opendc.experiments.sc20.runner.ExperimentDescriptor -import org.opendc.experiments.sc20.runner.ExperimentRunner -import org.opendc.experiments.sc20.runner.execution.ExperimentExecutionContext -import org.opendc.experiments.sc20.runner.execution.ExperimentExecutionListener -import org.opendc.experiments.sc20.runner.execution.ExperimentExecutionResult -import org.opendc.experiments.sc20.runner.execution.ExperimentScheduler -import java.util.concurrent.ConcurrentHashMap - -/** - * The default implementation of the [ExperimentRunner] interface. - * - * @param scheduler The scheduler to use. - */ -public class DefaultExperimentRunner(private val scheduler: ExperimentScheduler) : ExperimentRunner { - override val id: String = "default" - - override val version: String? = "1.0" - - override fun execute(root: ExperimentDescriptor, listener: ExperimentExecutionListener): Unit = runBlocking { - val context = object : ExperimentExecutionContext { - override val listener: ExperimentExecutionListener = listener - override val scheduler: ExperimentScheduler = this@DefaultExperimentRunner.scheduler - override val cache: MutableMap = ConcurrentHashMap() - } - - listener.descriptorRegistered(root) - context.listener.executionStarted(root) - try { - root(context) - context.listener.executionFinished(root, ExperimentExecutionResult.Success) - } catch (e: Throwable) { - context.listener.executionFinished(root, ExperimentExecutionResult.Failed(e)) - } - } -} diff --git a/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/telemetry/RunEvent.kt b/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/telemetry/RunEvent.kt index 3bcd10a1..4f4706f0 100644 --- a/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/telemetry/RunEvent.kt +++ b/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/telemetry/RunEvent.kt @@ -22,12 +22,13 @@ package org.opendc.experiments.sc20.telemetry -import org.opendc.experiments.sc20.experiment.Run +import org.opendc.experiments.sc20.experiment.Portfolio /** * A periodic report of the host machine metrics. */ public data class RunEvent( - public val run: Run, + val portfolio: Portfolio, + val repeat: Int, override val timestamp: Long ) : Event("run") diff --git a/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/telemetry/parquet/ParquetRunEventWriter.kt b/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/telemetry/parquet/ParquetRunEventWriter.kt index 74efb660..b50a698c 100644 --- a/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/telemetry/parquet/ParquetRunEventWriter.kt +++ b/simulator/opendc-experiments/opendc-experiments-sc20/src/main/kotlin/org/opendc/experiments/sc20/telemetry/parquet/ParquetRunEventWriter.kt @@ -38,33 +38,27 @@ public class ParquetRunEventWriter(path: File, bufferSize: Int) : public companion object { private val convert: (RunEvent, GenericData.Record) -> Unit = { event, record -> - val run = event.run - val scenario = run.parent - val portfolio = scenario.parent - record.put("portfolio_id", portfolio.id) + val portfolio = event.portfolio record.put("portfolio_name", portfolio.name) - record.put("scenario_id", scenario.id) - record.put("run_id", run.id) - record.put("repetitions", scenario.repetitions) - record.put("topology", scenario.topology.name) - record.put("workload_name", scenario.workload.name) - record.put("workload_fraction", scenario.workload.fraction) - record.put("workload_sampler", scenario.workload.samplingStrategy) - record.put("allocation_policy", scenario.allocationPolicy) - record.put("failure_frequency", scenario.operationalPhenomena.failureFrequency) - record.put("interference", scenario.operationalPhenomena.hasInterference) - record.put("seed", run.seed) + record.put("scenario_id", portfolio.id) + record.put("run_id", event.repeat) + record.put("topology", portfolio.topology.name) + record.put("workload_name", portfolio.workload.name) + record.put("workload_fraction", portfolio.workload.fraction) + record.put("workload_sampler", portfolio.workload.samplingStrategy) + record.put("allocation_policy", portfolio.allocationPolicy) + record.put("failure_frequency", portfolio.operationalPhenomena.failureFrequency) + record.put("interference", portfolio.operationalPhenomena.hasInterference) + record.put("seed", event.repeat) } private val schema: Schema = SchemaBuilder .record("runs") .namespace("org.opendc.experiments.sc20") .fields() - .name("portfolio_id").type().intType().noDefault() .name("portfolio_name").type().stringType().noDefault() .name("scenario_id").type().intType().noDefault() .name("run_id").type().intType().noDefault() - .name("repetitions").type().intType().noDefault() .name("topology").type().stringType().noDefault() .name("workload_name").type().stringType().noDefault() .name("workload_fraction").type().doubleType().noDefault() -- cgit v1.2.3