diff options
Diffstat (limited to 'opendc-harness/opendc-harness-engine')
27 files changed, 1371 insertions, 0 deletions
diff --git a/opendc-harness/opendc-harness-engine/build.gradle.kts b/opendc-harness/opendc-harness-engine/build.gradle.kts new file mode 100644 index 00000000..bffafdac --- /dev/null +++ b/opendc-harness/opendc-harness-engine/build.gradle.kts @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +description = "Engine for running experiments using the OpenDC Harness" + +/* Build configuration */ +plugins { + `kotlin-library-conventions` + `testing-conventions` + `jacoco-conventions` +} + +dependencies { + api(platform(project(":opendc-platform"))) + api(project(":opendc-harness:opendc-harness-api")) + api("org.jetbrains.kotlinx:kotlinx-coroutines-core") + + implementation("io.github.classgraph:classgraph:${versions["classgraph"]}") + implementation("io.github.microutils:kotlin-logging") +} diff --git a/opendc-harness/opendc-harness-engine/src/main/kotlin/org/opendc/harness/engine/ExperimentEngine.kt b/opendc-harness/opendc-harness-engine/src/main/kotlin/org/opendc/harness/engine/ExperimentEngine.kt new file mode 100644 index 00000000..a36f1f9b --- /dev/null +++ b/opendc-harness/opendc-harness-engine/src/main/kotlin/org/opendc/harness/engine/ExperimentEngine.kt @@ -0,0 +1,104 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.harness.engine + +import kotlinx.coroutines.* +import kotlinx.coroutines.flow.asFlow +import kotlinx.coroutines.flow.buffer +import kotlinx.coroutines.flow.collect +import kotlinx.coroutines.flow.map +import org.opendc.harness.api.ExperimentDefinition +import org.opendc.harness.api.Trial +import org.opendc.harness.engine.scheduler.ExperimentScheduler +import org.opendc.harness.engine.strategy.ExperimentStrategy + +/** + * The [ExperimentEngine] orchestrates the execution of experiments. + * + * @property strategy The [ExperimentStrategy] used to explore the experiment design space. + * @property scheduler The [ExperimentScheduler] to schedule the trials over compute resources. + * @property listener The [ExperimentExecutionListener] to observe the progress. + * @property repeats The number of repeats to perform. + */ +public class ExperimentEngine( + private val strategy: ExperimentStrategy, + private val scheduler: ExperimentScheduler, + private val listener: ExperimentExecutionListener, + private val repeats: Int +) { + /** + * Execute the specified [experiment][root]. + * + * @param root The experiment to execute. + */ + public suspend fun execute(root: ExperimentDefinition) { + listener.experimentStarted(root) + + try { + supervisorScope { + strategy.generate(root) + .asFlow() + .map { scenario -> + listener.scenarioStarted(scenario) + scenario + } + .buffer(100) + .collect { scenario -> + launch { + val jobs = (0 until repeats).map { repeat -> + val worker = scheduler.allocate() + launch { + val trial = Trial(scenario, repeat) + try { + listener.trialStarted(trial) + worker.dispatch(trial) + listener.trialFinished(trial, null) + } catch (e: Throwable) { + listener.trialFinished(trial, e) + throw e + } + } + } + + try { + jobs.joinAll() + listener.scenarioFinished(scenario, null) + } catch (e: CancellationException) { + listener.scenarioFinished(scenario, null) + throw e + } catch (e: Throwable) { + listener.scenarioFinished(scenario, e) + } + } + } + } + + listener.experimentFinished(root, null) + } catch (e: Throwable) { + listener.experimentFinished(root, e) + throw e + } + } + + override fun toString(): String = "ExperimentEngine" +} diff --git a/opendc-harness/opendc-harness-engine/src/main/kotlin/org/opendc/harness/engine/ExperimentEngineLauncher.kt b/opendc-harness/opendc-harness-engine/src/main/kotlin/org/opendc/harness/engine/ExperimentEngineLauncher.kt new file mode 100644 index 00000000..759e96f8 --- /dev/null +++ b/opendc-harness/opendc-harness-engine/src/main/kotlin/org/opendc/harness/engine/ExperimentEngineLauncher.kt @@ -0,0 +1,121 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.harness.engine + +import kotlinx.coroutines.flow.Flow +import kotlinx.coroutines.flow.collect +import kotlinx.coroutines.runBlocking +import org.opendc.harness.api.ExperimentDefinition +import org.opendc.harness.engine.internal.CompositeExperimentExecutionListener +import org.opendc.harness.engine.scheduler.ExperimentScheduler +import org.opendc.harness.engine.scheduler.ThreadPoolExperimentScheduler +import org.opendc.harness.engine.strategy.CartesianExperimentStrategy +import org.opendc.harness.engine.strategy.ExperimentStrategy + +/** + * A builder class for conducting experiments via the [ExperimentEngine]. + */ +public class ExperimentEngineLauncher private constructor( + private val strategy: ExperimentStrategy?, + private val scheduler: ExperimentScheduler?, + private val listeners: List<ExperimentExecutionListener>, + private val repeats: Int +) { + /** + * Construct an [ExperimentEngineLauncher] instance. + */ + public constructor() : this(null, null, emptyList(), 1) + + /** + * Create an [ExperimentEngineLauncher] with the specified [strategy]. + */ + public fun withScheduler(strategy: ExperimentStrategy): ExperimentEngineLauncher { + return ExperimentEngineLauncher(strategy, scheduler, listeners, repeats) + } + + /** + * Create an [ExperimentEngineLauncher] with the specified [scheduler]. + */ + public fun withScheduler(scheduler: ExperimentScheduler): ExperimentEngineLauncher { + return ExperimentEngineLauncher(strategy, scheduler, listeners, repeats) + } + + /** + * Create an [ExperimentEngineLauncher] with the specified [listener] added. + */ + public fun withListener(listener: ExperimentExecutionListener): ExperimentEngineLauncher { + return ExperimentEngineLauncher(strategy, scheduler, listeners + listener, repeats) + } + + /** + * Create an [ExperimentEngineLauncher] with the specified number of repeats. + */ + public fun withRepeats(repeats: Int): ExperimentEngineLauncher { + require(repeats > 0) { "Invalid number of repeats; must be greater than zero. " } + return ExperimentEngineLauncher(strategy, scheduler, listeners, repeats) + } + + /** + * Launch the specified experiments via the [ExperimentEngine] and block execution until finished. + */ + public suspend fun run(experiments: Flow<ExperimentDefinition>) { + val engine = ExperimentEngine(createStrategy(), createScheduler(), createListener(), repeats) + experiments.collect { experiment -> engine.execute(experiment) } + } + + /** + * Launch the specified experiments via the [ExperimentEngine] and block the current thread until finished. + */ + public fun runBlocking(experiments: Flow<ExperimentDefinition>) { + runBlocking { + run(experiments) + } + } + + /** + * Return a string representation of this instance. + */ + public override fun toString(): String = "ExperimentEngineLauncher" + + /** + * Create the [ExperimentStrategy] that explores the experiment design space. + */ + private fun createStrategy(): ExperimentStrategy { + return strategy ?: CartesianExperimentStrategy + } + + /** + * Create the [ExperimentScheduler] that schedules the trials over the compute resources. + */ + private fun createScheduler(): ExperimentScheduler { + return scheduler ?: ThreadPoolExperimentScheduler(Runtime.getRuntime().availableProcessors()) + } + + /** + * Create the [ExperimentExecutionListener] that listens the to the execution of the experiments. + */ + private fun createListener(): ExperimentExecutionListener { + require(listeners.isNotEmpty()) { "No listeners registered." } + return CompositeExperimentExecutionListener(listeners) + } +} diff --git a/opendc-harness/opendc-harness-engine/src/main/kotlin/org/opendc/harness/engine/ExperimentExecutionListener.kt b/opendc-harness/opendc-harness-engine/src/main/kotlin/org/opendc/harness/engine/ExperimentExecutionListener.kt new file mode 100644 index 00000000..9ef71863 --- /dev/null +++ b/opendc-harness/opendc-harness-engine/src/main/kotlin/org/opendc/harness/engine/ExperimentExecutionListener.kt @@ -0,0 +1,77 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.harness.engine + +import org.opendc.harness.api.ExperimentDefinition +import org.opendc.harness.api.Scenario +import org.opendc.harness.api.Trial + +/** + * Listener to be notified of experiment execution events by experiment runners. + */ +public interface ExperimentExecutionListener { + /** + * A method that is invoked when an experiment is started. + * + * @param experiment The [ExperimentDefinition] that started. + */ + public fun experimentStarted(experiment: ExperimentDefinition) {} + + /** + * A method that is invoked when an experiment is finished, regardless of the outcome. + * + * @param experiment The [ExperimentDefinition] that finished. + * @param throwable The exception that was thrown during execution or `null` if the execution completed successfully. + */ + public fun experimentFinished(experiment: ExperimentDefinition, throwable: Throwable?) {} + + /** + * A method that is invoked when a scenario is started. + * + * @param scenario The scenario that is started. + */ + public fun scenarioStarted(scenario: Scenario) {} + + /** + * A method that is invoked when a scenario is finished, regardless of the outcome. + * + * @param scenario The [Scenario] that has finished. + * @param throwable The exception that was thrown during execution or `null` if the execution completed successfully. + */ + public fun scenarioFinished(scenario: Scenario, throwable: Throwable?) {} + + /** + * A method that is invoked when a trial is started. + * + * @param trial The trial that is started. + */ + public fun trialStarted(trial: Trial) {} + + /** + * A method that is invoked when a scenario is finished, regardless of the outcome. + * + * @param trial The [Trial] that has finished. + * @param throwable The exception that was thrown during execution or `null` if the execution completed successfully. + */ + public fun trialFinished(trial: Trial, throwable: Throwable?) {} +} diff --git a/opendc-harness/opendc-harness-engine/src/main/kotlin/org/opendc/harness/engine/discovery/Discovery.kt b/opendc-harness/opendc-harness-engine/src/main/kotlin/org/opendc/harness/engine/discovery/Discovery.kt new file mode 100644 index 00000000..f7f73b38 --- /dev/null +++ b/opendc-harness/opendc-harness-engine/src/main/kotlin/org/opendc/harness/engine/discovery/Discovery.kt @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.harness.engine.discovery + +import kotlinx.coroutines.flow.Flow +import org.opendc.harness.api.ExperimentDefinition + +/** + * Component responsible for scanning for [ExperimentDefinition]s. + */ +public interface Discovery { + /** + * Start discovery of experiments. + * + * @param request The [DiscoveryRequest] to determine the experiments to discover. + * @return A flow of [ExperimentDefinition]s that have been discovered by the implementation. + */ + public fun discover(request: DiscoveryRequest): Flow<ExperimentDefinition> +} diff --git a/opendc-harness/opendc-harness-engine/src/main/kotlin/org/opendc/harness/engine/discovery/DiscoveryFilter.kt b/opendc-harness/opendc-harness-engine/src/main/kotlin/org/opendc/harness/engine/discovery/DiscoveryFilter.kt new file mode 100644 index 00000000..219d09cd --- /dev/null +++ b/opendc-harness/opendc-harness-engine/src/main/kotlin/org/opendc/harness/engine/discovery/DiscoveryFilter.kt @@ -0,0 +1,51 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.harness.engine.discovery + +import org.opendc.harness.api.ExperimentDefinition +import java.util.function.Predicate + +/** + * A [DiscoveryFilter] decides how the selected experiments are filtered. + */ +public sealed class DiscoveryFilter { + /** + * Test whether the specified [ExperimentDefinition] should be selected. + */ + public abstract fun test(definition: ExperimentDefinition): Boolean + + /** + * Filter an experiment based on its name. + */ + public data class Name(val predicate: Predicate<String>) : DiscoveryFilter() { + override fun test(definition: ExperimentDefinition): Boolean = predicate.test(definition.name) + } + + /** + * Filter an experiment based on its metadata. + */ + public data class Meta(val key: String, val predicate: Predicate<Any>) : DiscoveryFilter() { + override fun test(definition: ExperimentDefinition): Boolean = + definition.meta[key]?.let { predicate.test(it) } ?: false + } +} diff --git a/opendc-harness/opendc-harness-engine/src/main/kotlin/org/opendc/harness/engine/discovery/DiscoveryProvider.kt b/opendc-harness/opendc-harness-engine/src/main/kotlin/org/opendc/harness/engine/discovery/DiscoveryProvider.kt new file mode 100644 index 00000000..204de3fc --- /dev/null +++ b/opendc-harness/opendc-harness-engine/src/main/kotlin/org/opendc/harness/engine/discovery/DiscoveryProvider.kt @@ -0,0 +1,65 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.harness.engine.discovery + +import org.opendc.harness.engine.internal.CompositeDiscovery +import java.util.* + +/** + * A provider interface for the [Discovery] component. + */ +public interface DiscoveryProvider { + /** + * A unique identifier for this discovery implementation. + * + * Each discovery implementation must provide a unique ID, so that they can be selected by the user. + * When in doubt, you may use the fully qualified name of your custom [Discovery] implementation class. + */ + public val id: String + + /** + * Factory method for creating a new [Discovery] instance. + */ + public fun create(): Discovery + + public companion object { + /** + * The available [DiscoveryProvider]s. + */ + private val providers by lazy { ServiceLoader.load(DiscoveryProvider::class.java) } + + /** + * Obtain the [DiscoveryProvider] with the specified [id] or return `null`. + */ + public fun findById(id: String): DiscoveryProvider? { + return providers.find { it.id == id } + } + + /** + * Obtain a composite [Discovery] that combines the results of all available providers. + */ + public fun createComposite(): Discovery { + return CompositeDiscovery(providers) + } + } +} diff --git a/opendc-harness/opendc-harness-engine/src/main/kotlin/org/opendc/harness/engine/discovery/DiscoveryRequest.kt b/opendc-harness/opendc-harness-engine/src/main/kotlin/org/opendc/harness/engine/discovery/DiscoveryRequest.kt new file mode 100644 index 00000000..5bc08dac --- /dev/null +++ b/opendc-harness/opendc-harness-engine/src/main/kotlin/org/opendc/harness/engine/discovery/DiscoveryRequest.kt @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.harness.engine.discovery + +/** + * A request for discovering experiments according to the specified information. + * + * @param selectors The selectors for this discovery request. + * @param filters The filters for this discovery request. + */ +public data class DiscoveryRequest( + val selectors: List<DiscoverySelector> = emptyList(), + val filters: List<DiscoveryFilter> = emptyList(), +) diff --git a/opendc-harness/opendc-harness-engine/src/main/kotlin/org/opendc/harness/engine/discovery/DiscoverySelector.kt b/opendc-harness/opendc-harness-engine/src/main/kotlin/org/opendc/harness/engine/discovery/DiscoverySelector.kt new file mode 100644 index 00000000..67681303 --- /dev/null +++ b/opendc-harness/opendc-harness-engine/src/main/kotlin/org/opendc/harness/engine/discovery/DiscoverySelector.kt @@ -0,0 +1,49 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.harness.engine.discovery + +import org.opendc.harness.api.ExperimentDefinition + +/** + * A [DiscoverySelector] defines the properties used to discover experiments. + */ +public sealed class DiscoverySelector { + /** + * Test whether the specified [ExperimentDefinition] should be selected. + */ + public abstract fun test(definition: ExperimentDefinition): Boolean + + /** + * Select an experiment based on its name. + */ + public data class Name(val name: String) : DiscoverySelector() { + override fun test(definition: ExperimentDefinition): Boolean = definition.name == name + } + + /** + * Select an experiment based on its metadata. + */ + public data class Meta(val key: String, val value: Any) : DiscoverySelector() { + override fun test(definition: ExperimentDefinition): Boolean = definition.meta[key] == value + } +} diff --git a/opendc-harness/opendc-harness-engine/src/main/kotlin/org/opendc/harness/engine/internal/CompositeDiscovery.kt b/opendc-harness/opendc-harness-engine/src/main/kotlin/org/opendc/harness/engine/internal/CompositeDiscovery.kt new file mode 100644 index 00000000..8ebc485a --- /dev/null +++ b/opendc-harness/opendc-harness-engine/src/main/kotlin/org/opendc/harness/engine/internal/CompositeDiscovery.kt @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.harness.engine.internal + +import kotlinx.coroutines.FlowPreview +import kotlinx.coroutines.flow.* +import org.opendc.harness.api.ExperimentDefinition +import org.opendc.harness.engine.discovery.Discovery +import org.opendc.harness.engine.discovery.DiscoveryProvider +import org.opendc.harness.engine.discovery.DiscoveryRequest + +/** + * A composite [Discovery] instance that combines the results of multiple delegate instances. + */ +internal class CompositeDiscovery(providers: Iterable<DiscoveryProvider>) : Discovery { + /** + * The [Discovery] instances to delegate to. + */ + private val delegates = providers.map { it.create() } + + @OptIn(FlowPreview::class) + override fun discover(request: DiscoveryRequest): Flow<ExperimentDefinition> { + return delegates.asFlow() + .map { it.discover(request) } + .flattenMerge(delegates.size) + } +} diff --git a/opendc-harness/opendc-harness-engine/src/main/kotlin/org/opendc/harness/engine/internal/CompositeExperimentExecutionListener.kt b/opendc-harness/opendc-harness-engine/src/main/kotlin/org/opendc/harness/engine/internal/CompositeExperimentExecutionListener.kt new file mode 100644 index 00000000..62b4ca8b --- /dev/null +++ b/opendc-harness/opendc-harness-engine/src/main/kotlin/org/opendc/harness/engine/internal/CompositeExperimentExecutionListener.kt @@ -0,0 +1,57 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.harness.engine.internal + +import org.opendc.harness.api.ExperimentDefinition +import org.opendc.harness.api.Scenario +import org.opendc.harness.api.Trial +import org.opendc.harness.engine.ExperimentExecutionListener + +/** + * An [ExperimentExecutionListener] that composes multiple other listeners. + */ +public class CompositeExperimentExecutionListener(private val listeners: List<ExperimentExecutionListener>) : ExperimentExecutionListener { + override fun experimentStarted(experiment: ExperimentDefinition) { + listeners.forEach { it.experimentStarted(experiment) } + } + + override fun experimentFinished(experiment: ExperimentDefinition, throwable: Throwable?) { + listeners.forEach { it.experimentFinished(experiment, throwable) } + } + + override fun scenarioStarted(scenario: Scenario) { + listeners.forEach { it.scenarioStarted(scenario) } + } + + override fun scenarioFinished(scenario: Scenario, throwable: Throwable?) { + listeners.forEach { it.scenarioFinished(scenario, throwable) } + } + + override fun trialStarted(trial: Trial) { + listeners.forEach { it.trialStarted(trial) } + } + + override fun trialFinished(trial: Trial, throwable: Throwable?) { + listeners.forEach { it.trialFinished(trial, throwable) } + } +} diff --git a/opendc-harness/opendc-harness-engine/src/main/kotlin/org/opendc/harness/engine/internal/DslDiscovery.kt b/opendc-harness/opendc-harness-engine/src/main/kotlin/org/opendc/harness/engine/internal/DslDiscovery.kt new file mode 100644 index 00000000..20708230 --- /dev/null +++ b/opendc-harness/opendc-harness-engine/src/main/kotlin/org/opendc/harness/engine/internal/DslDiscovery.kt @@ -0,0 +1,101 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.harness.engine.internal + +import io.github.classgraph.ClassGraph +import io.github.classgraph.ScanResult +import kotlinx.coroutines.flow.Flow +import kotlinx.coroutines.flow.asFlow +import org.opendc.harness.api.ExperimentDefinition +import org.opendc.harness.dsl.Experiment +import org.opendc.harness.engine.discovery.Discovery +import org.opendc.harness.engine.discovery.DiscoveryFilter +import org.opendc.harness.engine.discovery.DiscoveryRequest +import org.opendc.harness.engine.discovery.DiscoverySelector + +/** + * A [Discovery] implementation that discovers [Experiment] instances on the classpath. + */ +internal class DslDiscovery : Discovery { + /* + * Lazily memoize the results of the classpath scan. + */ + private val scanResult by lazy { scan() } + + override fun discover(request: DiscoveryRequest): Flow<ExperimentDefinition> { + return findExperiments() + .map { cls -> + val exp = cls.constructors[0].newInstance() as Experiment + exp.toDefinition() + } + .filter(select(request.selectors)) + .filter(filter(request.filters)) + .asFlow() + } + + /** + * Find the classes on the classpath implementing the [Experiment] class. + */ + private fun findExperiments(): Sequence<Class<out Experiment>> { + return scanResult + .getSubclasses(Experiment::class.java.name) + .filter { !(it.isAbstract || it.isInterface) } + .map { it.loadClass() } + .filterIsInstance<Class<out Experiment>>() + .asSequence() + } + + /** + * Create a predicate for filtering the experiments based on the specified [filters]. + */ + private fun filter(filters: List<DiscoveryFilter>): (ExperimentDefinition) -> Boolean = { def -> + filters.isEmpty() || filters.all { it.test(def) } + } + + /** + * Create a predicate for selecting the experiments based on the specified [selectors]. + */ + private fun select(selectors: List<DiscoverySelector>): (ExperimentDefinition) -> Boolean = { def -> + selectors.isEmpty() || selectors.any { it.test(def) } + } + + /** + * Scan the classpath using [ClassGraph]. + */ + private fun scan(): ScanResult { + return ClassGraph() + .enableClassInfo() + .enableExternalClasses() + .ignoreClassVisibility() + .rejectPackages( + "java.*", + "javax.*", + "sun.*", + "com.sun.*", + "kotlin.*", + "androidx.*", + "org.jetbrains.kotlin.*", + "org.junit.*" + ).scan() + } +} diff --git a/opendc-harness/opendc-harness-engine/src/main/kotlin/org/opendc/harness/engine/internal/DslDiscoveryProvider.kt b/opendc-harness/opendc-harness-engine/src/main/kotlin/org/opendc/harness/engine/internal/DslDiscoveryProvider.kt new file mode 100644 index 00000000..987bc889 --- /dev/null +++ b/opendc-harness/opendc-harness-engine/src/main/kotlin/org/opendc/harness/engine/internal/DslDiscoveryProvider.kt @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.harness.engine.internal + +import org.opendc.harness.dsl.Experiment +import org.opendc.harness.engine.discovery.Discovery +import org.opendc.harness.engine.discovery.DiscoveryProvider + +/** + * A [DiscoveryProvider] for the [Experiment]s on the classpath. + */ +public class DslDiscoveryProvider : DiscoveryProvider { + override val id: String = "dsl" + + override fun create(): Discovery = DslDiscovery() +} diff --git a/opendc-harness/opendc-harness-engine/src/main/kotlin/org/opendc/harness/engine/internal/ScenarioImpl.kt b/opendc-harness/opendc-harness-engine/src/main/kotlin/org/opendc/harness/engine/internal/ScenarioImpl.kt new file mode 100644 index 00000000..65d19e87 --- /dev/null +++ b/opendc-harness/opendc-harness-engine/src/main/kotlin/org/opendc/harness/engine/internal/ScenarioImpl.kt @@ -0,0 +1,49 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.harness.engine.internal + +import org.opendc.harness.api.ExperimentDefinition +import org.opendc.harness.api.Parameter +import org.opendc.harness.api.Scenario + +/** + * Internal implementation of a [Scenario]. + */ +internal data class ScenarioImpl( + override val id: Int, + override val experiment: ExperimentDefinition, + val parameters: Map<Parameter<*>, Any?> +) : Scenario { + + override fun <T> get(param: Parameter<T>): T { + if (!parameters.containsKey(param)) { + throw IllegalArgumentException("Unknown parameter for this scenario.") + } + + // This cast should always succeed + @Suppress("UNCHECKED_CAST") + return parameters[param] as T + } + + override fun toString(): String = "Scenario" +} diff --git a/opendc-harness/opendc-harness-engine/src/main/kotlin/org/opendc/harness/engine/scheduler/ExperimentScheduler.kt b/opendc-harness/opendc-harness-engine/src/main/kotlin/org/opendc/harness/engine/scheduler/ExperimentScheduler.kt new file mode 100644 index 00000000..0265554a --- /dev/null +++ b/opendc-harness/opendc-harness-engine/src/main/kotlin/org/opendc/harness/engine/scheduler/ExperimentScheduler.kt @@ -0,0 +1,52 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.harness.engine.scheduler + +import org.opendc.harness.api.Trial + +/** + * The [ExperimentScheduler] is responsible for scheduling the execution of experiment runs over some set of compute + * resources (e.g., threads or even multiple machines). + */ +public interface ExperimentScheduler : AutoCloseable { + /** + * Allocate a [Worker] for executing an experiment trial. This method may suspend in case no resources are directly + * available at the moment. + * + * @return The available worker. + */ + public suspend fun allocate(): Worker + + /** + * An isolated worker of an [ExperimentScheduler] that is responsible for conducting a single experiment trial. + */ + public interface Worker { + /** + * Dispatch an experiment trial immediately to one of the available compute resources and block execution until + * the trial has finished. + * + * @param trial The trial to dispatch. + */ + public suspend fun dispatch(trial: Trial) + } +} diff --git a/opendc-harness/opendc-harness-engine/src/main/kotlin/org/opendc/harness/engine/scheduler/ExperimentSchedulerProvider.kt b/opendc-harness/opendc-harness-engine/src/main/kotlin/org/opendc/harness/engine/scheduler/ExperimentSchedulerProvider.kt new file mode 100644 index 00000000..a93d4bf6 --- /dev/null +++ b/opendc-harness/opendc-harness-engine/src/main/kotlin/org/opendc/harness/engine/scheduler/ExperimentSchedulerProvider.kt @@ -0,0 +1,57 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.harness.engine.scheduler + +import java.util.* + +/** + * A factory for constructing an [ExperimentScheduler]. + */ +public interface ExperimentSchedulerProvider { + /** + * A unique identifier for this scheduler implementation. + * + * Each experiment scheduler must provide a unique ID, so that they can be selected by the user. + * When in doubt, you may use the fully qualified name of your custom [ExperimentScheduler] implementation class. + */ + public val id: String + + /** + * Factory method for creating a new [ExperimentScheduler] instance. + */ + public fun create(): ExperimentScheduler + + public companion object { + /** + * The available [ExperimentSchedulerProvider]s. + */ + private val providers by lazy { ServiceLoader.load(ExperimentSchedulerProvider::class.java) } + + /** + * Obtain the [ExperimentScheduler] with the specified [id] or return `null`. + */ + public fun findById(id: String): ExperimentSchedulerProvider? { + return providers.find { it.id == id } + } + } +} diff --git a/opendc-harness/opendc-harness-engine/src/main/kotlin/org/opendc/harness/engine/scheduler/ThreadPoolExperimentScheduler.kt b/opendc-harness/opendc-harness-engine/src/main/kotlin/org/opendc/harness/engine/scheduler/ThreadPoolExperimentScheduler.kt new file mode 100644 index 00000000..1ae533cf --- /dev/null +++ b/opendc-harness/opendc-harness-engine/src/main/kotlin/org/opendc/harness/engine/scheduler/ThreadPoolExperimentScheduler.kt @@ -0,0 +1,58 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.harness.engine.scheduler + +import kotlinx.coroutines.asCoroutineDispatcher +import kotlinx.coroutines.sync.Semaphore +import kotlinx.coroutines.withContext +import org.opendc.harness.api.Trial +import java.util.concurrent.Executors + +/** + * An [ExperimentScheduler] that runs experiment trials using a local thread pool. + * + * @param parallelism The maximum amount of concurrent workers. + */ +public class ThreadPoolExperimentScheduler(parallelism: Int) : ExperimentScheduler { + private val dispatcher = Executors.newCachedThreadPool().asCoroutineDispatcher() + private val tickets = Semaphore(parallelism) + + override suspend fun allocate(): ExperimentScheduler.Worker { + tickets.acquire() + return object : ExperimentScheduler.Worker { + override suspend fun dispatch(trial: Trial) { + try { + withContext(dispatcher) { + trial.scenario.experiment.evaluator(trial) + } + } finally { + tickets.release() + } + } + } + } + + override fun close(): Unit = dispatcher.close() + + override fun toString(): String = "ThreadPoolScheduler" +} diff --git a/opendc-harness/opendc-harness-engine/src/main/kotlin/org/opendc/harness/engine/scheduler/ThreadPoolExperimentSchedulerProvider.kt b/opendc-harness/opendc-harness-engine/src/main/kotlin/org/opendc/harness/engine/scheduler/ThreadPoolExperimentSchedulerProvider.kt new file mode 100644 index 00000000..cf9a132f --- /dev/null +++ b/opendc-harness/opendc-harness-engine/src/main/kotlin/org/opendc/harness/engine/scheduler/ThreadPoolExperimentSchedulerProvider.kt @@ -0,0 +1,33 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.harness.engine.scheduler + +/** + * An [ExperimentSchedulerProvider] for constructing a [ThreadPoolExperimentScheduler]. + */ +public class ThreadPoolExperimentSchedulerProvider : ExperimentSchedulerProvider { + override val id: String = "thread-pool" + + override fun create(): ExperimentScheduler = + ThreadPoolExperimentScheduler(Runtime.getRuntime().availableProcessors()) +} diff --git a/opendc-harness/opendc-harness-engine/src/main/kotlin/org/opendc/harness/engine/strategy/CartesianExperimentStrategy.kt b/opendc-harness/opendc-harness-engine/src/main/kotlin/org/opendc/harness/engine/strategy/CartesianExperimentStrategy.kt new file mode 100644 index 00000000..733eca8f --- /dev/null +++ b/opendc-harness/opendc-harness-engine/src/main/kotlin/org/opendc/harness/engine/strategy/CartesianExperimentStrategy.kt @@ -0,0 +1,55 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.harness.engine.strategy + +import org.opendc.harness.api.ExperimentDefinition +import org.opendc.harness.api.Parameter +import org.opendc.harness.api.Scenario +import org.opendc.harness.engine.internal.ScenarioImpl + +/** + * An [ExperimentStrategy] that takes the cartesian product of the parameters and evaluates every combination. + */ +public object CartesianExperimentStrategy : ExperimentStrategy { + /** + * Build the trials of an experiment. + */ + override fun generate(experiment: ExperimentDefinition): Sequence<Scenario> { + return experiment.parameters + .asSequence() + .map { param -> mapParameter(param).map { value -> listOf(param to value) } } + .reduce { acc, param -> + acc.flatMap { x -> param.map { y -> x + y } } + } + .mapIndexed { id, values -> ScenarioImpl(id, experiment, values.toMap()) } + } + + /** + * Instantiate a parameter and return a sequence of possible values. + */ + private fun <T> mapParameter(param: Parameter<T>): Sequence<T> { + return when (param) { + is Parameter.Generic<T> -> param.values.asSequence() + } + } +} diff --git a/opendc-harness/opendc-harness-engine/src/main/kotlin/org/opendc/harness/engine/strategy/CartesianExperimentStrategyProvider.kt b/opendc-harness/opendc-harness-engine/src/main/kotlin/org/opendc/harness/engine/strategy/CartesianExperimentStrategyProvider.kt new file mode 100644 index 00000000..f18795a3 --- /dev/null +++ b/opendc-harness/opendc-harness-engine/src/main/kotlin/org/opendc/harness/engine/strategy/CartesianExperimentStrategyProvider.kt @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.harness.engine.strategy + +/** + * An [ExperimentStrategyProvider] for constructing a [CartesianExperimentStrategy]. + */ +public class CartesianExperimentStrategyProvider : ExperimentStrategyProvider { + override val id: String = "cartesian" + + override fun create(): ExperimentStrategy = CartesianExperimentStrategy +} diff --git a/opendc-harness/opendc-harness-engine/src/main/kotlin/org/opendc/harness/engine/strategy/ExperimentStrategy.kt b/opendc-harness/opendc-harness-engine/src/main/kotlin/org/opendc/harness/engine/strategy/ExperimentStrategy.kt new file mode 100644 index 00000000..3a0148ad --- /dev/null +++ b/opendc-harness/opendc-harness-engine/src/main/kotlin/org/opendc/harness/engine/strategy/ExperimentStrategy.kt @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.harness.engine.strategy + +import org.opendc.harness.api.ExperimentDefinition +import org.opendc.harness.api.Scenario + +/** + * The [ExperimentStrategy] is responsible for traversing the design space of an [ExperimentDefinition] based on its + * parameters, generating concrete points in the space represented as [Scenario]s. + */ +public interface ExperimentStrategy { + /** + * Generate the points in the design space of the specified [experiment] to explore. + * + * @param experiment The experiment design space to explore. + * @return A sequence of [Scenario]s which may be explored by the [ExperimentEngine]. + */ + public fun generate(experiment: ExperimentDefinition): Sequence<Scenario> +} diff --git a/opendc-harness/opendc-harness-engine/src/main/kotlin/org/opendc/harness/engine/strategy/ExperimentStrategyProvider.kt b/opendc-harness/opendc-harness-engine/src/main/kotlin/org/opendc/harness/engine/strategy/ExperimentStrategyProvider.kt new file mode 100644 index 00000000..7fa05f34 --- /dev/null +++ b/opendc-harness/opendc-harness-engine/src/main/kotlin/org/opendc/harness/engine/strategy/ExperimentStrategyProvider.kt @@ -0,0 +1,57 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.harness.engine.strategy + +import java.util.* + +/** + * A factory for constructing an [ExperimentStrategy]. + */ +public interface ExperimentStrategyProvider { + /** + * A unique identifier for this strategy implementation. + * + * Each experiment strategy must provide a unique ID, so that they can be selected by the user. + * When in doubt, you may use the fully qualified name of your custom [ExperimentStrategy] implementation class. + */ + public val id: String + + /** + * Factory method for creating a new [ExperimentStrategy] instance. + */ + public fun create(): ExperimentStrategy + + public companion object { + /** + * The available [ExperimentStrategyProvider]s. + */ + private val providers by lazy { ServiceLoader.load(ExperimentStrategyProvider::class.java) } + + /** + * Obtain the [ExperimentStrategy] with the specified [id] or return `null`. + */ + public fun findById(id: String): ExperimentStrategyProvider? { + return providers.find { it.id == id } + } + } +} diff --git a/opendc-harness/opendc-harness-engine/src/main/resources/META-INF/services/org.opendc.harness.engine.discovery.DiscoveryProvider b/opendc-harness/opendc-harness-engine/src/main/resources/META-INF/services/org.opendc.harness.engine.discovery.DiscoveryProvider new file mode 100644 index 00000000..83ac5015 --- /dev/null +++ b/opendc-harness/opendc-harness-engine/src/main/resources/META-INF/services/org.opendc.harness.engine.discovery.DiscoveryProvider @@ -0,0 +1 @@ +org.opendc.harness.engine.internal.DslDiscoveryProvider diff --git a/opendc-harness/opendc-harness-engine/src/main/resources/META-INF/services/org.opendc.harness.engine.scheduler.ExperimentSchedulerProvider b/opendc-harness/opendc-harness-engine/src/main/resources/META-INF/services/org.opendc.harness.engine.scheduler.ExperimentSchedulerProvider new file mode 100644 index 00000000..2ba3a7cb --- /dev/null +++ b/opendc-harness/opendc-harness-engine/src/main/resources/META-INF/services/org.opendc.harness.engine.scheduler.ExperimentSchedulerProvider @@ -0,0 +1 @@ +org.opendc.harness.engine.scheduler.ThreadPoolExperimentSchedulerProvider diff --git a/opendc-harness/opendc-harness-engine/src/main/resources/META-INF/services/org.opendc.harness.engine.strategy.ExperimentStrategyProvider b/opendc-harness/opendc-harness-engine/src/main/resources/META-INF/services/org.opendc.harness.engine.strategy.ExperimentStrategyProvider new file mode 100644 index 00000000..cb1c70ac --- /dev/null +++ b/opendc-harness/opendc-harness-engine/src/main/resources/META-INF/services/org.opendc.harness.engine.strategy.ExperimentStrategyProvider @@ -0,0 +1 @@ +org.opendc.harness.engine.strategy.CartesianExperimentStrategyProvider diff --git a/opendc-harness/opendc-harness-engine/src/test/kotlin/org/opendc/harness/EngineTest.kt b/opendc-harness/opendc-harness-engine/src/test/kotlin/org/opendc/harness/EngineTest.kt new file mode 100644 index 00000000..6f2989db --- /dev/null +++ b/opendc-harness/opendc-harness-engine/src/test/kotlin/org/opendc/harness/EngineTest.kt @@ -0,0 +1,61 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.harness + +import kotlinx.coroutines.flow.flowOf +import kotlinx.coroutines.flow.toList +import kotlinx.coroutines.runBlocking +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertNotNull +import org.junit.jupiter.api.Test +import org.opendc.harness.api.ExperimentDefinition +import org.opendc.harness.engine.ExperimentEngine +import org.opendc.harness.engine.ExperimentEngineLauncher +import org.opendc.harness.engine.ExperimentExecutionListener +import org.opendc.harness.engine.discovery.DiscoveryProvider +import org.opendc.harness.engine.discovery.DiscoveryRequest + +/** + * A test suite for the [ExperimentEngine]. + */ +internal class EngineTest { + @Test + fun test() { + val listener = object : ExperimentExecutionListener {} + ExperimentEngineLauncher() + .withListener(listener) + .runBlocking(flowOf(TestExperiment().toDefinition())) + } + + @Test + fun discovery() { + runBlocking { + val discovery = DiscoveryProvider.findById("dsl")?.create() + assertNotNull(discovery) + val res = mutableListOf<ExperimentDefinition>() + discovery?.discover(DiscoveryRequest())?.toList(res) + println(res) + assertEquals(1, res.size) + } + } +} diff --git a/opendc-harness/opendc-harness-engine/src/test/kotlin/org/opendc/harness/TestExperiment.kt b/opendc-harness/opendc-harness-engine/src/test/kotlin/org/opendc/harness/TestExperiment.kt new file mode 100644 index 00000000..bedd1c76 --- /dev/null +++ b/opendc-harness/opendc-harness-engine/src/test/kotlin/org/opendc/harness/TestExperiment.kt @@ -0,0 +1,54 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.harness + +import org.opendc.harness.dsl.Experiment +import org.opendc.harness.dsl.anyOf + +/** + * An experiment to test the harness' functionality. + */ +class TestExperiment : Experiment("Design Space Exploration") { + /** + * The cloud environment to use. + */ + private val environment: String by anyOf( + "../../traces/setup-small.json", + "../../traces/setup.json", + "../../traces/setup-large.json" + ) + + /** + * The trace to use. + */ + private val trace: String by anyOf( + "../../traces/gwf/askalon_workload_olde.gwf", + "../../traces/gwf/askalon_workload_ee.gwf", + "../../traces/gwf/chronos_exp_noscaler_ca.gwf" + ) + + override fun doRun(repeat: Int) { + println("Id $id, Run $repeat, Environment $environment, Trace $trace") + Thread.sleep(500 * id.toLong()) + } +} |