harness: Split harness into separate modules

This change splits the OpenDC Experiment Harness into separate modules.
This prevents users from pulling in unnecessary dependencies when
depending on the harness API.

4 files changed, 291 insertions, 0 deletions

diff --git a/opendc-harness/opendc-harness-junit5/build.gradle.kts b/opendc-harness/opendc-harness-junit5/build.gradle.kts
new file mode 100644
index 00000000..aa0b78d5
--- /dev/null
+++ b/opendc-harness/opendc-harness-junit5/build.gradle.kts
@@ -0,0 +1,36 @@
+/*
+ * Copyright (c) 2021 AtLarge Research
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+description = "JUnit Platform interface to the OpenDC Harness"
+
+/* Build configuration */
+plugins {
+    `kotlin-library-conventions`
+}
+
+dependencies {
+    api(platform(project(":opendc-platform")))
+    api(project(":opendc-harness:opendc-harness-engine"))
+
+    implementation("io.github.microutils:kotlin-logging")
+    implementation("org.junit.platform:junit-platform-engine:${versions.junitPlatform}")
+}
diff --git a/opendc-harness/opendc-harness-junit5/src/main/kotlin/org/opendc/harness/runner/junit5/JUnitExperimentExecutionListener.kt b/opendc-harness/opendc-harness-junit5/src/main/kotlin/org/opendc/harness/runner/junit5/JUnitExperimentExecutionListener.kt
new file mode 100644
index 00000000..9e2b629d
--- /dev/null
+++ b/opendc-harness/opendc-harness-junit5/src/main/kotlin/org/opendc/harness/runner/junit5/JUnitExperimentExecutionListener.kt
@@ -0,0 +1,162 @@
+/*
+ * Copyright (c) 2021 AtLarge Research
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+package org.opendc.harness.runner.junit5
+
+import org.junit.platform.engine.*
+import org.junit.platform.engine.support.descriptor.AbstractTestDescriptor
+import org.junit.platform.engine.support.descriptor.ClassSource
+import org.junit.platform.engine.support.descriptor.EngineDescriptor
+import org.opendc.harness.api.ExperimentDefinition
+import org.opendc.harness.api.Scenario
+import org.opendc.harness.api.Trial
+import org.opendc.harness.engine.ExperimentExecutionListener
+import java.util.*
+
+/**
+ * An [ExperimentExecutionListener] that notifies JUnit platform of the progress of the experiment trials.
+ */
+public class JUnitExperimentExecutionListener(
+    private val listener: EngineExecutionListener,
+    private val root: EngineDescriptor
+) : ExperimentExecutionListener {
+    /**
+     * The current active experiments.
+     */
+    private val experiments = mutableMapOf<ExperimentDefinition, TestDescriptor>()
+
+    /**
+     * The current active scenarios.
+     */
+    private val scenarios = mutableMapOf<Scenario, TestDescriptor>()
+
+    /**
+     * The current active trials.
+     */
+    private val trials = mutableMapOf<Trial, TestDescriptor>()
+
+    override fun experimentStarted(experiment: ExperimentDefinition) {
+        val descriptor = experiment.toDescriptor(root)
+        root.addChild(descriptor)
+        experiments[experiment] = descriptor
+
+        listener.dynamicTestRegistered(descriptor)
+        listener.executionStarted(descriptor)
+    }
+
+    override fun experimentFinished(experiment: ExperimentDefinition, throwable: Throwable?) {
+        val descriptor = experiments.remove(experiment)
+
+        if (throwable != null) {
+            listener.executionFinished(descriptor, TestExecutionResult.failed(throwable))
+        } else {
+            listener.executionFinished(descriptor, TestExecutionResult.successful())
+        }
+    }
+
+    override fun scenarioStarted(scenario: Scenario) {
+        val parent = experiments[scenario.experiment] ?: return
+        val descriptor = scenario.toDescriptor(parent)
+        parent.addChild(descriptor)
+        scenarios[scenario] = descriptor
+
+        listener.dynamicTestRegistered(descriptor)
+        listener.executionStarted(descriptor)
+    }
+
+    override fun scenarioFinished(scenario: Scenario, throwable: Throwable?) {
+        val descriptor = scenarios.remove(scenario)
+
+        if (throwable != null) {
+            listener.executionFinished(descriptor, TestExecutionResult.failed(throwable))
+        } else {
+            listener.executionFinished(descriptor, TestExecutionResult.successful())
+        }
+    }
+
+    override fun trialStarted(trial: Trial) {
+        val parent = scenarios[trial.scenario] ?: return
+        val descriptor = trial.toDescriptor(parent)
+        parent.addChild(descriptor)
+        trials[trial] = descriptor
+
+        listener.dynamicTestRegistered(descriptor)
+        listener.executionStarted(descriptor)
+    }
+
+    override fun trialFinished(trial: Trial, throwable: Throwable?) {
+        val descriptor = trials.remove(trial)
+
+        if (throwable != null) {
+            listener.executionFinished(descriptor, TestExecutionResult.failed(throwable))
+        } else {
+            listener.executionFinished(descriptor, TestExecutionResult.successful())
+        }
+    }
+
+    /**
+     * Create a [TestDescriptor] for an [ExperimentDefinition].
+     */
+    private fun ExperimentDefinition.toDescriptor(parent: TestDescriptor): TestDescriptor {
+        return object : AbstractTestDescriptor(parent.uniqueId.append("experiment", name), name) {
+            override fun getType(): TestDescriptor.Type = TestDescriptor.Type.CONTAINER
+
+            override fun mayRegisterTests(): Boolean = true
+
+            override fun toString(): String = "ExperimentDescriptor"
+
+            override fun getSource(): Optional<TestSource> {
+                val cls = meta["class.name"] as? String
+                return if (cls != null)
+                    Optional.of(ClassSource.from(cls))
+                else
+                    Optional.empty()
+            }
+        }
+    }
+
+    /**
+     * Create a [TestDescriptor] for a [Scenario].
+     */
+    private fun Scenario.toDescriptor(parent: TestDescriptor): TestDescriptor {
+        return object : AbstractTestDescriptor(parent.uniqueId.append("scenario", id.toString()), "Scenario $id") {
+            override fun getType(): TestDescriptor.Type = TestDescriptor.Type.CONTAINER_AND_TEST
+
+            override fun mayRegisterTests(): Boolean = true
+
+            override fun toString(): String = "ScenarioDescriptor"
+        }
+    }
+
+    /**
+     * Create a [TestDescriptor] for a [Trial].
+     */
+    private fun Trial.toDescriptor(parent: TestDescriptor): TestDescriptor {
+        return object : AbstractTestDescriptor(parent.uniqueId.append("repeat", repeat.toString()), "Repeat $repeat") {
+            override fun getType(): TestDescriptor.Type = TestDescriptor.Type.TEST
+
+            override fun mayRegisterTests(): Boolean = false
+
+            override fun toString(): String = "TrialDescriptor"
+        }
+    }
+}
diff --git a/opendc-harness/opendc-harness-junit5/src/main/kotlin/org/opendc/harness/runner/junit5/OpenDCTestEngine.kt b/opendc-harness/opendc-harness-junit5/src/main/kotlin/org/opendc/harness/runner/junit5/OpenDCTestEngine.kt
new file mode 100644
index 00000000..ab7367b8
--- /dev/null
+++ b/opendc-harness/opendc-harness-junit5/src/main/kotlin/org/opendc/harness/runner/junit5/OpenDCTestEngine.kt
@@ -0,0 +1,92 @@
+/*
+ * Copyright (c) 2021 AtLarge Research
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+package org.opendc.harness.runner.junit5
+
+import kotlinx.coroutines.flow.Flow
+import kotlinx.coroutines.flow.emptyFlow
+import mu.KotlinLogging
+import org.junit.platform.engine.*
+import org.junit.platform.engine.discovery.ClassNameFilter
+import org.junit.platform.engine.discovery.ClassSelector
+import org.junit.platform.engine.discovery.MethodSelector
+import org.junit.platform.engine.support.descriptor.EngineDescriptor
+import org.opendc.harness.api.ExperimentDefinition
+import org.opendc.harness.engine.ExperimentEngineLauncher
+import org.opendc.harness.engine.discovery.DiscoveryFilter
+import org.opendc.harness.engine.discovery.DiscoveryProvider
+import org.opendc.harness.engine.discovery.DiscoveryRequest
+import org.opendc.harness.engine.discovery.DiscoverySelector
+import java.util.*
+
+/**
+ * A [TestEngine] implementation that is able to run experiments defined using the harness.
+ */
+public class OpenDCTestEngine : TestEngine {
+    /**
+     * The logging instance for this engine.
+     */
+    private val logger = KotlinLogging.logger {}
+
+    override fun getId(): String = "opendc-harness"
+
+    override fun getGroupId(): Optional<String> = Optional.of("org.opendc")
+
+    override fun getArtifactId(): Optional<String> = Optional.of("opendc-harness")
+
+    override fun discover(request: EngineDiscoveryRequest, uniqueId: UniqueId): TestDescriptor {
+        // IntelliJ will pass a [MethodSelector] to run just a single method inside a file. In that
+        // case, no experiments should be discovered, since we support only experiments by class.
+        if (request.getSelectorsByType(MethodSelector::class.java).isNotEmpty()) {
+            return ExperimentEngineDescriptor(uniqueId, emptyFlow())
+        }
+
+        val classNames = request.getSelectorsByType(ClassSelector::class.java).map { DiscoverySelector.Meta("class.name", it.className) }
+        val classNameFilters = request.getFiltersByType(ClassNameFilter::class.java).map { DiscoveryFilter.Name(it.toPredicate()) }
+
+        val discovery = DiscoveryProvider.createComposite()
+        val definitions = discovery.discover(DiscoveryRequest(classNames, classNameFilters))
+
+        return ExperimentEngineDescriptor(uniqueId, definitions)
+    }
+
+    override fun execute(request: ExecutionRequest) {
+        logger.debug { "JUnit ExecutionRequest[${request::class.java.name}] [configurationParameters=${request.configurationParameters}; rootTestDescriptor=${request.rootTestDescriptor}]" }
+        val root = request.rootTestDescriptor as ExperimentEngineDescriptor
+        val listener = request.engineExecutionListener
+
+        listener.executionStarted(root)
+
+        try {
+            ExperimentEngineLauncher()
+                .withListener(JUnitExperimentExecutionListener(listener, root))
+                .runBlocking(root.experiments)
+            listener.executionFinished(root, TestExecutionResult.successful())
+        } catch (e: Throwable) {
+            listener.executionFinished(root, TestExecutionResult.failed(e))
+        }
+    }
+
+    private class ExperimentEngineDescriptor(id: UniqueId, val experiments: Flow<ExperimentDefinition>) : EngineDescriptor(id, "opendc") {
+        override fun mayRegisterTests(): Boolean = true
+    }
+}
diff --git a/opendc-harness/opendc-harness-junit5/src/main/resources/META-INF/services/org.junit.platform.engine.TestEngine b/opendc-harness/opendc-harness-junit5/src/main/resources/META-INF/services/org.junit.platform.engine.TestEngine
new file mode 100644
index 00000000..b83eec0c
--- /dev/null
+++ b/opendc-harness/opendc-harness-junit5/src/main/resources/META-INF/services/org.junit.platform.engine.TestEngine
@@ -0,0 +1 @@
+org.opendc.harness.runner.junit5.OpenDCTestEngine