harness: Improve OpenDC Experiment Harness (v1)

This pull request contains several improvements to the OpenDC Experiment Harness, 
which aims to improve user-experience of users when re-running experiments in the repository.

*  Split the harness into separate modules
*  Add support for adding additional classpath entries when running experiments via the console
*  Initial support for configuration of experiments using HOCON.

4 files changed, 318 insertions, 0 deletions

diff --git a/opendc-harness/opendc-harness-cli/build.gradle.kts b/opendc-harness/opendc-harness-cli/build.gradle.kts
new file mode 100644
index 00000000..3e169e2a
--- /dev/null
+++ b/opendc-harness/opendc-harness-cli/build.gradle.kts
@@ -0,0 +1,45 @@
+/*
+ * Copyright (c) 2021 AtLarge Research
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+description = "Command line interface of OpenDC Harness"
+
+/* Build configuration */
+plugins {
+    `kotlin-library-conventions`
+    application
+}
+
+application {
+    applicationName = "opendc-harness"
+    mainClass.set("org.opendc.harness.runner.cli.ConsoleRunnerKt")
+}
+
+dependencies {
+    api(platform(project(":opendc-platform")))
+    api(project(":opendc-harness:opendc-harness-engine"))
+
+    implementation("io.github.microutils:kotlin-logging")
+    implementation("com.github.ajalt.clikt:clikt:${versions["clikt"]}")
+    implementation("me.tongfei:progressbar:${versions["progressbar"]}")
+
+    runtimeOnly("org.apache.logging.log4j:log4j-slf4j-impl:${versions.log4j}")
+}
diff --git a/opendc-harness/opendc-harness-cli/src/main/kotlin/org/opendc/harness/runner/cli/ConsoleExperimentReporter.kt b/opendc-harness/opendc-harness-cli/src/main/kotlin/org/opendc/harness/runner/cli/ConsoleExperimentReporter.kt
new file mode 100644
index 00000000..7177467a
--- /dev/null
+++ b/opendc-harness/opendc-harness-cli/src/main/kotlin/org/opendc/harness/runner/cli/ConsoleExperimentReporter.kt
@@ -0,0 +1,79 @@
+/*
+ * Copyright (c) 2021 AtLarge Research
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+package org.opendc.harness.runner.cli
+
+import me.tongfei.progressbar.ProgressBar
+import me.tongfei.progressbar.ProgressBarBuilder
+import mu.KotlinLogging
+import org.opendc.harness.api.Trial
+import org.opendc.harness.engine.ExperimentExecutionListener
+
+/**
+ * A reporter that reports the experiment progress to the console.
+ */
+public class ConsoleExperimentReporter : ExperimentExecutionListener, AutoCloseable {
+    /**
+     * The active [Trial]s.
+     */
+    private val trials: MutableSet<Trial> = mutableSetOf()
+
+    /**
+     * The total number of runs.
+     */
+    private var total = 0
+
+    /**
+     * The logger for this reporter.
+     */
+    private val logger = KotlinLogging.logger {}
+
+    /**
+     * The progress bar to keep track of the progress.
+     */
+    private val pb: ProgressBar = ProgressBarBuilder()
+        .setTaskName("")
+        .setInitialMax(1)
+        .build()
+
+    override fun trialFinished(trial: Trial, throwable: Throwable?) {
+        trials -= trial
+
+        pb.stepTo(total - trials.size.toLong())
+        if (trials.isEmpty()) {
+            pb.close()
+        }
+
+        if (throwable != null) {
+            logger.warn(throwable) { "Trial $trial failed" }
+        }
+    }
+
+    override fun trialStarted(trial: Trial) {
+        trials += trial
+        pb.maxHint((++total).toLong())
+    }
+
+    override fun close() {
+        pb.close()
+    }
+}
diff --git a/opendc-harness/opendc-harness-cli/src/main/kotlin/org/opendc/harness/runner/cli/ConsoleRunner.kt b/opendc-harness/opendc-harness-cli/src/main/kotlin/org/opendc/harness/runner/cli/ConsoleRunner.kt
new file mode 100644
index 00000000..06800f0e
--- /dev/null
+++ b/opendc-harness/opendc-harness-cli/src/main/kotlin/org/opendc/harness/runner/cli/ConsoleRunner.kt
@@ -0,0 +1,156 @@
+/*
+ * Copyright (c) 2021 AtLarge Research
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+package org.opendc.harness.runner.cli
+
+import com.github.ajalt.clikt.core.CliktCommand
+import com.github.ajalt.clikt.parameters.arguments.argument
+import com.github.ajalt.clikt.parameters.arguments.multiple
+import com.github.ajalt.clikt.parameters.arguments.unique
+import com.github.ajalt.clikt.parameters.options.default
+import com.github.ajalt.clikt.parameters.options.multiple
+import com.github.ajalt.clikt.parameters.options.option
+import com.github.ajalt.clikt.parameters.options.unique
+import com.github.ajalt.clikt.parameters.types.file
+import com.github.ajalt.clikt.parameters.types.int
+import mu.KotlinLogging
+import org.junit.platform.commons.util.ClassLoaderUtils
+import org.opendc.harness.engine.ExperimentEngineLauncher
+import org.opendc.harness.engine.discovery.DiscoveryProvider
+import org.opendc.harness.engine.discovery.DiscoveryRequest
+import org.opendc.harness.engine.discovery.DiscoverySelector
+import org.opendc.harness.engine.scheduler.ThreadPoolExperimentScheduler
+import java.net.URLClassLoader
+
+/**
+ * The logger for this experiment runner.
+ */
+private val logger = KotlinLogging.logger {}
+
+/**
+ * The command line interface for the console experiment runner.
+ */
+public class ConsoleRunner : CliktCommand(name = "opendc-harness") {
+    /**
+     * The number of repeats per scenario.
+     */
+    private val repeats by option("-r", "--repeats", help = "Number of repeats per scenario")
+        .int()
+        .default(1)
+
+    /**
+     * The selected experiments to run by name.
+     */
+    private val experiments by argument(help = "Experiments to explore")
+        .multiple()
+        .unique()
+
+    /**
+     * The maximum number of worker threads to use.
+     */
+    private val parallelism by option("-p", "--parallelism", help = "Maximum number of concurrent simulation runs")
+        .int()
+        .default(Runtime.getRuntime().availableProcessors() - 1)
+
+    /**
+     * Additional classpath entries to load.
+     */
+    private val additionalClasspathEntries by option("--class-path", help = "Additional classpath entries to load")
+        .file(mustExist = true)
+        .multiple()
+        .unique()
+
+    /**
+     * Configuration file to load.
+     */
+    private val config by option("-c", "--config", help = "Configuration file for the experiments")
+        .file(mustExist = true, canBeDir = false, mustBeReadable = true)
+
+    override fun run() {
+        logger.info { "Starting OpenDC Console Experiment Runner" }
+
+        val config = config
+        if (config != null) {
+            System.setProperty("config.file", config.path)
+        }
+
+        val classLoader = createClassLoader()
+        // TODO: Add way to specify class loader for scheduler
+        Thread.currentThread().contextClassLoader = classLoader
+
+        val discovery = DiscoveryProvider.createComposite(classLoader)
+        val experiments = discovery.discover(
+            DiscoveryRequest(
+                selectors = experiments.flatMap {
+                    listOf(DiscoverySelector.Name(it), DiscoverySelector.Meta("class.name", it))
+                }
+            )
+        )
+
+        val reporter = ConsoleExperimentReporter()
+        val scheduler = ThreadPoolExperimentScheduler(parallelism)
+
+        try {
+            ExperimentEngineLauncher()
+                .withListener(reporter)
+                .withRepeats(repeats)
+                .withScheduler(scheduler)
+                .runBlocking(experiments)
+        } catch (e: Throwable) {
+            logger.error(e) { "Failed to finish experiments" }
+        } finally {
+            reporter.close()
+            scheduler.close()
+        }
+
+        logger.info { "Finished all experiments. Exiting." }
+    }
+
+    /**
+     * Create a [ClassLoader] that is used to load the
+     */
+    private fun createClassLoader(): ClassLoader {
+        val parent = ClassLoaderUtils.getDefaultClassLoader()
+
+        if (additionalClasspathEntries.isNotEmpty()) {
+            val urls = additionalClasspathEntries.flatMap { file ->
+                if (file.isDirectory) {
+                    file.walk()
+                        .filter { it.extension == "jar" }
+                        .map { it.toURI().toURL() }
+                        .toList()
+                } else {
+                    listOf(file.toURI().toURL())
+                }
+            }
+
+            return URLClassLoader.newInstance(urls.toTypedArray(), parent)
+        }
+
+        return parent
+    }
+}
+
+/**
+ * Main entry point of the experiment runner.
+ */
+public fun main(args: Array<String>): Unit = ConsoleRunner().main(args)
diff --git a/opendc-harness/opendc-harness-cli/src/main/resources/log4j2.xml b/opendc-harness/opendc-harness-cli/src/main/resources/log4j2.xml
new file mode 100644
index 00000000..32d81416
--- /dev/null
+++ b/opendc-harness/opendc-harness-cli/src/main/resources/log4j2.xml
@@ -0,0 +1,38 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ~ Copyright (c) 2021 AtLarge Research
+  ~
+  ~ Permission is hereby granted, free of charge, to any person obtaining a copy
+  ~ of this software and associated documentation files (the "Software"), to deal
+  ~ in the Software without restriction, including without limitation the rights
+  ~ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+  ~ copies of the Software, and to permit persons to whom the Software is
+  ~ furnished to do so, subject to the following conditions:
+  ~
+  ~ The above copyright notice and this permission notice shall be included in all
+  ~ copies or substantial portions of the Software.
+  ~
+  ~ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+  ~ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+  ~ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+  ~ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+  ~ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+  ~ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+  ~ SOFTWARE.
+  -->
+
+<Configuration status="WARN">
+    <Appenders>
+        <Console name="Console" target="SYSTEM_OUT">
+            <PatternLayout pattern="%d{HH:mm:ss.SSS} [%highlight{%-5level}] %logger{36} - %msg%n" disableAnsi="false"/>
+        </Console>
+    </Appenders>
+    <Loggers>
+        <Logger name="org.opendc" level="info" additivity="false">
+            <AppenderRef ref="Console"/>
+        </Logger>
+        <Root level="error">
+            <AppenderRef ref="Console"/>
+        </Root>
+    </Loggers>
+</Configuration>