Scenario and Portfolio update (#209)

* Initial commit

* Implemented a new systems of defining and running scenarios / portfolios. Scenarios and Portfolios can now be defined using JSON files similar to topologies. This allows user to define experiments without changing any KotLin code.

* Ran spotlessApply

19 files changed, 959 insertions, 66 deletions

diff --git a/opendc-experiments/opendc-experiments-base/src/main/kotlin/org/opendc/experiments/base/portfolio/Portfolio.kt b/opendc-experiments/opendc-experiments-base/src/main/kotlin/org/opendc/experiments/base/models/portfolio/Portfolio.kt
index 961ae106..7b0299c5 100644
--- a/opendc-experiments/opendc-experiments-base/src/main/kotlin/org/opendc/experiments/base/portfolio/Portfolio.kt
+++ b/opendc-experiments/opendc-experiments-base/src/main/kotlin/org/opendc/experiments/base/models/portfolio/Portfolio.kt
@@ -20,16 +20,13 @@
  * SOFTWARE.
  */
 
-package org.opendc.experiments.base.portfolio
+package org.opendc.experiments.base.models.portfolio
 
-import org.opendc.experiments.base.portfolio.model.Scenario
+import org.opendc.experiments.base.models.scenario.Scenario
 
 /**
  * A portfolio represents a collection of scenarios are tested for the work.
  */
-public interface Portfolio {
-    /**
-     * The scenarios that belong to this portfolio.
-     */
-    public val scenarios: Iterable<Scenario>
-}
+public class Portfolio(
+    public val scenarios: Iterable<Scenario>,
+)
diff --git a/opendc-experiments/opendc-experiments-base/src/main/kotlin/org/opendc/experiments/base/portfolio/model/OperationalPhenomena.kt b/opendc-experiments/opendc-experiments-base/src/main/kotlin/org/opendc/experiments/base/models/portfolio/PortfolioFactories.kt
index ea78e556..aee87814 100644
--- a/opendc-experiments/opendc-experiments-base/src/main/kotlin/org/opendc/experiments/base/portfolio/model/OperationalPhenomena.kt
+++ b/opendc-experiments/opendc-experiments-base/src/main/kotlin/org/opendc/experiments/base/models/portfolio/PortfolioFactories.kt
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2021 AtLarge Research
+ * Copyright (c) 2024 AtLarge Research
  *
  * Permission is hereby granted, free of charge, to any person obtaining a copy
  * of this software and associated documentation files (the "Software"), to deal
@@ -20,12 +20,26 @@
  * SOFTWARE.
  */
 
-package org.opendc.experiments.base.portfolio.model
+package org.opendc.experiments.base.models.portfolio
 
-/**
- * Operation phenomena during experiments.
- *
- * @param failureFrequency The average time between failures in hours.
- * @param hasInterference A flag to enable performance interference between VMs.
- */
-public data class OperationalPhenomena(val failureFrequency: Double, val hasInterference: Boolean)
+import org.opendc.experiments.base.models.scenario.getScenario
+import java.io.File
+
+private val porfolioReader = PortfolioReader()
+
+public fun getPortfolio(filePath: String): Portfolio {
+    return getPortfolio(File(filePath))
+}
+
+public fun getPortfolio(file: File): Portfolio {
+    return getPortfolio(porfolioReader.read(file))
+}
+
+public fun getPortfolio(portfolioSpec: PortfolioSpec): Portfolio {
+    return Portfolio(
+        portfolioSpec.scenarios.map {
+                scenario ->
+            getScenario(scenario)
+        },
+    )
+}
diff --git a/opendc-experiments/opendc-experiments-base/src/main/kotlin/org/opendc/experiments/base/models/portfolio/PortfolioReader.kt b/opendc-experiments/opendc-experiments-base/src/main/kotlin/org/opendc/experiments/base/models/portfolio/PortfolioReader.kt
new file mode 100644
index 00000000..767b61bb
--- /dev/null
+++ b/opendc-experiments/opendc-experiments-base/src/main/kotlin/org/opendc/experiments/base/models/portfolio/PortfolioReader.kt
@@ -0,0 +1,48 @@
+/*
+ * Copyright (c) 2024 AtLarge Research
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+package org.opendc.experiments.base.models.portfolio
+
+import kotlinx.serialization.ExperimentalSerializationApi
+import kotlinx.serialization.json.Json
+import kotlinx.serialization.json.decodeFromStream
+import java.io.File
+import java.io.InputStream
+
+public class PortfolioReader {
+    @OptIn(ExperimentalSerializationApi::class)
+    public fun read(file: File): PortfolioSpec {
+        val input = file.inputStream()
+        val obj = Json.decodeFromStream<PortfolioSpec>(input)
+
+        return obj
+    }
+
+    /**
+     * Read the specified [input].
+     */
+    @OptIn(ExperimentalSerializationApi::class)
+    public fun read(input: InputStream): PortfolioSpec {
+        val obj = Json.decodeFromStream<PortfolioSpec>(input)
+        return obj
+    }
+}
diff --git a/opendc-experiments/opendc-experiments-base/src/main/kotlin/org/opendc/experiments/base/portfolio/model/Topology.kt b/opendc-experiments/opendc-experiments-base/src/main/kotlin/org/opendc/experiments/base/models/portfolio/PortfolioSpec.kt
index 0053b541..554442b2 100644
--- a/opendc-experiments/opendc-experiments-base/src/main/kotlin/org/opendc/experiments/base/portfolio/model/Topology.kt
+++ b/opendc-experiments/opendc-experiments-base/src/main/kotlin/org/opendc/experiments/base/models/portfolio/PortfolioSpec.kt
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2021 AtLarge Research
+ * Copyright (c) 2024 AtLarge Research
  *
  * Permission is hereby granted, free of charge, to any person obtaining a copy
  * of this software and associated documentation files (the "Software"), to deal
@@ -20,9 +20,12 @@
  * SOFTWARE.
  */
 
-package org.opendc.experiments.base.portfolio.model
+package org.opendc.experiments.base.models.portfolio
 
-/**
- * The topology on which we simulate the workload.
- */
-public data class Topology(val name: String)
+import kotlinx.serialization.Serializable
+import org.opendc.experiments.base.models.scenario.ScenarioSpec
+
+@Serializable
+public data class PortfolioSpec(
+    val scenarios: List<ScenarioSpec>,
+)
diff --git a/opendc-experiments/opendc-experiments-base/src/main/kotlin/org/opendc/experiments/base/portfolio/model/Workload.kt b/opendc-experiments/opendc-experiments-base/src/main/kotlin/org/opendc/experiments/base/models/scenario/Scenario.kt
index 0dd9df09..192bba1e 100644
--- a/opendc-experiments/opendc-experiments-base/src/main/kotlin/org/opendc/experiments/base/portfolio/model/Workload.kt
+++ b/opendc-experiments/opendc-experiments-base/src/main/kotlin/org/opendc/experiments/base/models/scenario/Scenario.kt
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2021 AtLarge Research
+ * Copyright (c) 2024 AtLarge Research
  *
  * Permission is hereby granted, free of charge, to any person obtaining a copy
  * of this software and associated documentation files (the "Software"), to deal
@@ -20,14 +20,19 @@
  * SOFTWARE.
  */
 
-package org.opendc.experiments.base.portfolio.model
+package org.opendc.experiments.base.models.scenario
 
-import org.opendc.compute.workload.ComputeWorkload
+import org.opendc.compute.simulator.failure.FailureModel
+import org.opendc.compute.topology.specs.HostSpec
 
-/**
- * A single workload originating from a trace.
- *
- * @param name the name of the workload.
- * @param source The source of the workload data.
- */
-public data class Workload(val name: String, val source: ComputeWorkload)
+public data class Scenario(
+    val topology: List<HostSpec>,
+    val workload: WorkloadSpec,
+    val allocationPolicy: AllocationPolicySpec,
+    val failureModel: FailureModel?,
+    val exportModel: ExportSpec = ExportSpec(),
+    val outputFolder: String = "output",
+    val name: String = "",
+    val runs: Int = 1,
+    val initialSeed: Int = 0,
+)
diff --git a/opendc-experiments/opendc-experiments-base/src/main/kotlin/org/opendc/experiments/base/models/scenario/ScenarioFactories.kt b/opendc-experiments/opendc-experiments-base/src/main/kotlin/org/opendc/experiments/base/models/scenario/ScenarioFactories.kt
new file mode 100644
index 00000000..d806e95e
--- /dev/null
+++ b/opendc-experiments/opendc-experiments-base/src/main/kotlin/org/opendc/experiments/base/models/scenario/ScenarioFactories.kt
@@ -0,0 +1,57 @@
+/*
+ * Copyright (c) 2024 AtLarge Research
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+package org.opendc.experiments.base.models.scenario
+
+import org.opendc.compute.simulator.failure.getFailureModel
+import org.opendc.compute.topology.clusterTopology
+import java.io.File
+
+private val scenarioReader = ScenarioReader()
+
+public fun getScenario(filePath: String): Scenario {
+    return getScenario(File(filePath))
+}
+
+public fun getScenario(file: File): Scenario {
+    return getScenario(scenarioReader.read(file))
+}
+
+public fun getScenario(scenarioSpec: ScenarioSpec): Scenario {
+    val topology = clusterTopology(File(scenarioSpec.topology.pathToFile))
+    val workload = scenarioSpec.workload
+    val allocationPolicy = scenarioSpec.allocationPolicy
+    val failureModel = getFailureModel(scenarioSpec.failureModel.failureInterval)
+    val exportModel = scenarioSpec.exportModel
+
+    return Scenario(
+        topology,
+        workload,
+        allocationPolicy,
+        failureModel,
+        exportModel,
+        scenarioSpec.outputFolder,
+        scenarioSpec.name,
+        scenarioSpec.runs,
+        scenarioSpec.initialSeed,
+    )
+}
diff --git a/opendc-experiments/opendc-experiments-base/src/main/kotlin/org/opendc/experiments/base/portfolio/model/Scenario.kt b/opendc-experiments/opendc-experiments-base/src/main/kotlin/org/opendc/experiments/base/models/scenario/ScenarioReader.kt
index cf0f5320..e7c7b4ae 100644
--- a/opendc-experiments/opendc-experiments-base/src/main/kotlin/org/opendc/experiments/base/portfolio/model/Scenario.kt
+++ b/opendc-experiments/opendc-experiments-base/src/main/kotlin/org/opendc/experiments/base/models/scenario/ScenarioReader.kt
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2022 AtLarge Research
+ * Copyright (c) 2024 AtLarge Research
  *
  * Permission is hereby granted, free of charge, to any person obtaining a copy
  * of this software and associated documentation files (the "Software"), to deal
@@ -20,21 +20,29 @@
  * SOFTWARE.
  */
 
-package org.opendc.experiments.base.portfolio.model
+package org.opendc.experiments.base.models.scenario
 
-/**
- * A single scenario of a portfolio.
- *
- * @property topology The topology to test.
- * @property workload The workload to test.
- * @property operationalPhenomena The [OperationalPhenomena] to model.
- * @property allocationPolicy The allocation policy of the scheduler.
- * @property partitions The partition of the scenario.
- */
-public data class Scenario(
-    val topology: Topology,
-    val workload: Workload,
-    val operationalPhenomena: OperationalPhenomena,
-    val allocationPolicy: String,
-    val partitions: Map<String, String> = emptyMap(),
-)
+import kotlinx.serialization.ExperimentalSerializationApi
+import kotlinx.serialization.json.Json
+import kotlinx.serialization.json.decodeFromStream
+import java.io.File
+import java.io.InputStream
+
+public class ScenarioReader {
+    @OptIn(ExperimentalSerializationApi::class)
+    public fun read(file: File): ScenarioSpec {
+        val input = file.inputStream()
+        val obj = Json.decodeFromStream<ScenarioSpec>(input)
+
+        return obj
+    }
+
+    /**
+     * Read the specified [input].
+     */
+    @OptIn(ExperimentalSerializationApi::class)
+    public fun read(input: InputStream): ScenarioSpec {
+        val obj = Json.decodeFromStream<ScenarioSpec>(input)
+        return obj
+    }
+}
diff --git a/opendc-experiments/opendc-experiments-base/src/main/kotlin/org/opendc/experiments/base/models/scenario/ScenarioSpecs.kt b/opendc-experiments/opendc-experiments-base/src/main/kotlin/org/opendc/experiments/base/models/scenario/ScenarioSpecs.kt
new file mode 100644
index 00000000..20c8a6e0
--- /dev/null
+++ b/opendc-experiments/opendc-experiments-base/src/main/kotlin/org/opendc/experiments/base/models/scenario/ScenarioSpecs.kt
@@ -0,0 +1,165 @@
+/*
+ * Copyright (c) 2024 AtLarge Research
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+package org.opendc.experiments.base.models.scenario
+
+import kotlinx.serialization.Serializable
+import org.opendc.compute.service.scheduler.ComputeSchedulerEnum
+import org.opendc.compute.workload.ComputeWorkload
+import org.opendc.compute.workload.sampleByLoad
+import org.opendc.compute.workload.trace
+import java.io.File
+
+/**
+ * specification describing a scenario
+ *
+ * @property topology
+ * @property workload
+ * @property allocationPolicy
+ * @property failureModel
+ * @property exportModel
+ * @property outputFolder
+ * @property initialSeed
+ * @property runs
+ */
+@Serializable
+public data class ScenarioSpec(
+    val topology: TopologySpec,
+    val workload: WorkloadSpec,
+    val allocationPolicy: AllocationPolicySpec,
+    val failureModel: FailureModelSpec = FailureModelSpec(),
+    val exportModel: ExportSpec = ExportSpec(),
+    val outputFolder: String = "output",
+    val initialSeed: Int = 0,
+    val runs: Int = 1,
+    var name: String = "",
+) {
+    init {
+        require(runs > 0) { "The number of runs should always be positive" }
+
+        // generate name if not provided
+        if (name == "") {
+            name = "workload=${workload.name}_topology=${topology.name}_allocationPolicy=${allocationPolicy.name}"
+        }
+    }
+}
+
+/**
+ * specification describing a topology
+ *
+ * @property pathToFile
+ */
+@Serializable
+public data class TopologySpec(
+    val pathToFile: String,
+) {
+    public val name: String = File(pathToFile).nameWithoutExtension
+
+    init {
+        require(File(pathToFile).exists()) { "The provided path to the topology: $pathToFile does not exist " }
+    }
+}
+
+/**
+ * specification describing a workload
+ *
+ * @property pathToFile
+ * @property type
+ */
+@Serializable
+public data class WorkloadSpec(
+    val pathToFile: String,
+    val type: WorkloadTypes,
+) {
+    public val name: String = File(pathToFile).nameWithoutExtension
+
+    init {
+        require(File(pathToFile).exists()) { "The provided path to the workload: $pathToFile does not exist " }
+    }
+}
+
+/**
+ * specification describing a workload type
+ *
+ * @constructor Create empty Workload types
+ */
+public enum class WorkloadTypes {
+    /**
+     * Compute workload
+     *
+     * @constructor Create empty Compute workload
+     */
+    ComputeWorkload,
+}
+
+/**
+ *
+ *TODO: move to separate file
+ * @param type
+ */
+public fun getWorkloadType(type: WorkloadTypes): ComputeWorkload {
+    return when (type) {
+        WorkloadTypes.ComputeWorkload -> trace("trace").sampleByLoad(1.0)
+    }
+}
+
+/**
+ * specification describing how tasks are allocated
+ *
+ * @property policyType
+ *
+ * TODO: expand with more variables such as allowed over-subscription
+ */
+@Serializable
+public data class AllocationPolicySpec(
+    val policyType: ComputeSchedulerEnum,
+) {
+    public val name: String = policyType.toString()
+}
+
+/**
+ * specification describing the failure model
+ *
+ * @property failureInterval The interval between failures in s. Should be 0.0 or higher
+ */
+@Serializable
+public data class FailureModelSpec(
+    val failureInterval: Double = 0.0,
+) {
+    init {
+        require(failureInterval >= 0.0) { "failure frequency cannot be lower than 0" }
+    }
+}
+
+/**
+ * specification describing how the results should be exported
+ *
+ * @property exportInterval The interval of exporting results in s. Should be higher than 0.0
+ */
+@Serializable
+public data class ExportSpec(
+    val exportInterval: Long = 5 * 60,
+) {
+    init {
+        require(exportInterval > 0) { "The Export interval has to be higher than 0" }
+    }
+}
diff --git a/opendc-experiments/opendc-experiments-base/src/main/kotlin/org/opendc/experiments/base/runner/TraceHelpers.kt b/opendc-experiments/opendc-experiments-base/src/main/kotlin/org/opendc/experiments/base/runner/ScenarioHelpers.kt
index ddfa35cc..a6a05d78 100644
--- a/opendc-experiments/opendc-experiments-base/src/main/kotlin/org/opendc/experiments/base/runner/TraceHelpers.kt
+++ b/opendc-experiments/opendc-experiments-base/src/main/kotlin/org/opendc/experiments/base/runner/ScenarioHelpers.kt
@@ -77,7 +77,6 @@ public class RunningServerWatcher : ServerWatcher {
  * @param seed The seed to use for randomness.
  * @param submitImmediately A flag to indicate that the servers are scheduled immediately (so not at their start time).
  * @param failureModel A failure model to use for injecting failures.
- * @param interference A flag to indicate that VM interference needs to be enabled.
  */
 public suspend fun ComputeService.replay(
     clock: InstantSource,
@@ -85,7 +84,6 @@ public suspend fun ComputeService.replay(
     seed: Long,
     submitImmediately: Boolean = false,
     failureModel: FailureModel? = null,
-    interference: Boolean = false,
 ) {
     val injector = failureModel?.createInjector(coroutineContext, clock, this, Random(seed))
     val client = newClient()
@@ -120,11 +118,6 @@ public suspend fun ComputeService.replay(
                 val workload = entry.trace.createWorkload(start)
                 val meta = mutableMapOf<String, Any>("workload" to workload)
 
-                val interferenceProfile = entry.interferenceProfile
-                if (interference && interferenceProfile != null) {
-                    meta["interference-profile"] = interferenceProfile
-                }
-
                 launch {
                     val server =
                         client.newServer(
diff --git a/opendc-experiments/opendc-experiments-base/src/main/kotlin/org/opendc/experiments/base/runner/ScenarioRunner.kt b/opendc-experiments/opendc-experiments-base/src/main/kotlin/org/opendc/experiments/base/runner/ScenarioRunner.kt
new file mode 100644
index 00000000..3dce2bf1
--- /dev/null
+++ b/opendc-experiments/opendc-experiments-base/src/main/kotlin/org/opendc/experiments/base/runner/ScenarioRunner.kt
@@ -0,0 +1,142 @@
+/*
+ * Copyright (c) 2022 AtLarge Research
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+package org.opendc.experiments.base.runner
+
+import me.tongfei.progressbar.ProgressBarBuilder
+import me.tongfei.progressbar.ProgressBarStyle
+import org.opendc.compute.service.ComputeService
+import org.opendc.compute.service.scheduler.ComputeSchedulerEnum
+import org.opendc.compute.service.scheduler.createComputeScheduler
+import org.opendc.compute.simulator.provisioner.Provisioner
+import org.opendc.compute.simulator.provisioner.registerComputeMonitor
+import org.opendc.compute.simulator.provisioner.setupComputeService
+import org.opendc.compute.simulator.provisioner.setupHosts
+import org.opendc.compute.telemetry.export.parquet.ParquetComputeMonitor
+import org.opendc.compute.workload.ComputeWorkloadLoader
+import org.opendc.experiments.base.models.portfolio.Portfolio
+import org.opendc.experiments.base.models.scenario.Scenario
+import org.opendc.experiments.base.models.scenario.getWorkloadType
+import org.opendc.simulator.kotlin.runSimulation
+import java.io.File
+import java.time.Duration
+import java.util.Random
+import java.util.concurrent.ForkJoinPool
+import java.util.stream.LongStream
+
+public fun runPortfolio(
+    portfolio: Portfolio,
+    parallelism: Int,
+) {
+    val pool = ForkJoinPool(parallelism)
+
+    for (scenario in portfolio.scenarios) {
+        runScenario(scenario, pool)
+    }
+}
+
+/**
+ * Run scenario when no pool is available for parallel execution
+ *
+ * @param scenario The scenario to run
+ * @param parallelism The number of scenarios that can be run in parallel
+ */
+public fun runScenario(
+    scenario: Scenario,
+    parallelism: Int,
+) {
+    val pool = ForkJoinPool(parallelism)
+    runScenario(scenario, pool)
+}
+
+/**
+ * Run scenario when a pool is available for parallel execution
+ * The scenario is run multiple times based on the user input
+ *
+ * @param scenario The scenario to run
+ * @param pool The pool on which to run the scenarios
+ */
+public fun runScenario(
+    scenario: Scenario,
+    pool: ForkJoinPool,
+) {
+    val pb =
+        ProgressBarBuilder()
+            .setInitialMax(scenario.runs.toLong())
+            .setStyle(ProgressBarStyle.ASCII)
+            .setTaskName("Simulating...")
+            .build()
+
+    pool.submit {
+        LongStream.range(0, scenario.runs.toLong())
+            .parallel()
+            .forEach {
+                runScenario(scenario, scenario.initialSeed + it)
+                pb.step()
+            }
+
+        pb.close()
+    }.join()
+}
+
+/**
+ * Run a single scenario with a specific seed
+ *
+ * @param scenario The scenario to run
+ * @param seed The starting seed of the random generator.
+ */
+public fun runScenario(
+    scenario: Scenario,
+    seed: Long,
+): Unit =
+    runSimulation {
+        val serviceDomain = "compute.opendc.org"
+
+        Provisioner(dispatcher, seed).use { provisioner ->
+
+            provisioner.runSteps(
+                setupComputeService(serviceDomain, { createComputeScheduler(ComputeSchedulerEnum.Mem, Random(it.seeder.nextLong())) }),
+                setupHosts(serviceDomain, scenario.topology, optimize = true),
+            )
+
+            val partition = scenario.name + "/seed=$seed"
+
+            provisioner.runStep(
+                registerComputeMonitor(
+                    serviceDomain,
+                    ParquetComputeMonitor(
+                        File(scenario.outputFolder),
+                        partition,
+                        bufferSize = 4096,
+                    ),
+                    Duration.ofSeconds(scenario.exportModel.exportInterval),
+                ),
+            )
+
+            val service = provisioner.registry.resolve(serviceDomain, ComputeService::class.java)!!
+
+            val workloadLoader = ComputeWorkloadLoader(File(scenario.workload.pathToFile))
+            val vms = getWorkloadType(scenario.workload.type).resolve(workloadLoader, Random(seed))
+
+            service.replay(timeSource, vms, seed, failureModel = scenario.failureModel)
+        }
+    }
diff --git a/opendc-experiments/opendc-experiments-base/src/main/output/host/seed=0/data.parquet b/opendc-experiments/opendc-experiments-base/src/main/output/host/seed=0/data.parquet
deleted file mode 100644
index d3c19ab4..00000000
--- a/opendc-experiments/opendc-experiments-base/src/main/output/host/seed=0/data.parquet
+++ /dev/null
diff --git a/opendc-experiments/opendc-experiments-base/src/main/output/server/seed=0/data.parquet b/opendc-experiments/opendc-experiments-base/src/main/output/server/seed=0/data.parquet
deleted file mode 100644
index 6049e8cb..00000000
--- a/opendc-experiments/opendc-experiments-base/src/main/output/server/seed=0/data.parquet
+++ /dev/null
diff --git a/opendc-experiments/opendc-experiments-base/src/main/output/service/seed=0/data.parquet b/opendc-experiments/opendc-experiments-base/src/main/output/service/seed=0/data.parquet
deleted file mode 100644
index 969954bb..00000000
--- a/opendc-experiments/opendc-experiments-base/src/main/output/service/seed=0/data.parquet
+++ /dev/null
diff --git a/opendc-experiments/opendc-experiments-base/src/test/kotlin/org/opendc/experiments/base/ScenarioIntegrationTest.kt b/opendc-experiments/opendc-experiments-base/src/test/kotlin/org/opendc/experiments/base/ScenarioIntegrationTest.kt
new file mode 100644
index 00000000..d67ed727
--- /dev/null
+++ b/opendc-experiments/opendc-experiments-base/src/test/kotlin/org/opendc/experiments/base/ScenarioIntegrationTest.kt
@@ -0,0 +1,298 @@
+/*
+ * Copyright (c) 2020 AtLarge Research
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+package org.opendc.experiments.base
+
+import org.junit.jupiter.api.Assertions.assertEquals
+import org.junit.jupiter.api.BeforeEach
+import org.junit.jupiter.api.Test
+import org.junit.jupiter.api.assertAll
+import org.opendc.compute.service.ComputeService
+import org.opendc.compute.service.scheduler.FilterScheduler
+import org.opendc.compute.service.scheduler.filters.ComputeFilter
+import org.opendc.compute.service.scheduler.filters.RamFilter
+import org.opendc.compute.service.scheduler.filters.VCpuFilter
+import org.opendc.compute.service.scheduler.weights.CoreRamWeigher
+import org.opendc.compute.simulator.failure.getFailureModel
+import org.opendc.compute.simulator.provisioner.Provisioner
+import org.opendc.compute.simulator.provisioner.registerComputeMonitor
+import org.opendc.compute.simulator.provisioner.setupComputeService
+import org.opendc.compute.simulator.provisioner.setupHosts
+import org.opendc.compute.telemetry.ComputeMonitor
+import org.opendc.compute.telemetry.table.HostTableReader
+import org.opendc.compute.telemetry.table.ServiceTableReader
+import org.opendc.compute.topology.clusterTopology
+import org.opendc.compute.topology.specs.HostSpec
+import org.opendc.compute.workload.ComputeWorkloadLoader
+import org.opendc.compute.workload.VirtualMachine
+import org.opendc.compute.workload.sampleByLoad
+import org.opendc.compute.workload.trace
+import org.opendc.experiments.base.runner.replay
+import org.opendc.simulator.kotlin.runSimulation
+import java.io.File
+import java.util.Random
+
+/**
+ * An integration test suite for the Scenario experiments.
+ */
+class ScenarioIntegrationTest {
+    /**
+     * The monitor used to keep track of the metrics.
+     */
+    private lateinit var monitor: TestComputeMonitor
+
+    /**
+     * The [FilterScheduler] to use for all experiments.
+     */
+    private lateinit var computeScheduler: FilterScheduler
+
+    /**
+     * The [ComputeWorkloadLoader] responsible for loading the traces.
+     */
+    private lateinit var workloadLoader: ComputeWorkloadLoader
+
+    /**
+     * Set up the experimental environment.
+     */
+    @BeforeEach
+    fun setUp() {
+        monitor = TestComputeMonitor()
+        computeScheduler =
+            FilterScheduler(
+                filters = listOf(ComputeFilter(), VCpuFilter(16.0), RamFilter(1.0)),
+                weighers = listOf(CoreRamWeigher(multiplier = 1.0)),
+            )
+        workloadLoader = ComputeWorkloadLoader(File("src/test/resources/trace"))
+    }
+
+    /**
+     * Test a large simulation setup.
+     */
+    @Test
+    fun testLarge() =
+        runSimulation {
+            val seed = 0L
+            val workload = createTestWorkload(1.0, seed)
+            val topology = createTopology("multi.json")
+            val monitor = monitor
+
+            Provisioner(dispatcher, seed).use { provisioner ->
+                provisioner.runSteps(
+                    setupComputeService(serviceDomain = "compute.opendc.org", { computeScheduler }),
+                    registerComputeMonitor(serviceDomain = "compute.opendc.org", monitor),
+                    setupHosts(serviceDomain = "compute.opendc.org", topology),
+                )
+
+                val service = provisioner.registry.resolve("compute.opendc.org", ComputeService::class.java)!!
+                service.replay(timeSource, workload, seed)
+            }
+
+            println(
+                "Scheduler " +
+                    "Success=${monitor.attemptsSuccess} " +
+                    "Failure=${monitor.attemptsFailure} " +
+                    "Error=${monitor.attemptsError} " +
+                    "Pending=${monitor.serversPending} " +
+                    "Active=${monitor.serversActive}",
+            )
+
+            // Note that these values have been verified beforehand
+            assertAll(
+                { assertEquals(50, monitor.attemptsSuccess, "The scheduler should schedule 50 VMs") },
+                { assertEquals(0, monitor.serversActive, "All VMs should finish after a run") },
+                { assertEquals(0, monitor.attemptsFailure, "No VM should be unscheduled") },
+                { assertEquals(0, monitor.serversPending, "No VM should not be in the queue") },
+                { assertEquals(223379991650, monitor.idleTime) { "Incorrect idle time" } },
+                { assertEquals(66977091124, monitor.activeTime) { "Incorrect active time" } },
+                { assertEquals(3160267873, monitor.stealTime) { "Incorrect steal time" } },
+                { assertEquals(0, monitor.lostTime) { "Incorrect lost time" } },
+                { assertEquals(2.5892214E7, monitor.powerDraw, 1E4) { "Incorrect power draw" } },
+                { assertEquals(7.7672373E9, monitor.energyUsage, 1E4) { "Incorrect energy usage" } },
+            )
+        }
+
+    /**
+     * Test a small simulation setup.
+     */
+    @Test
+    fun testSmall() =
+        runSimulation {
+            val seed = 1L
+            val workload = createTestWorkload(0.25, seed)
+            val topology = createTopology("single.json")
+            val monitor = monitor
+
+            Provisioner(dispatcher, seed).use { provisioner ->
+                provisioner.runSteps(
+                    setupComputeService(serviceDomain = "compute.opendc.org", { computeScheduler }),
+                    registerComputeMonitor(serviceDomain = "compute.opendc.org", monitor),
+                    setupHosts(serviceDomain = "compute.opendc.org", topology),
+                )
+
+                val service = provisioner.registry.resolve("compute.opendc.org", ComputeService::class.java)!!
+                service.replay(timeSource, workload, seed)
+            }
+
+            println(
+                "Scheduler " +
+                    "Success=${monitor.attemptsSuccess} " +
+                    "Failure=${monitor.attemptsFailure} " +
+                    "Error=${monitor.attemptsError} " +
+                    "Pending=${monitor.serversPending} " +
+                    "Active=${monitor.serversActive}",
+            )
+
+            // Note that these values have been verified beforehand
+            assertAll(
+                { assertEquals(10996730092, monitor.idleTime) { "Idle time incorrect" } },
+                { assertEquals(9741285381, monitor.activeTime) { "Active time incorrect" } },
+                { assertEquals(152, monitor.stealTime) { "Steal time incorrect" } },
+                { assertEquals(0, monitor.lostTime) { "Lost time incorrect" } },
+                { assertEquals(2644612.0, monitor.powerDraw, 1E4) { "Incorrect power draw" } },
+                { assertEquals(7.9336867E8, monitor.energyUsage, 1E4) { "Incorrect energy usage" } },
+            )
+        }
+
+    /**
+     * Test a small simulation setup with interference.
+     * TODO: Interference is currently removed from OpenDC. Reactivate when interference is back in.
+     */
+    fun testInterference() =
+        runSimulation {
+            val seed = 0L
+            val workload = createTestWorkload(1.0, seed)
+            val topology = createTopology("single.json")
+
+            Provisioner(dispatcher, seed).use { provisioner ->
+                provisioner.runSteps(
+                    setupComputeService(serviceDomain = "compute.opendc.org", { computeScheduler }),
+                    registerComputeMonitor(serviceDomain = "compute.opendc.org", monitor),
+                    setupHosts(serviceDomain = "compute.opendc.org", topology),
+                )
+
+                val service = provisioner.registry.resolve("compute.opendc.org", ComputeService::class.java)!!
+                service.replay(timeSource, workload, seed)
+            }
+
+            println(
+                "Scheduler " +
+                    "Success=${monitor.attemptsSuccess} " +
+                    "Failure=${monitor.attemptsFailure} " +
+                    "Error=${monitor.attemptsError} " +
+                    "Pending=${monitor.serversPending} " +
+                    "Active=${monitor.serversActive}",
+            )
+
+            // Note that these values have been verified beforehand
+            assertAll(
+                { assertEquals(42814948316, monitor.idleTime) { "Idle time incorrect" } },
+                { assertEquals(40138266225, monitor.activeTime) { "Active time incorrect" } },
+                { assertEquals(23489356981, monitor.stealTime) { "Steal time incorrect" } },
+                { assertEquals(0, monitor.lostTime) { "Lost time incorrect" } },
+            )
+        }
+
+    /**
+     * Test a small simulation setup with failures.
+     */
+    @Test
+    fun testFailures() =
+        runSimulation {
+            val seed = 0L
+            val topology = createTopology("single.json")
+            val workload = createTestWorkload(0.25, seed)
+            val monitor = monitor
+
+            Provisioner(dispatcher, seed).use { provisioner ->
+                provisioner.runSteps(
+                    setupComputeService(serviceDomain = "compute.opendc.org", { computeScheduler }),
+                    registerComputeMonitor(serviceDomain = "compute.opendc.org", monitor),
+                    setupHosts(serviceDomain = "compute.opendc.org", topology),
+                )
+
+                val service = provisioner.registry.resolve("compute.opendc.org", ComputeService::class.java)!!
+                service.replay(timeSource, workload, seed, failureModel = getFailureModel(7.0 * 24 * 60 * 60))
+            }
+
+            // Note that these values have been verified beforehand
+            assertAll(
+                { assertEquals(1404277711, monitor.idleTime) { "Idle time incorrect" } },
+                { assertEquals(1478675712, monitor.activeTime) { "Active time incorrect" } },
+                { assertEquals(152, monitor.stealTime) { "Steal time incorrect" } },
+                { assertEquals(0, monitor.lostTime) { "Lost time incorrect" } },
+                { assertEquals(360369187, monitor.uptime) { "Uptime incorrect" } },
+            )
+        }
+
+    /**
+     * Obtain the trace reader for the test.
+     */
+    private fun createTestWorkload(
+        fraction: Double,
+        seed: Long,
+    ): List<VirtualMachine> {
+        val source = trace("bitbrains-small").sampleByLoad(fraction)
+        return source.resolve(workloadLoader, Random(seed))
+    }
+
+    /**
+     * Obtain the topology factory for the test.
+     */
+    private fun createTopology(name: String): List<HostSpec> {
+        val stream = checkNotNull(object {}.javaClass.getResourceAsStream("/env/$name"))
+        return stream.use { clusterTopology(stream) }
+    }
+
+    class TestComputeMonitor : ComputeMonitor {
+        var attemptsSuccess = 0
+        var attemptsFailure = 0
+        var attemptsError = 0
+        var serversPending = 0
+        var serversActive = 0
+
+        override fun record(reader: ServiceTableReader) {
+            attemptsSuccess = reader.attemptsSuccess
+            attemptsFailure = reader.attemptsFailure
+            attemptsError = reader.attemptsError
+            serversPending = reader.serversPending
+            serversActive = reader.serversActive
+        }
+
+        var idleTime = 0L
+        var activeTime = 0L
+        var stealTime = 0L
+        var lostTime = 0L
+        var powerDraw = 0.0
+        var energyUsage = 0.0
+        var uptime = 0L
+
+        override fun record(reader: HostTableReader) {
+            idleTime += reader.cpuIdleTime
+            activeTime += reader.cpuActiveTime
+            stealTime += reader.cpuStealTime
+            lostTime += reader.cpuLostTime
+            powerDraw += reader.powerDraw
+            energyUsage += reader.energyUsage
+            uptime += reader.uptime
+        }
+    }
+}
diff --git a/opendc-experiments/opendc-experiments-base/src/test/kotlin/org/opendc/experiments/base/ScenarioRunnerTest.kt b/opendc-experiments/opendc-experiments-base/src/test/kotlin/org/opendc/experiments/base/ScenarioRunnerTest.kt
new file mode 100644
index 00000000..f10ab310
--- /dev/null
+++ b/opendc-experiments/opendc-experiments-base/src/test/kotlin/org/opendc/experiments/base/ScenarioRunnerTest.kt
@@ -0,0 +1,79 @@
+/*
+ * Copyright (c) 2022 AtLarge Research
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+package org.opendc.experiments.base
+
+import java.io.File
+
+/**
+ * Test suite for [ScenarioRunner].
+ */
+class ScenarioRunnerTest {
+    /**
+     * The path to the environments.
+     */
+    private val envPath = File("src/test/resources/env")
+
+    /**
+     * The path to the traces.
+     */
+    private val tracePath = File("src/test/resources/trace")
+
+    /**
+     * Smoke test with output.
+     * fixme: Fix failures and enable
+     *
+     fun testSmoke() {
+     val outputPath = Files.createTempDirectory("output").toFile()
+
+     try {
+     val runner = ScenarioRunner(envPath, tracePath, outputPath)
+     val scenario = Scenario(
+     Topology("topology"),
+     Workload("bitbrains-small", trace("bitbrains-small")),
+     OperationalPhenomena(failureFrequency = 24.0 * 7, hasInterference = true),
+     "active-servers"
+     )
+
+     assertDoesNotThrow { runner.runScenario(scenario, seed = 0L) }
+     } finally {
+     outputPath.delete()
+     }
+     }
+
+     /**
+     * Smoke test without output.
+     * fixme: Fix failures and enable
+     */
+     fun testSmokeNoOutput() {
+     val runner = ScenarioRunner(envPath, tracePath, null)
+     val scenario = Scenario(
+     Topology("topology"),
+     Workload("bitbrains-small", trace("bitbrains-small")),
+     OperationalPhenomena(failureFrequency = 24.0 * 7, hasInterference = true),
+     "active-servers"
+     )
+
+     assertDoesNotThrow { runner.runScenario(scenario, seed = 0L) }
+     }
+     */
+}
diff --git a/opendc-experiments/opendc-experiments-base/src/test/resources/env/multi.json b/opendc-experiments/opendc-experiments-base/src/test/resources/env/multi.json
new file mode 100644
index 00000000..721005b0
--- /dev/null
+++ b/opendc-experiments/opendc-experiments-base/src/test/resources/env/multi.json
@@ -0,0 +1,66 @@
+{
+    "clusters":
+    [
+        {
+            "name": "C01",
+            "hosts" :
+            [
+                {
+                    "name": "H01",
+                    "cpus":
+                    [
+                        {
+                            "coreCount": 32,
+                            "coreSpeed": 3200
+                        }
+                    ],
+                    "memory": {
+                        "memorySize": 256000
+                    }
+                }
+            ]
+        },
+        {
+            "name": "C02",
+            "hosts" :
+            [
+                {
+                    "name": "H02",
+                    "count": 6,
+                    "cpus":
+                    [
+                        {
+                            "coreCount": 8,
+                            "coreSpeed": 2930
+                        }
+                    ],
+                    "memory": {
+                        "memorySize": 64000
+                    }
+                }
+            ]
+        },
+        {
+            "name": "C03",
+            "hosts" :
+            [
+                {
+                    "name": "H03",
+                    "count": 2,
+                    "cpus":
+                    [
+                        {
+                            "coreCount": 16,
+                            "coreSpeed": 3200
+                        }
+                    ],
+                    "memory": {
+                        "memorySize": 128000
+                    }
+                }
+            ]
+        }
+    ]
+}
+
+
diff --git a/opendc-experiments/opendc-experiments-base/src/test/resources/env/single.json b/opendc-experiments/opendc-experiments-base/src/test/resources/env/single.json
new file mode 100644
index 00000000..a1c8d95a
--- /dev/null
+++ b/opendc-experiments/opendc-experiments-base/src/test/resources/env/single.json
@@ -0,0 +1,26 @@
+{
+    "clusters":
+    [
+        {
+            "name": "C01",
+            "hosts" :
+            [
+                {
+                    "name": "H01",
+                    "cpus":
+                    [
+                        {
+                            "coreCount": 8,
+                            "coreSpeed": 3200
+                        }
+                    ],
+                    "memory": {
+                        "memorySize": 128000
+                    }
+                }
+            ]
+        }
+    ]
+}
+
+
diff --git a/opendc-experiments/opendc-experiments-base/src/test/resources/env/single.txt b/opendc-experiments/opendc-experiments-base/src/test/resources/env/single.txt
deleted file mode 100644
index 5642003d..00000000
--- a/opendc-experiments/opendc-experiments-base/src/test/resources/env/single.txt
+++ /dev/null
@@ -1,3 +0,0 @@
-ClusterID;ClusterName;Cores;Speed;Memory;numberOfHosts;memoryCapacityPerHost;coreCountPerHost
-A01;A01;8;3.2;128;1;128;8
-
diff --git a/opendc-experiments/opendc-experiments-base/src/test/resources/env/topology.txt b/opendc-experiments/opendc-experiments-base/src/test/resources/env/topology.txt
deleted file mode 100644
index 6b347bff..00000000
--- a/opendc-experiments/opendc-experiments-base/src/test/resources/env/topology.txt
+++ /dev/null
@@ -1,5 +0,0 @@
-ClusterID;ClusterName;Cores;Speed;Memory;numberOfHosts;memoryCapacityPerHost;coreCountPerHost
-A01;A01;32;3.2;2048;1;256;32
-B01;B01;48;2.93;1256;6;64;8
-C01;C01;32;3.2;2048;2;128;16
-