Added max number of failures (#254)

* Added a max failure for tasks. If tasks fail more times, they get cancelled

* Added maxNumFailures to the frontend

* Updated tests

6 files changed, 22 insertions, 21 deletions

diff --git a/opendc-experiments/opendc-experiments-base/src/main/kotlin/org/opendc/experiments/base/runner/ScenarioReplayer.kt b/opendc-experiments/opendc-experiments-base/src/main/kotlin/org/opendc/experiments/base/runner/ScenarioReplayer.kt
index f5829f6d..49fa409e 100644
--- a/opendc-experiments/opendc-experiments-base/src/main/kotlin/org/opendc/experiments/base/runner/ScenarioReplayer.kt
+++ b/opendc-experiments/opendc-experiments-base/src/main/kotlin/org/opendc/experiments/base/runner/ScenarioReplayer.kt
@@ -125,13 +125,6 @@ public suspend fun ComputeService.replay(
                 val checkpointDuration = checkpointModelSpec?.checkpointDuration ?: 0L
                 val checkpointIntervalScaling = checkpointModelSpec?.checkpointIntervalScaling ?: 1.0
 
-//                val workload = SimRuntimeWorkload(
-//                    entry.duration,
-//                    1.0,
-//                    checkpointTime,
-//                    checkpointWait
-//                )
-
                 val workload = entry.trace.createWorkload(start, checkpointInterval, checkpointDuration, checkpointIntervalScaling)
                 val meta = mutableMapOf<String, Any>("workload" to workload)
 
diff --git a/opendc-experiments/opendc-experiments-base/src/main/kotlin/org/opendc/experiments/base/runner/ScenarioRunner.kt b/opendc-experiments/opendc-experiments-base/src/main/kotlin/org/opendc/experiments/base/runner/ScenarioRunner.kt
index 074fe49c..2bd9dfa3 100644
--- a/opendc-experiments/opendc-experiments-base/src/main/kotlin/org/opendc/experiments/base/runner/ScenarioRunner.kt
+++ b/opendc-experiments/opendc-experiments-base/src/main/kotlin/org/opendc/experiments/base/runner/ScenarioRunner.kt
@@ -87,6 +87,7 @@ public fun runScenario(
                 setupComputeService(
                     serviceDomain,
                     { createComputeScheduler(scenario.allocationPolicySpec.policyType, Random(it.seeder.nextLong())) },
+                    maxNumFailures = scenario.maxNumFailures,
                 ),
                 setupHosts(serviceDomain, topology, optimize = true),
             )
diff --git a/opendc-experiments/opendc-experiments-base/src/main/kotlin/org/opendc/experiments/base/scenario/ExperimentFactories.kt b/opendc-experiments/opendc-experiments-base/src/main/kotlin/org/opendc/experiments/base/scenario/ExperimentFactories.kt
index b364ffe6..ca0578a2 100644
--- a/opendc-experiments/opendc-experiments-base/src/main/kotlin/org/opendc/experiments/base/scenario/ExperimentFactories.kt
+++ b/opendc-experiments/opendc-experiments-base/src/main/kotlin/org/opendc/experiments/base/scenario/ExperimentFactories.kt
@@ -71,18 +71,19 @@ public fun getExperiment(experimentSpec: ExperimentSpec): List<Scenario> {
         val scenario =
             Scenario(
                 id = scenarioID,
+                name = scenarioID.toString(),
+                outputFolder = outputFolder,
+                runs = experimentSpec.runs,
+                initialSeed = experimentSpec.initialSeed,
+                computeExportConfig = scenarioSpec.computeExportConfig,
                 topologySpec = scenarioSpec.topology,
                 workloadSpec = scenarioSpec.workload,
                 allocationPolicySpec = scenarioSpec.allocationPolicy,
+                exportModelSpec = scenarioSpec.exportModel,
                 failureModelSpec = scenarioSpec.failureModel,
                 checkpointModelSpec = scenarioSpec.checkpointModel,
                 carbonTracePath = scenarioSpec.carbonTracePath,
-                exportModelSpec = scenarioSpec.exportModel,
-                outputFolder = outputFolder,
-                name = scenarioID.toString(),
-                runs = experimentSpec.runs,
-                initialSeed = experimentSpec.initialSeed,
-                computeExportConfig = scenarioSpec.computeExportConfig,
+                maxNumFailures = scenarioSpec.maxNumFailures,
             )
         trackScenario(scenarioSpec, outputFolder)
         scenarios.add(scenario)
diff --git a/opendc-experiments/opendc-experiments-base/src/main/kotlin/org/opendc/experiments/base/scenario/Scenario.kt b/opendc-experiments/opendc-experiments-base/src/main/kotlin/org/opendc/experiments/base/scenario/Scenario.kt
index c31f0300..91cd09ba 100644
--- a/opendc-experiments/opendc-experiments-base/src/main/kotlin/org/opendc/experiments/base/scenario/Scenario.kt
+++ b/opendc-experiments/opendc-experiments-base/src/main/kotlin/org/opendc/experiments/base/scenario/Scenario.kt
@@ -46,16 +46,17 @@ import org.opendc.experiments.base.scenario.specs.WorkloadSpec
  */
 public data class Scenario(
     var id: Int = -1,
+    val name: String = "",
+    val outputFolder: String = "output",
+    val runs: Int = 1,
+    val initialSeed: Int = 0,
+    val computeExportConfig: ComputeExportConfig,
     val topologySpec: ScenarioTopologySpec,
     val workloadSpec: WorkloadSpec,
     val allocationPolicySpec: AllocationPolicySpec,
+    val exportModelSpec: ExportModelSpec = ExportModelSpec(),
     val failureModelSpec: FailureModelSpec?,
     val checkpointModelSpec: CheckpointModelSpec?,
     val carbonTracePath: String? = null,
-    val exportModelSpec: ExportModelSpec = ExportModelSpec(),
-    val outputFolder: String = "output",
-    val computeExportConfig: ComputeExportConfig,
-    val name: String = "",
-    val runs: Int = 1,
-    val initialSeed: Int = 0,
+    val maxNumFailures: Int = 10,
 )
diff --git a/opendc-experiments/opendc-experiments-base/src/main/kotlin/org/opendc/experiments/base/scenario/specs/ExperimentSpec.kt b/opendc-experiments/opendc-experiments-base/src/main/kotlin/org/opendc/experiments/base/scenario/specs/ExperimentSpec.kt
index db26849e..67d45e89 100644
--- a/opendc-experiments/opendc-experiments-base/src/main/kotlin/org/opendc/experiments/base/scenario/specs/ExperimentSpec.kt
+++ b/opendc-experiments/opendc-experiments-base/src/main/kotlin/org/opendc/experiments/base/scenario/specs/ExperimentSpec.kt
@@ -57,6 +57,7 @@ public data class ExperimentSpec(
     val checkpointModels: Set<CheckpointModelSpec?> = setOf(null),
     val carbonTracePaths: Set<String?> = setOf(null),
     val computeExportConfig: ComputeExportConfig = ComputeExportConfig.ALL_COLUMNS,
+    val maxNumFailures: Set<Int> = setOf(10),
 ) {
     init {
         require(runs > 0) { "The number of runs should always be positive" }
@@ -73,7 +74,8 @@ public data class ExperimentSpec(
 
     public fun getCartesian(): Sequence<ScenarioSpec> {
         return sequence {
-            val checkpointDiv = carbonTracePaths.size
+            val carbonTracePathDiv = maxNumFailures.size
+            val checkpointDiv = carbonTracePathDiv * carbonTracePaths.size
             val failureDiv = checkpointDiv * checkpointModels.size
             val exportDiv = failureDiv * failureModels.size
             val allocationDiv = exportDiv * exportModels.size
@@ -88,6 +90,7 @@ public data class ExperimentSpec(
             val failureModelList = failureModels.toList()
             val checkpointModelList = checkpointModels.toList()
             val carbonTracePathList = carbonTracePaths.toList()
+            val maxNumFailuresList = maxNumFailures.toList()
 
             for (i in 0 until numScenarios) {
                 yield(
@@ -102,7 +105,8 @@ public data class ExperimentSpec(
                         exportModelList[(i / exportDiv) % exportModelList.size],
                         failureModelList[(i / failureDiv) % failureModelList.size],
                         checkpointModelList[(i / checkpointDiv) % checkpointModelList.size],
-                        carbonTracePathList[i % carbonTracePathList.size],
+                        carbonTracePathList[(i / carbonTracePathDiv) % carbonTracePathList.size],
+                        maxNumFailuresList[i % maxNumFailuresList.size],
                     ),
                 )
             }
diff --git a/opendc-experiments/opendc-experiments-base/src/main/kotlin/org/opendc/experiments/base/scenario/specs/ScenarioSpec.kt b/opendc-experiments/opendc-experiments-base/src/main/kotlin/org/opendc/experiments/base/scenario/specs/ScenarioSpec.kt
index 41e9a885..d7fdb8f4 100644
--- a/opendc-experiments/opendc-experiments-base/src/main/kotlin/org/opendc/experiments/base/scenario/specs/ScenarioSpec.kt
+++ b/opendc-experiments/opendc-experiments-base/src/main/kotlin/org/opendc/experiments/base/scenario/specs/ScenarioSpec.kt
@@ -38,4 +38,5 @@ public data class ScenarioSpec(
     val failureModel: FailureModelSpec? = null,
     val checkpointModel: CheckpointModelSpec? = null,
     val carbonTracePath: String? = null,
+    val maxNumFailures: Int = 10,
 )