feat: Move to ERROR state on failure

3 files changed, 33 insertions, 16 deletions

diff --git a/opendc/opendc-compute/src/main/kotlin/com/atlarge/opendc/compute/metal/driver/SimpleBareMetalDriver.kt b/opendc/opendc-compute/src/main/kotlin/com/atlarge/opendc/compute/metal/driver/SimpleBareMetalDriver.kt
index 2d95b489..37ae9eb5 100644
--- a/opendc/opendc-compute/src/main/kotlin/com/atlarge/opendc/compute/metal/driver/SimpleBareMetalDriver.kt
+++ b/opendc/opendc-compute/src/main/kotlin/com/atlarge/opendc/compute/metal/driver/SimpleBareMetalDriver.kt
@@ -308,10 +308,16 @@ public class SimpleBareMetalDriver(
 
     override suspend fun fail() {
         serverContext?.unavailable = true
+
+        val server = nodeState.value.server?.copy(state = ServerState.ERROR)
+        setNode(nodeState.value.copy(state = NodeState.ERROR, server = server))
     }
 
     override suspend fun recover() {
         serverContext?.unavailable = false
+
+        val server = nodeState.value.server?.copy(state = ServerState.ACTIVE)
+        setNode(nodeState.value.copy(state = NodeState.ACTIVE, server = server))
     }
 
     override fun toString(): String = "SimpleBareMetalDriver(node = ${nodeState.value.uid})"
diff --git a/opendc/opendc-compute/src/main/kotlin/com/atlarge/opendc/compute/virt/service/SimpleVirtProvisioningService.kt b/opendc/opendc-compute/src/main/kotlin/com/atlarge/opendc/compute/virt/service/SimpleVirtProvisioningService.kt
index c7347783..85bdc438 100644
--- a/opendc/opendc-compute/src/main/kotlin/com/atlarge/opendc/compute/virt/service/SimpleVirtProvisioningService.kt
+++ b/opendc/opendc-compute/src/main/kotlin/com/atlarge/opendc/compute/virt/service/SimpleVirtProvisioningService.kt
@@ -122,7 +122,7 @@ class SimpleVirtProvisioningService(
             val requiredMemory = (imageInstance.image as VmImage).requiredMemory
             val selectedHv = allocationLogic.select(availableHypervisors, imageInstance) ?: break
             try {
-                log.info("Spawning ${imageInstance.image} on ${selectedHv.server}")
+                log.info("Spawning ${imageInstance.image} on ${selectedHv.server} ${availableHypervisors.size}")
                 incomingImages -= imageInstance
 
                 // Speculatively update the hypervisor view information to prevent other images in the queue from
@@ -139,6 +139,19 @@ class SimpleVirtProvisioningService(
                 imageInstance.server = server
                 imageInstance.continuation.resume(server)
                 activeImages += imageInstance
+
+                server.events
+                    .onEach { event ->
+                        when (event) {
+                            is ServerEvent.StateChanged -> {
+                                if (event.server.state == ServerState.SHUTOFF) {
+                                    activeImages -= imageInstance
+                                    selectedHv.provisionedCores -= server.flavor.cpuCount
+                                }
+                            }
+                        }
+                    }
+                    .launchIn(this)
             } catch (e: InsufficientMemoryOnServerException) {
                 println("Unable to deploy image due to insufficient memory")
 
@@ -152,18 +165,22 @@ class SimpleVirtProvisioningService(
     private fun stateChanged(server: Server) {
         when (server.state) {
             ServerState.ACTIVE -> {
-                val hv = HypervisorView(
-                    server.uid,
-                    server,
-                    0,
-                    server.flavor.memorySize,
-                    0
-                )
-                hypervisors[server] = hv
+                if (server in hypervisors) {
+                    // Corner case for when the hypervisor already exists
+                    availableHypervisors += hypervisors.getValue(server)
+                } else {
+                    val hv = HypervisorView(
+                        server.uid,
+                        server,
+                        0,
+                        server.flavor.memorySize,
+                        0
+                    )
+                    hypervisors[server] = hv
+                }
             }
             ServerState.SHUTOFF, ServerState.ERROR -> {
                 val hv = hypervisors[server] ?: return
-                hv.provisionedCores -= server.flavor.cpuCount
                 availableHypervisors -= hv
                 requestCycle()
             }
diff --git a/opendc/opendc-core/src/main/kotlin/com/atlarge/opendc/core/failure/CorrelatedFaultInjector.kt b/opendc/opendc-core/src/main/kotlin/com/atlarge/opendc/core/failure/CorrelatedFaultInjector.kt
index f46ce512..2904fbec 100644
--- a/opendc/opendc-core/src/main/kotlin/com/atlarge/opendc/core/failure/CorrelatedFaultInjector.kt
+++ b/opendc/opendc-core/src/main/kotlin/com/atlarge/opendc/core/failure/CorrelatedFaultInjector.kt
@@ -99,13 +99,9 @@ public class CorrelatedFaultInjector(
 
                 delay(d.toLong())
 
-
                 val n = lognvariate(sizeScale, sizeShape).toInt()
                 val targets = active.shuffled(random).take(n)
 
-                println("[${simulationContext.clock.instant()}] FAIL $targets")
-
-
                 for (failureDomain in targets) {
                     active -= failureDomain
                     failureDomain.fail()
@@ -120,8 +116,6 @@ public class CorrelatedFaultInjector(
 
                 delay(df.toLong())
 
-                println("[${simulationContext.clock.instant()}] RECOVER $targets")
-
                 for (failureDomain in targets) {
                     failureDomain.recover()