feat: Add support for failing hypervisors in provisioner

7 files changed, 34 insertions, 34 deletions

diff --git a/opendc/opendc-compute/src/main/kotlin/com/atlarge/opendc/compute/metal/service/SimpleProvisioningService.kt b/opendc/opendc-compute/src/main/kotlin/com/atlarge/opendc/compute/metal/service/SimpleProvisioningService.kt
index 117e502c..d8fe0dd9 100644
--- a/opendc/opendc-compute/src/main/kotlin/com/atlarge/opendc/compute/metal/service/SimpleProvisioningService.kt
+++ b/opendc/opendc-compute/src/main/kotlin/com/atlarge/opendc/compute/metal/service/SimpleProvisioningService.kt
@@ -62,7 +62,6 @@ public class SimpleProvisioningService(val domain: Domain) : ProvisioningService
 
     override suspend fun deploy(node: Node, image: Image, monitor: ServerMonitor): Node = withContext(domain.coroutineContext) {
         val driver = nodes[node]!!
-
         driver.setImage(image)
         val newNode = driver.reboot()
         monitors[newNode.server!!] = monitor
diff --git a/opendc/opendc-compute/src/main/kotlin/com/atlarge/opendc/compute/virt/service/NodeView.kt b/opendc/opendc-compute/src/main/kotlin/com/atlarge/opendc/compute/virt/service/HypervisorView.kt
index 41e67624..996bd8eb 100644
--- a/opendc/opendc-compute/src/main/kotlin/com/atlarge/opendc/compute/virt/service/NodeView.kt
+++ b/opendc/opendc-compute/src/main/kotlin/com/atlarge/opendc/compute/virt/service/HypervisorView.kt
@@ -1,10 +1,10 @@
 package com.atlarge.opendc.compute.virt.service
 
-import com.atlarge.opendc.compute.metal.Node
+import com.atlarge.opendc.compute.core.Server
 import com.atlarge.opendc.compute.virt.driver.hypervisor.HypervisorImage
 
-class NodeView(
-    val node: Node,
+class HypervisorView(
+    var server: Server,
     val hypervisor: HypervisorImage,
     var numberOfActiveServers: Int,
     var availableMemory: Long
diff --git a/opendc/opendc-compute/src/main/kotlin/com/atlarge/opendc/compute/virt/service/SimpleVirtProvisioningService.kt b/opendc/opendc-compute/src/main/kotlin/com/atlarge/opendc/compute/virt/service/SimpleVirtProvisioningService.kt
index 17960186..f0bb4e25 100644
--- a/opendc/opendc-compute/src/main/kotlin/com/atlarge/opendc/compute/virt/service/SimpleVirtProvisioningService.kt
+++ b/opendc/opendc-compute/src/main/kotlin/com/atlarge/opendc/compute/virt/service/SimpleVirtProvisioningService.kt
@@ -6,7 +6,6 @@ import com.atlarge.opendc.compute.core.Server
 import com.atlarge.opendc.compute.core.ServerState
 import com.atlarge.opendc.compute.core.image.Image
 import com.atlarge.opendc.compute.core.monitor.ServerMonitor
-import com.atlarge.opendc.compute.metal.Node
 import com.atlarge.opendc.compute.metal.service.ProvisioningService
 import com.atlarge.opendc.compute.virt.driver.VirtDriver
 import com.atlarge.opendc.compute.virt.driver.VirtDriverMonitor
@@ -24,24 +23,24 @@ class SimpleVirtProvisioningService(
     private val hypervisorMonitor: HypervisorMonitor
 ) : VirtProvisioningService, ServerMonitor {
     /**
-     * The nodes that are controlled by the service.
+     * The hypervisors that have been launched by the service.
      */
-    internal lateinit var nodes: List<Node>
+    private val hypervisors: MutableMap<Server, HypervisorView> = mutableMapOf()
 
     /**
-     * The available nodes.
+     * The available hypervisors.
      */
-    internal val availableNodes: MutableSet<NodeView> = mutableSetOf()
+    private val availableHypervisors: MutableSet<HypervisorView> = mutableSetOf()
 
     /**
      * The incoming images to be processed by the provisioner.
      */
-    internal val incomingImages: MutableSet<ImageView> = mutableSetOf()
+    private val incomingImages: MutableSet<ImageView> = mutableSetOf()
 
     /**
      * The active images in the system.
      */
-    internal val activeImages: MutableSet<ImageView> = mutableSetOf()
+    private val activeImages: MutableSet<ImageView> = mutableSetOf()
 
     init {
         ctx.domain.launch {
@@ -49,23 +48,23 @@ class SimpleVirtProvisioningService(
             val deployedNodes = provisionedNodes.map { node ->
                 val hypervisorImage = HypervisorImage(hypervisorMonitor)
                 val deployedNode = provisioningService.deploy(node, hypervisorImage, this@SimpleVirtProvisioningService)
-                val nodeView = NodeView(
-                    deployedNode,
+                val server = deployedNode.server!!
+                val hvView = HypervisorView(
+                    server,
                     hypervisorImage,
                     0,
-                    deployedNode.server!!.flavor.memorySize
+                    server.flavor.memorySize
                 )
                 yield()
-                deployedNode.server.serviceRegistry[VirtDriver.Key].addMonitor(object : VirtDriverMonitor {
+                server.serviceRegistry[VirtDriver.Key].addMonitor(object : VirtDriverMonitor {
                     override suspend fun onUpdate(numberOfActiveServers: Int, availableMemory: Long) {
-                        nodeView.numberOfActiveServers = numberOfActiveServers
-                        nodeView.availableMemory = availableMemory
+                        hvView.numberOfActiveServers = numberOfActiveServers
+                        hvView.availableMemory = availableMemory
                     }
                 })
-                nodeView
+                server to hvView
             }
-            nodes = deployedNodes.map { it.node }
-            availableNodes.addAll(deployedNodes)
+            hypervisors.putAll(deployedNodes)
         }
     }
 
@@ -86,11 +85,9 @@ class SimpleVirtProvisioningService(
 
         for (imageInstance in imagesToBeScheduled) {
             println("Spawning $imageInstance")
-
-            val selectedNode = availableNodes.minWith(allocationPolicy().thenBy { it.node.uid })
-
+            val selectedNode = availableHypervisors.minWith(allocationPolicy().thenBy { it.server.uid }) ?: break
             try {
-                imageInstance.server = selectedNode?.node!!.server!!.serviceRegistry[VirtDriver.Key].spawn(
+                imageInstance.server = selectedNode.server.serviceRegistry[VirtDriver.Key].spawn(
                     imageInstance.image,
                     imageInstance.monitor,
                     imageInstance.flavor
@@ -107,10 +104,14 @@ class SimpleVirtProvisioningService(
     override suspend fun onUpdate(server: Server, previousState: ServerState) {
         when (server.state) {
             ServerState.ACTIVE -> {
-                // TODO handle hypervisor server becoming active
+                val hv = hypervisors[server] ?: return
+                availableHypervisors += hv
+                requestCycle()
             }
             ServerState.SHUTOFF, ServerState.ERROR -> {
-                // TODO handle hypervisor server shutting down or failing
+                val hv = hypervisors[server] ?: return
+                availableHypervisors -= hv
+                requestCycle()
             }
             else -> throw IllegalStateException()
         }
diff --git a/opendc/opendc-compute/src/main/kotlin/com/atlarge/opendc/compute/virt/service/allocation/AllocationPolicy.kt b/opendc/opendc-compute/src/main/kotlin/com/atlarge/opendc/compute/virt/service/allocation/AllocationPolicy.kt
index a1c0ab9a..e2871cca 100644
--- a/opendc/opendc-compute/src/main/kotlin/com/atlarge/opendc/compute/virt/service/allocation/AllocationPolicy.kt
+++ b/opendc/opendc-compute/src/main/kotlin/com/atlarge/opendc/compute/virt/service/allocation/AllocationPolicy.kt
@@ -1,7 +1,7 @@
 package com.atlarge.opendc.compute.virt.service.allocation
 
 import com.atlarge.opendc.compute.metal.Node
-import com.atlarge.opendc.compute.virt.service.NodeView
+import com.atlarge.opendc.compute.virt.service.HypervisorView
 
 /**
  * A policy for selecting the [Node] an image should be deployed to,
@@ -10,5 +10,5 @@ interface AllocationPolicy {
     /**
      * Builds the logic of the policy.
      */
-    operator fun invoke(): Comparator<NodeView>
+    operator fun invoke(): Comparator<HypervisorView>
 }
diff --git a/opendc/opendc-compute/src/main/kotlin/com/atlarge/opendc/compute/virt/service/allocation/AvailableMemoryAllocationPolicy.kt b/opendc/opendc-compute/src/main/kotlin/com/atlarge/opendc/compute/virt/service/allocation/AvailableMemoryAllocationPolicy.kt
index b3e9d77e..f095849b 100644
--- a/opendc/opendc-compute/src/main/kotlin/com/atlarge/opendc/compute/virt/service/allocation/AvailableMemoryAllocationPolicy.kt
+++ b/opendc/opendc-compute/src/main/kotlin/com/atlarge/opendc/compute/virt/service/allocation/AvailableMemoryAllocationPolicy.kt
@@ -1,12 +1,12 @@
 package com.atlarge.opendc.compute.virt.service.allocation
 
-import com.atlarge.opendc.compute.virt.service.NodeView
+import com.atlarge.opendc.compute.virt.service.HypervisorView
 
 /**
  * Allocation policy that selects the node with the most available memory.
  */
 class AvailableMemoryAllocationPolicy : AllocationPolicy {
-    override fun invoke(): Comparator<NodeView> = Comparator { o1, o2 ->
+    override fun invoke(): Comparator<HypervisorView> = Comparator { o1, o2 ->
         compareValuesBy(o1, o2) { -it.availableMemory }
     }
 }
diff --git a/opendc/opendc-compute/src/main/kotlin/com/atlarge/opendc/compute/virt/service/allocation/NumberOfActiveServersAllocationPolicy.kt b/opendc/opendc-compute/src/main/kotlin/com/atlarge/opendc/compute/virt/service/allocation/NumberOfActiveServersAllocationPolicy.kt
index 9d6582dd..59e48465 100644
--- a/opendc/opendc-compute/src/main/kotlin/com/atlarge/opendc/compute/virt/service/allocation/NumberOfActiveServersAllocationPolicy.kt
+++ b/opendc/opendc-compute/src/main/kotlin/com/atlarge/opendc/compute/virt/service/allocation/NumberOfActiveServersAllocationPolicy.kt
@@ -1,13 +1,13 @@
 package com.atlarge.opendc.compute.virt.service.allocation
 
-import com.atlarge.opendc.compute.virt.service.NodeView
+import com.atlarge.opendc.compute.virt.service.HypervisorView
 import kotlinx.coroutines.runBlocking
 
 /**
  * Allocation policy that selects the node with the least amount of active servers.
  */
 class NumberOfActiveServersAllocationPolicy : AllocationPolicy {
-    override fun invoke(): Comparator<NodeView> = Comparator { o1, o2 ->
+    override fun invoke(): Comparator<HypervisorView> = Comparator { o1, o2 ->
         runBlocking {
             compareValuesBy(o1, o2) { it.numberOfActiveServers }
         }
diff --git a/opendc/opendc-compute/src/test/kotlin/com/atlarge/opendc/compute/metal/driver/SimpleBareMetalDriverTest.kt b/opendc/opendc-compute/src/test/kotlin/com/atlarge/opendc/compute/metal/driver/SimpleBareMetalDriverTest.kt
index 9378b5d7..1b7a47d5 100644
--- a/opendc/opendc-compute/src/test/kotlin/com/atlarge/opendc/compute/metal/driver/SimpleBareMetalDriverTest.kt
+++ b/opendc/opendc-compute/src/test/kotlin/com/atlarge/opendc/compute/metal/driver/SimpleBareMetalDriverTest.kt
@@ -35,6 +35,7 @@ import com.atlarge.opendc.compute.metal.Node
 import com.atlarge.opendc.compute.metal.NodeState
 import com.atlarge.opendc.compute.metal.monitor.NodeMonitor
 import com.atlarge.opendc.core.failure.FaultInjector
+import com.atlarge.opendc.core.failure.UncorrelatedFaultInjector
 import kotlinx.coroutines.launch
 import kotlinx.coroutines.runBlocking
 import kotlinx.coroutines.withContext
@@ -78,8 +79,7 @@ internal class SimpleBareMetalDriverTest {
                 driver.start()
             }
 
-
-            val injector = FaultInjector()
+            val injector = UncorrelatedFaultInjector()
             injector.enqueue(driver)
         }