feat: Revamp bare-metal state management

9 files changed, 155 insertions, 67 deletions

diff --git a/opendc/opendc-compute/src/main/kotlin/com/atlarge/opendc/compute/core/monitor/ServerMonitor.kt b/opendc/opendc-compute/src/main/kotlin/com/atlarge/opendc/compute/core/monitor/ServerMonitor.kt
index fbfd0ad6..26b94ba5 100644
--- a/opendc/opendc-compute/src/main/kotlin/com/atlarge/opendc/compute/core/monitor/ServerMonitor.kt
+++ b/opendc/opendc-compute/src/main/kotlin/com/atlarge/opendc/compute/core/monitor/ServerMonitor.kt
@@ -37,5 +37,5 @@ public interface ServerMonitor {
      * @param server The server which state was updated.
      * @param previousState The previous state of the server.
      */
-    public suspend fun onUpdate(server: Server, previousState: ServerState)
+    public suspend fun onUpdate(server: Server, previousState: ServerState) {}
 }
diff --git a/opendc/opendc-compute/src/main/kotlin/com/atlarge/opendc/compute/metal/Node.kt b/opendc/opendc-compute/src/main/kotlin/com/atlarge/opendc/compute/metal/Node.kt
index a43abfe9..7df5d99b 100644
--- a/opendc/opendc-compute/src/main/kotlin/com/atlarge/opendc/compute/metal/Node.kt
+++ b/opendc/opendc-compute/src/main/kotlin/com/atlarge/opendc/compute/metal/Node.kt
@@ -44,9 +44,9 @@ data class Node(
     public override val name: String,
 
     /**
-     * The power state of the node.
+     * The last known state of the compute node.
      */
-    public val powerState: PowerState,
+    public val state: NodeState,
 
     /**
      * The boot image of the node.
diff --git a/opendc/opendc-compute/src/main/kotlin/com/atlarge/opendc/compute/metal/PowerState.kt b/opendc/opendc-compute/src/main/kotlin/com/atlarge/opendc/compute/metal/NodeState.kt
index 5fce3f48..ca9cf509 100644
--- a/opendc/opendc-compute/src/main/kotlin/com/atlarge/opendc/compute/metal/PowerState.kt
+++ b/opendc/opendc-compute/src/main/kotlin/com/atlarge/opendc/compute/metal/NodeState.kt
@@ -25,16 +25,31 @@
 package com.atlarge.opendc.compute.metal
 
 /**
- * The power state of a compute node.
+ * An enumeration describing the possible states of a bare-metal compute node.
  */
-public enum class PowerState {
+public enum class NodeState {
     /**
-     * Node is powered on.
+     * The node is booting.
      */
-    POWER_ON,
+    BOOT,
 
     /**
-     * Node is powered off.
+     * The node is powered off.
      */
-    POWER_OFF,
+    SHUTOFF,
+
+    /**
+     * The node is active and running.
+     */
+    ACTIVE,
+
+    /**
+     * The node is in error.
+     */
+    ERROR,
+
+    /**
+     * The state of the node is unknown.
+     */
+    UNKNOWN,
 }
diff --git a/opendc/opendc-compute/src/main/kotlin/com/atlarge/opendc/compute/metal/driver/BareMetalDriver.kt b/opendc/opendc-compute/src/main/kotlin/com/atlarge/opendc/compute/metal/driver/BareMetalDriver.kt
index 1214dd36..fb2ff355 100644
--- a/opendc/opendc-compute/src/main/kotlin/com/atlarge/opendc/compute/metal/driver/BareMetalDriver.kt
+++ b/opendc/opendc-compute/src/main/kotlin/com/atlarge/opendc/compute/metal/driver/BareMetalDriver.kt
@@ -26,9 +26,8 @@ package com.atlarge.opendc.compute.metal.driver
 
 import com.atlarge.opendc.compute.core.Server
 import com.atlarge.opendc.compute.core.image.Image
-import com.atlarge.opendc.compute.core.monitor.ServerMonitor
 import com.atlarge.opendc.compute.metal.Node
-import com.atlarge.opendc.compute.metal.PowerState
+import com.atlarge.opendc.compute.metal.monitor.NodeMonitor
 import com.atlarge.opendc.core.power.Powerable
 import com.atlarge.opendc.core.services.AbstractServiceKey
 import kotlinx.coroutines.flow.Flow
@@ -47,12 +46,22 @@ public interface BareMetalDriver : Powerable {
     /**
      * Initialize the driver.
      */
-    public suspend fun init(monitor: ServerMonitor): Node
+    public suspend fun init(monitor: NodeMonitor): Node
 
     /**
-     * Update the power state of the compute node.
+     * Start the bare metal node with the specified boot disk image.
      */
-    public suspend fun setPower(powerState: PowerState): Node
+    public suspend fun start(): Node
+
+    /**
+     * Stop the bare metal node if it is running.
+     */
+    public suspend fun stop(): Node
+
+    /**
+     * Reboot the bare metal node.
+     */
+    public suspend fun reboot(): Node
 
     /**
      * Update the boot disk image of the compute node.
diff --git a/opendc/opendc-compute/src/main/kotlin/com/atlarge/opendc/compute/metal/driver/SimpleBareMetalDriver.kt b/opendc/opendc-compute/src/main/kotlin/com/atlarge/opendc/compute/metal/driver/SimpleBareMetalDriver.kt
index 6343bee5..2d803aa5 100644
--- a/opendc/opendc-compute/src/main/kotlin/com/atlarge/opendc/compute/metal/driver/SimpleBareMetalDriver.kt
+++ b/opendc/opendc-compute/src/main/kotlin/com/atlarge/opendc/compute/metal/driver/SimpleBareMetalDriver.kt
@@ -35,9 +35,9 @@ import com.atlarge.opendc.compute.core.ServerState
 import com.atlarge.opendc.compute.core.execution.ServerManagementContext
 import com.atlarge.opendc.compute.core.image.EmptyImage
 import com.atlarge.opendc.compute.core.image.Image
-import com.atlarge.opendc.compute.core.monitor.ServerMonitor
 import com.atlarge.opendc.compute.metal.Node
-import com.atlarge.opendc.compute.metal.PowerState
+import com.atlarge.opendc.compute.metal.NodeState
+import com.atlarge.opendc.compute.metal.monitor.NodeMonitor
 import com.atlarge.opendc.compute.metal.power.ConstantPowerModel
 import com.atlarge.opendc.core.power.PowerModel
 import kotlinx.coroutines.CancellationException
@@ -72,12 +72,27 @@ public class SimpleBareMetalDriver(
     /**
      * The monitor to use.
      */
-    private lateinit var monitor: ServerMonitor
+    private lateinit var monitor: NodeMonitor
 
     /**
      * The machine state.
      */
-    private var node: Node = Node(uid, name, PowerState.POWER_OFF, EmptyImage, null)
+    private var node: Node = Node(uid, name, NodeState.SHUTOFF, EmptyImage, null)
+        set(value) {
+            if (field.state != value.state) {
+                domain.launch {
+                    monitor.onUpdate(value, field.state)
+                }
+            }
+
+            if (field.server != null && value.server != null && field.server!!.state != value.server.state) {
+                domain.launch {
+                    monitor.onUpdate(value.server, field.server!!.state)
+                }
+            }
+
+            field = value
+        }
 
     /**
      * The flavor that corresponds to this machine.
@@ -98,42 +113,49 @@ public class SimpleBareMetalDriver(
 
     override val powerDraw: Flow<Double> = powerModel(this)
 
-    override suspend fun init(monitor: ServerMonitor): Node = withContext(domain.coroutineContext) {
+    override suspend fun init(monitor: NodeMonitor): Node = withContext(domain.coroutineContext) {
         this@SimpleBareMetalDriver.monitor = monitor
         return@withContext node
     }
 
-    override suspend fun setPower(powerState: PowerState): Node = withContext(domain.coroutineContext) {
-        val previousPowerState = node.powerState
-        val server = when (node.powerState to powerState) {
-            PowerState.POWER_OFF to PowerState.POWER_OFF -> null
-            PowerState.POWER_OFF to PowerState.POWER_ON -> Server(
-                UUID.randomUUID(),
-                node.name,
-                emptyMap(),
-                flavor,
-                node.image,
-                ServerState.BUILD
-            )
-            PowerState.POWER_ON to PowerState.POWER_OFF -> {
-                // We terminate the image running on the machine
-                job?.cancel()
-                job = null
-                null
-            }
-            PowerState.POWER_ON to PowerState.POWER_ON -> node.server
-            else -> throw IllegalStateException()
+    override suspend fun start(): Node = withContext(domain.coroutineContext) {
+        if (node.state != NodeState.SHUTOFF) {
+            return@withContext node
         }
-        server?.serviceRegistry?.set(BareMetalDriver.Key, this@SimpleBareMetalDriver)
-        node = node.copy(powerState = powerState, server = server)
 
-        if (powerState != previousPowerState && server != null) {
-            launch()
+        val server = Server(
+            UUID.randomUUID(),
+            node.name,
+            emptyMap(),
+            flavor,
+            node.image,
+            ServerState.BUILD
+        )
+
+        server.serviceRegistry[BareMetalDriver.Key] = this@SimpleBareMetalDriver
+        node = node.copy(state = NodeState.BOOT, server = server)
+        launch()
+        return@withContext node
+    }
+
+    override suspend fun stop(): Node = withContext(domain.coroutineContext) {
+        if (node.state == NodeState.SHUTOFF) {
+            return@withContext node
         }
 
+        // We terminate the image running on the machine
+        job?.cancel()
+        job = null
+
+        node = node.copy(state = NodeState.SHUTOFF, server = null)
         return@withContext node
     }
 
+    override suspend fun reboot(): Node = withContext(domain.coroutineContext) {
+        stop()
+        start()
+    }
+
     override suspend fun setImage(image: Image): Node = withContext(domain.coroutineContext) {
         node = node.copy(image = image)
         return@withContext node
@@ -163,29 +185,26 @@ public class SimpleBareMetalDriver(
 
         override val cpus: List<ProcessingUnit> = this@SimpleBareMetalDriver.cpus
 
-        override var server: Server
+        override val server: Server
             get() = node.server!!
-            set(value) {
-                node = node.copy(server = value)
-            }
 
         override suspend fun init() {
             if (initialized) {
                 throw IllegalStateException()
             }
-
-            val previousState = server.state
-            server = server.copy(state = ServerState.ACTIVE)
-            monitor.onUpdate(server, previousState)
             initialized = true
+
+            val server = server.copy(state = ServerState.ACTIVE)
+            node = node.copy(state = NodeState.ACTIVE, server = server)
         }
 
         override suspend fun exit(cause: Throwable?) {
-            val previousState = server.state
-            val state = if (cause == null) ServerState.SHUTOFF else ServerState.ERROR
-            server = server.copy(state = state)
             initialized = false
-            domain.launch { monitor.onUpdate(server, previousState) }
+
+            val serverState = if (cause == null) ServerState.SHUTOFF else ServerState.ERROR
+            val nodeState = if (cause == null) node.state else NodeState.ERROR
+            val server = server.copy(state = serverState)
+            node = node.copy(state = nodeState, server = server)
         }
 
         private var flush: Job? = null
diff --git a/opendc/opendc-compute/src/main/kotlin/com/atlarge/opendc/compute/metal/monitor/NodeMonitor.kt b/opendc/opendc-compute/src/main/kotlin/com/atlarge/opendc/compute/metal/monitor/NodeMonitor.kt
new file mode 100644
index 00000000..f35cf57b
--- /dev/null
+++ b/opendc/opendc-compute/src/main/kotlin/com/atlarge/opendc/compute/metal/monitor/NodeMonitor.kt
@@ -0,0 +1,42 @@
+/*
+ * MIT License
+ *
+ * Copyright (c) 2020 atlarge-research
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+package com.atlarge.opendc.compute.metal.monitor
+
+import com.atlarge.opendc.compute.core.monitor.ServerMonitor
+import com.atlarge.opendc.compute.metal.Node
+import com.atlarge.opendc.compute.metal.NodeState
+
+/**
+ * An interface for monitoring bare-metal nodes.
+ */
+public interface NodeMonitor : ServerMonitor {
+    /**
+     * This method is invoked when the state of a bare metal machine updates.
+     *
+     * @param node The node for which state was updated.
+     * @param previousState The previous state of the node.
+     */
+    public suspend fun onUpdate(node: Node, previousState: NodeState) {}
+}
diff --git a/opendc/opendc-compute/src/main/kotlin/com/atlarge/opendc/compute/metal/service/SimpleProvisioningService.kt b/opendc/opendc-compute/src/main/kotlin/com/atlarge/opendc/compute/metal/service/SimpleProvisioningService.kt
index b18a4006..117e502c 100644
--- a/opendc/opendc-compute/src/main/kotlin/com/atlarge/opendc/compute/metal/service/SimpleProvisioningService.kt
+++ b/opendc/opendc-compute/src/main/kotlin/com/atlarge/opendc/compute/metal/service/SimpleProvisioningService.kt
@@ -30,14 +30,14 @@ import com.atlarge.opendc.compute.core.ServerState
 import com.atlarge.opendc.compute.core.image.Image
 import com.atlarge.opendc.compute.core.monitor.ServerMonitor
 import com.atlarge.opendc.compute.metal.Node
-import com.atlarge.opendc.compute.metal.PowerState
 import com.atlarge.opendc.compute.metal.driver.BareMetalDriver
+import com.atlarge.opendc.compute.metal.monitor.NodeMonitor
 import kotlinx.coroutines.withContext
 
 /**
  * A very basic implementation of the [ProvisioningService].
  */
-public class SimpleProvisioningService(val domain: Domain) : ProvisioningService, ServerMonitor {
+public class SimpleProvisioningService(val domain: Domain) : ProvisioningService, NodeMonitor {
     /**
      * The active nodes in this service.
      */
@@ -64,8 +64,7 @@ public class SimpleProvisioningService(val domain: Domain) : ProvisioningService
         val driver = nodes[node]!!
 
         driver.setImage(image)
-        driver.setPower(PowerState.POWER_OFF)
-        val newNode = driver.setPower(PowerState.POWER_ON)
+        val newNode = driver.reboot()
         monitors[newNode.server!!] = monitor
         return@withContext newNode
     }
diff --git a/opendc/opendc-compute/src/test/kotlin/com/atlarge/opendc/compute/metal/driver/SimpleBareMetalDriverTest.kt b/opendc/opendc-compute/src/test/kotlin/com/atlarge/opendc/compute/metal/driver/SimpleBareMetalDriverTest.kt
index b8882eda..24a65b40 100644
--- a/opendc/opendc-compute/src/test/kotlin/com/atlarge/opendc/compute/metal/driver/SimpleBareMetalDriverTest.kt
+++ b/opendc/opendc-compute/src/test/kotlin/com/atlarge/opendc/compute/metal/driver/SimpleBareMetalDriverTest.kt
@@ -31,8 +31,9 @@ import com.atlarge.opendc.compute.core.ProcessingUnit
 import com.atlarge.opendc.compute.core.Server
 import com.atlarge.opendc.compute.core.ServerState
 import com.atlarge.opendc.compute.core.image.FlopsApplicationImage
-import com.atlarge.opendc.compute.core.monitor.ServerMonitor
-import com.atlarge.opendc.compute.metal.PowerState
+import com.atlarge.opendc.compute.metal.Node
+import com.atlarge.opendc.compute.metal.NodeState
+import com.atlarge.opendc.compute.metal.monitor.NodeMonitor
 import kotlinx.coroutines.launch
 import kotlinx.coroutines.runBlocking
 import kotlinx.coroutines.withContext
@@ -57,7 +58,11 @@ internal class SimpleBareMetalDriverTest {
             val cpus = List(4) { ProcessingUnit(cpuNode, it, 2400.0) }
             val driver = SimpleBareMetalDriver(dom, UUID.randomUUID(), "test", cpus, emptyList())
 
-            val monitor = object : ServerMonitor {
+            val monitor = object : NodeMonitor {
+                override suspend fun onUpdate(node: Node, previousState: NodeState) {
+                    println(node)
+                }
+
                 override suspend fun onUpdate(server: Server, previousState: ServerState) {
                     println("[${simulationContext.clock.millis()}] $server")
                     finalState = server.state
@@ -69,7 +74,7 @@ internal class SimpleBareMetalDriverTest {
             withContext(dom.coroutineContext) {
                 driver.init(monitor)
                 driver.setImage(image)
-                driver.setPower(PowerState.POWER_ON)
+                driver.start()
             }
         }
 
diff --git a/opendc/opendc-compute/src/test/kotlin/com/atlarge/opendc/compute/virt/driver/hypervisor/HypervisorTest.kt b/opendc/opendc-compute/src/test/kotlin/com/atlarge/opendc/compute/virt/driver/hypervisor/HypervisorTest.kt
index 254ad5fe..57a7150e 100644
--- a/opendc/opendc-compute/src/test/kotlin/com/atlarge/opendc/compute/virt/driver/hypervisor/HypervisorTest.kt
+++ b/opendc/opendc-compute/src/test/kotlin/com/atlarge/opendc/compute/virt/driver/hypervisor/HypervisorTest.kt
@@ -32,9 +32,8 @@ import com.atlarge.opendc.compute.core.Flavor
 import com.atlarge.opendc.compute.core.ProcessingNode
 import com.atlarge.opendc.compute.core.ServerState
 import com.atlarge.opendc.compute.core.image.FlopsApplicationImage
-import com.atlarge.opendc.compute.core.monitor.ServerMonitor
-import com.atlarge.opendc.compute.metal.PowerState
 import com.atlarge.opendc.compute.metal.driver.SimpleBareMetalDriver
+import com.atlarge.opendc.compute.metal.monitor.NodeMonitor
 import com.atlarge.opendc.compute.virt.driver.VirtDriver
 import com.atlarge.opendc.compute.virt.monitor.HypervisorMonitor
 import kotlinx.coroutines.delay
@@ -71,7 +70,7 @@ internal class HypervisorTest {
             })
             val workloadA = FlopsApplicationImage(UUID.randomUUID(), "<unnamed>", emptyMap(), 1_000, 1)
             val workloadB = FlopsApplicationImage(UUID.randomUUID(), "<unnamed>", emptyMap(), 2_000, 1)
-            val monitor = object : ServerMonitor {
+            val monitor = object : NodeMonitor {
                 override suspend fun onUpdate(server: Server, previousState: ServerState) {
                     println("[${simulationContext.clock.millis()}]: $server")
                 }
@@ -85,7 +84,7 @@ internal class HypervisorTest {
 
             metalDriver.init(monitor)
             metalDriver.setImage(vmm)
-            metalDriver.setPower(PowerState.POWER_ON)
+            metalDriver.start()
 
             delay(5)