compute: Improvements to cloud compute model (v1)

This is the first of the pull requests in an attempt to improve the existing cloud compute model (see #86). This pull request restructures the compute API and splits the consumer and service interfaces into different modules:

- opendc-compute-api now defines the API interface for the OpenDC Compute module, which can be used by consumers of the OpenDC Compute service.
- opendc-compute-service hosts the service implementation for OpenDC Compute and contains all business logic regarding the IaaS platform (such as scheduling).
- opendc-compute-simulator implements a "compute driver" for the OpenDC Compute platform that simulates submitted workloads.
- Image is now a data-class and does not specify itself the workload to simulate. Instead, the workload should be passed via its tags currently (with key "workload"). In the future, the simulation backend will accept a mapper interface that maps Images to Workloads.

9 files changed, 64 insertions, 55 deletions

diff --git a/simulator/opendc-format/src/main/kotlin/org/opendc/format/environment/sc18/Sc18EnvironmentReader.kt b/simulator/opendc-format/src/main/kotlin/org/opendc/format/environment/sc18/Sc18EnvironmentReader.kt
index 2e3e4a73..bbbbe87c 100644
--- a/simulator/opendc-format/src/main/kotlin/org/opendc/format/environment/sc18/Sc18EnvironmentReader.kt
+++ b/simulator/opendc-format/src/main/kotlin/org/opendc/format/environment/sc18/Sc18EnvironmentReader.kt
@@ -26,14 +26,14 @@ import com.fasterxml.jackson.databind.ObjectMapper
 import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper
 import com.fasterxml.jackson.module.kotlin.readValue
 import kotlinx.coroutines.CoroutineScope
-import org.opendc.compute.core.metal.service.ProvisioningService
-import org.opendc.compute.core.metal.service.SimpleProvisioningService
 import org.opendc.compute.simulator.SimBareMetalDriver
 import org.opendc.core.Environment
 import org.opendc.core.Platform
 import org.opendc.core.Zone
 import org.opendc.core.services.ServiceRegistry
 import org.opendc.format.environment.EnvironmentReader
+import org.opendc.metal.service.ProvisioningService
+import org.opendc.metal.service.SimpleProvisioningService
 import org.opendc.simulator.compute.SimMachineModel
 import org.opendc.simulator.compute.model.MemoryUnit
 import org.opendc.simulator.compute.model.ProcessingNode
diff --git a/simulator/opendc-format/src/main/kotlin/org/opendc/format/environment/sc20/Sc20ClusterEnvironmentReader.kt b/simulator/opendc-format/src/main/kotlin/org/opendc/format/environment/sc20/Sc20ClusterEnvironmentReader.kt
index 6ec8ba4a..998f9cd6 100644
--- a/simulator/opendc-format/src/main/kotlin/org/opendc/format/environment/sc20/Sc20ClusterEnvironmentReader.kt
+++ b/simulator/opendc-format/src/main/kotlin/org/opendc/format/environment/sc20/Sc20ClusterEnvironmentReader.kt
@@ -23,9 +23,6 @@
 package org.opendc.format.environment.sc20
 
 import kotlinx.coroutines.CoroutineScope
-import org.opendc.compute.core.metal.NODE_CLUSTER
-import org.opendc.compute.core.metal.service.ProvisioningService
-import org.opendc.compute.core.metal.service.SimpleProvisioningService
 import org.opendc.compute.simulator.SimBareMetalDriver
 import org.opendc.compute.simulator.power.models.LinearPowerModel
 import org.opendc.core.Environment
@@ -33,6 +30,9 @@ import org.opendc.core.Platform
 import org.opendc.core.Zone
 import org.opendc.core.services.ServiceRegistry
 import org.opendc.format.environment.EnvironmentReader
+import org.opendc.metal.NODE_CLUSTER
+import org.opendc.metal.service.ProvisioningService
+import org.opendc.metal.service.SimpleProvisioningService
 import org.opendc.simulator.compute.SimMachineModel
 import org.opendc.simulator.compute.model.MemoryUnit
 import org.opendc.simulator.compute.model.ProcessingNode
diff --git a/simulator/opendc-format/src/main/kotlin/org/opendc/format/environment/sc20/Sc20EnvironmentReader.kt b/simulator/opendc-format/src/main/kotlin/org/opendc/format/environment/sc20/Sc20EnvironmentReader.kt
index a58a2524..6cf65f7f 100644
--- a/simulator/opendc-format/src/main/kotlin/org/opendc/format/environment/sc20/Sc20EnvironmentReader.kt
+++ b/simulator/opendc-format/src/main/kotlin/org/opendc/format/environment/sc20/Sc20EnvironmentReader.kt
@@ -26,8 +26,6 @@ import com.fasterxml.jackson.databind.ObjectMapper
 import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper
 import com.fasterxml.jackson.module.kotlin.readValue
 import kotlinx.coroutines.CoroutineScope
-import org.opendc.compute.core.metal.service.ProvisioningService
-import org.opendc.compute.core.metal.service.SimpleProvisioningService
 import org.opendc.compute.simulator.SimBareMetalDriver
 import org.opendc.compute.simulator.power.models.LinearPowerModel
 import org.opendc.core.Environment
@@ -35,6 +33,8 @@ import org.opendc.core.Platform
 import org.opendc.core.Zone
 import org.opendc.core.services.ServiceRegistry
 import org.opendc.format.environment.EnvironmentReader
+import org.opendc.metal.service.ProvisioningService
+import org.opendc.metal.service.SimpleProvisioningService
 import org.opendc.simulator.compute.SimMachineModel
 import org.opendc.simulator.compute.model.MemoryUnit
 import org.opendc.simulator.compute.model.ProcessingNode
diff --git a/simulator/opendc-format/src/main/kotlin/org/opendc/format/trace/bitbrains/BitbrainsTraceReader.kt b/simulator/opendc-format/src/main/kotlin/org/opendc/format/trace/bitbrains/BitbrainsTraceReader.kt
index 90d751ea..1571b17d 100644
--- a/simulator/opendc-format/src/main/kotlin/org/opendc/format/trace/bitbrains/BitbrainsTraceReader.kt
+++ b/simulator/opendc-format/src/main/kotlin/org/opendc/format/trace/bitbrains/BitbrainsTraceReader.kt
@@ -22,8 +22,8 @@
 
 package org.opendc.format.trace.bitbrains
 
-import org.opendc.compute.core.workload.VmWorkload
-import org.opendc.compute.simulator.SimWorkloadImage
+import org.opendc.compute.api.ComputeWorkload
+import org.opendc.compute.api.Image
 import org.opendc.core.User
 import org.opendc.format.trace.TraceEntry
 import org.opendc.format.trace.TraceReader
@@ -45,17 +45,17 @@ import kotlin.math.min
 public class BitbrainsTraceReader(
     traceDirectory: File,
     performanceInterferenceModel: PerformanceInterferenceModel
-) : TraceReader<VmWorkload> {
+) : TraceReader<ComputeWorkload> {
     /**
      * The internal iterator to use for this reader.
      */
-    private val iterator: Iterator<TraceEntry<VmWorkload>>
+    private val iterator: Iterator<TraceEntry<ComputeWorkload>>
 
     /**
      * Initialize the reader.
      */
     init {
-        val entries = mutableMapOf<Long, TraceEntry<VmWorkload>>()
+        val entries = mutableMapOf<Long, TraceEntry<ComputeWorkload>>()
 
         var timestampCol = 0
         var coreCol = 0
@@ -131,19 +131,20 @@ public class BitbrainsTraceReader(
                             .toSortedSet()
                     )
 
-                val vmWorkload = VmWorkload(
+                val workload = SimTraceWorkload(flopsHistory.asSequence())
+                val vmWorkload = ComputeWorkload(
                     uuid,
                     "VM Workload $vmId",
                     UnnamedUser,
-                    SimWorkloadImage(
+                    Image(
                         uuid,
                         vmId.toString(),
                         mapOf(
                             IMAGE_PERF_INTERFERENCE_MODEL to relevantPerformanceInterferenceModelItems,
                             "cores" to cores,
-                            "required-memory" to requiredMemory
-                        ),
-                        SimTraceWorkload(flopsHistory.asSequence())
+                            "required-memory" to requiredMemory,
+                            "workload" to workload
+                        )
                     )
                 )
                 entries[vmId] = TraceEntryImpl(
@@ -158,7 +159,7 @@ public class BitbrainsTraceReader(
 
     override fun hasNext(): Boolean = iterator.hasNext()
 
-    override fun next(): TraceEntry<VmWorkload> = iterator.next()
+    override fun next(): TraceEntry<ComputeWorkload> = iterator.next()
 
     override fun close() {}
 
@@ -175,6 +176,6 @@ public class BitbrainsTraceReader(
      */
     private data class TraceEntryImpl(
         override var submissionTime: Long,
-        override val workload: VmWorkload
-    ) : TraceEntry<VmWorkload>
+        override val workload: ComputeWorkload
+    ) : TraceEntry<ComputeWorkload>
 }
diff --git a/simulator/opendc-format/src/main/kotlin/org/opendc/format/trace/gwf/GwfTraceReader.kt b/simulator/opendc-format/src/main/kotlin/org/opendc/format/trace/gwf/GwfTraceReader.kt
index c76889c8..cd7aff3c 100644
--- a/simulator/opendc-format/src/main/kotlin/org/opendc/format/trace/gwf/GwfTraceReader.kt
+++ b/simulator/opendc-format/src/main/kotlin/org/opendc/format/trace/gwf/GwfTraceReader.kt
@@ -22,7 +22,7 @@
 
 package org.opendc.format.trace.gwf
 
-import org.opendc.compute.simulator.SimWorkloadImage
+import org.opendc.compute.api.Image
 import org.opendc.core.User
 import org.opendc.format.trace.TraceEntry
 import org.opendc.format.trace.TraceReader
@@ -41,7 +41,6 @@ import kotlin.collections.List
 import kotlin.collections.MutableSet
 import kotlin.collections.component1
 import kotlin.collections.component2
-import kotlin.collections.emptyMap
 import kotlin.collections.filter
 import kotlin.collections.forEach
 import kotlin.collections.getOrPut
@@ -136,10 +135,11 @@ public class GwfTraceReader(reader: BufferedReader) : TraceReader<Job> {
                         TraceEntryImpl(submitTime, Job(UUID(0L, taskId), "<unnamed>", UnnamedUser, HashSet()))
                     }
                     val workflow = entry.workload
+                    val workload = SimFlopsWorkload(flops)
                     val task = Task(
                         UUID(0L, taskId),
                         "<unnamed>",
-                        SimWorkloadImage(UUID.randomUUID(), "<unnamed>", emptyMap(), SimFlopsWorkload(flops)),
+                        Image(UUID.randomUUID(), "<unnamed>", mapOf("workload" to workload)),
                         HashSet(),
                         mapOf(
                             WORKFLOW_TASK_CORES to cores,
diff --git a/simulator/opendc-format/src/main/kotlin/org/opendc/format/trace/sc20/Sc20TraceReader.kt b/simulator/opendc-format/src/main/kotlin/org/opendc/format/trace/sc20/Sc20TraceReader.kt
index 78f581ca..07785632 100644
--- a/simulator/opendc-format/src/main/kotlin/org/opendc/format/trace/sc20/Sc20TraceReader.kt
+++ b/simulator/opendc-format/src/main/kotlin/org/opendc/format/trace/sc20/Sc20TraceReader.kt
@@ -22,8 +22,8 @@
 
 package org.opendc.format.trace.sc20
 
-import org.opendc.compute.core.workload.VmWorkload
-import org.opendc.compute.simulator.SimWorkloadImage
+import org.opendc.compute.api.ComputeWorkload
+import org.opendc.compute.api.Image
 import org.opendc.core.User
 import org.opendc.format.trace.TraceEntry
 import org.opendc.format.trace.TraceReader
@@ -49,17 +49,17 @@ public class Sc20TraceReader(
     performanceInterferenceModel: PerformanceInterferenceModel,
     selectedVms: List<String>,
     random: Random
-) : TraceReader<VmWorkload> {
+) : TraceReader<ComputeWorkload> {
     /**
      * The internal iterator to use for this reader.
      */
-    private val iterator: Iterator<TraceEntry<VmWorkload>>
+    private val iterator: Iterator<TraceEntry<ComputeWorkload>>
 
     /**
      * Initialize the reader.
      */
     init {
-        val entries = mutableMapOf<UUID, TraceEntry<VmWorkload>>()
+        val entries = mutableMapOf<UUID, TraceEntry<ComputeWorkload>>()
 
         val timestampCol = 0
         val cpuUsageCol = 1
@@ -156,19 +156,20 @@ public class Sc20TraceReader(
                         performanceInterferenceModel.items.filter { it.workloadNames.contains(vmId) }.toSortedSet(),
                         Random(random.nextInt())
                     )
-                val vmWorkload = VmWorkload(
+                val workload = SimTraceWorkload(flopsFragments.asSequence())
+                val vmWorkload = ComputeWorkload(
                     uuid,
                     "VM Workload $vmId",
                     UnnamedUser,
-                    SimWorkloadImage(
+                    Image(
                         uuid,
                         vmId,
                         mapOf(
                             IMAGE_PERF_INTERFERENCE_MODEL to relevantPerformanceInterferenceModelItems,
                             "cores" to cores,
-                            "required-memory" to requiredMemory
-                        ),
-                        SimTraceWorkload(flopsFragments.asSequence())
+                            "required-memory" to requiredMemory,
+                            "workload" to workload
+                        )
                     )
                 )
                 entries[uuid] = TraceEntryImpl(
@@ -183,7 +184,7 @@ public class Sc20TraceReader(
 
     override fun hasNext(): Boolean = iterator.hasNext()
 
-    override fun next(): TraceEntry<VmWorkload> = iterator.next()
+    override fun next(): TraceEntry<ComputeWorkload> = iterator.next()
 
     override fun close() {}
 
@@ -200,6 +201,6 @@ public class Sc20TraceReader(
      */
     private data class TraceEntryImpl(
         override var submissionTime: Long,
-        override val workload: VmWorkload
-    ) : TraceEntry<VmWorkload>
+        override val workload: ComputeWorkload
+    ) : TraceEntry<ComputeWorkload>
 }
diff --git a/simulator/opendc-format/src/main/kotlin/org/opendc/format/trace/swf/SwfTraceReader.kt b/simulator/opendc-format/src/main/kotlin/org/opendc/format/trace/swf/SwfTraceReader.kt
index 80c54354..ead20c35 100644
--- a/simulator/opendc-format/src/main/kotlin/org/opendc/format/trace/swf/SwfTraceReader.kt
+++ b/simulator/opendc-format/src/main/kotlin/org/opendc/format/trace/swf/SwfTraceReader.kt
@@ -22,8 +22,8 @@
 
 package org.opendc.format.trace.swf
 
-import org.opendc.compute.core.workload.VmWorkload
-import org.opendc.compute.simulator.SimWorkloadImage
+import org.opendc.compute.api.ComputeWorkload
+import org.opendc.compute.api.Image
 import org.opendc.core.User
 import org.opendc.format.trace.TraceEntry
 import org.opendc.format.trace.TraceReader
@@ -43,17 +43,17 @@ import java.util.*
 public class SwfTraceReader(
     file: File,
     maxNumCores: Int = -1
-) : TraceReader<VmWorkload> {
+) : TraceReader<ComputeWorkload> {
     /**
      * The internal iterator to use for this reader.
      */
-    private val iterator: Iterator<TraceEntry<VmWorkload>>
+    private val iterator: Iterator<TraceEntry<ComputeWorkload>>
 
     /**
      * Initialize the reader.
      */
     init {
-        val entries = mutableMapOf<Long, TraceEntry<VmWorkload>>()
+        val entries = mutableMapOf<Long, TraceEntry<ComputeWorkload>>()
 
         val jobNumberCol = 0
         val submitTimeCol = 1 // seconds (begin of trace is 0)
@@ -154,18 +154,19 @@ public class SwfTraceReader(
                     }
 
                     val uuid = UUID(0L, jobNumber)
-                    val vmWorkload = VmWorkload(
+                    val workload = SimTraceWorkload(flopsHistory.asSequence())
+                    val vmWorkload = ComputeWorkload(
                         uuid,
                         "SWF Workload $jobNumber",
                         UnnamedUser,
-                        SimWorkloadImage(
+                        Image(
                             uuid,
                             jobNumber.toString(),
                             mapOf(
                                 "cores" to cores,
-                                "required-memory" to memory
-                            ),
-                            SimTraceWorkload(flopsHistory.asSequence())
+                                "required-memory" to memory,
+                                "workload" to workload
+                            )
                         )
                     )
 
@@ -179,7 +180,7 @@ public class SwfTraceReader(
 
     override fun hasNext(): Boolean = iterator.hasNext()
 
-    override fun next(): TraceEntry<VmWorkload> = iterator.next()
+    override fun next(): TraceEntry<ComputeWorkload> = iterator.next()
 
     override fun close() {}
 
@@ -196,6 +197,6 @@ public class SwfTraceReader(
      */
     private data class TraceEntryImpl(
         override var submissionTime: Long,
-        override val workload: VmWorkload
-    ) : TraceEntry<VmWorkload>
+        override val workload: ComputeWorkload
+    ) : TraceEntry<ComputeWorkload>
 }
diff --git a/simulator/opendc-format/src/main/kotlin/org/opendc/format/trace/wtf/WtfTraceReader.kt b/simulator/opendc-format/src/main/kotlin/org/opendc/format/trace/wtf/WtfTraceReader.kt
index d7dc09fa..5a271fab 100644
--- a/simulator/opendc-format/src/main/kotlin/org/opendc/format/trace/wtf/WtfTraceReader.kt
+++ b/simulator/opendc-format/src/main/kotlin/org/opendc/format/trace/wtf/WtfTraceReader.kt
@@ -25,7 +25,7 @@ package org.opendc.format.trace.wtf
 import org.apache.avro.generic.GenericRecord
 import org.apache.hadoop.fs.Path
 import org.apache.parquet.avro.AvroParquetReader
-import org.opendc.compute.simulator.SimWorkloadImage
+import org.opendc.compute.api.Image
 import org.opendc.core.User
 import org.opendc.format.trace.TraceEntry
 import org.opendc.format.trace.TraceReader
@@ -78,10 +78,17 @@ public class WtfTraceReader(path: String) : TraceReader<Job> {
                 TraceEntryImpl(submitTime, Job(UUID(0L, taskId), "<unnamed>", UnnamedUser, HashSet()))
             }
             val workflow = entry.workload
+            val workload = SimFlopsWorkload(flops)
             val task = Task(
                 UUID(0L, taskId),
                 "<unnamed>",
-                SimWorkloadImage(UUID.randomUUID(), "<unnamed>", emptyMap(), SimFlopsWorkload(flops)),
+                Image(
+                    UUID.randomUUID(),
+                    "<unnamed>",
+                    mapOf(
+                        "workload" to workload
+                    )
+                ),
                 HashSet(),
                 mapOf(
                     WORKFLOW_TASK_CORES to cores,
diff --git a/simulator/opendc-format/src/test/kotlin/org/opendc/format/trace/swf/SwfTraceReaderTest.kt b/simulator/opendc-format/src/test/kotlin/org/opendc/format/trace/swf/SwfTraceReaderTest.kt
index 45c125c4..7e3d2623 100644
--- a/simulator/opendc-format/src/test/kotlin/org/opendc/format/trace/swf/SwfTraceReaderTest.kt
+++ b/simulator/opendc-format/src/test/kotlin/org/opendc/format/trace/swf/SwfTraceReaderTest.kt
@@ -24,7 +24,6 @@ package org.opendc.format.trace.swf
 
 import org.junit.jupiter.api.Assertions.assertEquals
 import org.junit.jupiter.api.Test
-import org.opendc.compute.simulator.SimWorkloadImage
 import org.opendc.simulator.compute.workload.SimTraceWorkload
 import java.io.File
 
@@ -35,12 +34,12 @@ class SwfTraceReaderTest {
         var entry = reader.next()
         assertEquals(0, entry.submissionTime)
         // 1961 slices for waiting, 3 full and 1 partial running slices
-        assertEquals(1965, ((entry.workload.image as SimWorkloadImage).workload as SimTraceWorkload).trace.toList().size)
+        assertEquals(1965, (entry.workload.image.tags["workload"] as SimTraceWorkload).trace.toList().size)
 
         entry = reader.next()
         assertEquals(164472, entry.submissionTime)
         // 1188 slices for waiting, 0 full and 1 partial running slices
-        assertEquals(1189, ((entry.workload.image as SimWorkloadImage).workload as SimTraceWorkload).trace.toList().size)
-        assertEquals(0.25, ((entry.workload.image as SimWorkloadImage).workload as SimTraceWorkload).trace.toList().last().usage)
+        assertEquals(1189, (entry.workload.image.tags["workload"] as SimTraceWorkload).trace.toList().size)
+        assertEquals(0.25, (entry.workload.image.tags["workload"] as SimTraceWorkload).trace.toList().last().usage)
     }
 }