refactor(telemetry): Create separate MeterProvider per service/host

This change refactors the telemetry implementation by creating a
separate MeterProvider per service or host. This means we have to keep
track of multiple metric producers, but that we can attach resource
information to each of the MeterProviders like we would in a real world
scenario.

5 files changed, 93 insertions, 45 deletions

diff --git a/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/ComputeService.kt b/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/ComputeService.kt
index 1873eb99..2a1fbaa0 100644
--- a/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/ComputeService.kt
+++ b/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/ComputeService.kt
@@ -23,11 +23,13 @@
 package org.opendc.compute.service
 
 import io.opentelemetry.api.metrics.Meter
+import io.opentelemetry.api.metrics.MeterProvider
 import org.opendc.compute.api.ComputeClient
 import org.opendc.compute.service.driver.Host
 import org.opendc.compute.service.internal.ComputeServiceImpl
 import org.opendc.compute.service.scheduler.ComputeScheduler
 import java.time.Clock
+import java.time.Duration
 import kotlin.coroutines.CoroutineContext
 
 /**
@@ -70,16 +72,18 @@ public interface ComputeService : AutoCloseable {
          *
          * @param context The [CoroutineContext] to use in the service.
          * @param clock The clock instance to use.
+         * @param meterProvider The [MeterProvider] for creating a [Meter] for the service.
          * @param scheduler The scheduler implementation to use.
+         * @param schedulingQuantum The interval between scheduling cycles.
          */
         public operator fun invoke(
             context: CoroutineContext,
             clock: Clock,
-            meter: Meter,
+            meterProvider: MeterProvider,
             scheduler: ComputeScheduler,
-            schedulingQuantum: Long = 300000,
+            schedulingQuantum: Duration = Duration.ofMinutes(5),
         ): ComputeService {
-            return ComputeServiceImpl(context, clock, meter, scheduler, schedulingQuantum)
+            return ComputeServiceImpl(context, clock, meterProvider, scheduler, schedulingQuantum)
         }
     }
 }
diff --git a/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/internal/ComputeServiceImpl.kt b/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/internal/ComputeServiceImpl.kt
index f1c055d4..824becf4 100644
--- a/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/internal/ComputeServiceImpl.kt
+++ b/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/internal/ComputeServiceImpl.kt
@@ -22,9 +22,8 @@
 
 package org.opendc.compute.service.internal
 
-import io.opentelemetry.api.common.Attributes
 import io.opentelemetry.api.metrics.Meter
-import io.opentelemetry.semconv.resource.attributes.ResourceAttributes
+import io.opentelemetry.api.metrics.MeterProvider
 import kotlinx.coroutines.*
 import mu.KotlinLogging
 import org.opendc.compute.api.*
@@ -35,6 +34,7 @@ import org.opendc.compute.service.driver.HostState
 import org.opendc.compute.service.scheduler.ComputeScheduler
 import org.opendc.utils.TimerScheduler
 import java.time.Clock
+import java.time.Duration
 import java.util.*
 import kotlin.coroutines.CoroutineContext
 import kotlin.math.max
@@ -42,15 +42,18 @@ import kotlin.math.max
 /**
  * Internal implementation of the OpenDC Compute service.
  *
- * @param context The [CoroutineContext] to use.
- * @param clock The clock instance to keep track of time.
+ * @param context The [CoroutineContext] to use in the service.
+ * @param clock The clock instance to use.
+ * @param meterProvider The [MeterProvider] for creating a [Meter] for the service.
+ * @param scheduler The scheduler implementation to use.
+ * @param schedulingQuantum The interval between scheduling cycles.
  */
 internal class ComputeServiceImpl(
     private val context: CoroutineContext,
     private val clock: Clock,
-    private val meter: Meter,
+    meterProvider: MeterProvider,
     private val scheduler: ComputeScheduler,
-    private val schedulingQuantum: Long
+    private val schedulingQuantum: Duration
 ) : ComputeService, HostListener {
     /**
      * The [CoroutineScope] of the service bounded by the lifecycle of the service.
@@ -63,6 +66,11 @@ internal class ComputeServiceImpl(
     private val logger = KotlinLogging.logger {}
 
     /**
+     * The [Meter] to track metrics of the [ComputeService].
+     */
+    private val meter = meterProvider.get("org.opendc.compute.service")
+
+    /**
      * The [Random] instance used to generate unique identifiers for the objects.
      */
     private val random = Random(0)
@@ -365,10 +373,12 @@ internal class ComputeServiceImpl(
             return
         }
 
+        val quantum = schedulingQuantum.toMillis()
+
         // We assume that the provisioner runs at a fixed slot every time quantum (e.g t=0, t=60, t=120).
         // This is important because the slices of the VMs need to be aligned.
         // We calculate here the delay until the next scheduling slot.
-        val delay = schedulingQuantum - (clock.millis() % schedulingQuantum)
+        val delay = quantum - (clock.millis() % quantum)
 
         timerScheduler.startSingleTimer(Unit, delay) {
             doSchedule()
@@ -414,7 +424,7 @@ internal class ComputeServiceImpl(
             // Remove request from queue
             queue.poll()
             _waitingServers.add(-1)
-            _schedulerDuration.record(now - request.submitTime, Attributes.of(ResourceAttributes.HOST_ID, server.uid.toString()))
+            _schedulerDuration.record(now - request.submitTime, server.attributes)
 
             logger.info { "Assigned server $server to host $host." }
 
diff --git a/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/internal/InternalServer.kt b/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/internal/InternalServer.kt
index d9d0f3fc..05a7e1bf 100644
--- a/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/internal/InternalServer.kt
+++ b/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/internal/InternalServer.kt
@@ -22,6 +22,9 @@
 
 package org.opendc.compute.service.internal
 
+import io.opentelemetry.api.common.AttributeKey
+import io.opentelemetry.api.common.Attributes
+import io.opentelemetry.semconv.resource.attributes.ResourceAttributes
 import mu.KotlinLogging
 import org.opendc.compute.api.*
 import org.opendc.compute.service.driver.Host
@@ -50,6 +53,21 @@ internal class InternalServer(
     private val watchers = mutableListOf<ServerWatcher>()
 
     /**
+     * The attributes of a server.
+     */
+    internal val attributes: Attributes = Attributes.builder()
+        .put(ResourceAttributes.HOST_NAME, name)
+        .put(ResourceAttributes.HOST_ID, uid.toString())
+        .put(ResourceAttributes.HOST_TYPE, flavor.name)
+        .put(AttributeKey.longKey("host.num_cpus"), flavor.cpuCount.toLong())
+        .put(AttributeKey.longKey("host.mem_capacity"), flavor.memorySize)
+        .put(AttributeKey.stringArrayKey("host.labels"), labels.map { (k, v) -> "$k:$v" })
+        .put(ResourceAttributes.HOST_ARCH, ResourceAttributes.HostArchValues.AMD64)
+        .put(ResourceAttributes.HOST_IMAGE_NAME, image.name)
+        .put(ResourceAttributes.HOST_IMAGE_ID, image.uid.toString())
+        .build()
+
+    /**
      * The [Host] that has been assigned to host the server.
      */
     internal var host: Host? = null
diff --git a/opendc-compute/opendc-compute-service/src/test/kotlin/org/opendc/compute/service/ComputeServiceTest.kt b/opendc-compute/opendc-compute-service/src/test/kotlin/org/opendc/compute/service/ComputeServiceTest.kt
index d036ec00..564f9493 100644
--- a/opendc-compute/opendc-compute-service/src/test/kotlin/org/opendc/compute/service/ComputeServiceTest.kt
+++ b/opendc-compute/opendc-compute-service/src/test/kotlin/org/opendc/compute/service/ComputeServiceTest.kt
@@ -61,8 +61,7 @@ internal class ComputeServiceTest {
             filters = listOf(ComputeFilter(), VCpuFilter(allocationRatio = 1.0), RamFilter(allocationRatio = 1.0)),
             weighers = listOf(RamWeigher())
         )
-        val meter = MeterProvider.noop().get("opendc-compute")
-        service = ComputeService(scope.coroutineContext, clock, meter, computeScheduler)
+        service = ComputeService(scope.coroutineContext, clock, MeterProvider.noop(), computeScheduler)
     }
 
     @Test
diff --git a/opendc-compute/opendc-compute-service/src/test/kotlin/org/opendc/compute/service/InternalServerTest.kt b/opendc-compute/opendc-compute-service/src/test/kotlin/org/opendc/compute/service/InternalServerTest.kt
index 28fd8217..dfd3bc67 100644
--- a/opendc-compute/opendc-compute-service/src/test/kotlin/org/opendc/compute/service/InternalServerTest.kt
+++ b/opendc-compute/opendc-compute-service/src/test/kotlin/org/opendc/compute/service/InternalServerTest.kt
@@ -47,8 +47,9 @@ class InternalServerTest {
     fun testEquality() {
         val service = mockk<ComputeServiceImpl>()
         val uid = UUID.randomUUID()
-        val flavor = mockk<InternalFlavor>()
-        val image = mockk<InternalImage>()
+        val flavor = mockFlavor()
+        val image = mockImage()
+
         val a = InternalServer(service, uid, "test", flavor, image, mutableMapOf(), mutableMapOf())
         val b = InternalServer(service, uid, "test", flavor, image, mutableMapOf(), mutableMapOf())
 
@@ -59,8 +60,8 @@ class InternalServerTest {
     fun testEqualityWithDifferentType() {
         val service = mockk<ComputeServiceImpl>()
         val uid = UUID.randomUUID()
-        val flavor = mockk<InternalFlavor>()
-        val image = mockk<InternalImage>()
+        val flavor = mockFlavor()
+        val image = mockImage()
         val a = InternalServer(service, uid, "test", flavor, image, mutableMapOf(), mutableMapOf())
 
         val b = mockk<Server>(relaxUnitFun = true)
@@ -73,8 +74,8 @@ class InternalServerTest {
     fun testInequalityWithDifferentType() {
         val service = mockk<ComputeServiceImpl>()
         val uid = UUID.randomUUID()
-        val flavor = mockk<InternalFlavor>()
-        val image = mockk<InternalImage>()
+        val flavor = mockFlavor()
+        val image = mockImage()
         val a = InternalServer(service, uid, "test", flavor, image, mutableMapOf(), mutableMapOf())
 
         val b = mockk<Server>(relaxUnitFun = true)
@@ -87,8 +88,8 @@ class InternalServerTest {
     fun testInequalityWithIncorrectType() {
         val service = mockk<ComputeServiceImpl>()
         val uid = UUID.randomUUID()
-        val flavor = mockk<InternalFlavor>()
-        val image = mockk<InternalImage>()
+        val flavor = mockFlavor()
+        val image = mockImage()
         val a = InternalServer(service, uid, "test", flavor, image, mutableMapOf(), mutableMapOf())
 
         assertNotEquals(a, Unit)
@@ -98,8 +99,8 @@ class InternalServerTest {
     fun testStartTerminatedServer() = runBlockingSimulation {
         val service = mockk<ComputeServiceImpl>()
         val uid = UUID.randomUUID()
-        val flavor = mockk<InternalFlavor>()
-        val image = mockk<InternalImage>()
+        val flavor = mockFlavor()
+        val image = mockImage()
         val server = InternalServer(service, uid, "test", flavor, image, mutableMapOf(), mutableMapOf())
 
         every { service.schedule(any()) } answers { ComputeServiceImpl.SchedulingRequest(it.invocation.args[0] as InternalServer, 0) }
@@ -114,8 +115,8 @@ class InternalServerTest {
     fun testStartDeletedServer() = runBlockingSimulation {
         val service = mockk<ComputeServiceImpl>()
         val uid = UUID.randomUUID()
-        val flavor = mockk<InternalFlavor>()
-        val image = mockk<InternalImage>()
+        val flavor = mockFlavor()
+        val image = mockImage()
         val server = InternalServer(service, uid, "test", flavor, image, mutableMapOf(), mutableMapOf())
 
         server.state = ServerState.DELETED
@@ -127,8 +128,8 @@ class InternalServerTest {
     fun testStartProvisioningServer() = runBlockingSimulation {
         val service = mockk<ComputeServiceImpl>()
         val uid = UUID.randomUUID()
-        val flavor = mockk<InternalFlavor>()
-        val image = mockk<InternalImage>()
+        val flavor = mockFlavor()
+        val image = mockImage()
         val server = InternalServer(service, uid, "test", flavor, image, mutableMapOf(), mutableMapOf())
 
         server.state = ServerState.PROVISIONING
@@ -142,8 +143,8 @@ class InternalServerTest {
     fun testStartRunningServer() = runBlockingSimulation {
         val service = mockk<ComputeServiceImpl>()
         val uid = UUID.randomUUID()
-        val flavor = mockk<InternalFlavor>()
-        val image = mockk<InternalImage>()
+        val flavor = mockFlavor()
+        val image = mockImage()
         val server = InternalServer(service, uid, "test", flavor, image, mutableMapOf(), mutableMapOf())
 
         server.state = ServerState.RUNNING
@@ -157,8 +158,8 @@ class InternalServerTest {
     fun testStopProvisioningServer() = runBlockingSimulation {
         val service = mockk<ComputeServiceImpl>()
         val uid = UUID.randomUUID()
-        val flavor = mockk<InternalFlavor>()
-        val image = mockk<InternalImage>()
+        val flavor = mockFlavor()
+        val image = mockImage()
         val server = InternalServer(service, uid, "test", flavor, image, mutableMapOf(), mutableMapOf())
         val request = ComputeServiceImpl.SchedulingRequest(server, 0)
 
@@ -175,8 +176,8 @@ class InternalServerTest {
     fun testStopTerminatedServer() = runBlockingSimulation {
         val service = mockk<ComputeServiceImpl>()
         val uid = UUID.randomUUID()
-        val flavor = mockk<InternalFlavor>()
-        val image = mockk<InternalImage>()
+        val flavor = mockFlavor()
+        val image = mockImage()
         val server = InternalServer(service, uid, "test", flavor, image, mutableMapOf(), mutableMapOf())
 
         server.state = ServerState.TERMINATED
@@ -189,8 +190,8 @@ class InternalServerTest {
     fun testStopDeletedServer() = runBlockingSimulation {
         val service = mockk<ComputeServiceImpl>()
         val uid = UUID.randomUUID()
-        val flavor = mockk<InternalFlavor>()
-        val image = mockk<InternalImage>()
+        val flavor = mockFlavor()
+        val image = mockImage()
         val server = InternalServer(service, uid, "test", flavor, image, mutableMapOf(), mutableMapOf())
 
         server.state = ServerState.DELETED
@@ -203,8 +204,8 @@ class InternalServerTest {
     fun testStopRunningServer() = runBlockingSimulation {
         val service = mockk<ComputeServiceImpl>()
         val uid = UUID.randomUUID()
-        val flavor = mockk<InternalFlavor>()
-        val image = mockk<InternalImage>()
+        val flavor = mockFlavor()
+        val image = mockImage()
         val server = InternalServer(service, uid, "test", flavor, image, mutableMapOf(), mutableMapOf())
         val host = mockk<Host>(relaxUnitFun = true)
 
@@ -220,8 +221,8 @@ class InternalServerTest {
     fun testDeleteProvisioningServer() = runBlockingSimulation {
         val service = mockk<ComputeServiceImpl>(relaxUnitFun = true)
         val uid = UUID.randomUUID()
-        val flavor = mockk<InternalFlavor>()
-        val image = mockk<InternalImage>()
+        val flavor = mockFlavor()
+        val image = mockImage()
         val server = InternalServer(service, uid, "test", flavor, image, mutableMapOf(), mutableMapOf())
         val request = ComputeServiceImpl.SchedulingRequest(server, 0)
 
@@ -239,8 +240,8 @@ class InternalServerTest {
     fun testDeleteTerminatedServer() = runBlockingSimulation {
         val service = mockk<ComputeServiceImpl>(relaxUnitFun = true)
         val uid = UUID.randomUUID()
-        val flavor = mockk<InternalFlavor>()
-        val image = mockk<InternalImage>()
+        val flavor = mockFlavor()
+        val image = mockImage()
         val server = InternalServer(service, uid, "test", flavor, image, mutableMapOf(), mutableMapOf())
 
         server.state = ServerState.TERMINATED
@@ -255,8 +256,8 @@ class InternalServerTest {
     fun testDeleteDeletedServer() = runBlockingSimulation {
         val service = mockk<ComputeServiceImpl>(relaxUnitFun = true)
         val uid = UUID.randomUUID()
-        val flavor = mockk<InternalFlavor>()
-        val image = mockk<InternalImage>()
+        val flavor = mockFlavor()
+        val image = mockImage()
         val server = InternalServer(service, uid, "test", flavor, image, mutableMapOf(), mutableMapOf())
 
         server.state = ServerState.DELETED
@@ -269,8 +270,8 @@ class InternalServerTest {
     fun testDeleteRunningServer() = runBlockingSimulation {
         val service = mockk<ComputeServiceImpl>(relaxUnitFun = true)
         val uid = UUID.randomUUID()
-        val flavor = mockk<InternalFlavor>()
-        val image = mockk<InternalImage>()
+        val flavor = mockFlavor()
+        val image = mockImage()
         val server = InternalServer(service, uid, "test", flavor, image, mutableMapOf(), mutableMapOf())
         val host = mockk<Host>(relaxUnitFun = true)
 
@@ -282,4 +283,20 @@ class InternalServerTest {
         coVerify { host.delete(server) }
         verify { service.delete(server) }
     }
+
+    private fun mockFlavor(): InternalFlavor {
+        val flavor = mockk<InternalFlavor>()
+        every { flavor.name } returns "c5.large"
+        every { flavor.uid } returns UUID.randomUUID()
+        every { flavor.cpuCount } returns 2
+        every { flavor.memorySize } returns 4096
+        return flavor
+    }
+
+    private fun mockImage(): InternalImage {
+        val image = mockk<InternalImage>()
+        every { image.name } returns "ubuntu-20.04"
+        every { image.uid } returns UUID.randomUUID()
+        return image
+    }
 }