feat(telemetry): Report provisioning time of virtual machines

This change adds support for collecting the provisioning time of virtual
machines in addition to their boot time.

6 files changed, 46 insertions, 24 deletions

diff --git a/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/internal/ComputeServiceImpl.kt b/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/internal/ComputeServiceImpl.kt
index 2a07a208..292feabe 100644
--- a/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/internal/ComputeServiceImpl.kt
+++ b/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/internal/ComputeServiceImpl.kt
@@ -26,6 +26,7 @@ import io.opentelemetry.api.common.AttributeKey
 import io.opentelemetry.api.common.Attributes
 import io.opentelemetry.api.metrics.Meter
 import io.opentelemetry.api.metrics.MeterProvider
+import io.opentelemetry.api.metrics.ObservableLongMeasurement
 import kotlinx.coroutines.*
 import mu.KotlinLogging
 import org.opendc.compute.api.*
@@ -173,6 +174,12 @@ internal class ComputeServiceImpl(
                 result.observe(available, upState)
                 result.observe(total - available, downState)
             }
+
+        meter.gaugeBuilder("system.time.provision")
+            .setDescription("The most recent timestamp where the server entered a provisioned state")
+            .setUnit("1")
+            .ofLongs()
+            .buildWithCallback(::collectProvisionTime)
     }
 
     override fun newClient(): ComputeClient {
@@ -324,8 +331,10 @@ internal class ComputeServiceImpl(
 
     internal fun schedule(server: InternalServer): SchedulingRequest {
         logger.debug { "Enqueueing server ${server.uid} to be assigned to host." }
+        val now = clock.millis()
+        val request = SchedulingRequest(server, now)
 
-        val request = SchedulingRequest(server, clock.millis())
+        server.lastProvisioningTimestamp = now
         queue.add(request)
         _serversPending.add(1)
         requestSchedulingCycle()
@@ -501,4 +510,13 @@ internal class ComputeServiceImpl(
             requestSchedulingCycle()
         }
     }
+
+    /**
+     * Collect the timestamp when each server entered its provisioning state most recently.
+     */
+    private fun collectProvisionTime(result: ObservableLongMeasurement) {
+        for ((_, server) in servers) {
+            result.observe(server.lastProvisioningTimestamp, server.attributes)
+        }
+    }
 }
diff --git a/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/internal/InternalServer.kt b/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/internal/InternalServer.kt
index 05a7e1bf..f1b92c66 100644
--- a/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/internal/InternalServer.kt
+++ b/opendc-compute/opendc-compute-service/src/main/kotlin/org/opendc/compute/service/internal/InternalServer.kt
@@ -55,7 +55,7 @@ internal class InternalServer(
     /**
      * The attributes of a server.
      */
-    internal val attributes: Attributes = Attributes.builder()
+    @JvmField internal val attributes: Attributes = Attributes.builder()
         .put(ResourceAttributes.HOST_NAME, name)
         .put(ResourceAttributes.HOST_ID, uid.toString())
         .put(ResourceAttributes.HOST_TYPE, flavor.name)
@@ -70,7 +70,12 @@ internal class InternalServer(
     /**
      * The [Host] that has been assigned to host the server.
      */
-    internal var host: Host? = null
+    @JvmField internal var host: Host? = null
+
+    /**
+     * The most recent timestamp when the server entered a provisioning state.
+     */
+    @JvmField internal var lastProvisioningTimestamp: Long = Long.MIN_VALUE
 
     /**
      * The current scheduling request.
diff --git a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/internal/Guest.kt b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/internal/Guest.kt
index 5ea1860d..61b3214e 100644
--- a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/internal/Guest.kt
+++ b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/internal/Guest.kt
@@ -248,7 +248,7 @@ internal class Guest(
      */
     fun collectBootTime(result: ObservableLongMeasurement) {
         if (_bootTime != Long.MIN_VALUE) {
-            result.observe(_bootTime)
+            result.observe(_bootTime, attributes)
         }
     }
 
diff --git a/opendc-compute/opendc-compute-workload/src/main/kotlin/org/opendc/compute/workload/export/parquet/ParquetServerDataWriter.kt b/opendc-compute/opendc-compute-workload/src/main/kotlin/org/opendc/compute/workload/export/parquet/ParquetServerDataWriter.kt
index 0d11ec23..4ebf8c62 100644
--- a/opendc-compute/opendc-compute-workload/src/main/kotlin/org/opendc/compute/workload/export/parquet/ParquetServerDataWriter.kt
+++ b/opendc-compute/opendc-compute-workload/src/main/kotlin/org/opendc/compute/workload/export/parquet/ParquetServerDataWriter.kt
@@ -55,9 +55,8 @@ public class ParquetServerDataWriter(path: File, bufferSize: Int) :
 
         builder["uptime"] = data.uptime
         builder["downtime"] = data.downtime
-        val bootTime = data.bootTime
-        builder["boot_time"] = bootTime?.toEpochMilli()
-        builder["scheduling_latency"] = data.schedulingLatency
+        builder["boot_time"] = data.bootTime?.toEpochMilli()
+        builder["provision_time"] = data.provisionTime?.toEpochMilli()
 
         builder["cpu_count"] = data.server.cpuCount
         builder["cpu_limit"] = data.cpuLimit
@@ -81,8 +80,8 @@ public class ParquetServerDataWriter(path: File, bufferSize: Int) :
             .name("host_id").type(UUID_SCHEMA.optional()).noDefault()
             .requiredLong("uptime")
             .requiredLong("downtime")
+            .name("provision_time").type(TIMESTAMP_SCHEMA.optional()).noDefault()
             .name("boot_time").type(TIMESTAMP_SCHEMA.optional()).noDefault()
-            .requiredLong("scheduling_latency")
             .requiredInt("cpu_count")
             .requiredDouble("cpu_limit")
             .requiredLong("cpu_time_active")
diff --git a/opendc-telemetry/opendc-telemetry-compute/src/main/kotlin/org/opendc/telemetry/compute/ComputeMetricAggregator.kt b/opendc-telemetry/opendc-telemetry-compute/src/main/kotlin/org/opendc/telemetry/compute/ComputeMetricAggregator.kt
index d5257bbd..b293f7b5 100644
--- a/opendc-telemetry/opendc-telemetry-compute/src/main/kotlin/org/opendc/telemetry/compute/ComputeMetricAggregator.kt
+++ b/opendc-telemetry/opendc-telemetry-compute/src/main/kotlin/org/opendc/telemetry/compute/ComputeMetricAggregator.kt
@@ -30,7 +30,6 @@ import io.opentelemetry.sdk.resources.Resource
 import io.opentelemetry.semconv.resource.attributes.ResourceAttributes
 import org.opendc.telemetry.compute.table.*
 import java.time.Instant
-import kotlin.math.roundToLong
 
 /**
  * Helper class responsible for aggregating [MetricData] into [ServiceData], [HostData] and [ServerData].
@@ -81,12 +80,6 @@ public class ComputeMetricAggregator {
                         }
                     }
                 }
-                "scheduler.latency" -> {
-                    for (point in metric.doubleHistogramData.points) {
-                        val server = getServer(servers, point) ?: continue
-                        server.schedulingLatency = (point.sum / point.count).roundToLong()
-                    }
-                }
 
                 // SimHost
                 "system.guests" -> {
@@ -190,13 +183,20 @@ public class ComputeMetricAggregator {
                         val server = getServer(servers, point)
 
                         if (server != null) {
-                            server.bootTime = point.value
+                            server.bootTime = Instant.ofEpochMilli(point.value)
                             server.host = agg.host
                         } else {
-                            agg.bootTime = point.value
+                            agg.bootTime = Instant.ofEpochMilli(point.value)
                         }
                     }
                 }
+                "system.time.provision" -> {
+                    for (point in metric.longGaugeData.points) {
+                        val server = getServer(servers, point) ?: continue
+                        server.recordTimestamp(point)
+                        server.provisionTime = Instant.ofEpochMilli(point.value)
+                    }
+                }
             }
         }
     }
@@ -323,7 +323,7 @@ public class ComputeMetricAggregator {
         private var previousUptime = 0L
         @JvmField var downtime = 0L
         private var previousDowntime = 0L
-        @JvmField var bootTime = Long.MIN_VALUE
+        @JvmField var bootTime: Instant? = null
 
         /**
          * Finish the aggregation for this cycle.
@@ -379,7 +379,7 @@ public class ComputeMetricAggregator {
                 powerTotal - previousPowerTotal,
                 uptime - previousUptime,
                 downtime - previousDowntime,
-                if (bootTime != Long.MIN_VALUE) Instant.ofEpochMilli(bootTime) else null
+                bootTime
             )
         }
 
@@ -419,8 +419,8 @@ public class ComputeMetricAggregator {
         private var previousUptime = 0L
         @JvmField var downtime: Long = 0
         private var previousDowntime = 0L
-        @JvmField var bootTime: Long = 0
-        @JvmField var schedulingLatency = 0L
+        @JvmField var provisionTime: Instant? = null
+        @JvmField var bootTime: Instant? = null
         @JvmField var cpuLimit = 0.0
         @JvmField var cpuActiveTime = 0L
         @JvmField var cpuIdleTime = 0L
@@ -461,8 +461,8 @@ public class ComputeMetricAggregator {
                 host,
                 uptime - previousUptime,
                 downtime - previousDowntime,
-                if (bootTime != Long.MIN_VALUE) Instant.ofEpochMilli(bootTime) else null,
-                schedulingLatency,
+                provisionTime,
+                bootTime,
                 cpuLimit,
                 cpuActiveTime - previousCpuActiveTime,
                 cpuIdleTime - previousCpuIdleTime,
diff --git a/opendc-telemetry/opendc-telemetry-compute/src/main/kotlin/org/opendc/telemetry/compute/table/ServerData.kt b/opendc-telemetry/opendc-telemetry-compute/src/main/kotlin/org/opendc/telemetry/compute/table/ServerData.kt
index c48bff3a..6fd2a81b 100644
--- a/opendc-telemetry/opendc-telemetry-compute/src/main/kotlin/org/opendc/telemetry/compute/table/ServerData.kt
+++ b/opendc-telemetry/opendc-telemetry-compute/src/main/kotlin/org/opendc/telemetry/compute/table/ServerData.kt
@@ -33,8 +33,8 @@ public data class ServerData(
     val host: HostInfo?,
     val uptime: Long,
     val downtime: Long,
+    val provisionTime: Instant?,
     val bootTime: Instant?,
-    val schedulingLatency: Long,
     val cpuLimit: Double,
     val cpuActiveTime: Long,
     val cpuIdleTime: Long,