refactor(telemetry): Standardize SimHost metrics

This change standardizes the metrics emitted by SimHost instances and
their guests based on the OpenTelemetry semantic conventions. We now
also report CPU time as opposed to CPU work as this metric is more
commonly used.

5 files changed, 67 insertions, 33 deletions

diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/Portfolio.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/Portfolio.kt
index 3ec424f1..6261ebbf 100644
--- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/Portfolio.kt
+++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/Portfolio.kt
@@ -149,9 +149,10 @@ abstract class Portfolio(name: String) : Experiment(name) {
         } finally {
             simulator.close()
             metricReader.close()
+            monitor.close()
         }
 
-        val monitorResults = collectServiceMetrics(clock.millis(), simulator.producers[0])
+        val monitorResults = collectServiceMetrics(clock.instant(), simulator.producers[0])
         logger.debug {
             "Scheduler " +
                 "Success=${monitorResults.attemptsSuccess} " +
diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/export/parquet/ParquetDataWriter.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/export/parquet/ParquetDataWriter.kt
index 5684bde9..e3d15c3b 100644
--- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/export/parquet/ParquetDataWriter.kt
+++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/export/parquet/ParquetDataWriter.kt
@@ -27,7 +27,6 @@ import org.apache.avro.Schema
 import org.apache.avro.generic.GenericData
 import org.apache.avro.generic.GenericRecordBuilder
 import org.apache.parquet.avro.AvroParquetWriter
-import org.apache.parquet.example.Paper.schema
 import org.apache.parquet.hadoop.ParquetFileWriter
 import org.apache.parquet.hadoop.ParquetWriter
 import org.apache.parquet.hadoop.metadata.CompressionCodecName
diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/export/parquet/ParquetHostDataWriter.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/export/parquet/ParquetHostDataWriter.kt
index fa00fc35..36207045 100644
--- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/export/parquet/ParquetHostDataWriter.kt
+++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/export/parquet/ParquetHostDataWriter.kt
@@ -44,20 +44,31 @@ public class ParquetHostDataWriter(path: File, bufferSize: Int) :
     }
 
     override fun convert(builder: GenericRecordBuilder, data: HostData) {
-        builder["timestamp"] = data.timestamp
+        builder["timestamp"] = data.timestamp.toEpochMilli()
+
         builder["host_id"] = data.host.id
-        builder["powered_on"] = true
+        builder["num_cpus"] = data.host.cpuCount
+        builder["mem_capacity"] = data.host.memCapacity
+
         builder["uptime"] = data.uptime
         builder["downtime"] = data.downtime
-        builder["total_work"] = data.totalWork
-        builder["granted_work"] = data.grantedWork
-        builder["overcommitted_work"] = data.overcommittedWork
-        builder["interfered_work"] = data.interferedWork
-        builder["cpu_usage"] = data.cpuUsage
-        builder["cpu_demand"] = data.cpuDemand
-        builder["power_draw"] = data.powerDraw
-        builder["num_instances"] = data.instanceCount
-        builder["num_cpus"] = data.host.cpuCount
+        val bootTime = data.bootTime
+        if (bootTime != null) {
+            builder["boot_time"] = bootTime.toEpochMilli()
+        }
+
+        builder["cpu_limit"] = data.cpuLimit
+        builder["cpu_time_active"] = data.cpuActiveTime
+        builder["cpu_time_idle"] = data.cpuIdleTime
+        builder["cpu_time_steal"] = data.cpuStealTime
+        builder["cpu_time_lost"] = data.cpuLostTime
+
+        builder["power_total"] = data.powerTotal
+
+        builder["guests_terminated"] = data.guestsTerminated
+        builder["guests_running"] = data.guestsRunning
+        builder["guests_error"] = data.guestsError
+        builder["guests_invalid"] = data.guestsInvalid
     }
 
     override fun toString(): String = "host-writer"
@@ -69,18 +80,21 @@ public class ParquetHostDataWriter(path: File, bufferSize: Int) :
             .fields()
             .requiredLong("timestamp")
             .requiredString("host_id")
-            .requiredBoolean("powered_on")
+            .requiredInt("num_cpus")
+            .requiredLong("mem_capacity")
             .requiredLong("uptime")
             .requiredLong("downtime")
-            .requiredDouble("total_work")
-            .requiredDouble("granted_work")
-            .requiredDouble("overcommitted_work")
-            .requiredDouble("interfered_work")
-            .requiredDouble("cpu_usage")
-            .requiredDouble("cpu_demand")
-            .requiredDouble("power_draw")
-            .requiredInt("num_instances")
-            .requiredInt("num_cpus")
+            .optionalLong("boot_time")
+            .requiredDouble("cpu_limit")
+            .requiredLong("cpu_time_active")
+            .requiredLong("cpu_time_idle")
+            .requiredLong("cpu_time_steal")
+            .requiredLong("cpu_time_lost")
+            .requiredDouble("power_total")
+            .requiredInt("guests_terminated")
+            .requiredInt("guests_running")
+            .requiredInt("guests_error")
+            .requiredInt("guests_invalid")
             .endRecord()
     }
 }
diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/export/parquet/ParquetServerDataWriter.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/export/parquet/ParquetServerDataWriter.kt
index bb2db4b7..c5a5e7c0 100644
--- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/export/parquet/ParquetServerDataWriter.kt
+++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/export/parquet/ParquetServerDataWriter.kt
@@ -40,18 +40,31 @@ public class ParquetServerDataWriter(path: File, bufferSize: Int) :
     override fun buildWriter(builder: AvroParquetWriter.Builder<GenericData.Record>): ParquetWriter<GenericData.Record> {
         return builder
             .withDictionaryEncoding("server_id", true)
-            .withDictionaryEncoding("state", true)
+            .withDictionaryEncoding("host_id", true)
             .build()
     }
 
     override fun convert(builder: GenericRecordBuilder, data: ServerData) {
-        builder["timestamp"] = data.timestamp
-        builder["server_id"] = data.server
-        // builder["state"] = data.server.state
+        builder["timestamp"] = data.timestamp.toEpochMilli()
+
+        builder["server_id"] = data.server.id
+        builder["host_id"] = data.host?.id
+        builder["num_vcpus"] = data.server.cpuCount
+        builder["mem_capacity"] = data.server.memCapacity
+
         builder["uptime"] = data.uptime
         builder["downtime"] = data.downtime
-        // builder["num_vcpus"] = data.server.flavor.cpuCount
-        // builder["mem_capacity"] = data.server.flavor.memorySize
+        val bootTime = data.bootTime
+        if (bootTime != null) {
+            builder["boot_time"] = bootTime.toEpochMilli()
+        }
+        builder["scheduling_latency"] = data.schedulingLatency
+
+        builder["cpu_limit"] = data.cpuLimit
+        builder["cpu_time_active"] = data.cpuActiveTime
+        builder["cpu_time_idle"] = data.cpuIdleTime
+        builder["cpu_time_steal"] = data.cpuStealTime
+        builder["cpu_time_lost"] = data.cpuLostTime
     }
 
     override fun toString(): String = "server-writer"
@@ -63,11 +76,18 @@ public class ParquetServerDataWriter(path: File, bufferSize: Int) :
             .fields()
             .requiredLong("timestamp")
             .requiredString("server_id")
-            .requiredString("state")
-            .requiredLong("uptime")
-            .requiredLong("downtime")
+            .optionalString("host_id")
             .requiredInt("num_vcpus")
             .requiredLong("mem_capacity")
+            .requiredLong("uptime")
+            .requiredLong("downtime")
+            .optionalLong("boot_time")
+            .requiredLong("scheduling_latency")
+            .requiredDouble("cpu_limit")
+            .requiredLong("cpu_time_active")
+            .requiredLong("cpu_time_idle")
+            .requiredLong("cpu_time_steal")
+            .requiredLong("cpu_time_lost")
             .endRecord()
     }
 }
diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/export/parquet/ParquetServiceDataWriter.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/export/parquet/ParquetServiceDataWriter.kt
index 29b48878..d9ca55cb 100644
--- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/export/parquet/ParquetServiceDataWriter.kt
+++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/export/parquet/ParquetServiceDataWriter.kt
@@ -35,7 +35,7 @@ public class ParquetServiceDataWriter(path: File, bufferSize: Int) :
     ParquetDataWriter<ServiceData>(path, SCHEMA, bufferSize) {
 
     override fun convert(builder: GenericRecordBuilder, data: ServiceData) {
-        builder["timestamp"] = data.timestamp
+        builder["timestamp"] = data.timestamp.toEpochMilli()
         builder["hosts_up"] = data.hostsUp
         builder["hosts_down"] = data.hostsDown
         builder["servers_pending"] = data.serversPending