From b0806dcf21ab811c46b715cfdff8a6307e117810 Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Wed, 1 Sep 2021 18:28:34 +0200 Subject: feat(trace): Add API for trace reading This change introduces a new OpenDC API for reading various trace formats in a streaming manner. --- .../org/opendc/trace/ResourceStateColumns.kt | 86 ++++++++++++++++++++++ 1 file changed, 86 insertions(+) create mode 100644 opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/ResourceStateColumns.kt (limited to 'opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/ResourceStateColumns.kt') diff --git a/opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/ResourceStateColumns.kt b/opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/ResourceStateColumns.kt new file mode 100644 index 00000000..17f52ab6 --- /dev/null +++ b/opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/ResourceStateColumns.kt @@ -0,0 +1,86 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +@file:JvmName("ResourceStateColumns") +package org.opendc.trace + +import java.time.Instant + +/** + * Identifier of the resource. + */ +public val RESOURCE_STATE_ID: TableColumn = stringColumn("resource_state:id") + +/** + * Timestamp for the state. + */ +public val RESOURCE_STATE_TIMESTAMP: TableColumn = TableColumn("resource_state:timestamp", Instant::class.java) + +/** + * Number of CPUs for the resource. + */ +public val RESOURCE_STATE_NCPUS: TableColumn = intColumn("resource_state:ncpus") + +/** + * Total CPU capacity of the resource in MHz. + */ +public val RESOURCE_STATE_CPU_CAPACITY: TableColumn = doubleColumn("resource_state:cpu_capacity") + +/** + * Total CPU usage of the resource in MHz. + */ +public val RESOURCE_STATE_CPU_USAGE: TableColumn = doubleColumn("resource_state:cpu_usage") + +/** + * Total CPU usage of the resource in percentage. + */ +public val RESOURCE_STATE_CPU_USAGE_PCT: TableColumn = doubleColumn("resource_state:cpu_usage_pct") + +/** + * Memory capacity of the resource in KB. + */ +public val RESOURCE_STATE_MEM_CAPACITY: TableColumn = doubleColumn("resource_state:mem_capacity") + +/** + * Memory usage of the resource in KB. + */ +public val RESOURCE_STATE_MEM_USAGE: TableColumn = doubleColumn("resource_state:mem_usage") + +/** + * Disk read throughput of the resource in KB/s. + */ +public val RESOURCE_STATE_DISK_READ: TableColumn = doubleColumn("resource_state:disk_read") + +/** + * Disk write throughput of the resource in KB/s. + */ +public val RESOURCE_STATE_DISK_WRITE: TableColumn = doubleColumn("resource_state:disk_write") + +/** + * Network receive throughput of the resource in KB/s. + */ +public val RESOURCE_STATE_NET_RX: TableColumn = doubleColumn("resource_state:net_rx") + +/** + * Network transmit throughput of the resource in KB/s. + */ +public val RESOURCE_STATE_NET_TX: TableColumn = doubleColumn("resource_state:net_tx") -- cgit v1.2.3 From 8bae0f3053a53aac9d483ae97d99f2e7e80b42ef Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Wed, 1 Sep 2021 22:30:39 +0200 Subject: refactor(capelin): Migrate trace reader to new trace API This change updates the trace reading classes in the Capelin experiment to use the new trace API in order to re-use many of the trace reading parts. --- .../org/opendc/trace/ResourceStateColumns.kt | 26 ++++++++++++++++++++++ 1 file changed, 26 insertions(+) (limited to 'opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/ResourceStateColumns.kt') diff --git a/opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/ResourceStateColumns.kt b/opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/ResourceStateColumns.kt index 17f52ab6..c2d896a8 100644 --- a/opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/ResourceStateColumns.kt +++ b/opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/ResourceStateColumns.kt @@ -23,6 +23,7 @@ @file:JvmName("ResourceStateColumns") package org.opendc.trace +import java.time.Duration import java.time.Instant /** @@ -30,11 +31,26 @@ import java.time.Instant */ public val RESOURCE_STATE_ID: TableColumn = stringColumn("resource_state:id") +/** + * The cluster to which the resource belongs. + */ +public val RESOURCE_STATE_CLUSTER_ID: TableColumn = stringColumn("resource_state:cluster_id") + /** * Timestamp for the state. */ public val RESOURCE_STATE_TIMESTAMP: TableColumn = TableColumn("resource_state:timestamp", Instant::class.java) +/** + * Duration for the state. + */ +public val RESOURCE_STATE_DURATION: TableColumn = TableColumn("resource_state:duration", Duration::class.java) + +/** + * A flag to indicate that the resource is powered on. + */ +public val RESOURCE_STATE_POWERED_ON: TableColumn = booleanColumn("resource_state:powered_on") + /** * Number of CPUs for the resource. */ @@ -55,6 +71,16 @@ public val RESOURCE_STATE_CPU_USAGE: TableColumn = doubleColumn("resourc */ public val RESOURCE_STATE_CPU_USAGE_PCT: TableColumn = doubleColumn("resource_state:cpu_usage_pct") +/** + * Total CPU demand of the resource in MHz. + */ +public val RESOURCE_STATE_CPU_DEMAND: TableColumn = doubleColumn("resource_state:cpu_demand") + +/** + * CPU ready percentage. + */ +public val RESOURCE_STATE_CPU_READY_PCT: TableColumn = doubleColumn("resource_state:cpu_ready_pct") + /** * Memory capacity of the resource in KB. */ -- cgit v1.2.3 From 5935531137a22fdb920921580d491f86adec65c9 Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Thu, 2 Sep 2021 11:11:50 +0200 Subject: perf(trace): Improve performance of column lookup --- .../kotlin/org/opendc/trace/ResourceStateColumns.kt | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) (limited to 'opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/ResourceStateColumns.kt') diff --git a/opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/ResourceStateColumns.kt b/opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/ResourceStateColumns.kt index c2d896a8..1933967e 100644 --- a/opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/ResourceStateColumns.kt +++ b/opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/ResourceStateColumns.kt @@ -29,84 +29,101 @@ import java.time.Instant /** * Identifier of the resource. */ +@JvmField public val RESOURCE_STATE_ID: TableColumn = stringColumn("resource_state:id") /** * The cluster to which the resource belongs. */ +@JvmField public val RESOURCE_STATE_CLUSTER_ID: TableColumn = stringColumn("resource_state:cluster_id") /** * Timestamp for the state. */ +@JvmField public val RESOURCE_STATE_TIMESTAMP: TableColumn = TableColumn("resource_state:timestamp", Instant::class.java) /** * Duration for the state. */ +@JvmField public val RESOURCE_STATE_DURATION: TableColumn = TableColumn("resource_state:duration", Duration::class.java) /** * A flag to indicate that the resource is powered on. */ +@JvmField public val RESOURCE_STATE_POWERED_ON: TableColumn = booleanColumn("resource_state:powered_on") /** * Number of CPUs for the resource. */ +@JvmField public val RESOURCE_STATE_NCPUS: TableColumn = intColumn("resource_state:ncpus") /** * Total CPU capacity of the resource in MHz. */ +@JvmField public val RESOURCE_STATE_CPU_CAPACITY: TableColumn = doubleColumn("resource_state:cpu_capacity") /** * Total CPU usage of the resource in MHz. */ +@JvmField public val RESOURCE_STATE_CPU_USAGE: TableColumn = doubleColumn("resource_state:cpu_usage") /** * Total CPU usage of the resource in percentage. */ +@JvmField public val RESOURCE_STATE_CPU_USAGE_PCT: TableColumn = doubleColumn("resource_state:cpu_usage_pct") /** * Total CPU demand of the resource in MHz. */ +@JvmField public val RESOURCE_STATE_CPU_DEMAND: TableColumn = doubleColumn("resource_state:cpu_demand") /** * CPU ready percentage. */ +@JvmField public val RESOURCE_STATE_CPU_READY_PCT: TableColumn = doubleColumn("resource_state:cpu_ready_pct") /** * Memory capacity of the resource in KB. */ +@JvmField public val RESOURCE_STATE_MEM_CAPACITY: TableColumn = doubleColumn("resource_state:mem_capacity") /** * Memory usage of the resource in KB. */ +@JvmField public val RESOURCE_STATE_MEM_USAGE: TableColumn = doubleColumn("resource_state:mem_usage") /** * Disk read throughput of the resource in KB/s. */ +@JvmField public val RESOURCE_STATE_DISK_READ: TableColumn = doubleColumn("resource_state:disk_read") /** * Disk write throughput of the resource in KB/s. */ +@JvmField public val RESOURCE_STATE_DISK_WRITE: TableColumn = doubleColumn("resource_state:disk_write") /** * Network receive throughput of the resource in KB/s. */ +@JvmField public val RESOURCE_STATE_NET_RX: TableColumn = doubleColumn("resource_state:net_rx") /** * Network transmit throughput of the resource in KB/s. */ +@JvmField public val RESOURCE_STATE_NET_TX: TableColumn = doubleColumn("resource_state:net_tx") -- cgit v1.2.3 From 474044649a67cfcc857615b6a0f8387a2954abbd Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Thu, 16 Sep 2021 12:34:53 +0200 Subject: feat(trace): Update OpenDC VM trace format This change optimizes the OpenDC VM trace format by removing unnecessary columns as well as optimizing the writer settings. The new implementation still supports reading the old trace format in case users run OpenDC with older workload traces. --- .../src/main/kotlin/org/opendc/trace/ResourceStateColumns.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/ResourceStateColumns.kt') diff --git a/opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/ResourceStateColumns.kt b/opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/ResourceStateColumns.kt index 1933967e..b683923b 100644 --- a/opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/ResourceStateColumns.kt +++ b/opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/ResourceStateColumns.kt @@ -60,7 +60,7 @@ public val RESOURCE_STATE_POWERED_ON: TableColumn = booleanColumn("reso * Number of CPUs for the resource. */ @JvmField -public val RESOURCE_STATE_NCPUS: TableColumn = intColumn("resource_state:ncpus") +public val RESOURCE_STATE_CPU_COUNT: TableColumn = intColumn("resource_state:cpu_count") /** * Total CPU capacity of the resource in MHz. -- cgit v1.2.3 From 55a4c8208cc44ac626f7b8c61a19d5ec725ec936 Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Mon, 20 Sep 2021 11:48:18 +0200 Subject: refactor(trace): Unify columns of different tables This change unifies columns of different tables used by trace formats. This concretely means that instead of having columns specific per table (e.g., RESOURCE_ID and RESOURCE_STATE_ID), with this changes these columns are shared between the tables with a single definition (RESOURCE_ID). --- .../org/opendc/trace/ResourceStateColumns.kt | 54 +++++----------------- 1 file changed, 12 insertions(+), 42 deletions(-) (limited to 'opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/ResourceStateColumns.kt') diff --git a/opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/ResourceStateColumns.kt b/opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/ResourceStateColumns.kt index b683923b..44762da5 100644 --- a/opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/ResourceStateColumns.kt +++ b/opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/ResourceStateColumns.kt @@ -26,104 +26,74 @@ package org.opendc.trace import java.time.Duration import java.time.Instant -/** - * Identifier of the resource. - */ -@JvmField -public val RESOURCE_STATE_ID: TableColumn = stringColumn("resource_state:id") - -/** - * The cluster to which the resource belongs. - */ -@JvmField -public val RESOURCE_STATE_CLUSTER_ID: TableColumn = stringColumn("resource_state:cluster_id") - /** * Timestamp for the state. */ @JvmField -public val RESOURCE_STATE_TIMESTAMP: TableColumn = TableColumn("resource_state:timestamp", Instant::class.java) +public val RESOURCE_STATE_TIMESTAMP: TableColumn = column("resource_state:timestamp") /** * Duration for the state. */ @JvmField -public val RESOURCE_STATE_DURATION: TableColumn = TableColumn("resource_state:duration", Duration::class.java) +public val RESOURCE_STATE_DURATION: TableColumn = column("resource_state:duration") /** * A flag to indicate that the resource is powered on. */ @JvmField -public val RESOURCE_STATE_POWERED_ON: TableColumn = booleanColumn("resource_state:powered_on") - -/** - * Number of CPUs for the resource. - */ -@JvmField -public val RESOURCE_STATE_CPU_COUNT: TableColumn = intColumn("resource_state:cpu_count") - -/** - * Total CPU capacity of the resource in MHz. - */ -@JvmField -public val RESOURCE_STATE_CPU_CAPACITY: TableColumn = doubleColumn("resource_state:cpu_capacity") +public val RESOURCE_STATE_POWERED_ON: TableColumn = column("resource_state:powered_on") /** * Total CPU usage of the resource in MHz. */ @JvmField -public val RESOURCE_STATE_CPU_USAGE: TableColumn = doubleColumn("resource_state:cpu_usage") +public val RESOURCE_STATE_CPU_USAGE: TableColumn = column("resource_state:cpu_usage") /** * Total CPU usage of the resource in percentage. */ @JvmField -public val RESOURCE_STATE_CPU_USAGE_PCT: TableColumn = doubleColumn("resource_state:cpu_usage_pct") +public val RESOURCE_STATE_CPU_USAGE_PCT: TableColumn = column("resource_state:cpu_usage_pct") /** * Total CPU demand of the resource in MHz. */ @JvmField -public val RESOURCE_STATE_CPU_DEMAND: TableColumn = doubleColumn("resource_state:cpu_demand") +public val RESOURCE_STATE_CPU_DEMAND: TableColumn = column("resource_state:cpu_demand") /** * CPU ready percentage. */ @JvmField -public val RESOURCE_STATE_CPU_READY_PCT: TableColumn = doubleColumn("resource_state:cpu_ready_pct") - -/** - * Memory capacity of the resource in KB. - */ -@JvmField -public val RESOURCE_STATE_MEM_CAPACITY: TableColumn = doubleColumn("resource_state:mem_capacity") +public val RESOURCE_STATE_CPU_READY_PCT: TableColumn = column("resource_state:cpu_ready_pct") /** * Memory usage of the resource in KB. */ @JvmField -public val RESOURCE_STATE_MEM_USAGE: TableColumn = doubleColumn("resource_state:mem_usage") +public val RESOURCE_STATE_MEM_USAGE: TableColumn = column("resource_state:mem_usage") /** * Disk read throughput of the resource in KB/s. */ @JvmField -public val RESOURCE_STATE_DISK_READ: TableColumn = doubleColumn("resource_state:disk_read") +public val RESOURCE_STATE_DISK_READ: TableColumn = column("resource_state:disk_read") /** * Disk write throughput of the resource in KB/s. */ @JvmField -public val RESOURCE_STATE_DISK_WRITE: TableColumn = doubleColumn("resource_state:disk_write") +public val RESOURCE_STATE_DISK_WRITE: TableColumn = column("resource_state:disk_write") /** * Network receive throughput of the resource in KB/s. */ @JvmField -public val RESOURCE_STATE_NET_RX: TableColumn = doubleColumn("resource_state:net_rx") +public val RESOURCE_STATE_NET_RX: TableColumn = column("resource_state:net_rx") /** * Network transmit throughput of the resource in KB/s. */ @JvmField -public val RESOURCE_STATE_NET_TX: TableColumn = doubleColumn("resource_state:net_tx") +public val RESOURCE_STATE_NET_TX: TableColumn = column("resource_state:net_tx") -- cgit v1.2.3