From 9e5e830e15b74f040708e98c09ea41cd96d13871 Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Thu, 27 May 2021 16:34:06 +0200 Subject: simulator: Centralize resource logic in SimResourceInterpreter This change introduces the SimResourceInterpreter which centralizes the logic for scheduling and interpreting the communication between resource consumer and provider. This approach offers better performance due to avoiding invalidating the state of the resource context when not necessary. Benchmarks show in the best case a 5x performance improvement and at worst a 2x improvement. --- opendc-experiments/opendc-experiments-capelin/build.gradle.kts | 2 ++ 1 file changed, 2 insertions(+) (limited to 'opendc-experiments/opendc-experiments-capelin/build.gradle.kts') diff --git a/opendc-experiments/opendc-experiments-capelin/build.gradle.kts b/opendc-experiments/opendc-experiments-capelin/build.gradle.kts index 7c7f0dad..0dade513 100644 --- a/opendc-experiments/opendc-experiments-capelin/build.gradle.kts +++ b/opendc-experiments/opendc-experiments-capelin/build.gradle.kts @@ -48,4 +48,6 @@ dependencies { exclude(group = "org.slf4j", module = "slf4j-log4j12") exclude(group = "log4j") } + + testImplementation(libs.log4j.slf4j) } -- cgit v1.2.3 From 0eb4fa604efe4e0b84d69749f688a79c2249c8b3 Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Wed, 9 Jun 2021 13:38:56 +0200 Subject: build: Eliminate most Hadoop dependencies This change eliminates all Hadoop dependencies that are not necessary for Parquet to work correctly. As a result, the number of dependencies should now be greatly reduced, which in turn leads to less artifacts that need to be retrieved at build time. --- opendc-experiments/opendc-experiments-capelin/build.gradle.kts | 5 ----- 1 file changed, 5 deletions(-) (limited to 'opendc-experiments/opendc-experiments-capelin/build.gradle.kts') diff --git a/opendc-experiments/opendc-experiments-capelin/build.gradle.kts b/opendc-experiments/opendc-experiments-capelin/build.gradle.kts index 0dade513..324cae3e 100644 --- a/opendc-experiments/opendc-experiments-capelin/build.gradle.kts +++ b/opendc-experiments/opendc-experiments-capelin/build.gradle.kts @@ -44,10 +44,5 @@ dependencies { implementation(libs.clikt) implementation(libs.parquet) - implementation(libs.hadoop.client) { - exclude(group = "org.slf4j", module = "slf4j-log4j12") - exclude(group = "log4j") - } - testImplementation(libs.log4j.slf4j) } -- cgit v1.2.3 From f111081627280d4e7e1d7147c56cdce708e32433 Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Wed, 25 Aug 2021 14:06:39 +0200 Subject: build: Upgrade to OpenTelemetry 1.5 This change upgrades the OpenTelemetry dependency to version 1.5, which contains various breaking changes in the metrics API. --- opendc-experiments/opendc-experiments-capelin/build.gradle.kts | 1 + 1 file changed, 1 insertion(+) (limited to 'opendc-experiments/opendc-experiments-capelin/build.gradle.kts') diff --git a/opendc-experiments/opendc-experiments-capelin/build.gradle.kts b/opendc-experiments/opendc-experiments-capelin/build.gradle.kts index 324cae3e..53643aba 100644 --- a/opendc-experiments/opendc-experiments-capelin/build.gradle.kts +++ b/opendc-experiments/opendc-experiments-capelin/build.gradle.kts @@ -37,6 +37,7 @@ dependencies { implementation(projects.opendcSimulator.opendcSimulatorFailures) implementation(projects.opendcCompute.opendcComputeSimulator) implementation(projects.opendcTelemetry.opendcTelemetrySdk) + implementation(libs.opentelemetry.semconv) implementation(libs.kotlin.logging) implementation(libs.config) -- cgit v1.2.3 From e8cdfbcec3f75b3f303ce52bac5f5595a94555e4 Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Tue, 31 Aug 2021 15:14:46 +0200 Subject: refactor(trace): Extract Parquet helpers into separate module This change extracts the Parquet helpers outside format module into a new module, in order to improve re-usability of these helpers. --- opendc-experiments/opendc-experiments-capelin/build.gradle.kts | 1 + 1 file changed, 1 insertion(+) (limited to 'opendc-experiments/opendc-experiments-capelin/build.gradle.kts') diff --git a/opendc-experiments/opendc-experiments-capelin/build.gradle.kts b/opendc-experiments/opendc-experiments-capelin/build.gradle.kts index 53643aba..e597c5ad 100644 --- a/opendc-experiments/opendc-experiments-capelin/build.gradle.kts +++ b/opendc-experiments/opendc-experiments-capelin/build.gradle.kts @@ -32,6 +32,7 @@ dependencies { api(platform(projects.opendcPlatform)) api(projects.opendcHarness.opendcHarnessApi) implementation(projects.opendcFormat) + implementation(projects.opendcTrace.opendcTraceParquet) implementation(projects.opendcSimulator.opendcSimulatorCore) implementation(projects.opendcSimulator.opendcSimulatorCompute) implementation(projects.opendcSimulator.opendcSimulatorFailures) -- cgit v1.2.3 From 214480d154771f0b783829b6e5ec82b837304ad2 Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Tue, 31 Aug 2021 16:18:56 +0200 Subject: refactor(trace): Move Bitbrains format into separate module This change moves Bitbrains trace support into a separate module and adds support for the new trace api. --- opendc-experiments/opendc-experiments-capelin/build.gradle.kts | 1 + 1 file changed, 1 insertion(+) (limited to 'opendc-experiments/opendc-experiments-capelin/build.gradle.kts') diff --git a/opendc-experiments/opendc-experiments-capelin/build.gradle.kts b/opendc-experiments/opendc-experiments-capelin/build.gradle.kts index e597c5ad..97ca97ec 100644 --- a/opendc-experiments/opendc-experiments-capelin/build.gradle.kts +++ b/opendc-experiments/opendc-experiments-capelin/build.gradle.kts @@ -33,6 +33,7 @@ dependencies { api(projects.opendcHarness.opendcHarnessApi) implementation(projects.opendcFormat) implementation(projects.opendcTrace.opendcTraceParquet) + implementation(projects.opendcTrace.opendcTraceBitbrains) implementation(projects.opendcSimulator.opendcSimulatorCore) implementation(projects.opendcSimulator.opendcSimulatorCompute) implementation(projects.opendcSimulator.opendcSimulatorFailures) -- cgit v1.2.3 From b2308e1077dc60ec6a4dc646613a4be5b59695a6 Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Wed, 1 Sep 2021 11:29:27 +0200 Subject: refactor(trace): Implement trace API for WTF reader This change updates the WTF trace reader to support the new streaming trace API. --- opendc-experiments/opendc-experiments-capelin/build.gradle.kts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'opendc-experiments/opendc-experiments-capelin/build.gradle.kts') diff --git a/opendc-experiments/opendc-experiments-capelin/build.gradle.kts b/opendc-experiments/opendc-experiments-capelin/build.gradle.kts index 97ca97ec..65cebe1b 100644 --- a/opendc-experiments/opendc-experiments-capelin/build.gradle.kts +++ b/opendc-experiments/opendc-experiments-capelin/build.gradle.kts @@ -31,7 +31,6 @@ plugins { dependencies { api(platform(projects.opendcPlatform)) api(projects.opendcHarness.opendcHarnessApi) - implementation(projects.opendcFormat) implementation(projects.opendcTrace.opendcTraceParquet) implementation(projects.opendcTrace.opendcTraceBitbrains) implementation(projects.opendcSimulator.opendcSimulatorCore) @@ -45,6 +44,9 @@ dependencies { implementation(libs.config) implementation(libs.progressbar) implementation(libs.clikt) + implementation(libs.jackson.module.kotlin) { + exclude(group = "org.jetbrains.kotlin", module = "kotlin-reflect") + } implementation(libs.parquet) testImplementation(libs.log4j.slf4j) -- cgit v1.2.3 From aaedd4f3eed83d0c3ebc829fec08a1749a2bfba4 Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Fri, 27 Aug 2021 16:41:55 +0200 Subject: refactor(capelin): Move metric collection outside Capelin code This change moves the metric collection outside the Capelin codebase in a separate module so other modules can also benefit from the compute metric collection code. --- opendc-experiments/opendc-experiments-capelin/build.gradle.kts | 1 + 1 file changed, 1 insertion(+) (limited to 'opendc-experiments/opendc-experiments-capelin/build.gradle.kts') diff --git a/opendc-experiments/opendc-experiments-capelin/build.gradle.kts b/opendc-experiments/opendc-experiments-capelin/build.gradle.kts index 65cebe1b..1a4caf91 100644 --- a/opendc-experiments/opendc-experiments-capelin/build.gradle.kts +++ b/opendc-experiments/opendc-experiments-capelin/build.gradle.kts @@ -38,6 +38,7 @@ dependencies { implementation(projects.opendcSimulator.opendcSimulatorFailures) implementation(projects.opendcCompute.opendcComputeSimulator) implementation(projects.opendcTelemetry.opendcTelemetrySdk) + implementation(projects.opendcTelemetry.opendcTelemetryCompute) implementation(libs.opentelemetry.semconv) implementation(libs.kotlin.logging) -- cgit v1.2.3 From 18ff316a6b6ab984ebf8283ea48ed98ec69d8295 Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Thu, 2 Sep 2021 13:20:05 +0200 Subject: refactor(capelin): Restructure input reading classes --- opendc-experiments/opendc-experiments-capelin/build.gradle.kts | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) (limited to 'opendc-experiments/opendc-experiments-capelin/build.gradle.kts') diff --git a/opendc-experiments/opendc-experiments-capelin/build.gradle.kts b/opendc-experiments/opendc-experiments-capelin/build.gradle.kts index 1a4caf91..b2330af0 100644 --- a/opendc-experiments/opendc-experiments-capelin/build.gradle.kts +++ b/opendc-experiments/opendc-experiments-capelin/build.gradle.kts @@ -45,9 +45,8 @@ dependencies { implementation(libs.config) implementation(libs.progressbar) implementation(libs.clikt) - implementation(libs.jackson.module.kotlin) { - exclude(group = "org.jetbrains.kotlin", module = "kotlin-reflect") - } + implementation(libs.jackson.module.kotlin) + implementation(kotlin("reflect")) implementation(libs.parquet) testImplementation(libs.log4j.slf4j) -- cgit v1.2.3 From d24cc0cc9c4fe2145f0337d65e9a75f631365973 Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Fri, 10 Sep 2021 10:59:44 +0200 Subject: refactor(compute): Integrate fault injection into compute simulator This change moves the fault injection logic directly into the opendc-compute-simulator module, so that it can operate at a higher abstraction. In the future, we might again split the module if we can re-use some of its logic. --- opendc-experiments/opendc-experiments-capelin/build.gradle.kts | 1 - 1 file changed, 1 deletion(-) (limited to 'opendc-experiments/opendc-experiments-capelin/build.gradle.kts') diff --git a/opendc-experiments/opendc-experiments-capelin/build.gradle.kts b/opendc-experiments/opendc-experiments-capelin/build.gradle.kts index b2330af0..036d0638 100644 --- a/opendc-experiments/opendc-experiments-capelin/build.gradle.kts +++ b/opendc-experiments/opendc-experiments-capelin/build.gradle.kts @@ -35,7 +35,6 @@ dependencies { implementation(projects.opendcTrace.opendcTraceBitbrains) implementation(projects.opendcSimulator.opendcSimulatorCore) implementation(projects.opendcSimulator.opendcSimulatorCompute) - implementation(projects.opendcSimulator.opendcSimulatorFailures) implementation(projects.opendcCompute.opendcComputeSimulator) implementation(projects.opendcTelemetry.opendcTelemetrySdk) implementation(projects.opendcTelemetry.opendcTelemetryCompute) -- cgit v1.2.3 From 9e8ea96270701e643f95b18d2b91583d9fca08d2 Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Sat, 11 Sep 2021 11:10:38 +0200 Subject: feat(capelin): Implement trace API for Azure VM trace format This change adds a trace API implementation for the Azure VM traces. --- opendc-experiments/opendc-experiments-capelin/build.gradle.kts | 1 + 1 file changed, 1 insertion(+) (limited to 'opendc-experiments/opendc-experiments-capelin/build.gradle.kts') diff --git a/opendc-experiments/opendc-experiments-capelin/build.gradle.kts b/opendc-experiments/opendc-experiments-capelin/build.gradle.kts index 036d0638..7dadd14d 100644 --- a/opendc-experiments/opendc-experiments-capelin/build.gradle.kts +++ b/opendc-experiments/opendc-experiments-capelin/build.gradle.kts @@ -45,6 +45,7 @@ dependencies { implementation(libs.progressbar) implementation(libs.clikt) implementation(libs.jackson.module.kotlin) + implementation(libs.jackson.dataformat.csv) implementation(kotlin("reflect")) implementation(libs.parquet) -- cgit v1.2.3 From 859ce303f0b9110c7110b918e5957c2156fa8b26 Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Fri, 17 Sep 2021 17:48:02 +0200 Subject: refactor(capelin): Extract common code out of Capelin experiments This change creates a new module for doing simulations with virtual machine workloads. We have found that a lot of code in the Capelin experiments code is being re-used by non-experiment modules. --- .../opendc-experiments-capelin/build.gradle.kts | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) (limited to 'opendc-experiments/opendc-experiments-capelin/build.gradle.kts') diff --git a/opendc-experiments/opendc-experiments-capelin/build.gradle.kts b/opendc-experiments/opendc-experiments-capelin/build.gradle.kts index 7dadd14d..010d18b0 100644 --- a/opendc-experiments/opendc-experiments-capelin/build.gradle.kts +++ b/opendc-experiments/opendc-experiments-capelin/build.gradle.kts @@ -31,6 +31,8 @@ plugins { dependencies { api(platform(projects.opendcPlatform)) api(projects.opendcHarness.opendcHarnessApi) + api(projects.opendcCompute.opendcComputeWorkload) + implementation(projects.opendcTrace.opendcTraceParquet) implementation(projects.opendcTrace.opendcTraceBitbrains) implementation(projects.opendcSimulator.opendcSimulatorCore) @@ -38,16 +40,13 @@ dependencies { implementation(projects.opendcCompute.opendcComputeSimulator) implementation(projects.opendcTelemetry.opendcTelemetrySdk) implementation(projects.opendcTelemetry.opendcTelemetryCompute) - implementation(libs.opentelemetry.semconv) - implementation(libs.kotlin.logging) implementation(libs.config) - implementation(libs.progressbar) - implementation(libs.clikt) + implementation(libs.kotlin.logging) + implementation(libs.jackson.databind) implementation(libs.jackson.module.kotlin) - implementation(libs.jackson.dataformat.csv) implementation(kotlin("reflect")) + implementation(libs.opentelemetry.semconv) - implementation(libs.parquet) testImplementation(libs.log4j.slf4j) } -- cgit v1.2.3 From b0ece0533825f5cd7983752330847071f4e438c4 Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Wed, 15 Sep 2021 23:06:08 +0200 Subject: refactor(capelin): Support flexible topology creation This change adds support for creating flexible topologies by creating a TopologyFactory interface that is responsible for configuring the hosts of a compute service. --- opendc-experiments/opendc-experiments-capelin/build.gradle.kts | 1 + 1 file changed, 1 insertion(+) (limited to 'opendc-experiments/opendc-experiments-capelin/build.gradle.kts') diff --git a/opendc-experiments/opendc-experiments-capelin/build.gradle.kts b/opendc-experiments/opendc-experiments-capelin/build.gradle.kts index 010d18b0..4bcbaf61 100644 --- a/opendc-experiments/opendc-experiments-capelin/build.gradle.kts +++ b/opendc-experiments/opendc-experiments-capelin/build.gradle.kts @@ -45,6 +45,7 @@ dependencies { implementation(libs.kotlin.logging) implementation(libs.jackson.databind) implementation(libs.jackson.module.kotlin) + implementation(libs.jackson.dataformat.csv) implementation(kotlin("reflect")) implementation(libs.opentelemetry.semconv) -- cgit v1.2.3 From 68ef3700ed2f69bcf0118bb69eda71e6b1f4d54f Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Tue, 21 Sep 2021 11:34:34 +0200 Subject: feat(trace): Add support for writing traces This change adds a new API for writing traces in a trace format. Currently, writing is only supported by the OpenDC VM format, but over time the other formats will also have support for writing added. --- opendc-experiments/opendc-experiments-capelin/build.gradle.kts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'opendc-experiments/opendc-experiments-capelin/build.gradle.kts') diff --git a/opendc-experiments/opendc-experiments-capelin/build.gradle.kts b/opendc-experiments/opendc-experiments-capelin/build.gradle.kts index 4bcbaf61..23a3e4a7 100644 --- a/opendc-experiments/opendc-experiments-capelin/build.gradle.kts +++ b/opendc-experiments/opendc-experiments-capelin/build.gradle.kts @@ -33,8 +33,6 @@ dependencies { api(projects.opendcHarness.opendcHarnessApi) api(projects.opendcCompute.opendcComputeWorkload) - implementation(projects.opendcTrace.opendcTraceParquet) - implementation(projects.opendcTrace.opendcTraceBitbrains) implementation(projects.opendcSimulator.opendcSimulatorCore) implementation(projects.opendcSimulator.opendcSimulatorCompute) implementation(projects.opendcCompute.opendcComputeSimulator) @@ -49,5 +47,7 @@ dependencies { implementation(kotlin("reflect")) implementation(libs.opentelemetry.semconv) + runtimeOnly(projects.opendcTrace.opendcTraceOpendc) + testImplementation(libs.log4j.slf4j) } -- cgit v1.2.3 From 557797c63c19e80c908eccc96472f215eab0c2f3 Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Mon, 4 Oct 2021 19:26:29 +0200 Subject: perf(experiments): Add benchmark for Capelin experiment --- opendc-experiments/opendc-experiments-capelin/build.gradle.kts | 1 + 1 file changed, 1 insertion(+) (limited to 'opendc-experiments/opendc-experiments-capelin/build.gradle.kts') diff --git a/opendc-experiments/opendc-experiments-capelin/build.gradle.kts b/opendc-experiments/opendc-experiments-capelin/build.gradle.kts index 23a3e4a7..c20556b5 100644 --- a/opendc-experiments/opendc-experiments-capelin/build.gradle.kts +++ b/opendc-experiments/opendc-experiments-capelin/build.gradle.kts @@ -26,6 +26,7 @@ description = "Experiments for the Capelin work" plugins { `experiment-conventions` `testing-conventions` + `benchmark-conventions` } dependencies { -- cgit v1.2.3