From b6acc74b38643615df02ef2131380c5e8eba00dd Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Wed, 9 Jun 2021 21:59:20 +0200 Subject: simulator: Add module for datacenter power components This change adds a new module for simulating power components in datacenters such as PDUs and UPSes. This module will serve as the basis for the power monitoring framework in OpenDC and will future integrate with the other simulation components (such as compute). --- settings.gradle.kts | 1 + 1 file changed, 1 insertion(+) (limited to 'settings.gradle.kts') diff --git a/settings.gradle.kts b/settings.gradle.kts index d09e2912..3b0571b8 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -40,6 +40,7 @@ include(":opendc-web:opendc-web-ui") include(":opendc-web:opendc-web-runner") include(":opendc-simulator:opendc-simulator-core") include(":opendc-simulator:opendc-simulator-resources") +include(":opendc-simulator:opendc-simulator-power") include(":opendc-simulator:opendc-simulator-compute") include(":opendc-simulator:opendc-simulator-failures") include(":opendc-telemetry:opendc-telemetry-api") -- cgit v1.2.3 From 0fd7b3116fbace7deb8202d1849cece7146462a9 Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Thu, 17 Jun 2021 14:15:45 +0200 Subject: faas: Rename opendc-serverless to opendc-faas This change renames the opendc-serverless module to opendc-faas to better distinguish between the two terms (Serverless and FaaS) and be clearer about the intent of the module. The opendc-faas module holds the code for the FaaS platform on top of OpenDC. Although this is one approach of doing serverless, serverless can also entail other services that will not be covered by this module. --- settings.gradle.kts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'settings.gradle.kts') diff --git a/settings.gradle.kts b/settings.gradle.kts index 3b0571b8..34b9bf36 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -27,9 +27,9 @@ include(":opendc-compute:opendc-compute-service") include(":opendc-compute:opendc-compute-simulator") include(":opendc-workflow:opendc-workflow-api") include(":opendc-workflow:opendc-workflow-service") -include(":opendc-serverless:opendc-serverless-api") -include(":opendc-serverless:opendc-serverless-service") -include(":opendc-serverless:opendc-serverless-simulator") +include(":opendc-faas:opendc-faas-api") +include(":opendc-faas:opendc-faas-service") +include(":opendc-faas:opendc-faas-simulator") include(":opendc-format") include(":opendc-experiments:opendc-experiments-capelin") include(":opendc-experiments:opendc-experiments-energy21") -- cgit v1.2.3 From f28cc9964ad1ca1c074331ed54053d469e7373e5 Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Sat, 19 Jun 2021 12:58:07 +0200 Subject: simulator: Add core module for network simulation This change adds a basic framework as basis for network simulation support in OpenDC. It is modelled similarly to the power system that has been added recently. --- settings.gradle.kts | 1 + 1 file changed, 1 insertion(+) (limited to 'settings.gradle.kts') diff --git a/settings.gradle.kts b/settings.gradle.kts index 34b9bf36..d85c08f8 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -41,6 +41,7 @@ include(":opendc-web:opendc-web-runner") include(":opendc-simulator:opendc-simulator-core") include(":opendc-simulator:opendc-simulator-resources") include(":opendc-simulator:opendc-simulator-power") +include(":opendc-simulator:opendc-simulator-network") include(":opendc-simulator:opendc-simulator-compute") include(":opendc-simulator:opendc-simulator-failures") include(":opendc-telemetry:opendc-telemetry-api") -- cgit v1.2.3 From b0806dcf21ab811c46b715cfdff8a6307e117810 Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Wed, 1 Sep 2021 18:28:34 +0200 Subject: feat(trace): Add API for trace reading This change introduces a new OpenDC API for reading various trace formats in a streaming manner. --- settings.gradle.kts | 1 + 1 file changed, 1 insertion(+) (limited to 'settings.gradle.kts') diff --git a/settings.gradle.kts b/settings.gradle.kts index d85c08f8..36c1d9e0 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -46,6 +46,7 @@ include(":opendc-simulator:opendc-simulator-compute") include(":opendc-simulator:opendc-simulator-failures") include(":opendc-telemetry:opendc-telemetry-api") include(":opendc-telemetry:opendc-telemetry-sdk") +include(":opendc-trace:opendc-trace-api") include(":opendc-harness:opendc-harness-api") include(":opendc-harness:opendc-harness-engine") include(":opendc-harness:opendc-harness-cli") -- cgit v1.2.3 From 23c1502c2668305fd5f4c38c6c794c985d2037e3 Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Tue, 31 Aug 2021 14:56:08 +0200 Subject: refactor(trace): Move GWF trace reader into separate module This change starts the process of moving the different trace formats into separate modules. This change in particular moves the GWF trace format into a new module, opendc-trace-gwf. Furthermore, this change also implements the trace API for the GWF module. --- settings.gradle.kts | 1 + 1 file changed, 1 insertion(+) (limited to 'settings.gradle.kts') diff --git a/settings.gradle.kts b/settings.gradle.kts index 36c1d9e0..e941356e 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -47,6 +47,7 @@ include(":opendc-simulator:opendc-simulator-failures") include(":opendc-telemetry:opendc-telemetry-api") include(":opendc-telemetry:opendc-telemetry-sdk") include(":opendc-trace:opendc-trace-api") +include(":opendc-trace:opendc-trace-gwf") include(":opendc-harness:opendc-harness-api") include(":opendc-harness:opendc-harness-engine") include(":opendc-harness:opendc-harness-cli") -- cgit v1.2.3 From e8cdfbcec3f75b3f303ce52bac5f5595a94555e4 Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Tue, 31 Aug 2021 15:14:46 +0200 Subject: refactor(trace): Extract Parquet helpers into separate module This change extracts the Parquet helpers outside format module into a new module, in order to improve re-usability of these helpers. --- settings.gradle.kts | 1 + 1 file changed, 1 insertion(+) (limited to 'settings.gradle.kts') diff --git a/settings.gradle.kts b/settings.gradle.kts index e941356e..ec697d80 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -48,6 +48,7 @@ include(":opendc-telemetry:opendc-telemetry-api") include(":opendc-telemetry:opendc-telemetry-sdk") include(":opendc-trace:opendc-trace-api") include(":opendc-trace:opendc-trace-gwf") +include(":opendc-trace:opendc-trace-parquet") include(":opendc-harness:opendc-harness-api") include(":opendc-harness:opendc-harness-engine") include(":opendc-harness:opendc-harness-cli") -- cgit v1.2.3 From 214480d154771f0b783829b6e5ec82b837304ad2 Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Tue, 31 Aug 2021 16:18:56 +0200 Subject: refactor(trace): Move Bitbrains format into separate module This change moves Bitbrains trace support into a separate module and adds support for the new trace api. --- settings.gradle.kts | 1 + 1 file changed, 1 insertion(+) (limited to 'settings.gradle.kts') diff --git a/settings.gradle.kts b/settings.gradle.kts index ec697d80..fd1d404a 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -48,6 +48,7 @@ include(":opendc-telemetry:opendc-telemetry-api") include(":opendc-telemetry:opendc-telemetry-sdk") include(":opendc-trace:opendc-trace-api") include(":opendc-trace:opendc-trace-gwf") +include(":opendc-trace:opendc-trace-bitbrains") include(":opendc-trace:opendc-trace-parquet") include(":opendc-harness:opendc-harness-api") include(":opendc-harness:opendc-harness-engine") -- cgit v1.2.3 From 5c6bf9739aa0ffd9651df4fcb4cd46a8545144f0 Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Tue, 31 Aug 2021 18:08:14 +0200 Subject: refactor(trace): Implement trace API for SWF reader This change updates the SWF trace reader to support the new streaming trace API. --- settings.gradle.kts | 1 + 1 file changed, 1 insertion(+) (limited to 'settings.gradle.kts') diff --git a/settings.gradle.kts b/settings.gradle.kts index fd1d404a..73d9a151 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -48,6 +48,7 @@ include(":opendc-telemetry:opendc-telemetry-api") include(":opendc-telemetry:opendc-telemetry-sdk") include(":opendc-trace:opendc-trace-api") include(":opendc-trace:opendc-trace-gwf") +include(":opendc-trace:opendc-trace-swf") include(":opendc-trace:opendc-trace-bitbrains") include(":opendc-trace:opendc-trace-parquet") include(":opendc-harness:opendc-harness-api") -- cgit v1.2.3 From b2308e1077dc60ec6a4dc646613a4be5b59695a6 Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Wed, 1 Sep 2021 11:29:27 +0200 Subject: refactor(trace): Implement trace API for WTF reader This change updates the WTF trace reader to support the new streaming trace API. --- settings.gradle.kts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'settings.gradle.kts') diff --git a/settings.gradle.kts b/settings.gradle.kts index 73d9a151..5cae0a31 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -30,7 +30,6 @@ include(":opendc-workflow:opendc-workflow-service") include(":opendc-faas:opendc-faas-api") include(":opendc-faas:opendc-faas-service") include(":opendc-faas:opendc-faas-simulator") -include(":opendc-format") include(":opendc-experiments:opendc-experiments-capelin") include(":opendc-experiments:opendc-experiments-energy21") include(":opendc-experiments:opendc-experiments-serverless20") @@ -49,6 +48,7 @@ include(":opendc-telemetry:opendc-telemetry-sdk") include(":opendc-trace:opendc-trace-api") include(":opendc-trace:opendc-trace-gwf") include(":opendc-trace:opendc-trace-swf") +include(":opendc-trace:opendc-trace-wtf") include(":opendc-trace:opendc-trace-bitbrains") include(":opendc-trace:opendc-trace-parquet") include(":opendc-harness:opendc-harness-api") -- cgit v1.2.3 From aaedd4f3eed83d0c3ebc829fec08a1749a2bfba4 Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Fri, 27 Aug 2021 16:41:55 +0200 Subject: refactor(capelin): Move metric collection outside Capelin code This change moves the metric collection outside the Capelin codebase in a separate module so other modules can also benefit from the compute metric collection code. --- settings.gradle.kts | 1 + 1 file changed, 1 insertion(+) (limited to 'settings.gradle.kts') diff --git a/settings.gradle.kts b/settings.gradle.kts index 5cae0a31..cee8887b 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -45,6 +45,7 @@ include(":opendc-simulator:opendc-simulator-compute") include(":opendc-simulator:opendc-simulator-failures") include(":opendc-telemetry:opendc-telemetry-api") include(":opendc-telemetry:opendc-telemetry-sdk") +include(":opendc-telemetry:opendc-telemetry-compute") include(":opendc-trace:opendc-trace-api") include(":opendc-trace:opendc-trace-gwf") include(":opendc-trace:opendc-trace-swf") -- cgit v1.2.3 From d24cc0cc9c4fe2145f0337d65e9a75f631365973 Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Fri, 10 Sep 2021 10:59:44 +0200 Subject: refactor(compute): Integrate fault injection into compute simulator This change moves the fault injection logic directly into the opendc-compute-simulator module, so that it can operate at a higher abstraction. In the future, we might again split the module if we can re-use some of its logic. --- settings.gradle.kts | 1 - 1 file changed, 1 deletion(-) (limited to 'settings.gradle.kts') diff --git a/settings.gradle.kts b/settings.gradle.kts index cee8887b..427cdb52 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -42,7 +42,6 @@ include(":opendc-simulator:opendc-simulator-resources") include(":opendc-simulator:opendc-simulator-power") include(":opendc-simulator:opendc-simulator-network") include(":opendc-simulator:opendc-simulator-compute") -include(":opendc-simulator:opendc-simulator-failures") include(":opendc-telemetry:opendc-telemetry-api") include(":opendc-telemetry:opendc-telemetry-sdk") include(":opendc-telemetry:opendc-telemetry-compute") -- cgit v1.2.3 From b7be3400bb4b21d0cd7021e2baf1f6ce43aba189 Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Fri, 10 Sep 2021 22:10:22 +0200 Subject: feat(trace): Add support for WfCommons (WorkflowHub) traces This change adds support for reading WfCommons workflow traces in OpenDC. This functionality is available in the new `opendc-trace-wfformat` module. --- settings.gradle.kts | 1 + 1 file changed, 1 insertion(+) (limited to 'settings.gradle.kts') diff --git a/settings.gradle.kts b/settings.gradle.kts index 427cdb52..60e67e2b 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -49,6 +49,7 @@ include(":opendc-trace:opendc-trace-api") include(":opendc-trace:opendc-trace-gwf") include(":opendc-trace:opendc-trace-swf") include(":opendc-trace:opendc-trace-wtf") +include(":opendc-trace:opendc-trace-wfformat") include(":opendc-trace:opendc-trace-bitbrains") include(":opendc-trace:opendc-trace-parquet") include(":opendc-harness:opendc-harness-api") -- cgit v1.2.3 From 7960791430b0536df4704493c01d32e38f37f022 Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Tue, 14 Sep 2021 12:52:17 +0200 Subject: refactor(experiments): Remove energy experiments shell This change removes the energy experiments. The experiments only provided a setup for the original experiments and is not able to reproduce the results without further worker. --- settings.gradle.kts | 1 - 1 file changed, 1 deletion(-) (limited to 'settings.gradle.kts') diff --git a/settings.gradle.kts b/settings.gradle.kts index 60e67e2b..933febe0 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -31,7 +31,6 @@ include(":opendc-faas:opendc-faas-api") include(":opendc-faas:opendc-faas-service") include(":opendc-faas:opendc-faas-simulator") include(":opendc-experiments:opendc-experiments-capelin") -include(":opendc-experiments:opendc-experiments-energy21") include(":opendc-experiments:opendc-experiments-serverless20") include(":opendc-experiments:opendc-experiments-tf20") include(":opendc-web:opendc-web-api") -- cgit v1.2.3 From 859ce303f0b9110c7110b918e5957c2156fa8b26 Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Fri, 17 Sep 2021 17:48:02 +0200 Subject: refactor(capelin): Extract common code out of Capelin experiments This change creates a new module for doing simulations with virtual machine workloads. We have found that a lot of code in the Capelin experiments code is being re-used by non-experiment modules. --- settings.gradle.kts | 1 + 1 file changed, 1 insertion(+) (limited to 'settings.gradle.kts') diff --git a/settings.gradle.kts b/settings.gradle.kts index 933febe0..7bfc8fc6 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -25,6 +25,7 @@ include(":opendc-platform") include(":opendc-compute:opendc-compute-api") include(":opendc-compute:opendc-compute-service") include(":opendc-compute:opendc-compute-simulator") +include(":opendc-compute:opendc-compute-workload") include(":opendc-workflow:opendc-workflow-api") include(":opendc-workflow:opendc-workflow-service") include(":opendc-faas:opendc-faas-api") -- cgit v1.2.3 From 736aef9e56d149d54be16b735daf6784339071de Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Fri, 17 Sep 2021 23:04:00 +0200 Subject: feat(trace): Add support for Azure VM trace format This change adds support in the trace library for the Azure VM trace format. --- settings.gradle.kts | 1 + 1 file changed, 1 insertion(+) (limited to 'settings.gradle.kts') diff --git a/settings.gradle.kts b/settings.gradle.kts index 7bfc8fc6..f77fb676 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -51,6 +51,7 @@ include(":opendc-trace:opendc-trace-swf") include(":opendc-trace:opendc-trace-wtf") include(":opendc-trace:opendc-trace-wfformat") include(":opendc-trace:opendc-trace-bitbrains") +include(":opendc-trace:opendc-trace-azure") include(":opendc-trace:opendc-trace-parquet") include(":opendc-harness:opendc-harness-api") include(":opendc-harness:opendc-harness-engine") -- cgit v1.2.3 From 9b25eef67911d0aec6a36c82a34cd0e39b13b073 Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Sun, 19 Sep 2021 12:56:26 +0200 Subject: feat(trace): Add support for internal OpenDC VM trace format This change adds official support to the trace library for the internal VM trace format used by OpenDC for its experiments. This is a compact format that uses Parquet to store the virtual machine trace data in two Parquet files. --- settings.gradle.kts | 1 + 1 file changed, 1 insertion(+) (limited to 'settings.gradle.kts') diff --git a/settings.gradle.kts b/settings.gradle.kts index f77fb676..26bd0b11 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -52,6 +52,7 @@ include(":opendc-trace:opendc-trace-wtf") include(":opendc-trace:opendc-trace-wfformat") include(":opendc-trace:opendc-trace-bitbrains") include(":opendc-trace:opendc-trace-azure") +include(":opendc-trace:opendc-trace-opendc") include(":opendc-trace:opendc-trace-parquet") include(":opendc-harness:opendc-harness-api") include(":opendc-harness:opendc-harness-engine") -- cgit v1.2.3 From 6502fb752a6f80695c024b8904d7523c420ebdda Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Sun, 19 Sep 2021 13:42:26 +0200 Subject: feat(trace): Add tool for converting workload traces This change adds an initial implementation to the trace library for converting between workload trace formats. Currently the tool supports only converting to the OpenDC VM trace format. However, in the future, we will add support for converting between other formats as well. --- settings.gradle.kts | 1 + 1 file changed, 1 insertion(+) (limited to 'settings.gradle.kts') diff --git a/settings.gradle.kts b/settings.gradle.kts index 26bd0b11..587f1cb2 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -54,6 +54,7 @@ include(":opendc-trace:opendc-trace-bitbrains") include(":opendc-trace:opendc-trace-azure") include(":opendc-trace:opendc-trace-opendc") include(":opendc-trace:opendc-trace-parquet") +include(":opendc-trace:opendc-trace-tools") include(":opendc-harness:opendc-harness-api") include(":opendc-harness:opendc-harness-engine") include(":opendc-harness:opendc-harness-cli") -- cgit v1.2.3 From 4cc1d40d421c8736f8b21b360b61d6b065158b7a Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Wed, 29 Sep 2021 23:56:16 +0200 Subject: refactor(simulator): Migrate to flow-based simulation This change renames the `opendc-simulator-resources` module into the `opendc-simulator-flow` module to indicate that the core simulation model of OpenDC is based around modelling and simulating flows. Previously, the distinction between resource consumer and provider, and input and output caused some confusion. By switching to a flow-based model, this distinction is now clear (as in, the water flows from source to consumer/sink). --- settings.gradle.kts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'settings.gradle.kts') diff --git a/settings.gradle.kts b/settings.gradle.kts index 587f1cb2..d9b3d940 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -38,7 +38,7 @@ include(":opendc-web:opendc-web-api") include(":opendc-web:opendc-web-ui") include(":opendc-web:opendc-web-runner") include(":opendc-simulator:opendc-simulator-core") -include(":opendc-simulator:opendc-simulator-resources") +include(":opendc-simulator:opendc-simulator-flow") include(":opendc-simulator:opendc-simulator-power") include(":opendc-simulator:opendc-simulator-network") include(":opendc-simulator:opendc-simulator-compute") -- cgit v1.2.3