From cc9310efad6177909ff2f7415384d7c393383106 Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Wed, 5 May 2021 23:14:56 +0200 Subject: chore: Address deprecations due to Kotlin 1.5 This change addresses the deprecations that were caused by the migration to Kotlin 1.5. --- .../src/main/kotlin/org/opendc/experiments/tf20/keras/Sequential.kt | 4 ++-- .../kotlin/org/opendc/experiments/tf20/network/NetworkController.kt | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) (limited to 'opendc-experiments/opendc-experiments-tf20') diff --git a/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/keras/Sequential.kt b/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/keras/Sequential.kt index 411ddb59..83995fa1 100644 --- a/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/keras/Sequential.kt +++ b/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/keras/Sequential.kt @@ -49,10 +49,10 @@ public class Sequential(vararg layers: Layer) : TrainableModel(*layers) { } override fun forward(): Double { - return layers.sumByDouble { it.forward() } + return layers.sumOf { it.forward() } } override fun backward(): Double { - return layers.sumByDouble { it.backward() } + return layers.sumOf { it.backward() } } } diff --git a/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/network/NetworkController.kt b/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/network/NetworkController.kt index 75b11423..9771cc20 100644 --- a/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/network/NetworkController.kt +++ b/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/network/NetworkController.kt @@ -82,7 +82,7 @@ public class NetworkController(context: CoroutineContext, clock: Clock) : AutoCl val target = channels[to] ?: return // Drop if destination not found - scheduler.startSingleTimer(message, delayTime) { target.offer(message) } + scheduler.startSingleTimer(message, delayTime) { target.trySend(message) } } /** -- cgit v1.2.3 From 9e5e830e15b74f040708e98c09ea41cd96d13871 Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Thu, 27 May 2021 16:34:06 +0200 Subject: simulator: Centralize resource logic in SimResourceInterpreter This change introduces the SimResourceInterpreter which centralizes the logic for scheduling and interpreting the communication between resource consumer and provider. This approach offers better performance due to avoiding invalidating the state of the resource context when not necessary. Benchmarks show in the best case a 5x performance improvement and at worst a 2x improvement. --- .../main/kotlin/org/opendc/experiments/tf20/core/SimTFDevice.kt | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) (limited to 'opendc-experiments/opendc-experiments-tf20') diff --git a/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/core/SimTFDevice.kt b/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/core/SimTFDevice.kt index f4c18ff1..f345a4a5 100644 --- a/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/core/SimTFDevice.kt +++ b/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/core/SimTFDevice.kt @@ -35,10 +35,7 @@ import org.opendc.simulator.compute.model.MemoryUnit import org.opendc.simulator.compute.model.ProcessingUnit import org.opendc.simulator.compute.power.PowerModel import org.opendc.simulator.compute.workload.SimWorkload -import org.opendc.simulator.resources.SimResourceCommand -import org.opendc.simulator.resources.SimResourceConsumer -import org.opendc.simulator.resources.SimResourceContext -import org.opendc.simulator.resources.SimResourceEvent +import org.opendc.simulator.resources.* import java.time.Clock import java.util.* import kotlin.coroutines.Continuation @@ -67,7 +64,7 @@ public class SimTFDevice( * The [SimMachine] representing the device. */ private val machine = SimBareMetalMachine( - scope.coroutineContext, clock, SimMachineModel(listOf(pu), listOf(memory)), + SimResourceInterpreter(scope.coroutineContext, clock), SimMachineModel(listOf(pu), listOf(memory)), PerformanceScalingGovernor(), SimpleScalingDriver(powerModel) ) -- cgit v1.2.3 From 1ee02377aca356a99506ac247b376d7cf070048d Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Wed, 2 Jun 2021 23:06:36 +0200 Subject: simulator: Start consumers directly from workload This change updates the SimWorkload interfaces to allow implementations to start consumers for the machine resource providers directly. --- .../main/kotlin/org/opendc/experiments/tf20/core/SimTFDevice.kt | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) (limited to 'opendc-experiments/opendc-experiments-tf20') diff --git a/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/core/SimTFDevice.kt b/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/core/SimTFDevice.kt index f345a4a5..db131e82 100644 --- a/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/core/SimTFDevice.kt +++ b/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/core/SimTFDevice.kt @@ -116,9 +116,11 @@ public class SimTFDevice( */ private var activeWork: Work? = null - override fun onStart(ctx: SimMachineContext) {} - - override fun getConsumer(ctx: SimMachineContext, cpu: ProcessingUnit): SimResourceConsumer = this + override fun onStart(ctx: SimMachineContext) { + for (cpu in ctx.cpus) { + cpu.startConsumer(this) + } + } override fun onNext(ctx: SimResourceContext): SimResourceCommand { val activeWork = activeWork -- cgit v1.2.3 From 84468f4e3a331d7ea073ac7033b3d9937168ed9f Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Wed, 2 Jun 2021 23:23:55 +0200 Subject: simulator: Split CPUFreq subsystem in compute simulator This change splits the functionality present in the CPUFreq subsystem of the compute simulation. Currently, the DVFS functionality is embedded in SimBareMetalMachine. However, this functionality should not exist within the firmware layer of a machine. Instead, the operating system should perform this logic (in OpenDC this should be the hypervisor). Furthermore, this change moves the scaling driver into the power package. The power driver is a machine/firmware specific implementation that computes the power consumption of a machine. --- .../src/main/kotlin/org/opendc/experiments/tf20/core/SimTFDevice.kt | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) (limited to 'opendc-experiments/opendc-experiments-tf20') diff --git a/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/core/SimTFDevice.kt b/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/core/SimTFDevice.kt index db131e82..001547ef 100644 --- a/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/core/SimTFDevice.kt +++ b/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/core/SimTFDevice.kt @@ -29,11 +29,10 @@ import org.opendc.simulator.compute.SimBareMetalMachine import org.opendc.simulator.compute.SimMachine import org.opendc.simulator.compute.SimMachineContext import org.opendc.simulator.compute.SimMachineModel -import org.opendc.simulator.compute.cpufreq.PerformanceScalingGovernor -import org.opendc.simulator.compute.cpufreq.SimpleScalingDriver import org.opendc.simulator.compute.model.MemoryUnit import org.opendc.simulator.compute.model.ProcessingUnit import org.opendc.simulator.compute.power.PowerModel +import org.opendc.simulator.compute.power.SimplePowerDriver import org.opendc.simulator.compute.workload.SimWorkload import org.opendc.simulator.resources.* import java.time.Clock @@ -65,7 +64,7 @@ public class SimTFDevice( */ private val machine = SimBareMetalMachine( SimResourceInterpreter(scope.coroutineContext, clock), SimMachineModel(listOf(pu), listOf(memory)), - PerformanceScalingGovernor(), SimpleScalingDriver(powerModel) + SimplePowerDriver(powerModel) ) /** -- cgit v1.2.3 From 0eb4fa604efe4e0b84d69749f688a79c2249c8b3 Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Wed, 9 Jun 2021 13:38:56 +0200 Subject: build: Eliminate most Hadoop dependencies This change eliminates all Hadoop dependencies that are not necessary for Parquet to work correctly. As a result, the number of dependencies should now be greatly reduced, which in turn leads to less artifacts that need to be retrieved at build time. --- opendc-experiments/opendc-experiments-tf20/build.gradle.kts | 5 ----- 1 file changed, 5 deletions(-) (limited to 'opendc-experiments/opendc-experiments-tf20') diff --git a/opendc-experiments/opendc-experiments-tf20/build.gradle.kts b/opendc-experiments/opendc-experiments-tf20/build.gradle.kts index 64483bd4..b088045b 100644 --- a/opendc-experiments/opendc-experiments-tf20/build.gradle.kts +++ b/opendc-experiments/opendc-experiments-tf20/build.gradle.kts @@ -38,9 +38,4 @@ dependencies { implementation(projects.opendcUtils) implementation(libs.kotlin.logging) - implementation(libs.parquet) - implementation(libs.hadoop.client) { - exclude(group = "org.slf4j", module = "slf4j-log4j12") - exclude(group = "log4j") - } } -- cgit v1.2.3 From 1768292251957da5ce6411ecc7d2dffebf8709c8 Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Thu, 10 Jun 2021 17:32:43 +0200 Subject: simulator: Integrate power subsystem with compute subsystem This change integrates the power subsystem of the simulator with the compute subsystem by exposing a new field on a SimBareMetalMachine, psu, which provides access to the machine's PSU, which in turn can be connected to a SimPowerOutlet. --- .../src/main/kotlin/org/opendc/experiments/tf20/core/SimTFDevice.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'opendc-experiments/opendc-experiments-tf20') diff --git a/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/core/SimTFDevice.kt b/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/core/SimTFDevice.kt index 001547ef..92e7080a 100644 --- a/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/core/SimTFDevice.kt +++ b/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/core/SimTFDevice.kt @@ -153,7 +153,7 @@ public class SimTFDevice( } SimResourceEvent.Run -> { _usage.record(ctx.speed) - _power.record(machine.powerDraw) + _power.record(machine.psu.powerDraw) } else -> {} } -- cgit v1.2.3 From b29f90e5ad5bcac29cde86e56c06e0b65a52cedc Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Mon, 21 Jun 2021 20:57:06 +0200 Subject: simulator: Re-organize compute simulator module This change re-organizes the classes of the compute simulator module to make a clearer distinction between the hardware, firmware and software interfaces in this module. --- .../src/main/kotlin/org/opendc/experiments/tf20/core/SimTFDevice.kt | 4 ++-- .../kotlin/org/opendc/experiments/tf20/util/MLEnvironmentReader.kt | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) (limited to 'opendc-experiments/opendc-experiments-tf20') diff --git a/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/core/SimTFDevice.kt b/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/core/SimTFDevice.kt index 92e7080a..d8f92155 100644 --- a/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/core/SimTFDevice.kt +++ b/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/core/SimTFDevice.kt @@ -28,7 +28,7 @@ import kotlinx.coroutines.* import org.opendc.simulator.compute.SimBareMetalMachine import org.opendc.simulator.compute.SimMachine import org.opendc.simulator.compute.SimMachineContext -import org.opendc.simulator.compute.SimMachineModel +import org.opendc.simulator.compute.model.MachineModel import org.opendc.simulator.compute.model.MemoryUnit import org.opendc.simulator.compute.model.ProcessingUnit import org.opendc.simulator.compute.power.PowerModel @@ -63,7 +63,7 @@ public class SimTFDevice( * The [SimMachine] representing the device. */ private val machine = SimBareMetalMachine( - SimResourceInterpreter(scope.coroutineContext, clock), SimMachineModel(listOf(pu), listOf(memory)), + SimResourceInterpreter(scope.coroutineContext, clock), MachineModel(listOf(pu), listOf(memory)), SimplePowerDriver(powerModel) ) diff --git a/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/util/MLEnvironmentReader.kt b/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/util/MLEnvironmentReader.kt index eea079fb..3e61f508 100644 --- a/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/util/MLEnvironmentReader.kt +++ b/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/util/MLEnvironmentReader.kt @@ -27,7 +27,7 @@ import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper import com.fasterxml.jackson.module.kotlin.readValue import org.opendc.format.environment.EnvironmentReader import org.opendc.format.environment.MachineDef -import org.opendc.simulator.compute.SimMachineModel +import org.opendc.simulator.compute.model.MachineModel import org.opendc.simulator.compute.model.MemoryUnit import org.opendc.simulator.compute.model.ProcessingNode import org.opendc.simulator.compute.model.ProcessingUnit @@ -100,7 +100,7 @@ public class MLEnvironmentReader(input: InputStream, mapper: ObjectMapper = jack UUID(0, counter.toLong()), "node-${counter++}", mapOf("gpu" to isGpuFlag), - SimMachineModel(cores, memories), + MachineModel(cores, memories), LinearPowerModel(maxPower, minPower) ) } -- cgit v1.2.3 From f111081627280d4e7e1d7147c56cdce708e32433 Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Wed, 25 Aug 2021 14:06:39 +0200 Subject: build: Upgrade to OpenTelemetry 1.5 This change upgrades the OpenTelemetry dependency to version 1.5, which contains various breaking changes in the metrics API. --- .../org/opendc/experiments/tf20/core/SimTFDevice.kt | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) (limited to 'opendc-experiments/opendc-experiments-tf20') diff --git a/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/core/SimTFDevice.kt b/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/core/SimTFDevice.kt index d8f92155..0873aac9 100644 --- a/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/core/SimTFDevice.kt +++ b/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/core/SimTFDevice.kt @@ -22,8 +22,9 @@ package org.opendc.experiments.tf20.core +import io.opentelemetry.api.common.AttributeKey +import io.opentelemetry.api.common.Attributes import io.opentelemetry.api.metrics.Meter -import io.opentelemetry.api.metrics.common.Labels import kotlinx.coroutines.* import org.opendc.simulator.compute.SimBareMetalMachine import org.opendc.simulator.compute.SimMachine @@ -67,23 +68,28 @@ public class SimTFDevice( SimplePowerDriver(powerModel) ) + /** + * The identifier of a device. + */ + private val deviceId = AttributeKey.stringKey("device.id") + /** * The usage of the device. */ - private val _usage = meter.doubleValueRecorderBuilder("device.usage") + private val _usage = meter.histogramBuilder("device.usage") .setDescription("The amount of device resources used") .setUnit("MHz") .build() - .bind(Labels.of("device", uid.toString())) + .bind(Attributes.of(deviceId, uid.toString())) /** * The power draw of the device. */ - private val _power = meter.doubleValueRecorderBuilder("device.power") + private val _power = meter.histogramBuilder("device.power") .setDescription("The power draw of the device") .setUnit("W") .build() - .bind(Labels.of("device", uid.toString())) + .bind(Attributes.of(deviceId, uid.toString())) /** * The workload that will be run by the device. -- cgit v1.2.3 From 9fcce6ade8714f7f0a9073fe5b7ddd3f0b35c375 Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Tue, 31 Aug 2021 15:44:47 +0200 Subject: refactor(format): Remove environment reader from format library This change removes the environment reader from the format library since they are highly specific for the particular experiment. In the future, we hope to have a single format to setup the entire datacenter (perhaps similar to the format used by the web runner). --- .../opendc-experiments-tf20/build.gradle.kts | 5 ++- .../experiments/tf20/TensorFlowExperiment.kt | 3 +- .../experiments/tf20/util/MLEnvironmentReader.kt | 17 +++++----- .../org/opendc/experiments/tf20/util/MachineDef.kt | 38 ++++++++++++++++++++++ 4 files changed, 52 insertions(+), 11 deletions(-) create mode 100644 opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/util/MachineDef.kt (limited to 'opendc-experiments/opendc-experiments-tf20') diff --git a/opendc-experiments/opendc-experiments-tf20/build.gradle.kts b/opendc-experiments/opendc-experiments-tf20/build.gradle.kts index b088045b..882c4894 100644 --- a/opendc-experiments/opendc-experiments-tf20/build.gradle.kts +++ b/opendc-experiments/opendc-experiments-tf20/build.gradle.kts @@ -34,8 +34,11 @@ dependencies { implementation(projects.opendcSimulator.opendcSimulatorCore) implementation(projects.opendcSimulator.opendcSimulatorCompute) implementation(projects.opendcTelemetry.opendcTelemetrySdk) - implementation(projects.opendcFormat) implementation(projects.opendcUtils) implementation(libs.kotlin.logging) + implementation(libs.jackson.module.kotlin) { + exclude(group = "org.jetbrains.kotlin", module = "kotlin-reflect") + } + implementation("org.jetbrains.kotlin:kotlin-reflect:1.5.30") } diff --git a/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/TensorFlowExperiment.kt b/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/TensorFlowExperiment.kt index 9a48aced..2153a862 100644 --- a/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/TensorFlowExperiment.kt +++ b/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/TensorFlowExperiment.kt @@ -55,7 +55,8 @@ public class TensorFlowExperiment : Experiment(name = "tf20") { .build() val meter = meterProvider.get("opendc-tf20") - val def = MLEnvironmentReader(TensorFlowExperiment::class.java.getResourceAsStream(environmentFile)).read().first() + val envInput = checkNotNull(TensorFlowExperiment::class.java.getResourceAsStream(environmentFile)) + val def = MLEnvironmentReader().readEnvironment(envInput).first() val device = SimTFDevice( def.uid, def.meta["gpu"] as Boolean, coroutineContext, clock, meter, def.model.cpus[0], def.model.memory[0], LinearPowerModel(250.0, 60.0) diff --git a/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/util/MLEnvironmentReader.kt b/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/util/MLEnvironmentReader.kt index 3e61f508..3cdf28fd 100644 --- a/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/util/MLEnvironmentReader.kt +++ b/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/util/MLEnvironmentReader.kt @@ -25,8 +25,6 @@ package org.opendc.experiments.tf20.util import com.fasterxml.jackson.databind.ObjectMapper import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper import com.fasterxml.jackson.module.kotlin.readValue -import org.opendc.format.environment.EnvironmentReader -import org.opendc.format.environment.MachineDef import org.opendc.simulator.compute.model.MachineModel import org.opendc.simulator.compute.model.MemoryUnit import org.opendc.simulator.compute.model.ProcessingNode @@ -36,13 +34,16 @@ import java.io.InputStream import java.util.* /** - * An [EnvironmentReader] for the TensorFlow experiments. + * An environment reader for the TensorFlow experiments. */ -public class MLEnvironmentReader(input: InputStream, mapper: ObjectMapper = jacksonObjectMapper()) : EnvironmentReader { +public class MLEnvironmentReader { + /** + * The [ObjectMapper] to convert the format. + */ + private val mapper = jacksonObjectMapper() - private val setup: Setup = mapper.readValue(input) - - override fun read(): List { + public fun readEnvironment(input: InputStream): List { + val setup: Setup = mapper.readValue(input) var counter = 0 return setup.rooms.flatMap { room -> room.objects.flatMap { roomObject -> @@ -109,6 +110,4 @@ public class MLEnvironmentReader(input: InputStream, mapper: ObjectMapper = jack } } } - - override fun close() {} } diff --git a/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/util/MachineDef.kt b/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/util/MachineDef.kt new file mode 100644 index 00000000..271f5923 --- /dev/null +++ b/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/util/MachineDef.kt @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2021 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.experiments.tf20.util + +import org.opendc.simulator.compute.model.MachineModel +import org.opendc.simulator.compute.power.PowerModel +import java.util.* + +/** + * A definition of a machine in a cluster. + */ +public data class MachineDef( + val uid: UUID, + val name: String, + val meta: Map, + val model: MachineModel, + val powerModel: PowerModel +) -- cgit v1.2.3 From d575bed5418be222e1d3ad39af862e2390596d61 Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Sun, 26 Sep 2021 13:11:10 +0200 Subject: refactor(simulator): Combine work and deadline to duration This change removes the work and deadline properties from the SimResourceCommand.Consume class and introduces a new property duration. This property is now used in conjunction with the limit to compute the amount of work processed by a resource provider. Previously, we used both work and deadline to compute the duration and the amount of remaining work at the end of a consumption. However, with this change, we ensure that a resource consumption always runs at the same speed once establishing, drastically simplifying the computation for the amount of work processed during the consumption. --- .../org/opendc/experiments/tf20/core/SimTFDevice.kt | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) (limited to 'opendc-experiments/opendc-experiments-tf20') diff --git a/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/core/SimTFDevice.kt b/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/core/SimTFDevice.kt index 0873aac9..bfc5fc6f 100644 --- a/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/core/SimTFDevice.kt +++ b/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/core/SimTFDevice.kt @@ -41,6 +41,7 @@ import java.util.* import kotlin.coroutines.Continuation import kotlin.coroutines.CoroutineContext import kotlin.coroutines.resume +import kotlin.math.roundToLong /** * A [TFDevice] implementation using simulated components. @@ -127,13 +128,16 @@ public class SimTFDevice( } } - override fun onNext(ctx: SimResourceContext): SimResourceCommand { + override fun onNext(ctx: SimResourceContext, now: Long, delta: Long): SimResourceCommand { + val consumedWork = ctx.speed * delta / 1000.0 + val activeWork = activeWork if (activeWork != null) { - if (activeWork.consume(activeWork.flops - ctx.remainingWork)) { + if (activeWork.consume(consumedWork)) { this.activeWork = null } else { - return SimResourceCommand.Consume(activeWork.flops, ctx.capacity) + val duration = (activeWork.flops / ctx.capacity * 1000).roundToLong() + return SimResourceCommand.Consume(ctx.capacity, duration) } } @@ -141,9 +145,10 @@ public class SimTFDevice( val head = queue.poll() return if (head != null) { this.activeWork = head - SimResourceCommand.Consume(head.flops, ctx.capacity) + val duration = (head.flops / ctx.capacity * 1000).roundToLong() + SimResourceCommand.Consume(ctx.capacity, duration) } else { - SimResourceCommand.Idle() + SimResourceCommand.Consume(0.0) } } -- cgit v1.2.3 From 02fa44c0b116ff51c4cbe2876d8b2a225ed68553 Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Tue, 28 Sep 2021 11:58:19 +0200 Subject: refactor(simulator): Add support for pushing flow from context This change adds a new method to `SimResourceContext` called `push` which allows users to change the requested flow rate directly without having to interrupt the consumer. --- .../kotlin/org/opendc/experiments/tf20/core/SimTFDevice.kt | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) (limited to 'opendc-experiments/opendc-experiments-tf20') diff --git a/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/core/SimTFDevice.kt b/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/core/SimTFDevice.kt index bfc5fc6f..1fbd3b6a 100644 --- a/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/core/SimTFDevice.kt +++ b/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/core/SimTFDevice.kt @@ -128,7 +128,7 @@ public class SimTFDevice( } } - override fun onNext(ctx: SimResourceContext, now: Long, delta: Long): SimResourceCommand { + override fun onNext(ctx: SimResourceContext, now: Long, delta: Long): Long { val consumedWork = ctx.speed * delta / 1000.0 val activeWork = activeWork @@ -137,7 +137,8 @@ public class SimTFDevice( this.activeWork = null } else { val duration = (activeWork.flops / ctx.capacity * 1000).roundToLong() - return SimResourceCommand.Consume(ctx.capacity, duration) + ctx.push(ctx.capacity) + return duration } } @@ -146,9 +147,11 @@ public class SimTFDevice( return if (head != null) { this.activeWork = head val duration = (head.flops / ctx.capacity * 1000).roundToLong() - SimResourceCommand.Consume(ctx.capacity, duration) + ctx.push(ctx.capacity) + duration } else { - SimResourceCommand.Consume(0.0) + ctx.push(0.0) + Long.MAX_VALUE } } -- cgit v1.2.3 From 4cc1d40d421c8736f8b21b360b61d6b065158b7a Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Wed, 29 Sep 2021 23:56:16 +0200 Subject: refactor(simulator): Migrate to flow-based simulation This change renames the `opendc-simulator-resources` module into the `opendc-simulator-flow` module to indicate that the core simulation model of OpenDC is based around modelling and simulating flows. Previously, the distinction between resource consumer and provider, and input and output caused some confusion. By switching to a flow-based model, this distinction is now clear (as in, the water flows from source to consumer/sink). --- .../opendc/experiments/tf20/core/SimTFDevice.kt | 42 +++++++++++----------- .../opendc/experiments/tf20/distribute/Strategy.kt | 2 +- 2 files changed, 22 insertions(+), 22 deletions(-) (limited to 'opendc-experiments/opendc-experiments-tf20') diff --git a/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/core/SimTFDevice.kt b/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/core/SimTFDevice.kt index 1fbd3b6a..2ba65e90 100644 --- a/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/core/SimTFDevice.kt +++ b/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/core/SimTFDevice.kt @@ -35,7 +35,7 @@ import org.opendc.simulator.compute.model.ProcessingUnit import org.opendc.simulator.compute.power.PowerModel import org.opendc.simulator.compute.power.SimplePowerDriver import org.opendc.simulator.compute.workload.SimWorkload -import org.opendc.simulator.resources.* +import org.opendc.simulator.flow.* import java.time.Clock import java.util.* import kotlin.coroutines.Continuation @@ -65,7 +65,7 @@ public class SimTFDevice( * The [SimMachine] representing the device. */ private val machine = SimBareMetalMachine( - SimResourceInterpreter(scope.coroutineContext, clock), MachineModel(listOf(pu), listOf(memory)), + FlowEngine(scope.coroutineContext, clock), MachineModel(listOf(pu), listOf(memory)), SimplePowerDriver(powerModel) ) @@ -95,11 +95,11 @@ public class SimTFDevice( /** * The workload that will be run by the device. */ - private val workload = object : SimWorkload, SimResourceConsumer { + private val workload = object : SimWorkload, FlowSource { /** * The resource context to interrupt the workload with. */ - var ctx: SimResourceContext? = null + var ctx: FlowConnection? = null /** * The capacity of the device. @@ -128,16 +128,16 @@ public class SimTFDevice( } } - override fun onNext(ctx: SimResourceContext, now: Long, delta: Long): Long { - val consumedWork = ctx.speed * delta / 1000.0 + override fun onPull(conn: FlowConnection, now: Long, delta: Long): Long { + val consumedWork = conn.rate * delta / 1000.0 val activeWork = activeWork if (activeWork != null) { if (activeWork.consume(consumedWork)) { this.activeWork = null } else { - val duration = (activeWork.flops / ctx.capacity * 1000).roundToLong() - ctx.push(ctx.capacity) + val duration = (activeWork.flops / conn.capacity * 1000).roundToLong() + conn.push(conn.capacity) return duration } } @@ -146,27 +146,27 @@ public class SimTFDevice( val head = queue.poll() return if (head != null) { this.activeWork = head - val duration = (head.flops / ctx.capacity * 1000).roundToLong() - ctx.push(ctx.capacity) + val duration = (head.flops / conn.capacity * 1000).roundToLong() + conn.push(conn.capacity) duration } else { - ctx.push(0.0) + conn.push(0.0) Long.MAX_VALUE } } - override fun onEvent(ctx: SimResourceContext, event: SimResourceEvent) { + override fun onEvent(conn: FlowConnection, now: Long, event: FlowEvent) { when (event) { - SimResourceEvent.Start -> { - this.ctx = ctx - this.capacity = ctx.capacity + FlowEvent.Start -> { + this.ctx = conn + this.capacity = conn.capacity } - SimResourceEvent.Capacity -> { - this.capacity = ctx.capacity - ctx.interrupt() + FlowEvent.Capacity -> { + this.capacity = conn.capacity + conn.pull() } - SimResourceEvent.Run -> { - _usage.record(ctx.speed) + FlowEvent.Converge -> { + _usage.record(conn.rate) _power.record(machine.psu.powerDraw) } else -> {} @@ -188,7 +188,7 @@ public class SimTFDevice( override suspend fun compute(flops: Double) = suspendCancellableCoroutine { cont -> workload.queue.add(Work(flops, cont)) if (workload.isIdle) { - workload.ctx?.interrupt() + workload.ctx?.pull() } } diff --git a/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/distribute/Strategy.kt b/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/distribute/Strategy.kt index 5839c0df..3e755b56 100644 --- a/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/distribute/Strategy.kt +++ b/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/distribute/Strategy.kt @@ -27,7 +27,7 @@ package org.opendc.experiments.tf20.distribute */ public interface Strategy { /** - * Run the specified batch using the given strategy. + * Converge the specified batch using the given strategy. */ public suspend fun run(forward: Double, backward: Double, batchSize: Int) } -- cgit v1.2.3 From 4f5a1f88d0c6aa19ce4cab0ec7b9b13a24c92fbe Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Thu, 30 Sep 2021 14:45:42 +0200 Subject: refactor(simulator): Remove capacity event This change removes the Capacity entry from FlowEvent. Since the source is always pulled on a capacity change, we do not need a separate event for this. --- .../kotlin/org/opendc/experiments/tf20/core/SimTFDevice.kt | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) (limited to 'opendc-experiments/opendc-experiments-tf20') diff --git a/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/core/SimTFDevice.kt b/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/core/SimTFDevice.kt index 2ba65e90..6f460ef7 100644 --- a/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/core/SimTFDevice.kt +++ b/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/core/SimTFDevice.kt @@ -131,6 +131,8 @@ public class SimTFDevice( override fun onPull(conn: FlowConnection, now: Long, delta: Long): Long { val consumedWork = conn.rate * delta / 1000.0 + capacity = conn.capacity + val activeWork = activeWork if (activeWork != null) { if (activeWork.consume(consumedWork)) { @@ -158,12 +160,8 @@ public class SimTFDevice( override fun onEvent(conn: FlowConnection, now: Long, event: FlowEvent) { when (event) { FlowEvent.Start -> { - this.ctx = conn - this.capacity = conn.capacity - } - FlowEvent.Capacity -> { - this.capacity = conn.capacity - conn.pull() + ctx = conn + capacity = conn.capacity } FlowEvent.Converge -> { _usage.record(conn.rate) -- cgit v1.2.3 From a2ce07026bf3ef17326e72f395dfa2dd9d9b17be Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Thu, 30 Sep 2021 15:37:35 +0200 Subject: refactor(simulator): Create separate callbacks for remaining events This change creates separate callbacks for the remaining events: onStart, onStop and onConverge. --- .../org/opendc/experiments/tf20/core/SimTFDevice.kt | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) (limited to 'opendc-experiments/opendc-experiments-tf20') diff --git a/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/core/SimTFDevice.kt b/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/core/SimTFDevice.kt index 6f460ef7..017bca59 100644 --- a/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/core/SimTFDevice.kt +++ b/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/core/SimTFDevice.kt @@ -128,6 +128,11 @@ public class SimTFDevice( } } + override fun onStart(conn: FlowConnection, now: Long) { + ctx = conn + capacity = conn.capacity + } + override fun onPull(conn: FlowConnection, now: Long, delta: Long): Long { val consumedWork = conn.rate * delta / 1000.0 @@ -157,18 +162,9 @@ public class SimTFDevice( } } - override fun onEvent(conn: FlowConnection, now: Long, event: FlowEvent) { - when (event) { - FlowEvent.Start -> { - ctx = conn - capacity = conn.capacity - } - FlowEvent.Converge -> { - _usage.record(conn.rate) - _power.record(machine.psu.powerDraw) - } - else -> {} - } + override fun onConverge(conn: FlowConnection, now: Long, delta: Long) { + _usage.record(conn.rate) + _power.record(machine.psu.powerDraw) } } -- cgit v1.2.3 From 559ac2327b8aa319fb8ab4558d4f4aa3382349f4 Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Thu, 30 Sep 2021 16:27:45 +0200 Subject: perf(simulator): Make convergence callback optional This change adds two new properties for controlling whether the convergence callbacks of the source and consumer respectively should be invoked. This saves a lot of unnecessary calls for stages that do not have any implementation of the `onConvergence` method. --- .../src/main/kotlin/org/opendc/experiments/tf20/core/SimTFDevice.kt | 2 ++ 1 file changed, 2 insertions(+) (limited to 'opendc-experiments/opendc-experiments-tf20') diff --git a/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/core/SimTFDevice.kt b/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/core/SimTFDevice.kt index 017bca59..fb36d2c7 100644 --- a/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/core/SimTFDevice.kt +++ b/opendc-experiments/opendc-experiments-tf20/src/main/kotlin/org/opendc/experiments/tf20/core/SimTFDevice.kt @@ -131,6 +131,8 @@ public class SimTFDevice( override fun onStart(conn: FlowConnection, now: Long) { ctx = conn capacity = conn.capacity + + conn.shouldSourceConverge = true } override fun onPull(conn: FlowConnection, now: Long, delta: Long): Long { -- cgit v1.2.3