From e85a11645a2262e2e6fd1e3570ad001eb805c85f Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Tue, 2 Mar 2021 21:15:38 +0100 Subject: compute: Separate cloud compute layer from bare-metal layer This change separates the cloud compute layer in OpenDC (e.g., Server) from the bare-metal layer (e.g., Node), such that Node and BareMetalDriver are unaware of the existence of Server and co. --- .../org/opendc/runner/web/WebExperimentMonitor.kt | 57 +++++++++++----------- 1 file changed, 29 insertions(+), 28 deletions(-) (limited to 'simulator/opendc-runner-web') diff --git a/simulator/opendc-runner-web/src/main/kotlin/org/opendc/runner/web/WebExperimentMonitor.kt b/simulator/opendc-runner-web/src/main/kotlin/org/opendc/runner/web/WebExperimentMonitor.kt index f16f9b90..b7a26d34 100644 --- a/simulator/opendc-runner-web/src/main/kotlin/org/opendc/runner/web/WebExperimentMonitor.kt +++ b/simulator/opendc-runner-web/src/main/kotlin/org/opendc/runner/web/WebExperimentMonitor.kt @@ -24,7 +24,8 @@ package org.opendc.runner.web import mu.KotlinLogging import org.opendc.compute.core.Server -import org.opendc.compute.core.ServerState +import org.opendc.compute.core.metal.Node +import org.opendc.compute.core.metal.NodeState import org.opendc.compute.core.virt.driver.VirtDriver import org.opendc.compute.core.virt.service.VirtProvisioningEvent import org.opendc.experiments.capelin.monitor.ExperimentMonitor @@ -36,7 +37,7 @@ import kotlin.math.max */ public class WebExperimentMonitor : ExperimentMonitor { private val logger = KotlinLogging.logger {} - private val currentHostEvent = mutableMapOf() + private val currentHostEvent = mutableMapOf() private var startTime = -1L override fun reportVmStateChange(time: Long, server: Server) { @@ -51,11 +52,11 @@ public class WebExperimentMonitor : ExperimentMonitor { override fun reportHostStateChange( time: Long, driver: VirtDriver, - server: Server + host: Node ) { - logger.debug { "Host ${server.uid} changed state ${server.state} [$time]" } + logger.debug { "Host ${host.uid} changed state ${host.state} [$time]" } - val previousEvent = currentHostEvent[server] + val previousEvent = currentHostEvent[host] val roundedTime = previousEvent?.let { val duration = time - it.timestamp @@ -78,13 +79,13 @@ public class WebExperimentMonitor : ExperimentMonitor { 0.0, 0.0, 0, - server + host ) } - private val lastPowerConsumption = mutableMapOf() + private val lastPowerConsumption = mutableMapOf() - override fun reportPowerConsumption(host: Server, draw: Double) { + override fun reportPowerConsumption(host: Node, draw: Double) { lastPowerConsumption[host] = draw } @@ -97,16 +98,16 @@ public class WebExperimentMonitor : ExperimentMonitor { cpuUsage: Double, cpuDemand: Double, numberOfDeployedImages: Int, - hostServer: Server, + host: Node, duration: Long ) { - val previousEvent = currentHostEvent[hostServer] + val previousEvent = currentHostEvent[host] when { previousEvent == null -> { val event = HostEvent( time, 5 * 60 * 1000L, - hostServer, + host, numberOfDeployedImages, requestedBurst, grantedBurst, @@ -114,17 +115,17 @@ public class WebExperimentMonitor : ExperimentMonitor { interferedBurst, cpuUsage, cpuDemand, - lastPowerConsumption[hostServer] ?: 200.0, - hostServer.flavor.cpuCount + lastPowerConsumption[host] ?: 200.0, + host.flavor.cpuCount ) - currentHostEvent[hostServer] = event + currentHostEvent[host] = event } previousEvent.timestamp == time -> { val event = HostEvent( time, previousEvent.duration, - hostServer, + host, numberOfDeployedImages, requestedBurst, grantedBurst, @@ -132,11 +133,11 @@ public class WebExperimentMonitor : ExperimentMonitor { interferedBurst, cpuUsage, cpuDemand, - lastPowerConsumption[hostServer] ?: 200.0, - hostServer.flavor.cpuCount + lastPowerConsumption[host] ?: 200.0, + host.flavor.cpuCount ) - currentHostEvent[hostServer] = event + currentHostEvent[host] = event } else -> { processHostEvent(previousEvent) @@ -144,7 +145,7 @@ public class WebExperimentMonitor : ExperimentMonitor { val event = HostEvent( time, time - previousEvent.timestamp, - hostServer, + host, numberOfDeployedImages, requestedBurst, grantedBurst, @@ -152,17 +153,17 @@ public class WebExperimentMonitor : ExperimentMonitor { interferedBurst, cpuUsage, cpuDemand, - lastPowerConsumption[hostServer] ?: 200.0, - hostServer.flavor.cpuCount + lastPowerConsumption[host] ?: 200.0, + host.flavor.cpuCount ) - currentHostEvent[hostServer] = event + currentHostEvent[host] = event } } } private var hostAggregateMetrics: AggregateHostMetrics = AggregateHostMetrics() - private val hostMetrics: MutableMap = mutableMapOf() + private val hostMetrics: MutableMap = mutableMapOf() private fun processHostEvent(event: HostEvent) { val slices = event.duration / SLICE_LENGTH @@ -173,14 +174,14 @@ public class WebExperimentMonitor : ExperimentMonitor { hostAggregateMetrics.totalOvercommittedBurst + event.overcommissionedBurst, hostAggregateMetrics.totalInterferedBurst + event.interferedBurst, hostAggregateMetrics.totalPowerDraw + (slices * (event.powerDraw / 12)), - hostAggregateMetrics.totalFailureSlices + if (event.host.state != ServerState.ACTIVE) slices.toLong() else 0, - hostAggregateMetrics.totalFailureVmSlices + if (event.host.state != ServerState.ACTIVE) event.vmCount * slices.toLong() else 0 + hostAggregateMetrics.totalFailureSlices + if (event.node.state != NodeState.ACTIVE) slices.toLong() else 0, + hostAggregateMetrics.totalFailureVmSlices + if (event.node.state != NodeState.ACTIVE) event.vmCount * slices.toLong() else 0 ) - hostMetrics.compute(event.host) { key, prev -> + hostMetrics.compute(event.node) { _, prev -> HostMetrics( - (event.cpuUsage.takeIf { event.host.state == ServerState.ACTIVE } ?: 0.0) + (prev?.cpuUsage ?: 0.0), - (event.cpuDemand.takeIf { event.host.state == ServerState.ACTIVE } ?: 0.0) + (prev?.cpuDemand ?: 0.0), + (event.cpuUsage.takeIf { event.node.state == NodeState.ACTIVE } ?: 0.0) + (prev?.cpuUsage ?: 0.0), + (event.cpuDemand.takeIf { event.node.state == NodeState.ACTIVE } ?: 0.0) + (prev?.cpuDemand ?: 0.0), event.vmCount + (prev?.vmCount ?: 0), 1 + (prev?.count ?: 0) ) -- cgit v1.2.3 From a0c0657dc867db61951edff24ddc944bed132ac0 Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Fri, 5 Mar 2021 14:44:30 +0100 Subject: compute: Make VirtProvisoningService responsible for Server lifecycle This change refactors the OpenDC Compute module so that the VirtProvisioningService is now responsible for managing the lifecycle of Server objects as opposed to the VirtDriver and BareMetalDriver previously. --- .../src/main/kotlin/org/opendc/runner/web/WebExperimentMonitor.kt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'simulator/opendc-runner-web') diff --git a/simulator/opendc-runner-web/src/main/kotlin/org/opendc/runner/web/WebExperimentMonitor.kt b/simulator/opendc-runner-web/src/main/kotlin/org/opendc/runner/web/WebExperimentMonitor.kt index b7a26d34..2a41be65 100644 --- a/simulator/opendc-runner-web/src/main/kotlin/org/opendc/runner/web/WebExperimentMonitor.kt +++ b/simulator/opendc-runner-web/src/main/kotlin/org/opendc/runner/web/WebExperimentMonitor.kt @@ -26,7 +26,7 @@ import mu.KotlinLogging import org.opendc.compute.core.Server import org.opendc.compute.core.metal.Node import org.opendc.compute.core.metal.NodeState -import org.opendc.compute.core.virt.driver.VirtDriver +import org.opendc.compute.core.virt.Host import org.opendc.compute.core.virt.service.VirtProvisioningEvent import org.opendc.experiments.capelin.monitor.ExperimentMonitor import org.opendc.experiments.capelin.telemetry.HostEvent @@ -51,7 +51,7 @@ public class WebExperimentMonitor : ExperimentMonitor { override fun reportHostStateChange( time: Long, - driver: VirtDriver, + driver: Host, host: Node ) { logger.debug { "Host ${host.uid} changed state ${host.state} [$time]" } -- cgit v1.2.3 From 10f71541cd2c72e12f1b2325ee4f25e38a10e0ef Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Fri, 5 Mar 2021 16:26:06 +0100 Subject: compute: Convert Server to stateful interface This change converts the Server data class which can be used as a stateful object to control an instance running in the cloud. --- .../src/main/kotlin/org/opendc/runner/web/WebExperimentMonitor.kt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'simulator/opendc-runner-web') diff --git a/simulator/opendc-runner-web/src/main/kotlin/org/opendc/runner/web/WebExperimentMonitor.kt b/simulator/opendc-runner-web/src/main/kotlin/org/opendc/runner/web/WebExperimentMonitor.kt index 2a41be65..6a5b5e32 100644 --- a/simulator/opendc-runner-web/src/main/kotlin/org/opendc/runner/web/WebExperimentMonitor.kt +++ b/simulator/opendc-runner-web/src/main/kotlin/org/opendc/runner/web/WebExperimentMonitor.kt @@ -24,6 +24,7 @@ package org.opendc.runner.web import mu.KotlinLogging import org.opendc.compute.core.Server +import org.opendc.compute.core.ServerState import org.opendc.compute.core.metal.Node import org.opendc.compute.core.metal.NodeState import org.opendc.compute.core.virt.Host @@ -40,7 +41,7 @@ public class WebExperimentMonitor : ExperimentMonitor { private val currentHostEvent = mutableMapOf() private var startTime = -1L - override fun reportVmStateChange(time: Long, server: Server) { + override fun reportVmStateChange(time: Long, server: Server, newState: ServerState) { if (startTime < 0) { startTime = time -- cgit v1.2.3 From dfbca195cbe1d6c4eebe7ccd4cc707c84ac43e79 Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Fri, 5 Mar 2021 16:48:59 +0100 Subject: compute: Extract API from compute service This change extracts the API for the OpenDC Compute service into a separate module to establish a clearer boundary between the interface meant for consumers and interfaces meant for the the serve implementation. --- .../src/main/kotlin/org/opendc/runner/web/WebExperimentMonitor.kt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'simulator/opendc-runner-web') diff --git a/simulator/opendc-runner-web/src/main/kotlin/org/opendc/runner/web/WebExperimentMonitor.kt b/simulator/opendc-runner-web/src/main/kotlin/org/opendc/runner/web/WebExperimentMonitor.kt index 6a5b5e32..f0c6297f 100644 --- a/simulator/opendc-runner-web/src/main/kotlin/org/opendc/runner/web/WebExperimentMonitor.kt +++ b/simulator/opendc-runner-web/src/main/kotlin/org/opendc/runner/web/WebExperimentMonitor.kt @@ -23,8 +23,8 @@ package org.opendc.runner.web import mu.KotlinLogging -import org.opendc.compute.core.Server -import org.opendc.compute.core.ServerState +import org.opendc.compute.api.Server +import org.opendc.compute.api.ServerState import org.opendc.compute.core.metal.Node import org.opendc.compute.core.metal.NodeState import org.opendc.compute.core.virt.Host -- cgit v1.2.3 From 9bb91897404bbeac1d5f7a7f890abd3a9d5d9084 Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Sun, 7 Mar 2021 15:44:36 +0100 Subject: compute: Move ComputeService implementation in service module This change introduces the ComputeService interface (previously VirtProvisioningService) and provides a central implementation in opendc-compute-service. Previously, the implementation of this interface was bound to the simulator package, which meant that independent business logic could not be re-used without importing the simulator code. --- .../src/main/kotlin/org/opendc/runner/web/Main.kt | 10 ++++++++-- .../main/kotlin/org/opendc/runner/web/WebExperimentMonitor.kt | 6 +++--- 2 files changed, 11 insertions(+), 5 deletions(-) (limited to 'simulator/opendc-runner-web') diff --git a/simulator/opendc-runner-web/src/main/kotlin/org/opendc/runner/web/Main.kt b/simulator/opendc-runner-web/src/main/kotlin/org/opendc/runner/web/Main.kt index 533bf321..482fe754 100644 --- a/simulator/opendc-runner-web/src/main/kotlin/org/opendc/runner/web/Main.kt +++ b/simulator/opendc-runner-web/src/main/kotlin/org/opendc/runner/web/Main.kt @@ -40,6 +40,11 @@ import kotlinx.coroutines.test.TestCoroutineScope import mu.KotlinLogging import org.bson.Document import org.bson.types.ObjectId +import org.opendc.compute.service.scheduler.AvailableCoreMemoryAllocationPolicy +import org.opendc.compute.service.scheduler.AvailableMemoryAllocationPolicy +import org.opendc.compute.service.scheduler.NumberOfActiveServersAllocationPolicy +import org.opendc.compute.service.scheduler.ProvisionedCoresAllocationPolicy +import org.opendc.compute.service.scheduler.RandomAllocationPolicy import org.opendc.compute.simulator.allocation.* import org.opendc.experiments.capelin.experiment.attachMonitor import org.opendc.experiments.capelin.experiment.createFailureDomain @@ -242,7 +247,7 @@ public class RunnerCli : CliktCommand(name = "runner") { val tracer = EventTracer(clock) testScope.launch { - val (bareMetalProvisioner, scheduler) = createProvisioner( + val (bareMetalProvisioner, provisioner, scheduler) = createProvisioner( this, clock, environment, @@ -281,7 +286,8 @@ public class RunnerCli : CliktCommand(name = "runner") { logger.debug("FINISHED=${scheduler.finishedVms}") failureDomain?.cancel() - scheduler.terminate() + scheduler.close() + provisioner.close() } try { diff --git a/simulator/opendc-runner-web/src/main/kotlin/org/opendc/runner/web/WebExperimentMonitor.kt b/simulator/opendc-runner-web/src/main/kotlin/org/opendc/runner/web/WebExperimentMonitor.kt index f0c6297f..6209cff2 100644 --- a/simulator/opendc-runner-web/src/main/kotlin/org/opendc/runner/web/WebExperimentMonitor.kt +++ b/simulator/opendc-runner-web/src/main/kotlin/org/opendc/runner/web/WebExperimentMonitor.kt @@ -27,8 +27,8 @@ import org.opendc.compute.api.Server import org.opendc.compute.api.ServerState import org.opendc.compute.core.metal.Node import org.opendc.compute.core.metal.NodeState -import org.opendc.compute.core.virt.Host -import org.opendc.compute.core.virt.service.VirtProvisioningEvent +import org.opendc.compute.service.ComputeServiceEvent +import org.opendc.compute.service.driver.Host import org.opendc.experiments.capelin.monitor.ExperimentMonitor import org.opendc.experiments.capelin.telemetry.HostEvent import kotlin.math.max @@ -210,7 +210,7 @@ public class WebExperimentMonitor : ExperimentMonitor { private var provisionerMetrics: AggregateProvisionerMetrics = AggregateProvisionerMetrics() - override fun reportProvisionerMetrics(time: Long, event: VirtProvisioningEvent.MetricsAvailable) { + override fun reportProvisionerMetrics(time: Long, event: ComputeServiceEvent.MetricsAvailable) { provisionerMetrics = AggregateProvisionerMetrics( max(event.totalVmCount, provisionerMetrics.vmTotalCount), max(event.waitingVmCount, provisionerMetrics.vmWaitingCount), -- cgit v1.2.3 From 75751865179c6cd5a05abb4a0641193595f59b45 Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Sun, 7 Mar 2021 15:58:06 +0100 Subject: compute: Move bare-metal provisioning in separate module This change moves the bare-metal provisioning packages outside the compute module since these modules represent different layers in the ecosystem and should not be mixed. --- .../src/main/kotlin/org/opendc/runner/web/TopologyParser.kt | 6 +++--- .../src/main/kotlin/org/opendc/runner/web/WebExperimentMonitor.kt | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) (limited to 'simulator/opendc-runner-web') diff --git a/simulator/opendc-runner-web/src/main/kotlin/org/opendc/runner/web/TopologyParser.kt b/simulator/opendc-runner-web/src/main/kotlin/org/opendc/runner/web/TopologyParser.kt index f43d0869..2f11347d 100644 --- a/simulator/opendc-runner-web/src/main/kotlin/org/opendc/runner/web/TopologyParser.kt +++ b/simulator/opendc-runner-web/src/main/kotlin/org/opendc/runner/web/TopologyParser.kt @@ -32,9 +32,6 @@ import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.launch import org.bson.Document import org.bson.types.ObjectId -import org.opendc.compute.core.metal.NODE_CLUSTER -import org.opendc.compute.core.metal.service.ProvisioningService -import org.opendc.compute.core.metal.service.SimpleProvisioningService import org.opendc.compute.simulator.SimBareMetalDriver import org.opendc.compute.simulator.power.models.LinearPowerModel import org.opendc.core.Environment @@ -42,6 +39,9 @@ import org.opendc.core.Platform import org.opendc.core.Zone import org.opendc.core.services.ServiceRegistry import org.opendc.format.environment.EnvironmentReader +import org.opendc.metal.NODE_CLUSTER +import org.opendc.metal.service.ProvisioningService +import org.opendc.metal.service.SimpleProvisioningService import org.opendc.simulator.compute.SimMachineModel import org.opendc.simulator.compute.model.MemoryUnit import org.opendc.simulator.compute.model.ProcessingNode diff --git a/simulator/opendc-runner-web/src/main/kotlin/org/opendc/runner/web/WebExperimentMonitor.kt b/simulator/opendc-runner-web/src/main/kotlin/org/opendc/runner/web/WebExperimentMonitor.kt index 6209cff2..fe814c76 100644 --- a/simulator/opendc-runner-web/src/main/kotlin/org/opendc/runner/web/WebExperimentMonitor.kt +++ b/simulator/opendc-runner-web/src/main/kotlin/org/opendc/runner/web/WebExperimentMonitor.kt @@ -25,12 +25,12 @@ package org.opendc.runner.web import mu.KotlinLogging import org.opendc.compute.api.Server import org.opendc.compute.api.ServerState -import org.opendc.compute.core.metal.Node -import org.opendc.compute.core.metal.NodeState import org.opendc.compute.service.ComputeServiceEvent import org.opendc.compute.service.driver.Host import org.opendc.experiments.capelin.monitor.ExperimentMonitor import org.opendc.experiments.capelin.telemetry.HostEvent +import org.opendc.metal.Node +import org.opendc.metal.NodeState import kotlin.math.max /** -- cgit v1.2.3