From c27c43fd0f34b8ae95ce2a0939d827530b9e2bf2 Mon Sep 17 00:00:00 2001 From: Georgios Andreadis Date: Tue, 5 May 2020 18:16:40 +0200 Subject: Add basic replay policy --- .../service/allocation/ReplayAllocationPolicy.kt | 19 ++++++++ .../opendc/experiments/sc20/TestExperiment.kt | 11 +++++ .../opendc/format/trace/VmPlacementReader.kt | 37 ++++++++++++++++ .../format/trace/sc20/Sc20VmPlacementReader.kt | 51 ++++++++++++++++++++++ 4 files changed, 118 insertions(+) create mode 100644 opendc/opendc-compute/src/main/kotlin/com/atlarge/opendc/compute/virt/service/allocation/ReplayAllocationPolicy.kt create mode 100644 opendc/opendc-format/src/main/kotlin/com/atlarge/opendc/format/trace/VmPlacementReader.kt create mode 100644 opendc/opendc-format/src/main/kotlin/com/atlarge/opendc/format/trace/sc20/Sc20VmPlacementReader.kt diff --git a/opendc/opendc-compute/src/main/kotlin/com/atlarge/opendc/compute/virt/service/allocation/ReplayAllocationPolicy.kt b/opendc/opendc-compute/src/main/kotlin/com/atlarge/opendc/compute/virt/service/allocation/ReplayAllocationPolicy.kt new file mode 100644 index 00000000..31731256 --- /dev/null +++ b/opendc/opendc-compute/src/main/kotlin/com/atlarge/opendc/compute/virt/service/allocation/ReplayAllocationPolicy.kt @@ -0,0 +1,19 @@ +package com.atlarge.opendc.compute.virt.service.allocation + +import com.atlarge.opendc.compute.virt.service.HypervisorView +import com.atlarge.opendc.compute.virt.service.SimpleVirtProvisioningService + +class ReplayAllocationPolicy(val vmPlacements: Map) : AllocationPolicy { + override fun invoke(): AllocationPolicy.Logic = object : AllocationPolicy.Logic { + override fun select( + hypervisors: Set, + image: SimpleVirtProvisioningService.ImageView + ): HypervisorView? { + val clusterName = vmPlacements[image.name] + ?: throw IllegalArgumentException("Could not find placement data in VM placement file for VM ${image.name}") + val machinesInCluster = hypervisors.filter { it.server.name.contains(clusterName) } + return machinesInCluster.minBy { it.numberOfActiveServers } + ?: throw IllegalArgumentException("Cloud not find any machines belonging to cluster $clusterName for image ${image.name}") + } + } +} diff --git a/opendc/opendc-experiments-sc20/src/main/kotlin/com/atlarge/opendc/experiments/sc20/TestExperiment.kt b/opendc/opendc-experiments-sc20/src/main/kotlin/com/atlarge/opendc/experiments/sc20/TestExperiment.kt index 028cfb9a..54af8a4b 100644 --- a/opendc/opendc-experiments-sc20/src/main/kotlin/com/atlarge/opendc/experiments/sc20/TestExperiment.kt +++ b/opendc/opendc-experiments-sc20/src/main/kotlin/com/atlarge/opendc/experiments/sc20/TestExperiment.kt @@ -39,11 +39,13 @@ import com.atlarge.opendc.compute.virt.service.allocation.AvailableMemoryAllocat import com.atlarge.opendc.compute.virt.service.allocation.NumberOfActiveServersAllocationPolicy import com.atlarge.opendc.compute.virt.service.allocation.ProvisionedCoresAllocationPolicy import com.atlarge.opendc.compute.virt.service.allocation.RandomAllocationPolicy +import com.atlarge.opendc.compute.virt.service.allocation.ReplayAllocationPolicy import com.atlarge.opendc.core.failure.CorrelatedFaultInjector import com.atlarge.opendc.core.failure.FailureDomain import com.atlarge.opendc.core.failure.FaultInjector import com.atlarge.opendc.format.environment.sc20.Sc20ClusterEnvironmentReader import com.atlarge.opendc.format.trace.sc20.Sc20PerformanceInterferenceReader +import com.atlarge.opendc.format.trace.sc20.Sc20VmPlacementReader import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper import com.fasterxml.jackson.module.kotlin.readValue import com.xenomachina.argparser.ArgParser @@ -67,6 +69,7 @@ class ExperimentParameters(parser: ArgParser) { val traceDirectory by parser.storing("path to the trace directory") val environmentFile by parser.storing("path to the environment file") val performanceInterferenceFile by parser.storing("path to the performance interference file").default { null } + val vmPlacementFile by parser.storing("path to the VM placement file").default { null } val outputFile by parser.storing("path to where the output should be stored") .default { "data/results-${System.currentTimeMillis()}.parquet" } val selectedVms by parser.storing("the VMs to run") { parseVMs(this) } @@ -121,6 +124,7 @@ fun main(args: Array) { println("trace-directory: $traceDirectory") println("environment-file: $environmentFile") println("performance-interference-file: $performanceInterferenceFile") + println("vm-placement-file: $vmPlacementFile") println("selected-vms-file: $selectedVmsFile") println("seed: $seed") println("failures: $failures") @@ -134,6 +138,12 @@ fun main(args: Array) { val root = system.newDomain("root") val chan = Channel(Channel.CONFLATED) + val vmPlacements = if (vmPlacementFile == null) { + emptyMap() + } else { + Sc20VmPlacementReader(File(vmPlacementFile!!).inputStream().buffered()).construct() + } + val allocationPolicies = mapOf( "mem" to AvailableMemoryAllocationPolicy(), "mem-inv" to AvailableMemoryAllocationPolicy(true), @@ -143,6 +153,7 @@ fun main(args: Array) { "active-servers-inv" to NumberOfActiveServersAllocationPolicy(true), "provisioned-cores" to ProvisionedCoresAllocationPolicy(), "provisioned-cores-inv" to ProvisionedCoresAllocationPolicy(true), + "replay" to ReplayAllocationPolicy(vmPlacements), "random" to RandomAllocationPolicy(Random(seed)) ) diff --git a/opendc/opendc-format/src/main/kotlin/com/atlarge/opendc/format/trace/VmPlacementReader.kt b/opendc/opendc-format/src/main/kotlin/com/atlarge/opendc/format/trace/VmPlacementReader.kt new file mode 100644 index 00000000..7caebb76 --- /dev/null +++ b/opendc/opendc-format/src/main/kotlin/com/atlarge/opendc/format/trace/VmPlacementReader.kt @@ -0,0 +1,37 @@ +/* + * MIT License + * + * Copyright (c) 2019 atlarge-research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package com.atlarge.opendc.format.trace + +import java.io.Closeable + +/** + * An interface for reading VM placement data into memory. + */ +interface VmPlacementReader : Closeable { + /** + * Construct a map of VMs to clusters. + */ + fun construct(): Map +} diff --git a/opendc/opendc-format/src/main/kotlin/com/atlarge/opendc/format/trace/sc20/Sc20VmPlacementReader.kt b/opendc/opendc-format/src/main/kotlin/com/atlarge/opendc/format/trace/sc20/Sc20VmPlacementReader.kt new file mode 100644 index 00000000..f8ce4380 --- /dev/null +++ b/opendc/opendc-format/src/main/kotlin/com/atlarge/opendc/format/trace/sc20/Sc20VmPlacementReader.kt @@ -0,0 +1,51 @@ +/* + * MIT License + * + * Copyright (c) 2019 atlarge-research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package com.atlarge.opendc.format.trace.sc20 + +import com.atlarge.opendc.format.trace.VmPlacementReader +import com.fasterxml.jackson.databind.ObjectMapper +import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper +import com.fasterxml.jackson.module.kotlin.readValue +import java.io.InputStream + +/** + * A parser for the JSON VM placement data files used for the SC20 paper. + * + * @param input The input stream to read from. + * @param mapper The Jackson object mapper to use. + */ +class Sc20VmPlacementReader(input: InputStream, mapper: ObjectMapper = jacksonObjectMapper()) : + VmPlacementReader { + /** + * The environment that was read from the file. + */ + private val placements = mapper.readValue>(input) + + override fun construct(): Map { + return placements.mapKeys { "vm__workload__${it.key}.txt" } + } + + override fun close() {} +} -- cgit v1.2.3 From d6b3ecfc6737638af48caca3c369087e8c3da3db Mon Sep 17 00:00:00 2001 From: Georgios Andreadis Date: Tue, 5 May 2020 18:17:52 +0200 Subject: Add docs --- .../compute/virt/service/allocation/ReplayAllocationPolicy.kt | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/opendc/opendc-compute/src/main/kotlin/com/atlarge/opendc/compute/virt/service/allocation/ReplayAllocationPolicy.kt b/opendc/opendc-compute/src/main/kotlin/com/atlarge/opendc/compute/virt/service/allocation/ReplayAllocationPolicy.kt index 31731256..790121c4 100644 --- a/opendc/opendc-compute/src/main/kotlin/com/atlarge/opendc/compute/virt/service/allocation/ReplayAllocationPolicy.kt +++ b/opendc/opendc-compute/src/main/kotlin/com/atlarge/opendc/compute/virt/service/allocation/ReplayAllocationPolicy.kt @@ -3,6 +3,12 @@ package com.atlarge.opendc.compute.virt.service.allocation import com.atlarge.opendc.compute.virt.service.HypervisorView import com.atlarge.opendc.compute.virt.service.SimpleVirtProvisioningService +/** + * Policy replaying VM-cluster assignnment. + * + * Within each cluster, the active servers on each node determine which node gets + * assigned the VM image. + */ class ReplayAllocationPolicy(val vmPlacements: Map) : AllocationPolicy { override fun invoke(): AllocationPolicy.Logic = object : AllocationPolicy.Logic { override fun select( -- cgit v1.2.3 From b8637f4d54dd228f08029f66f45cfe4d2240ddff Mon Sep 17 00:00:00 2001 From: Georgios Andreadis Date: Tue, 5 May 2020 19:01:39 +0200 Subject: Fix cluster format --- .../opendc/compute/virt/service/allocation/ReplayAllocationPolicy.kt | 4 ++-- .../com/atlarge/opendc/format/trace/sc20/Sc20VmPlacementReader.kt | 4 +++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/opendc/opendc-compute/src/main/kotlin/com/atlarge/opendc/compute/virt/service/allocation/ReplayAllocationPolicy.kt b/opendc/opendc-compute/src/main/kotlin/com/atlarge/opendc/compute/virt/service/allocation/ReplayAllocationPolicy.kt index 790121c4..280f3028 100644 --- a/opendc/opendc-compute/src/main/kotlin/com/atlarge/opendc/compute/virt/service/allocation/ReplayAllocationPolicy.kt +++ b/opendc/opendc-compute/src/main/kotlin/com/atlarge/opendc/compute/virt/service/allocation/ReplayAllocationPolicy.kt @@ -16,10 +16,10 @@ class ReplayAllocationPolicy(val vmPlacements: Map) : Allocation image: SimpleVirtProvisioningService.ImageView ): HypervisorView? { val clusterName = vmPlacements[image.name] - ?: throw IllegalArgumentException("Could not find placement data in VM placement file for VM ${image.name}") + ?: throw RuntimeException("Could not find placement data in VM placement file for VM ${image.name}") val machinesInCluster = hypervisors.filter { it.server.name.contains(clusterName) } return machinesInCluster.minBy { it.numberOfActiveServers } - ?: throw IllegalArgumentException("Cloud not find any machines belonging to cluster $clusterName for image ${image.name}") + ?: throw RuntimeException("Cloud not find any machines belonging to cluster $clusterName for image ${image.name}") } } } diff --git a/opendc/opendc-format/src/main/kotlin/com/atlarge/opendc/format/trace/sc20/Sc20VmPlacementReader.kt b/opendc/opendc-format/src/main/kotlin/com/atlarge/opendc/format/trace/sc20/Sc20VmPlacementReader.kt index f8ce4380..5295ae03 100644 --- a/opendc/opendc-format/src/main/kotlin/com/atlarge/opendc/format/trace/sc20/Sc20VmPlacementReader.kt +++ b/opendc/opendc-format/src/main/kotlin/com/atlarge/opendc/format/trace/sc20/Sc20VmPlacementReader.kt @@ -44,7 +44,9 @@ class Sc20VmPlacementReader(input: InputStream, mapper: ObjectMapper = jacksonOb private val placements = mapper.readValue>(input) override fun construct(): Map { - return placements.mapKeys { "vm__workload__${it.key}.txt" } + return placements + .mapKeys { "vm__workload__${it.key}.txt" } + .mapValues { it.value.split("/")[1] } // Clusters have format XX0 / X00 } override fun close() {} -- cgit v1.2.3 From 9892e3d9a6755e4b9ec213adc30e7539696248f6 Mon Sep 17 00:00:00 2001 From: Georgios Andreadis Date: Tue, 5 May 2020 19:38:09 +0200 Subject: Add check for VMs not in placement data --- .../virt/service/allocation/ReplayAllocationPolicy.kt | 2 +- .../atlarge/opendc/experiments/sc20/TestExperiment.kt | 17 +++++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/opendc/opendc-compute/src/main/kotlin/com/atlarge/opendc/compute/virt/service/allocation/ReplayAllocationPolicy.kt b/opendc/opendc-compute/src/main/kotlin/com/atlarge/opendc/compute/virt/service/allocation/ReplayAllocationPolicy.kt index 280f3028..1276a973 100644 --- a/opendc/opendc-compute/src/main/kotlin/com/atlarge/opendc/compute/virt/service/allocation/ReplayAllocationPolicy.kt +++ b/opendc/opendc-compute/src/main/kotlin/com/atlarge/opendc/compute/virt/service/allocation/ReplayAllocationPolicy.kt @@ -18,7 +18,7 @@ class ReplayAllocationPolicy(val vmPlacements: Map) : Allocation val clusterName = vmPlacements[image.name] ?: throw RuntimeException("Could not find placement data in VM placement file for VM ${image.name}") val machinesInCluster = hypervisors.filter { it.server.name.contains(clusterName) } - return machinesInCluster.minBy { it.numberOfActiveServers } + return machinesInCluster.maxBy { it.availableMemory } ?: throw RuntimeException("Cloud not find any machines belonging to cluster $clusterName for image ${image.name}") } } diff --git a/opendc/opendc-experiments-sc20/src/main/kotlin/com/atlarge/opendc/experiments/sc20/TestExperiment.kt b/opendc/opendc-experiments-sc20/src/main/kotlin/com/atlarge/opendc/experiments/sc20/TestExperiment.kt index 54af8a4b..424f411c 100644 --- a/opendc/opendc-experiments-sc20/src/main/kotlin/com/atlarge/opendc/experiments/sc20/TestExperiment.kt +++ b/opendc/opendc-experiments-sc20/src/main/kotlin/com/atlarge/opendc/experiments/sc20/TestExperiment.kt @@ -253,6 +253,23 @@ fun main(args: Array) { val reader = Sc20ParquetTraceReader(File(traceDirectory), performanceInterferenceModel, getSelectedVmList(), Random(seed)) while (reader.hasNext()) { val (time, workload) = reader.next() + + if (vmPlacements.isNotEmpty()) { + val vmId = workload.name.replace("VM Workload ", "") + // Check if VM in topology + val clusterName = vmPlacements[vmId] + if (clusterName == null) { + println("Could not find placement data in VM placement file for VM ${vmId}") + continue + } + val machinesInCluster = + hypervisors.filter { (it as SimpleVirtDriver).server.name.contains(clusterName) } + if (machinesInCluster.isEmpty()) { + println("Ignored VM") + continue + } + } + submitted++ delay(max(0, time - simulationContext.clock.millis())) launch { -- cgit v1.2.3 From 18c80d362703f6d96920ed3362dc9c389d723ae9 Mon Sep 17 00:00:00 2001 From: Georgios Andreadis Date: Tue, 5 May 2020 19:47:41 +0200 Subject: Fix redundant braces --- .../main/kotlin/com/atlarge/opendc/experiments/sc20/TestExperiment.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opendc/opendc-experiments-sc20/src/main/kotlin/com/atlarge/opendc/experiments/sc20/TestExperiment.kt b/opendc/opendc-experiments-sc20/src/main/kotlin/com/atlarge/opendc/experiments/sc20/TestExperiment.kt index 424f411c..a2f609a5 100644 --- a/opendc/opendc-experiments-sc20/src/main/kotlin/com/atlarge/opendc/experiments/sc20/TestExperiment.kt +++ b/opendc/opendc-experiments-sc20/src/main/kotlin/com/atlarge/opendc/experiments/sc20/TestExperiment.kt @@ -259,7 +259,7 @@ fun main(args: Array) { // Check if VM in topology val clusterName = vmPlacements[vmId] if (clusterName == null) { - println("Could not find placement data in VM placement file for VM ${vmId}") + println("Could not find placement data in VM placement file for VM $vmId") continue } val machinesInCluster = -- cgit v1.2.3 From c4ba00bdb3a777dea866a043820e2df89ff81f86 Mon Sep 17 00:00:00 2001 From: Georgios Andreadis Date: Tue, 5 May 2020 22:23:11 +0200 Subject: Change exception --- .../opendc/compute/virt/service/allocation/ReplayAllocationPolicy.kt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/opendc/opendc-compute/src/main/kotlin/com/atlarge/opendc/compute/virt/service/allocation/ReplayAllocationPolicy.kt b/opendc/opendc-compute/src/main/kotlin/com/atlarge/opendc/compute/virt/service/allocation/ReplayAllocationPolicy.kt index 1276a973..f88eaed8 100644 --- a/opendc/opendc-compute/src/main/kotlin/com/atlarge/opendc/compute/virt/service/allocation/ReplayAllocationPolicy.kt +++ b/opendc/opendc-compute/src/main/kotlin/com/atlarge/opendc/compute/virt/service/allocation/ReplayAllocationPolicy.kt @@ -16,10 +16,10 @@ class ReplayAllocationPolicy(val vmPlacements: Map) : Allocation image: SimpleVirtProvisioningService.ImageView ): HypervisorView? { val clusterName = vmPlacements[image.name] - ?: throw RuntimeException("Could not find placement data in VM placement file for VM ${image.name}") + ?: throw IllegalStateException("Could not find placement data in VM placement file for VM ${image.name}") val machinesInCluster = hypervisors.filter { it.server.name.contains(clusterName) } return machinesInCluster.maxBy { it.availableMemory } - ?: throw RuntimeException("Cloud not find any machines belonging to cluster $clusterName for image ${image.name}") + ?: throw IllegalStateException("Cloud not find any machines belonging to cluster $clusterName for image ${image.name}") } } } -- cgit v1.2.3