From db3b8a63b66f5a34d17483bcb29a85bdd8b75598 Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Thu, 23 Jul 2020 20:55:25 +0200 Subject: Make HPC sampling strategy consistent --- .../experiments/sc20/trace/WorkloadSampler.kt | 51 ++++++++++++++++------ 1 file changed, 38 insertions(+), 13 deletions(-) diff --git a/simulator/opendc/opendc-experiments-sc20/src/main/kotlin/com/atlarge/opendc/experiments/sc20/trace/WorkloadSampler.kt b/simulator/opendc/opendc-experiments-sc20/src/main/kotlin/com/atlarge/opendc/experiments/sc20/trace/WorkloadSampler.kt index b24d6de1..d03d556b 100644 --- a/simulator/opendc/opendc-experiments-sc20/src/main/kotlin/com/atlarge/opendc/experiments/sc20/trace/WorkloadSampler.kt +++ b/simulator/opendc/opendc-experiments-sc20/src/main/kotlin/com/atlarge/opendc/experiments/sc20/trace/WorkloadSampler.kt @@ -109,6 +109,24 @@ fun sampleHpcWorkload( name.matches(pattern) } + val hpcSequence = generateSequence(0) { it + 1 } + .map { index -> + val res = mutableListOf>() + hpc.mapTo(res) { sample(it, index) } + res.shuffle(random) + res + } + .flatten() + + val nonHpcSequence = generateSequence(0) { it + 1 } + .map { index -> + val res = mutableListOf>() + nonHpc.mapTo(res) { sample(it, index) } + res.shuffle(random) + res + } + .flatten() + logger.debug { "Found ${hpc.size} HPC workloads and ${nonHpc.size} non-HPC workloads" } val totalLoad = if (workload is CompositeWorkload) { @@ -117,15 +135,14 @@ fun sampleHpcWorkload( trace.sumByDouble { it.workload.image.tags.getValue("total-load") as Double } } + logger.debug { "Total trace load: $totalLoad" } + val res = mutableListOf>() if (sampleOnLoad) { var currentLoad = 0.0 var i = 0 - while (true) { - // Sample random HPC entry with replacement - val entry = sample(hpc.random(random), i++) - + for (entry in hpcSequence) { val entryLoad = entry.workload.image.tags.getValue("total-load") as Double if ((currentLoad + entryLoad) / totalLoad > fraction || res.size > trace.size) { break @@ -135,8 +152,7 @@ fun sampleHpcWorkload( res += entry } - (nonHpc as MutableList>).shuffle(random) - for (entry in nonHpc) { + for (entry in nonHpcSequence) { val entryLoad = entry.workload.image.tags.getValue("total-load") as Double if ((currentLoad + entryLoad) / totalLoad > 1 || res.size > trace.size) { break @@ -146,14 +162,23 @@ fun sampleHpcWorkload( res += entry } } else { - repeat((fraction * trace.size).toInt()) { i -> - // Sample random HPC entry with replacement - val entry = sample(hpc.random(random), i) - res.add(entry) - } + var hpcLoad = 0.0 + hpcSequence + .take((fraction * trace.size).toInt()) + .forEach { entry -> + hpcLoad += entry.workload.image.tags.getValue("total-load") as Double + res.add(entry) + } + + var nonHpcLoad = 0.0 + nonHpcSequence + .take(((1 - fraction) * trace.size).toInt()) + .forEach { entry -> + nonHpcLoad += entry.workload.image.tags.getValue("total-load") as Double + res.add(entry) + } - (nonHpc as MutableList>).shuffle(random) - res.addAll(nonHpc.subList(0, ((1 - fraction) * trace.size).toInt())) + logger.debug { "HPC load $hpcLoad and non-HPC load $nonHpcLoad" } } logger.info { "Sampled ${trace.size} VMs (fraction $fraction) into subset of ${res.size} VMs" } -- cgit v1.2.3 From 7b76fd389bf6e5f19487dcf3033efc3508d1f9b5 Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Thu, 23 Jul 2020 21:00:18 +0200 Subject: Add missing HPC scenarios --- .../kotlin/com/atlarge/opendc/experiments/sc20/experiment/Portfolios.kt | 1 + 1 file changed, 1 insertion(+) diff --git a/simulator/opendc/opendc-experiments-sc20/src/main/kotlin/com/atlarge/opendc/experiments/sc20/experiment/Portfolios.kt b/simulator/opendc/opendc-experiments-sc20/src/main/kotlin/com/atlarge/opendc/experiments/sc20/experiment/Portfolios.kt index b8dfb1be..09a6ce40 100644 --- a/simulator/opendc/opendc-experiments-sc20/src/main/kotlin/com/atlarge/opendc/experiments/sc20/experiment/Portfolios.kt +++ b/simulator/opendc/opendc-experiments-sc20/src/main/kotlin/com/atlarge/opendc/experiments/sc20/experiment/Portfolios.kt @@ -203,6 +203,7 @@ public class MoreHpcPortfolio(parent: Experiment, id: Int) : Portfolio(parent, i ) override val workloads = listOf( + Workload("solvinity", 0.0, samplingStrategy = SamplingStrategy.HPC), Workload("solvinity", 0.25, samplingStrategy = SamplingStrategy.HPC), Workload("solvinity", 0.5, samplingStrategy = SamplingStrategy.HPC), Workload("solvinity", 1.0, samplingStrategy = SamplingStrategy.HPC), -- cgit v1.2.3 From 77c195945c89187addb14c2b9273813687abae95 Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Thu, 23 Jul 2020 21:23:30 +0200 Subject: Report additional workload sampling information --- simulator/opendc/opendc-experiments-sc20/src/main/resources/log4j2.xml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/simulator/opendc/opendc-experiments-sc20/src/main/resources/log4j2.xml b/simulator/opendc/opendc-experiments-sc20/src/main/resources/log4j2.xml index f47a6da8..6906bfc3 100644 --- a/simulator/opendc/opendc-experiments-sc20/src/main/resources/log4j2.xml +++ b/simulator/opendc/opendc-experiments-sc20/src/main/resources/log4j2.xml @@ -39,6 +39,9 @@ + + + -- cgit v1.2.3 From ab98dfe71ddda7b5cd2b57a24c118097ba5db0ac Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Fri, 24 Jul 2020 11:56:48 +0200 Subject: Allow load sampling result to exceed original trace size --- .../opendc/experiments/sc20/trace/WorkloadSampler.kt | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/simulator/opendc/opendc-experiments-sc20/src/main/kotlin/com/atlarge/opendc/experiments/sc20/trace/WorkloadSampler.kt b/simulator/opendc/opendc-experiments-sc20/src/main/kotlin/com/atlarge/opendc/experiments/sc20/trace/WorkloadSampler.kt index d03d556b..99830717 100644 --- a/simulator/opendc/opendc-experiments-sc20/src/main/kotlin/com/atlarge/opendc/experiments/sc20/trace/WorkloadSampler.kt +++ b/simulator/opendc/opendc-experiments-sc20/src/main/kotlin/com/atlarge/opendc/experiments/sc20/trace/WorkloadSampler.kt @@ -136,6 +136,10 @@ fun sampleHpcWorkload( } logger.debug { "Total trace load: $totalLoad" } + var hpcCount = 0 + var hpcLoad = 0.0 + var nonHpcCount = 0 + var nonHpcLoad = 0.0 val res = mutableListOf>() @@ -144,43 +148,47 @@ fun sampleHpcWorkload( var i = 0 for (entry in hpcSequence) { val entryLoad = entry.workload.image.tags.getValue("total-load") as Double - if ((currentLoad + entryLoad) / totalLoad > fraction || res.size > trace.size) { + if ((currentLoad + entryLoad) / totalLoad > fraction) { break } + hpcLoad += entryLoad + hpcCount += 1 currentLoad += entryLoad res += entry } for (entry in nonHpcSequence) { val entryLoad = entry.workload.image.tags.getValue("total-load") as Double - if ((currentLoad + entryLoad) / totalLoad > 1 || res.size > trace.size) { + if ((currentLoad + entryLoad) / totalLoad > 1) { break } + nonHpcLoad += entryLoad + nonHpcCount += 1 currentLoad += entryLoad res += entry } } else { - var hpcLoad = 0.0 hpcSequence .take((fraction * trace.size).toInt()) .forEach { entry -> hpcLoad += entry.workload.image.tags.getValue("total-load") as Double + hpcCount += 1 res.add(entry) } - var nonHpcLoad = 0.0 nonHpcSequence .take(((1 - fraction) * trace.size).toInt()) .forEach { entry -> nonHpcLoad += entry.workload.image.tags.getValue("total-load") as Double + nonHpcCount += 1 res.add(entry) } - - logger.debug { "HPC load $hpcLoad and non-HPC load $nonHpcLoad" } } + logger.debug { "HPC $hpcCount (load $hpcLoad) and non-HPC $nonHpcCount (load $nonHpcLoad)" } + logger.debug { "Total sampled load: ${hpcLoad + nonHpcLoad}" } logger.info { "Sampled ${trace.size} VMs (fraction $fraction) into subset of ${res.size} VMs" } return res -- cgit v1.2.3 From bbf77c59f5b2532ebca8daf8e67012205d764b97 Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Fri, 24 Jul 2020 13:31:22 +0200 Subject: Fix performance interference in HPC experiments --- .../compute/core/workload/PerformanceInterferenceModel.kt | 11 +++++------ .../atlarge/opendc/experiments/sc20/trace/WorkloadSampler.kt | 4 ++-- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/simulator/opendc/opendc-compute/src/main/kotlin/com/atlarge/opendc/compute/core/workload/PerformanceInterferenceModel.kt b/simulator/opendc/opendc-compute/src/main/kotlin/com/atlarge/opendc/compute/core/workload/PerformanceInterferenceModel.kt index f458877b..e1f03d21 100644 --- a/simulator/opendc/opendc-compute/src/main/kotlin/com/atlarge/opendc/compute/core/workload/PerformanceInterferenceModel.kt +++ b/simulator/opendc/opendc-compute/src/main/kotlin/com/atlarge/opendc/compute/core/workload/PerformanceInterferenceModel.kt @@ -25,8 +25,7 @@ package com.atlarge.opendc.compute.core.workload import com.atlarge.opendc.compute.core.Server -import java.util.SortedSet -import java.util.TreeSet +import java.util.* import kotlin.random.Random /** @@ -44,21 +43,21 @@ class PerformanceInterferenceModel( val random: Random = Random(0) ) { private var intersectingItems: List = emptyList() - private val colocatedWorkloads = TreeSet() + private val colocatedWorkloads = TreeMap() fun vmStarted(server: Server) { - colocatedWorkloads.add(server.image.name) + colocatedWorkloads.merge(server.image.name, 1, Int::plus) intersectingItems = items.filter { item -> doesMatch(item) } } fun vmStopped(server: Server) { - colocatedWorkloads.remove(server.image.name) + colocatedWorkloads.computeIfPresent(server.image.name) { _, v -> (v - 1).takeUnless { it == 0 } } intersectingItems = items.filter { item -> doesMatch(item) } } private fun doesMatch(item: PerformanceInterferenceModelItem): Boolean { var count = 0 - for (name in item.workloadNames.subSet(colocatedWorkloads.first(), colocatedWorkloads.last() + "\u0000")) { + for (name in item.workloadNames.subSet(colocatedWorkloads.firstKey(), colocatedWorkloads.lastKey() + "\u0000")) { if (name in colocatedWorkloads) count++ if (count > 1) diff --git a/simulator/opendc/opendc-experiments-sc20/src/main/kotlin/com/atlarge/opendc/experiments/sc20/trace/WorkloadSampler.kt b/simulator/opendc/opendc-experiments-sc20/src/main/kotlin/com/atlarge/opendc/experiments/sc20/trace/WorkloadSampler.kt index 99830717..a46bb3e6 100644 --- a/simulator/opendc/opendc-experiments-sc20/src/main/kotlin/com/atlarge/opendc/experiments/sc20/trace/WorkloadSampler.kt +++ b/simulator/opendc/opendc-experiments-sc20/src/main/kotlin/com/atlarge/opendc/experiments/sc20/trace/WorkloadSampler.kt @@ -201,13 +201,13 @@ private fun sample(entry: TraceEntry, i: Int): TraceEntry Date: Fri, 24 Jul 2020 13:36:55 +0200 Subject: Consider duplicate VMs for performance interference --- .../opendc/compute/core/workload/PerformanceInterferenceModel.kt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/simulator/opendc/opendc-compute/src/main/kotlin/com/atlarge/opendc/compute/core/workload/PerformanceInterferenceModel.kt b/simulator/opendc/opendc-compute/src/main/kotlin/com/atlarge/opendc/compute/core/workload/PerformanceInterferenceModel.kt index e1f03d21..3f885f89 100644 --- a/simulator/opendc/opendc-compute/src/main/kotlin/com/atlarge/opendc/compute/core/workload/PerformanceInterferenceModel.kt +++ b/simulator/opendc/opendc-compute/src/main/kotlin/com/atlarge/opendc/compute/core/workload/PerformanceInterferenceModel.kt @@ -58,8 +58,7 @@ class PerformanceInterferenceModel( private fun doesMatch(item: PerformanceInterferenceModelItem): Boolean { var count = 0 for (name in item.workloadNames.subSet(colocatedWorkloads.firstKey(), colocatedWorkloads.lastKey() + "\u0000")) { - if (name in colocatedWorkloads) - count++ + count += colocatedWorkloads.getOrDefault(name, 0) if (count > 1) return true } -- cgit v1.2.3