From 24fd4828d6798c19476543fa16df87d45811b54e Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Tue, 12 May 2020 20:35:46 +0200 Subject: refactor: Restructure experiment setup --- .../com/atlarge/opendc/experiments/sc20/Sc20IntegrationTest.kt | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) (limited to 'opendc/opendc-experiments-sc20/src/test') diff --git a/opendc/opendc-experiments-sc20/src/test/kotlin/com/atlarge/opendc/experiments/sc20/Sc20IntegrationTest.kt b/opendc/opendc-experiments-sc20/src/test/kotlin/com/atlarge/opendc/experiments/sc20/Sc20IntegrationTest.kt index 239d018a..5177c04a 100644 --- a/opendc/opendc-experiments-sc20/src/test/kotlin/com/atlarge/opendc/experiments/sc20/Sc20IntegrationTest.kt +++ b/opendc/opendc-experiments-sc20/src/test/kotlin/com/atlarge/opendc/experiments/sc20/Sc20IntegrationTest.kt @@ -31,6 +31,7 @@ import com.atlarge.opendc.compute.core.Server import com.atlarge.opendc.compute.core.workload.VmWorkload import com.atlarge.opendc.compute.virt.service.SimpleVirtProvisioningService import com.atlarge.opendc.compute.virt.service.allocation.AvailableCoreMemoryAllocationPolicy +import com.atlarge.opendc.experiments.sc20.reporter.ExperimentReporter import com.atlarge.opendc.format.environment.EnvironmentReader import com.atlarge.opendc.format.environment.sc20.Sc20ClusterEnvironmentReader import com.atlarge.opendc.format.trace.TraceReader @@ -63,7 +64,7 @@ class Sc20IntegrationTest { /** * The monitor used to keep track of the metrics. */ - private lateinit var monitor: TestSc20Reporter + private lateinit var monitor: TestExperimentReporter /** * Setup the experimental environment. @@ -73,7 +74,7 @@ class Sc20IntegrationTest { val provider = ServiceLoader.load(SimulationEngineProvider::class.java).first() simulationEngine = provider("test") root = simulationEngine.newDomain("root") - monitor = TestSc20Reporter() + monitor = TestExperimentReporter() } /** @@ -151,7 +152,7 @@ class Sc20IntegrationTest { return Sc20ClusterEnvironmentReader(stream) } - class TestSc20Reporter : Sc20Reporter { + class TestExperimentReporter : ExperimentReporter { var totalRequestedBurst = 0L var totalGrantedBurst = 0L var totalOvercommissionedBurst = 0L -- cgit v1.2.3 From 189001983350cbcc7f3524ea5983df48c873709b Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Thu, 14 May 2020 02:28:33 +0200 Subject: feat: Persist provisioner events --- .../com/atlarge/opendc/experiments/sc20/Sc20IntegrationTest.kt | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) (limited to 'opendc/opendc-experiments-sc20/src/test') diff --git a/opendc/opendc-experiments-sc20/src/test/kotlin/com/atlarge/opendc/experiments/sc20/Sc20IntegrationTest.kt b/opendc/opendc-experiments-sc20/src/test/kotlin/com/atlarge/opendc/experiments/sc20/Sc20IntegrationTest.kt index 5177c04a..1edb7bc2 100644 --- a/opendc/opendc-experiments-sc20/src/test/kotlin/com/atlarge/opendc/experiments/sc20/Sc20IntegrationTest.kt +++ b/opendc/opendc-experiments-sc20/src/test/kotlin/com/atlarge/opendc/experiments/sc20/Sc20IntegrationTest.kt @@ -158,7 +158,7 @@ class Sc20IntegrationTest { var totalOvercommissionedBurst = 0L var totalInterferedBurst = 0L - override suspend fun reportHostSlice( + override fun reportHostSlice( time: Long, requestedBurst: Long, grantedBurst: Long, @@ -168,10 +168,6 @@ class Sc20IntegrationTest { cpuDemand: Double, numberOfDeployedImages: Int, hostServer: Server, - submittedVms: Long, - queuedVms: Long, - runningVms: Long, - finishedVms: Long, duration: Long ) { totalRequestedBurst += requestedBurst -- cgit v1.2.3 From ac8c2bd72cf6408a4f8325529a9d67be47637b9f Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Thu, 14 May 2020 20:01:49 +0200 Subject: feat: Add option for specifying failure interval --- .../kotlin/com/atlarge/opendc/experiments/sc20/Sc20IntegrationTest.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'opendc/opendc-experiments-sc20/src/test') diff --git a/opendc/opendc-experiments-sc20/src/test/kotlin/com/atlarge/opendc/experiments/sc20/Sc20IntegrationTest.kt b/opendc/opendc-experiments-sc20/src/test/kotlin/com/atlarge/opendc/experiments/sc20/Sc20IntegrationTest.kt index 1edb7bc2..fd617115 100644 --- a/opendc/opendc-experiments-sc20/src/test/kotlin/com/atlarge/opendc/experiments/sc20/Sc20IntegrationTest.kt +++ b/opendc/opendc-experiments-sc20/src/test/kotlin/com/atlarge/opendc/experiments/sc20/Sc20IntegrationTest.kt @@ -102,7 +102,7 @@ class Sc20IntegrationTest { val failureDomain = if (failures) { println("ENABLING failures") - createFailureDomain(seed, 24 * 7, bareMetalProvisioner, chan) + createFailureDomain(seed, 24.0 * 7, bareMetalProvisioner, chan) } else { null } -- cgit v1.2.3 From 287d85732a8bcd5d85a8628006828fa460baaff9 Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Fri, 15 May 2020 02:18:45 +0200 Subject: refactor: Move entirely to Parquet --- .../opendc/experiments/sc20/Sc20IntegrationTest.kt | 36 ++++++++++++++++++---- 1 file changed, 30 insertions(+), 6 deletions(-) (limited to 'opendc/opendc-experiments-sc20/src/test') diff --git a/opendc/opendc-experiments-sc20/src/test/kotlin/com/atlarge/opendc/experiments/sc20/Sc20IntegrationTest.kt b/opendc/opendc-experiments-sc20/src/test/kotlin/com/atlarge/opendc/experiments/sc20/Sc20IntegrationTest.kt index fd617115..ae3d6db1 100644 --- a/opendc/opendc-experiments-sc20/src/test/kotlin/com/atlarge/opendc/experiments/sc20/Sc20IntegrationTest.kt +++ b/opendc/opendc-experiments-sc20/src/test/kotlin/com/atlarge/opendc/experiments/sc20/Sc20IntegrationTest.kt @@ -31,7 +31,12 @@ import com.atlarge.opendc.compute.core.Server import com.atlarge.opendc.compute.core.workload.VmWorkload import com.atlarge.opendc.compute.virt.service.SimpleVirtProvisioningService import com.atlarge.opendc.compute.virt.service.allocation.AvailableCoreMemoryAllocationPolicy -import com.atlarge.opendc.experiments.sc20.reporter.ExperimentReporter +import com.atlarge.opendc.experiments.sc20.experiment.attachMonitor +import com.atlarge.opendc.experiments.sc20.experiment.createFailureDomain +import com.atlarge.opendc.experiments.sc20.experiment.createProvisioner +import com.atlarge.opendc.experiments.sc20.experiment.createTraceReader +import com.atlarge.opendc.experiments.sc20.experiment.monitor.ExperimentMonitor +import com.atlarge.opendc.experiments.sc20.experiment.processTrace import com.atlarge.opendc.format.environment.EnvironmentReader import com.atlarge.opendc.format.environment.sc20.Sc20ClusterEnvironmentReader import com.atlarge.opendc.format.trace.TraceReader @@ -96,19 +101,33 @@ class Sc20IntegrationTest { lateinit var scheduler: SimpleVirtProvisioningService root.launch { - val res = createProvisioner(root, environmentReader, allocationPolicy) + val res = createProvisioner( + root, + environmentReader, + allocationPolicy + ) val bareMetalProvisioner = res.first scheduler = res.second val failureDomain = if (failures) { println("ENABLING failures") - createFailureDomain(seed, 24.0 * 7, bareMetalProvisioner, chan) + createFailureDomain( + seed, + 24.0 * 7, + bareMetalProvisioner, + chan + ) } else { null } attachMonitor(scheduler, monitor) - processTrace(traceReader, scheduler, chan, monitor) + processTrace( + traceReader, + scheduler, + chan, + monitor + ) println("Finish SUBMIT=${scheduler.submittedVms} FAIL=${scheduler.unscheduledVms} QUEUE=${scheduler.queuedVms} RUNNING=${scheduler.runningVms} FINISH=${scheduler.finishedVms}") @@ -141,7 +160,12 @@ class Sc20IntegrationTest { val performanceInterferenceStream = object {}.javaClass.getResourceAsStream("/env/performance-interference.json") val performanceInterferenceModel = Sc20PerformanceInterferenceReader(performanceInterferenceStream) .construct() - return createTraceReader(File("src/test/resources/trace"), performanceInterferenceModel, emptyList(), 0) + return createTraceReader( + File("src/test/resources/trace"), + performanceInterferenceModel, + emptyList(), + 0 + ) } /** @@ -152,7 +176,7 @@ class Sc20IntegrationTest { return Sc20ClusterEnvironmentReader(stream) } - class TestExperimentReporter : ExperimentReporter { + class TestExperimentReporter : ExperimentMonitor { var totalRequestedBurst = 0L var totalGrantedBurst = 0L var totalOvercommissionedBurst = 0L -- cgit v1.2.3 From 269860ba2616c32ca8a81ac66b6fbf95c2f1c77d Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Fri, 15 May 2020 13:29:25 +0200 Subject: perf: Reduce memory consumption of perf interference model --- .../opendc/experiments/sc20/Sc20IntegrationTest.kt | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) (limited to 'opendc/opendc-experiments-sc20/src/test') diff --git a/opendc/opendc-experiments-sc20/src/test/kotlin/com/atlarge/opendc/experiments/sc20/Sc20IntegrationTest.kt b/opendc/opendc-experiments-sc20/src/test/kotlin/com/atlarge/opendc/experiments/sc20/Sc20IntegrationTest.kt index ae3d6db1..abd5c961 100644 --- a/opendc/opendc-experiments-sc20/src/test/kotlin/com/atlarge/opendc/experiments/sc20/Sc20IntegrationTest.kt +++ b/opendc/opendc-experiments-sc20/src/test/kotlin/com/atlarge/opendc/experiments/sc20/Sc20IntegrationTest.kt @@ -34,13 +34,14 @@ import com.atlarge.opendc.compute.virt.service.allocation.AvailableCoreMemoryAll import com.atlarge.opendc.experiments.sc20.experiment.attachMonitor import com.atlarge.opendc.experiments.sc20.experiment.createFailureDomain import com.atlarge.opendc.experiments.sc20.experiment.createProvisioner -import com.atlarge.opendc.experiments.sc20.experiment.createTraceReader +import com.atlarge.opendc.experiments.sc20.experiment.model.Workload import com.atlarge.opendc.experiments.sc20.experiment.monitor.ExperimentMonitor import com.atlarge.opendc.experiments.sc20.experiment.processTrace +import com.atlarge.opendc.experiments.sc20.trace.Sc20ParquetTraceReader +import com.atlarge.opendc.experiments.sc20.trace.Sc20RawParquetTraceReader import com.atlarge.opendc.format.environment.EnvironmentReader import com.atlarge.opendc.format.environment.sc20.Sc20ClusterEnvironmentReader import com.atlarge.opendc.format.trace.TraceReader -import com.atlarge.opendc.format.trace.sc20.Sc20PerformanceInterferenceReader import kotlinx.coroutines.cancel import kotlinx.coroutines.channels.Channel import kotlinx.coroutines.launch @@ -141,8 +142,8 @@ class Sc20IntegrationTest { assertEquals(50, scheduler.submittedVms, "The trace contains 50 VMs") assertEquals(50, scheduler.finishedVms, "All VMs should finish after a run") assertEquals(207379117949, monitor.totalRequestedBurst) - assertEquals(207378478631, monitor.totalGrantedBurst) - assertEquals(639360, monitor.totalOvercommissionedBurst) + assertEquals(207102919834, monitor.totalGrantedBurst) + assertEquals(276198896, monitor.totalOvercommissionedBurst) assertEquals(0, monitor.totalInterferedBurst) } @@ -157,13 +158,10 @@ class Sc20IntegrationTest { * Obtain the trace reader for the test. */ private fun createTestTraceReader(): TraceReader { - val performanceInterferenceStream = object {}.javaClass.getResourceAsStream("/env/performance-interference.json") - val performanceInterferenceModel = Sc20PerformanceInterferenceReader(performanceInterferenceStream) - .construct() - return createTraceReader( - File("src/test/resources/trace"), - performanceInterferenceModel, - emptyList(), + return Sc20ParquetTraceReader( + Sc20RawParquetTraceReader(File("src/test/resources/trace")), + emptyMap(), + Workload("test", 1.0), 0 ) } -- cgit v1.2.3