From 06391c0af88f2907071ae1c52ee7b843b054c57e Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Fri, 14 Feb 2020 12:55:12 +0100 Subject: refactor: Move experiments from TPDS module to SC18 module These experiments were originally designed for the SC18 paper. --- opendc/opendc-experiments-sc18/build.gradle.kts | 47 ++++++++ .../opendc/experiments/sc18/TestExperiment.kt | 120 +++++++++++++++++++++ .../src/main/resources/env/setup-test.json | 36 +++++++ opendc/opendc-experiments-tpds/build.gradle.kts | 47 -------- .../opendc/experiments/tpds/TestExperiment.kt | 119 -------------------- .../src/main/resources/env/setup-test.json | 36 ------- settings.gradle.kts | 2 +- 7 files changed, 204 insertions(+), 203 deletions(-) create mode 100644 opendc/opendc-experiments-sc18/build.gradle.kts create mode 100644 opendc/opendc-experiments-sc18/src/main/kotlin/com/atlarge/opendc/experiments/sc18/TestExperiment.kt create mode 100644 opendc/opendc-experiments-sc18/src/main/resources/env/setup-test.json delete mode 100644 opendc/opendc-experiments-tpds/build.gradle.kts delete mode 100644 opendc/opendc-experiments-tpds/src/main/kotlin/com/atlarge/opendc/experiments/tpds/TestExperiment.kt delete mode 100644 opendc/opendc-experiments-tpds/src/main/resources/env/setup-test.json diff --git a/opendc/opendc-experiments-sc18/build.gradle.kts b/opendc/opendc-experiments-sc18/build.gradle.kts new file mode 100644 index 00000000..2e366a43 --- /dev/null +++ b/opendc/opendc-experiments-sc18/build.gradle.kts @@ -0,0 +1,47 @@ +/* + * MIT License + * + * Copyright (c) 2019 atlarge-research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +description = "Experiments for the TPDS paper" + +/* Build configuration */ +plugins { + `kotlin-library-convention` + application +} + +application { + mainClassName = "com.atlarge.opendc.experiments.sc18.TestExperiment" +} + +dependencies { + api(project(":opendc:opendc-core")) + implementation(project(":opendc:opendc-format")) + implementation(project(":opendc:opendc-workflows")) + implementation(kotlin("stdlib")) + + runtimeOnly(project(":odcsim:odcsim-engine-omega")) + testImplementation("org.junit.jupiter:junit-jupiter-api:${Library.JUNIT_JUPITER}") + testRuntimeOnly("org.junit.jupiter:junit-jupiter-engine:${Library.JUNIT_JUPITER}") + testImplementation("org.junit.platform:junit-platform-launcher:${Library.JUNIT_PLATFORM}") +} diff --git a/opendc/opendc-experiments-sc18/src/main/kotlin/com/atlarge/opendc/experiments/sc18/TestExperiment.kt b/opendc/opendc-experiments-sc18/src/main/kotlin/com/atlarge/opendc/experiments/sc18/TestExperiment.kt new file mode 100644 index 00000000..9ed4e369 --- /dev/null +++ b/opendc/opendc-experiments-sc18/src/main/kotlin/com/atlarge/opendc/experiments/sc18/TestExperiment.kt @@ -0,0 +1,120 @@ +/* + * MIT License + * + * Copyright (c) 2019 atlarge-research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package com.atlarge.opendc.experiments.sc18 + +import com.atlarge.odcsim.SimulationEngineProvider +import com.atlarge.opendc.compute.metal.service.ProvisioningService +import com.atlarge.opendc.format.environment.sc18.Sc18EnvironmentReader +import com.atlarge.opendc.format.trace.gwf.GwfTraceReader +import com.atlarge.opendc.workflows.monitor.WorkflowMonitor +import com.atlarge.opendc.workflows.service.StageWorkflowService +import com.atlarge.opendc.workflows.service.WorkflowSchedulerMode +import com.atlarge.opendc.workflows.service.stage.job.FifoJobSortingPolicy +import com.atlarge.opendc.workflows.service.stage.job.NullJobAdmissionPolicy +import com.atlarge.opendc.workflows.service.stage.resource.FirstFitResourceSelectionPolicy +import com.atlarge.opendc.workflows.service.stage.resource.FunctionalResourceDynamicFilterPolicy +import com.atlarge.opendc.workflows.service.stage.task.FifoTaskSortingPolicy +import com.atlarge.opendc.workflows.service.stage.task.FunctionalTaskEligibilityPolicy +import com.atlarge.opendc.workflows.workload.Job +import com.atlarge.opendc.workflows.workload.Task +import java.io.File +import java.util.ServiceLoader +import kotlin.math.max +import kotlinx.coroutines.channels.Channel +import kotlinx.coroutines.delay +import kotlinx.coroutines.runBlocking + +/** + * Main entry point of the experiment. + */ +fun main(args: Array) { + if (args.isEmpty()) { + println("error: Please provide path to GWF trace") + return + } + + val environment = Sc18EnvironmentReader(object {}.javaClass.getResourceAsStream("/env/setup-test.json")) + .use { it.read() } + + var total = 0 + var finished = 0 + + val token = Channel() + + val monitor = object : WorkflowMonitor { + override suspend fun onJobStart(job: Job, time: Long) { + println("Job ${job.uid} started") + } + + override suspend fun onJobFinish(job: Job, time: Long) { + finished += 1 + println("Jobs $finished/$total finished (${job.tasks.size} tasks)") + + if (finished == total) { + token.send(true) + } + } + + override suspend fun onTaskStart(job: Job, task: Task, time: Long) { + } + + override suspend fun onTaskFinish(job: Job, task: Task, status: Int, time: Long) { + } + } + + val provider = ServiceLoader.load(SimulationEngineProvider::class.java).first() + val system = provider({ ctx -> + println(ctx.clock.instant()) + val scheduler = StageWorkflowService( + ctx, + environment.platforms[0].zones[0].services[ProvisioningService.Key], + mode = WorkflowSchedulerMode.Batch(100), + jobAdmissionPolicy = NullJobAdmissionPolicy, + jobSortingPolicy = FifoJobSortingPolicy(), + taskEligibilityPolicy = FunctionalTaskEligibilityPolicy(), + taskSortingPolicy = FifoTaskSortingPolicy(), + resourceDynamicFilterPolicy = FunctionalResourceDynamicFilterPolicy(), + resourceSelectionPolicy = FirstFitResourceSelectionPolicy() + ) + + val reader = GwfTraceReader(File(args[0])) + + while (reader.hasNext()) { + val (time, job) = reader.next() + total += 1 + delay(max(0, time * 1000 - ctx.clock.millis())) + scheduler.submit(job, monitor) + } + + token.receive() + + println(ctx.clock.instant()) + }, name = "sim") + + runBlocking { + system.run() + system.terminate() + } +} diff --git a/opendc/opendc-experiments-sc18/src/main/resources/env/setup-test.json b/opendc/opendc-experiments-sc18/src/main/resources/env/setup-test.json new file mode 100644 index 00000000..0965b250 --- /dev/null +++ b/opendc/opendc-experiments-sc18/src/main/resources/env/setup-test.json @@ -0,0 +1,36 @@ +{ + "name": "Experimental Setup 2", + "rooms": [ + { + "type": "SERVER", + "objects": [ + { + "type": "RACK", + "machines": [ + { "cpus": [2] }, { "cpus": [2]}, + { "cpus": [2] }, { "cpus": [2]}, + { "cpus": [2] }, { "cpus": [2]}, + { "cpus": [2] }, { "cpus": [2]}, + { "cpus": [2] }, { "cpus": [2]}, + { "cpus": [2] }, { "cpus": [2]}, + { "cpus": [2] }, { "cpus": [2]}, + { "cpus": [2] }, { "cpus": [2]} + ] + }, + { + "type": "RACK", + "machines": [ + { "cpus": [1] }, { "cpus": [1]}, + { "cpus": [1] }, { "cpus": [1]}, + { "cpus": [1] }, { "cpus": [1]}, + { "cpus": [1] }, { "cpus": [1]}, + { "cpus": [1] }, { "cpus": [1]}, + { "cpus": [1] }, { "cpus": [1]}, + { "cpus": [1] }, { "cpus": [1]}, + { "cpus": [1] }, { "cpus": [1]} + ] + } + ] + } + ] +} diff --git a/opendc/opendc-experiments-tpds/build.gradle.kts b/opendc/opendc-experiments-tpds/build.gradle.kts deleted file mode 100644 index 297e1d94..00000000 --- a/opendc/opendc-experiments-tpds/build.gradle.kts +++ /dev/null @@ -1,47 +0,0 @@ -/* - * MIT License - * - * Copyright (c) 2019 atlarge-research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -description = "Experiments for the TPDS paper" - -/* Build configuration */ -plugins { - `kotlin-library-convention` - application -} - -application { - mainClassName = "com.atlarge.opendc.experiments.tpds.TestExperiment" -} - -dependencies { - api(project(":opendc:opendc-core")) - implementation(project(":opendc:opendc-format")) - implementation(project(":opendc:opendc-workflows")) - implementation(kotlin("stdlib")) - - runtimeOnly(project(":odcsim:odcsim-engine-omega")) - testImplementation("org.junit.jupiter:junit-jupiter-api:${Library.JUNIT_JUPITER}") - testRuntimeOnly("org.junit.jupiter:junit-jupiter-engine:${Library.JUNIT_JUPITER}") - testImplementation("org.junit.platform:junit-platform-launcher:${Library.JUNIT_PLATFORM}") -} diff --git a/opendc/opendc-experiments-tpds/src/main/kotlin/com/atlarge/opendc/experiments/tpds/TestExperiment.kt b/opendc/opendc-experiments-tpds/src/main/kotlin/com/atlarge/opendc/experiments/tpds/TestExperiment.kt deleted file mode 100644 index 3dcea99d..00000000 --- a/opendc/opendc-experiments-tpds/src/main/kotlin/com/atlarge/opendc/experiments/tpds/TestExperiment.kt +++ /dev/null @@ -1,119 +0,0 @@ -/* - * MIT License - * - * Copyright (c) 2019 atlarge-research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package com.atlarge.opendc.experiments.tpds - -import com.atlarge.odcsim.SimulationEngineProvider -import com.atlarge.opendc.compute.metal.service.ProvisioningService -import com.atlarge.opendc.format.environment.sc18.Sc18EnvironmentReader -import com.atlarge.opendc.format.trace.gwf.GwfTraceReader -import com.atlarge.opendc.workflows.monitor.WorkflowMonitor -import com.atlarge.opendc.workflows.service.StageWorkflowService -import com.atlarge.opendc.workflows.service.WorkflowSchedulerMode -import com.atlarge.opendc.workflows.service.stage.job.FifoJobSortingPolicy -import com.atlarge.opendc.workflows.service.stage.job.NullJobAdmissionPolicy -import com.atlarge.opendc.workflows.service.stage.resource.FirstFitResourceSelectionPolicy -import com.atlarge.opendc.workflows.service.stage.resource.FunctionalResourceDynamicFilterPolicy -import com.atlarge.opendc.workflows.service.stage.task.FifoTaskSortingPolicy -import com.atlarge.opendc.workflows.service.stage.task.FunctionalTaskEligibilityPolicy -import com.atlarge.opendc.workflows.workload.Job -import com.atlarge.opendc.workflows.workload.Task -import java.io.File -import java.util.ServiceLoader -import kotlin.math.max -import kotlinx.coroutines.channels.Channel -import kotlinx.coroutines.delay -import kotlinx.coroutines.runBlocking - -/** - * Main entry point of the experiment. - */ -fun main(args: Array) { - if (args.isEmpty()) { - println("error: Please provide path to GWF trace") - return - } - - val environment = Sc18EnvironmentReader(object {}.javaClass.getResourceAsStream("/env/setup-test.json")) - .use { it.read() } - - var total = 0 - var finished = 0 - - val token = Channel() - - val monitor = object : WorkflowMonitor { - override suspend fun onJobStart(job: Job, time: Long) { - println("Job ${job.uid} submitted") - total += 1 - } - - override suspend fun onJobFinish(job: Job, time: Long) { - finished += 1 - println("Jobs $finished/$total finished (${job.tasks.size} tasks)") - - if (finished == total) { - token.send(true) - } - } - - override suspend fun onTaskStart(job: Job, task: Task, time: Long) { - println("Task started ${task.uid}") - } - - override suspend fun onTaskFinish(job: Job, task: Task, status: Int, time: Long) { - println("Task finished ${task.uid}") - } - } - - val provider = ServiceLoader.load(SimulationEngineProvider::class.java).first() - val system = provider({ ctx -> - val scheduler = StageWorkflowService( - ctx, - environment.platforms[0].zones[0].services[ProvisioningService.Key], - mode = WorkflowSchedulerMode.Batch(100), - jobAdmissionPolicy = NullJobAdmissionPolicy, - jobSortingPolicy = FifoJobSortingPolicy(), - taskEligibilityPolicy = FunctionalTaskEligibilityPolicy(), - taskSortingPolicy = FifoTaskSortingPolicy(), - resourceDynamicFilterPolicy = FunctionalResourceDynamicFilterPolicy(), - resourceSelectionPolicy = FirstFitResourceSelectionPolicy() - ) - - val reader = GwfTraceReader(File(args[0])) - - while (reader.hasNext()) { - val (time, job) = reader.next() - delay(max(0, time - ctx.clock.millis())) - scheduler.submit(job, monitor) - } - - token.receive() - }, name = "sim") - - runBlocking { - system.run() - system.terminate() - } -} diff --git a/opendc/opendc-experiments-tpds/src/main/resources/env/setup-test.json b/opendc/opendc-experiments-tpds/src/main/resources/env/setup-test.json deleted file mode 100644 index 0965b250..00000000 --- a/opendc/opendc-experiments-tpds/src/main/resources/env/setup-test.json +++ /dev/null @@ -1,36 +0,0 @@ -{ - "name": "Experimental Setup 2", - "rooms": [ - { - "type": "SERVER", - "objects": [ - { - "type": "RACK", - "machines": [ - { "cpus": [2] }, { "cpus": [2]}, - { "cpus": [2] }, { "cpus": [2]}, - { "cpus": [2] }, { "cpus": [2]}, - { "cpus": [2] }, { "cpus": [2]}, - { "cpus": [2] }, { "cpus": [2]}, - { "cpus": [2] }, { "cpus": [2]}, - { "cpus": [2] }, { "cpus": [2]}, - { "cpus": [2] }, { "cpus": [2]} - ] - }, - { - "type": "RACK", - "machines": [ - { "cpus": [1] }, { "cpus": [1]}, - { "cpus": [1] }, { "cpus": [1]}, - { "cpus": [1] }, { "cpus": [1]}, - { "cpus": [1] }, { "cpus": [1]}, - { "cpus": [1] }, { "cpus": [1]}, - { "cpus": [1] }, { "cpus": [1]}, - { "cpus": [1] }, { "cpus": [1]}, - { "cpus": [1] }, { "cpus": [1]} - ] - } - ] - } - ] -} diff --git a/settings.gradle.kts b/settings.gradle.kts index b591c654..c7dc5f1b 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -29,4 +29,4 @@ include(":opendc:opendc-core") include(":opendc:opendc-compute") include(":opendc:opendc-format") include(":opendc:opendc-workflows") -include(":opendc:opendc-experiments-tpds") +include(":opendc:opendc-experiments-sc18") -- cgit v1.2.3