From 1e35c61cd31b8bfb33a6ccbb46b08c0466518e6c Mon Sep 17 00:00:00 2001 From: Sacheendra Talluri Date: Thu, 20 Mar 2025 10:16:46 +0100 Subject: Adds load shifting over time (#319) * Start time shifting * Existing experiments work with new columns * Remove unused traces dir * Update java to 21 LTS and jacoco to be compatible * Minimal working timeshifting * Timeshift scheduler linked as carbon receiver * Add basic tests for timeshift scheduler * Run spotless apply * Modify tarce format tests to support new fields * Change all mentions of java 19 to 21 * Add a deferAll option to workload to make all tasks deferrable * Run spotless apply * Copy traces from resources in web dockerfile --- .github/workflows/build.yml | 4 +- .github/workflows/publish.yml | 2 +- .github/workflows/release.yml | 2 +- .../src/main/kotlin/jacoco-conventions.gradle.kts | 2 +- gradle/libs.versions.toml | 2 +- .../compute/simulator/service/ComputeService.java | 9 +- .../compute/simulator/service/ServiceTask.java | 25 ++++ .../compute/simulator/service/TaskNature.java | 32 ++++ .../compute/simulator/provisioner/ComputeSteps.kt | 4 +- .../simulator/provisioner/HostsProvisioningStep.kt | 5 + .../simulator/scheduler/ComputeSchedulers.kt | 14 +- .../compute/simulator/scheduler/FilterScheduler.kt | 1 - .../simulator/scheduler/MemorizingScheduler.kt | 1 + .../simulator/scheduler/TimeshiftScheduler.kt | 161 +++++++++++++++++++++ .../simulator/scheduler/TimeshiftSchedulerTest.kt | 89 ++++++++++++ .../compute/workload/ComputeWorkloadLoader.kt | 13 ++ .../kotlin/org/opendc/compute/workload/Task.kt | 4 +- .../org/opendc/compute/workload/WorkloadLoader.kt | 1 + .../opendc-experiments-base/build.gradle.kts | 2 +- .../base/experiment/specs/WorkloadSpec.kt | 3 + .../experiments/base/runner/ScenarioReplayer.kt | 13 ++ .../experiments/base/runner/ScenarioRunner.kt | 20 ++- .../org/opendc/experiments/base/TestingUtils.kt | 4 +- .../org/opendc/trace/conv/ResourceColumns.kt | 14 +- .../formats/opendc/OdcVmResourceTableReader.kt | 20 ++- .../formats/opendc/OdcVmResourceTableWriter.kt | 25 +++- .../trace/formats/opendc/OdcVmTraceFormat.kt | 4 + .../trace/formats/opendc/parquet/Resource.kt | 2 + .../formats/opendc/parquet/ResourceReadSupport.kt | 18 +++ .../opendc/parquet/ResourceRecordMaterializer.kt | 18 +++ .../formats/opendc/parquet/ResourceWriteSupport.kt | 19 +++ .../test/resources/opendc/trace-v2.1/tasks.parquet | Bin 4597 -> 5407 bytes opendc-web/opendc-web-runner/Dockerfile | 7 +- .../kotlin/org/opendc/web/runner/OpenDCRunner.kt | 4 +- opendc-web/opendc-web-server/Dockerfile | 4 +- site/docs/getting-started/0-installation.md | 2 +- .../docs/getting-started/4-start-using-intellij.md | 2 +- traces/bitbrains-small/meta.parquet | Bin 2723 -> 0 bytes traces/bitbrains-small/trace.parquet | Bin 2163354 -> 0 bytes 39 files changed, 521 insertions(+), 31 deletions(-) create mode 100644 opendc-compute/opendc-compute-simulator/src/main/java/org/opendc/compute/simulator/service/TaskNature.java create mode 100644 opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/scheduler/TimeshiftScheduler.kt create mode 100644 opendc-compute/opendc-compute-simulator/src/test/kotlin/org/opendc/compute/simulator/scheduler/TimeshiftSchedulerTest.kt delete mode 100644 traces/bitbrains-small/meta.parquet delete mode 100644 traces/bitbrains-small/trace.parquet diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e008127f..6d9cca16 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -14,10 +14,10 @@ jobs: strategy: matrix: os: [ ubuntu-22.04 ] - java: [ 19 ] + java: [ 21 ] include: - os: windows-2022 - java: 19 + java: 21 steps: - name: Checkout repository uses: actions/checkout@v4 diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 9b83de14..294be1d1 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -96,7 +96,7 @@ jobs: uses: actions/setup-java@v4 with: distribution: 'zulu' - java-version: 19 + java-version: 21 - name: Prepare id: prep run: | diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 1efb1f7f..05cf997c 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -19,7 +19,7 @@ jobs: uses: actions/setup-java@v4 with: distribution: 'zulu' - java-version: 19 + java-version: 21 - name: Publish with Gradle uses: gradle/actions/setup-gradle@v3 with: diff --git a/buildSrc/src/main/kotlin/jacoco-conventions.gradle.kts b/buildSrc/src/main/kotlin/jacoco-conventions.gradle.kts index 57ad1b9d..144fb1aa 100644 --- a/buildSrc/src/main/kotlin/jacoco-conventions.gradle.kts +++ b/buildSrc/src/main/kotlin/jacoco-conventions.gradle.kts @@ -26,7 +26,7 @@ plugins { } jacoco { - toolVersion = "0.8.8" + toolVersion = "0.8.11" } tasks.jacocoTestReport { diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index bec3c35c..44cd4003 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -8,7 +8,7 @@ hadoop = "3.3.6" hypersistence-utils = "3.7.3" jackson = "2.16.1" jandex-gradle = "1.1.0" -java = "19" +java = "21" jline = "3.25.1" jmh-gradle = "0.7.0" jakarta = "3.0.2" diff --git a/opendc-compute/opendc-compute-simulator/src/main/java/org/opendc/compute/simulator/service/ComputeService.java b/opendc-compute/opendc-compute-simulator/src/main/java/org/opendc/compute/simulator/service/ComputeService.java index eb8d3377..69625306 100644 --- a/opendc-compute/opendc-compute-simulator/src/main/java/org/opendc/compute/simulator/service/ComputeService.java +++ b/opendc-compute/opendc-compute-simulator/src/main/java/org/opendc/compute/simulator/service/ComputeService.java @@ -25,6 +25,7 @@ package org.opendc.compute.simulator.service; import java.time.Duration; import java.time.Instant; import java.time.InstantSource; +import java.time.temporal.TemporalAmount; import java.util.ArrayDeque; import java.util.ArrayList; import java.util.Collections; @@ -499,7 +500,6 @@ public final class ComputeService implements AutoCloseable { SimHost host = hv.getHost(); // Remove request from queue - taskQueue.remove(req); tasksPending--; LOGGER.info("Assigned task {} to host {}", task, host); @@ -642,6 +642,9 @@ public final class ComputeService implements AutoCloseable { @NotNull public ServiceTask newTask( @NotNull String name, + @NotNull TaskNature nature, + @NotNull TemporalAmount duration, + @NotNull Long deadline, @NotNull ServiceFlavor flavor, @NotNull Workload workload, @NotNull Map meta) { @@ -652,9 +655,9 @@ public final class ComputeService implements AutoCloseable { // final ServiceFlavor internalFlavor = // Objects.requireNonNull(service.flavorById.get(flavor.getUid()), "Unknown flavor"); - // ServiceTask task = new ServiceTask(service, uid, name, internalFlavor, workload, meta); - ServiceTask task = new ServiceTask(service, uid, name, flavor, workload, meta); + + ServiceTask task = new ServiceTask(service, uid, name, nature, duration, deadline, flavor, workload, meta); service.taskById.put(uid, task); diff --git a/opendc-compute/opendc-compute-simulator/src/main/java/org/opendc/compute/simulator/service/ServiceTask.java b/opendc-compute/opendc-compute-simulator/src/main/java/org/opendc/compute/simulator/service/ServiceTask.java index dac65d67..4d5611a8 100644 --- a/opendc-compute/opendc-compute-simulator/src/main/java/org/opendc/compute/simulator/service/ServiceTask.java +++ b/opendc-compute/opendc-compute-simulator/src/main/java/org/opendc/compute/simulator/service/ServiceTask.java @@ -23,6 +23,7 @@ package org.opendc.compute.simulator.service; import java.time.Instant; +import java.time.temporal.TemporalAmount; import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -49,6 +50,9 @@ public class ServiceTask { private final UUID uid; private final String name; + private final TaskNature nature; + private final TemporalAmount duration; + private final Long deadline; private ServiceFlavor flavor; public Workload workload; @@ -68,12 +72,18 @@ public class ServiceTask { ComputeService service, UUID uid, String name, + TaskNature nature, + TemporalAmount duration, + Long deadline, ServiceFlavor flavor, Workload workload, Map meta) { this.service = service; this.uid = uid; this.name = name; + this.nature = nature; + this.duration = duration; + this.deadline = deadline; this.flavor = flavor; this.workload = workload; this.meta = meta; @@ -86,6 +96,21 @@ public class ServiceTask { return uid; } + @NotNull + public TaskNature getNature() { + return nature; + } + + @NotNull + public TemporalAmount getDuration() { + return duration; + } + + @NotNull + public Long getDeadline() { + return deadline; + } + @NotNull public String getName() { return name; diff --git a/opendc-compute/opendc-compute-simulator/src/main/java/org/opendc/compute/simulator/service/TaskNature.java b/opendc-compute/opendc-compute-simulator/src/main/java/org/opendc/compute/simulator/service/TaskNature.java new file mode 100644 index 00000000..ffb49143 --- /dev/null +++ b/opendc-compute/opendc-compute-simulator/src/main/java/org/opendc/compute/simulator/service/TaskNature.java @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2025 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.compute.simulator.service; + +public class TaskNature { + + public final boolean deferrable; + + public TaskNature(boolean deferrable) { + this.deferrable = deferrable; + } +} diff --git a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/provisioner/ComputeSteps.kt b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/provisioner/ComputeSteps.kt index a18856f8..1145c15b 100644 --- a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/provisioner/ComputeSteps.kt +++ b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/provisioner/ComputeSteps.kt @@ -29,6 +29,7 @@ import org.opendc.compute.simulator.telemetry.ComputeMonitor import org.opendc.compute.simulator.telemetry.OutputFiles import org.opendc.compute.topology.specs.ClusterSpec import org.opendc.compute.topology.specs.HostSpec +import org.opendc.simulator.compute.power.CarbonReceiver import java.time.Duration /** @@ -84,7 +85,8 @@ public fun registerComputeMonitor( public fun setupHosts( serviceDomain: String, specs: List, + carbonReceivers: List, startTime: Long = 0L, ): ProvisioningStep { - return HostsProvisioningStep(serviceDomain, specs, startTime) + return HostsProvisioningStep(serviceDomain, specs, carbonReceivers, startTime) } diff --git a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/provisioner/HostsProvisioningStep.kt b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/provisioner/HostsProvisioningStep.kt index 68e263f8..c347e866 100644 --- a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/provisioner/HostsProvisioningStep.kt +++ b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/provisioner/HostsProvisioningStep.kt @@ -29,6 +29,7 @@ import org.opendc.compute.topology.specs.ClusterSpec import org.opendc.compute.topology.specs.HostSpec import org.opendc.compute.topology.specs.createSimBatteryPolicy import org.opendc.simulator.compute.power.CarbonModel +import org.opendc.simulator.compute.power.CarbonReceiver import org.opendc.simulator.compute.power.SimPowerSource import org.opendc.simulator.compute.power.batteries.BatteryAggregator import org.opendc.simulator.compute.power.batteries.SimBattery @@ -46,6 +47,7 @@ import org.opendc.simulator.engine.graph.FlowEdge public class HostsProvisioningStep internal constructor( private val serviceDomain: String, private val clusterSpecs: List, + private val carbonReceivers: List, private val startTime: Long = 0L, ) : ProvisioningStep { override fun apply(ctx: ProvisioningContext): AutoCloseable { @@ -75,6 +77,9 @@ public class HostsProvisioningStep internal constructor( if (carbonFragments != null) { carbonModel = CarbonModel(engine, carbonFragments, startTime) carbonModel.addReceiver(simPowerSource) + for (receiver in carbonReceivers) { + carbonModel.addReceiver(receiver) + } } if (cluster.battery != null) { diff --git a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/scheduler/ComputeSchedulers.kt b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/scheduler/ComputeSchedulers.kt index 8f2369dc..b83eafdd 100644 --- a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/scheduler/ComputeSchedulers.kt +++ b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/scheduler/ComputeSchedulers.kt @@ -31,6 +31,7 @@ import org.opendc.compute.simulator.scheduler.weights.CoreRamWeigher import org.opendc.compute.simulator.scheduler.weights.InstanceCountWeigher import org.opendc.compute.simulator.scheduler.weights.RamWeigher import org.opendc.compute.simulator.scheduler.weights.VCpuWeigher +import java.time.InstantSource import java.util.SplittableRandom import java.util.random.RandomGenerator @@ -45,13 +46,15 @@ public enum class ComputeSchedulerEnum { ProvisionedCoresInv, Random, TaskNumMemorizing, + Timeshift, } public fun createComputeScheduler( name: String, seeder: RandomGenerator, + clock: InstantSource, ): ComputeScheduler { - return createComputeScheduler(ComputeSchedulerEnum.valueOf(name.uppercase()), seeder) + return createComputeScheduler(ComputeSchedulerEnum.valueOf(name.uppercase()), seeder, clock) } /** @@ -60,6 +63,7 @@ public fun createComputeScheduler( public fun createComputeScheduler( name: ComputeSchedulerEnum, seeder: RandomGenerator, + clock: InstantSource, ): ComputeScheduler { val cpuAllocationRatio = 1.0 val ramAllocationRatio = 1.5 @@ -116,5 +120,13 @@ public fun createComputeScheduler( filters = listOf(ComputeFilter(), VCpuFilter(cpuAllocationRatio), RamFilter(ramAllocationRatio)), random = SplittableRandom(seeder.nextLong()), ) + ComputeSchedulerEnum.Timeshift -> + TimeshiftScheduler( + filters = listOf(ComputeFilter(), VCpuFilter(cpuAllocationRatio), RamFilter(ramAllocationRatio)), + weighers = listOf(RamWeigher(multiplier = 1.0)), + windowSize = 168, + clock = clock, + random = SplittableRandom(seeder.nextLong()), + ) } } diff --git a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/scheduler/FilterScheduler.kt b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/scheduler/FilterScheduler.kt index 832482eb..386e0d63 100644 --- a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/scheduler/FilterScheduler.kt +++ b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/scheduler/FilterScheduler.kt @@ -79,7 +79,6 @@ public class FilterScheduler( } val task = req.task - val hosts = hosts val filteredHosts = hosts.filter { host -> filters.all { filter -> filter.test(host, task) } } val subset = diff --git a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/scheduler/MemorizingScheduler.kt b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/scheduler/MemorizingScheduler.kt index c48e6fb0..faa1be6b 100644 --- a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/scheduler/MemorizingScheduler.kt +++ b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/scheduler/MemorizingScheduler.kt @@ -88,6 +88,7 @@ public class MemorizingScheduler( taskloop@ for (req in iter) { if (req.isCancelled) { iter.remove() + continue } for (chosenListIndex in minAvailableHost until hostsQueue.size) { diff --git a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/scheduler/TimeshiftScheduler.kt b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/scheduler/TimeshiftScheduler.kt new file mode 100644 index 00000000..a856f366 --- /dev/null +++ b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/scheduler/TimeshiftScheduler.kt @@ -0,0 +1,161 @@ +/* + * Copyright (c) 2025 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.compute.simulator.scheduler + +import org.opendc.compute.simulator.scheduler.filters.HostFilter +import org.opendc.compute.simulator.scheduler.weights.HostWeigher +import org.opendc.compute.simulator.service.HostView +import org.opendc.compute.simulator.service.ServiceTask +import org.opendc.simulator.compute.power.CarbonModel +import org.opendc.simulator.compute.power.CarbonReceiver +import java.time.Instant +import java.time.InstantSource +import java.util.LinkedList +import java.util.SplittableRandom +import java.util.random.RandomGenerator +import kotlin.math.min + +public class TimeshiftScheduler( + private val filters: List, + private val weighers: List, + private val windowSize: Int, + private val clock: InstantSource, + private val subsetSize: Int = 1, + private val random: RandomGenerator = SplittableRandom(0), +) : ComputeScheduler, CarbonReceiver { + /** + * The pool of hosts available to the scheduler. + */ + private val hosts = mutableListOf() + + init { + require(subsetSize >= 1) { "Subset size must be one or greater" } + } + + private val pastCarbonIntensities = LinkedList() + private var carbonRunningSum = 0.0 + + private var carbonIntensity = 0.0 + +// private var lowerCarbonIntensity = 0.0 + private var thresholdCarbonIntensity = 0.0 + + override fun addHost(host: HostView) { + hosts.add(host) + } + + override fun removeHost(host: HostView) { + hosts.remove(host) + } + + override fun select(iter: MutableIterator): SchedulingResult { + var result: SchedulingResult? = null + for (req in iter) { + if (req.isCancelled) { + iter.remove() + continue + } + + val task = req.task + + if (carbonIntensity > thresholdCarbonIntensity) { + if (task.nature.deferrable) { + val currentTime = clock.instant() + val estimatedCompletion = currentTime.plus(task.duration) + val deadline = Instant.ofEpochMilli(task.deadline) + if (estimatedCompletion.isBefore(deadline)) { + // No need to schedule this task in a high carbon intensity period + continue + } + } + } + + val filteredHosts = hosts.filter { host -> filters.all { filter -> filter.test(host, task) } } + + val subset = + if (weighers.isNotEmpty()) { + val filterResults = weighers.map { it.getWeights(filteredHosts, task) } + val weights = DoubleArray(filteredHosts.size) + + for (fr in filterResults) { + val min = fr.min + val range = (fr.max - min) + + // Skip result if all weights are the same + if (range == 0.0) { + continue + } + + val multiplier = fr.multiplier + val factor = multiplier / range + + for ((i, weight) in fr.weights.withIndex()) { + weights[i] += factor * (weight - min) + } + } + + weights.indices + .asSequence() + .sortedByDescending { weights[it] } + .map { filteredHosts[it] } + .take(subsetSize) + .toList() + } else { + filteredHosts + } + + val maxSize = min(subsetSize, subset.size) + if (maxSize == 0) { + result = SchedulingResult(SchedulingResultType.FAILURE, null, req) + break + } else { + iter.remove() + result = SchedulingResult(SchedulingResultType.SUCCESS, subset[random.nextInt(maxSize)], req) + break + } + } + + if (result == null) return SchedulingResult(SchedulingResultType.EMPTY) + + return result + } + + override fun removeTask( + task: ServiceTask, + host: HostView?, + ) {} + + override fun updateCarbonIntensity(newCarbonIntensity: Double) { + this.carbonIntensity = newCarbonIntensity + this.pastCarbonIntensities.addLast(newCarbonIntensity) + this.carbonRunningSum += newCarbonIntensity + if (this.pastCarbonIntensities.size > this.windowSize) { + this.carbonRunningSum -= this.pastCarbonIntensities.removeFirst() + } + this.thresholdCarbonIntensity = this.carbonRunningSum / this.pastCarbonIntensities.size + } + + override fun setCarbonModel(carbonModel: CarbonModel?) {} + + override fun removeCarbonModel(carbonModel: CarbonModel?) {} +} diff --git a/opendc-compute/opendc-compute-simulator/src/test/kotlin/org/opendc/compute/simulator/scheduler/TimeshiftSchedulerTest.kt b/opendc-compute/opendc-compute-simulator/src/test/kotlin/org/opendc/compute/simulator/scheduler/TimeshiftSchedulerTest.kt new file mode 100644 index 00000000..b893c1aa --- /dev/null +++ b/opendc-compute/opendc-compute-simulator/src/test/kotlin/org/opendc/compute/simulator/scheduler/TimeshiftSchedulerTest.kt @@ -0,0 +1,89 @@ +/* + * Copyright (c) 2025 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.compute.simulator.scheduler + +import io.mockk.every +import io.mockk.mockk +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Test +import org.opendc.compute.simulator.service.TaskNature +import java.time.Duration +import java.time.Instant +import java.time.InstantSource + +class TimeshiftSchedulerTest { + @Test + fun testBasicDeferring() { + val clock = mockk() + every { clock.instant() } returns Instant.ofEpochMilli(10) + + val scheduler = + TimeshiftScheduler( + filters = emptyList(), + weighers = emptyList(), + windowSize = 2, + clock = clock, + ) + + val req = mockk() + every { req.task.flavor.coreCount } returns 2 + every { req.task.flavor.memorySize } returns 1024 + every { req.isCancelled } returns false + every { req.task.nature } returns TaskNature(true) + every { req.task.duration } returns Duration.ofMillis(10) + every { req.task.deadline } returns 50 + + scheduler.updateCarbonIntensity(100.0) + scheduler.updateCarbonIntensity(200.0) + + assertEquals(SchedulingResultType.EMPTY, scheduler.select(mutableListOf(req).iterator()).resultType) + } + + @Test + fun testRespectDeadline() { + val clock = mockk() + every { clock.instant() } returns Instant.ofEpochMilli(10) + + val scheduler = + TimeshiftScheduler( + filters = emptyList(), + weighers = emptyList(), + windowSize = 2, + clock = clock, + ) + + val req = mockk() + every { req.task.flavor.coreCount } returns 2 + every { req.task.flavor.memorySize } returns 1024 + every { req.isCancelled } returns false + every { req.task.nature } returns TaskNature(true) + every { req.task.duration } returns Duration.ofMillis(10) + every { req.task.deadline } returns 20 + + scheduler.updateCarbonIntensity(100.0) + scheduler.updateCarbonIntensity(200.0) + + // The scheduler tries to schedule the task, but fails as there are no hosts. + assertEquals(SchedulingResultType.FAILURE, scheduler.select(mutableListOf(req).iterator()).resultType) + } +} diff --git a/opendc-compute/opendc-compute-workload/src/main/kotlin/org/opendc/compute/workload/ComputeWorkloadLoader.kt b/opendc-compute/opendc-compute-workload/src/main/kotlin/org/opendc/compute/workload/ComputeWorkloadLoader.kt index cbff5fb9..8e774acb 100644 --- a/opendc-compute/opendc-compute-workload/src/main/kotlin/org/opendc/compute/workload/ComputeWorkloadLoader.kt +++ b/opendc-compute/opendc-compute-workload/src/main/kotlin/org/opendc/compute/workload/ComputeWorkloadLoader.kt @@ -31,9 +31,11 @@ import org.opendc.trace.conv.TABLE_RESOURCES import org.opendc.trace.conv.TABLE_RESOURCE_STATES import org.opendc.trace.conv.resourceCpuCapacity import org.opendc.trace.conv.resourceCpuCount +import org.opendc.trace.conv.resourceDeadline import org.opendc.trace.conv.resourceDuration import org.opendc.trace.conv.resourceID import org.opendc.trace.conv.resourceMemCapacity +import org.opendc.trace.conv.resourceNature import org.opendc.trace.conv.resourceStateCpuUsage import org.opendc.trace.conv.resourceStateDuration import org.opendc.trace.conv.resourceSubmissionTime @@ -56,6 +58,7 @@ public class ComputeWorkloadLoader( private val checkpointDuration: Long = 0L, private val checkpointIntervalScaling: Double = 1.0, private val scalingPolicy: ScalingPolicy = NoDelayScaling(), + private val deferAll: Boolean = false, ) : WorkloadLoader(subMissionTime) { /** * The logger for this instance. @@ -115,6 +118,8 @@ public class ComputeWorkloadLoader( val cpuCountCol = reader.resolve(resourceCpuCount) val cpuCapacityCol = reader.resolve(resourceCpuCapacity) val memCol = reader.resolve(resourceMemCapacity) + val natureCol = reader.resolve(resourceNature) + val deadlineCol = reader.resolve(resourceDeadline) var counter = 0 val entries = mutableListOf() @@ -132,6 +137,12 @@ public class ComputeWorkloadLoader( val cpuCapacity = reader.getDouble(cpuCapacityCol) val memCapacity = reader.getDouble(memCol) / 1000.0 // Convert from KB to MB val uid = UUID.nameUUIDFromBytes("$id-${counter++}".toByteArray()) + var nature = reader.getString(natureCol) + var deadline = reader.getLong(deadlineCol) + if (deferAll) { + nature = "deferrable" + deadline = submissionTime + (3 * duration) + } val builder = fragments.getValue(id) // Get all fragments related to this VM val totalLoad = builder.totalLoad @@ -146,6 +157,8 @@ public class ComputeWorkloadLoader( totalLoad, submissionTime, duration, + nature, + deadline, builder.build(), ), ) diff --git a/opendc-compute/opendc-compute-workload/src/main/kotlin/org/opendc/compute/workload/Task.kt b/opendc-compute/opendc-compute-workload/src/main/kotlin/org/opendc/compute/workload/Task.kt index d32c84f6..787f271e 100644 --- a/opendc-compute/opendc-compute-workload/src/main/kotlin/org/opendc/compute/workload/Task.kt +++ b/opendc-compute/opendc-compute-workload/src/main/kotlin/org/opendc/compute/workload/Task.kt @@ -46,5 +46,7 @@ public data class Task( val totalLoad: Double, var submissionTime: Long, val duration: Long, - var trace: TraceWorkload, + val nature: String?, + var deadline: Long, + val trace: TraceWorkload, ) diff --git a/opendc-compute/opendc-compute-workload/src/main/kotlin/org/opendc/compute/workload/WorkloadLoader.kt b/opendc-compute/opendc-compute-workload/src/main/kotlin/org/opendc/compute/workload/WorkloadLoader.kt index 6b9c3753..c00bc768 100644 --- a/opendc-compute/opendc-compute-workload/src/main/kotlin/org/opendc/compute/workload/WorkloadLoader.kt +++ b/opendc-compute/opendc-compute-workload/src/main/kotlin/org/opendc/compute/workload/WorkloadLoader.kt @@ -40,6 +40,7 @@ public abstract class WorkloadLoader(private val submissionTime: String? = null) for (task in workload) { task.submissionTime += timeShift + task.deadline = if (task.deadline == -1L) -1L else task.deadline + timeShift } } diff --git a/opendc-experiments/opendc-experiments-base/build.gradle.kts b/opendc-experiments/opendc-experiments-base/build.gradle.kts index 36073418..8a77f0c4 100644 --- a/opendc-experiments/opendc-experiments-base/build.gradle.kts +++ b/opendc-experiments/opendc-experiments-base/build.gradle.kts @@ -55,7 +55,7 @@ val createScenarioApp by tasks.creating(CreateStartScripts::class) { applicationName = "OpenDCExperimentRunner" mainClass.set("org.opendc.experiments.base.runner.ExperimentCli") classpath = tasks.jar.get().outputs.files + configurations["runtimeClasspath"] - outputDir = project.buildDir.resolve("scripts") + outputDir = layout.buildDirectory.dir("scripts").get().asFile } // Create custom Scenario distribution diff --git a/opendc-experiments/opendc-experiments-base/src/main/kotlin/org/opendc/experiments/base/experiment/specs/WorkloadSpec.kt b/opendc-experiments/opendc-experiments-base/src/main/kotlin/org/opendc/experiments/base/experiment/specs/WorkloadSpec.kt index cf40d88d..4d2e932f 100644 --- a/opendc-experiments/opendc-experiments-base/src/main/kotlin/org/opendc/experiments/base/experiment/specs/WorkloadSpec.kt +++ b/opendc-experiments/opendc-experiments-base/src/main/kotlin/org/opendc/experiments/base/experiment/specs/WorkloadSpec.kt @@ -44,6 +44,7 @@ public data class WorkloadSpec( val type: WorkloadTypes, val sampleFraction: Double = 1.0, val submissionTime: String? = null, + val deferAll: Boolean = false, val scalingPolicy: ScalingPolicyEnum = ScalingPolicyEnum.NoDelay, ) { public val name: String = File(pathToFile).nameWithoutExtension @@ -74,6 +75,7 @@ public fun getWorkloadLoader( checkpointDuration: Long, checkpointIntervalScaling: Double, scalingPolicy: ScalingPolicy, + deferAll: Boolean, ): WorkloadLoader { return when (type) { WorkloadTypes.ComputeWorkload -> @@ -84,6 +86,7 @@ public fun getWorkloadLoader( checkpointDuration, checkpointIntervalScaling, scalingPolicy, + deferAll, ) } } diff --git a/opendc-experiments/opendc-experiments-base/src/main/kotlin/org/opendc/experiments/base/runner/ScenarioReplayer.kt b/opendc-experiments/opendc-experiments-base/src/main/kotlin/org/opendc/experiments/base/runner/ScenarioReplayer.kt index 5664e0cb..d56e4e4b 100644 --- a/opendc-experiments/opendc-experiments-base/src/main/kotlin/org/opendc/experiments/base/runner/ScenarioReplayer.kt +++ b/opendc-experiments/opendc-experiments-base/src/main/kotlin/org/opendc/experiments/base/runner/ScenarioReplayer.kt @@ -34,9 +34,11 @@ import org.opendc.compute.failure.models.FailureModel import org.opendc.compute.simulator.TaskWatcher import org.opendc.compute.simulator.service.ComputeService import org.opendc.compute.simulator.service.ServiceTask +import org.opendc.compute.simulator.service.TaskNature import org.opendc.compute.workload.Task import org.opendc.experiments.base.experiment.specs.FailureModelSpec import org.opendc.experiments.base.experiment.specs.createFailureModel +import java.time.Duration import java.time.InstantSource import java.util.Random import kotlin.coroutines.coroutineContext @@ -114,15 +116,26 @@ public suspend fun ComputeService.replay( // Delay the task based on the startTime given by the trace. if (!submitImmediately) { delay(max(0, (start - now - simulationOffset))) + entry.deadline -= simulationOffset } val workload = entry.trace val meta = mutableMapOf("workload" to workload) + val nature = + if (entry.nature == "deferrable") { + TaskNature(true) + } else { + TaskNature(false) + } + launch { val task = client.newTask( entry.name, + nature, + Duration.ofMillis(entry.duration), + entry.deadline, client.newFlavor( entry.name, entry.cpuCount, diff --git a/opendc-experiments/opendc-experiments-base/src/main/kotlin/org/opendc/experiments/base/runner/ScenarioRunner.kt b/opendc-experiments/opendc-experiments-base/src/main/kotlin/org/opendc/experiments/base/runner/ScenarioRunner.kt index 4da035ec..f8cbb4fd 100644 --- a/opendc-experiments/opendc-experiments-base/src/main/kotlin/org/opendc/experiments/base/runner/ScenarioRunner.kt +++ b/opendc-experiments/opendc-experiments-base/src/main/kotlin/org/opendc/experiments/base/runner/ScenarioRunner.kt @@ -35,6 +35,7 @@ import org.opendc.compute.topology.clusterTopology import org.opendc.experiments.base.experiment.Scenario import org.opendc.experiments.base.experiment.specs.getScalingPolicy import org.opendc.experiments.base.experiment.specs.getWorkloadLoader +import org.opendc.simulator.compute.power.CarbonReceiver import org.opendc.simulator.kotlin.runSimulation import java.io.File import java.time.Duration @@ -92,20 +93,35 @@ public fun runScenario( checkpointDuration, checkpointIntervalScaling, scalingPolicy, + scenario.workloadSpec.deferAll, ) var workload = workloadLoader.sampleByLoad(scenario.workloadSpec.sampleFraction) val startTimeLong = workload.minOf { it.submissionTime } val startTime = Duration.ofMillis(startTimeLong) + val carbonReceivers = mutableListOf() val topology = clusterTopology(scenario.topologySpec.pathToFile) provisioner.runSteps( setupComputeService( serviceDomain, - { createComputeScheduler(scenario.allocationPolicySpec.policyType, Random(it.seeder.nextLong())) }, + { + val computeScheduler = + createComputeScheduler( + scenario.allocationPolicySpec.policyType, + Random(it.seeder.nextLong()), + timeSource, + ) + + if (computeScheduler is CarbonReceiver) { + carbonReceivers.add(computeScheduler) + } + + return@setupComputeService computeScheduler + }, maxNumFailures = scenario.maxNumFailures, ), - setupHosts(serviceDomain, topology, startTimeLong), + setupHosts(serviceDomain, topology, carbonReceivers, startTimeLong), ) addExportModel(provisioner, serviceDomain, scenario, seed, startTime, scenario.id) diff --git a/opendc-experiments/opendc-experiments-base/src/test/kotlin/org/opendc/experiments/base/TestingUtils.kt b/opendc-experiments/opendc-experiments-base/src/test/kotlin/org/opendc/experiments/base/TestingUtils.kt index 68de1773..b2887de3 100644 --- a/opendc-experiments/opendc-experiments-base/src/test/kotlin/org/opendc/experiments/base/TestingUtils.kt +++ b/opendc-experiments/opendc-experiments-base/src/test/kotlin/org/opendc/experiments/base/TestingUtils.kt @@ -82,6 +82,8 @@ fun createTestTask( 1800000.0, LocalDateTime.parse(submissionTime).toInstant(ZoneOffset.UTC).toEpochMilli(), duration, + "", + -1, TraceWorkload( fragments, checkpointInterval, @@ -114,7 +116,7 @@ fun runTest( provisioner.runSteps( setupComputeService(serviceDomain = "compute.opendc.org", { computeScheduler }), registerComputeMonitor(serviceDomain = "compute.opendc.org", monitor, exportInterval = Duration.ofMinutes(1), startTime), - setupHosts(serviceDomain = "compute.opendc.org", topology, startTimeLong), + setupHosts(serviceDomain = "compute.opendc.org", topology, listOf(), startTimeLong), ) val service = provisioner.registry.resolve("compute.opendc.org", ComputeService::class.java)!! diff --git a/opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/conv/ResourceColumns.kt b/opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/conv/ResourceColumns.kt index 9a826418..d0f56bff 100644 --- a/opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/conv/ResourceColumns.kt +++ b/opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/conv/ResourceColumns.kt @@ -43,7 +43,7 @@ public val resourceClusterID: String = "cluster_id" public val resourceSubmissionTime: String = "submission_time" /** - * Start time for the resource. + * Carbon intensity of the resource. */ @JvmField public val resourceCarbonIntensity: String = "carbon_intensity" @@ -71,3 +71,15 @@ public val resourceCpuCapacity: String = "cpu_capacity" */ @JvmField public val resourceMemCapacity: String = "mem_capacity" + +/** + * Nature of the task. Delayable, interruptible, etc. + */ +@JvmField +public val resourceNature: String = "nature" + +/** + * Deadline of the task. + */ +@JvmField +public val resourceDeadline: String = "deadline" diff --git a/opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/formats/opendc/OdcVmResourceTableReader.kt b/opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/formats/opendc/OdcVmResourceTableReader.kt index 9c489bfd..10f60658 100644 --- a/opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/formats/opendc/OdcVmResourceTableReader.kt +++ b/opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/formats/opendc/OdcVmResourceTableReader.kt @@ -25,9 +25,11 @@ package org.opendc.trace.formats.opendc import org.opendc.trace.TableReader import org.opendc.trace.conv.resourceCpuCapacity import org.opendc.trace.conv.resourceCpuCount +import org.opendc.trace.conv.resourceDeadline import org.opendc.trace.conv.resourceDuration import org.opendc.trace.conv.resourceID import org.opendc.trace.conv.resourceMemCapacity +import org.opendc.trace.conv.resourceNature import org.opendc.trace.conv.resourceSubmissionTime import org.opendc.trace.formats.opendc.parquet.Resource import org.opendc.trace.util.parquet.LocalParquetReader @@ -62,6 +64,8 @@ internal class OdcVmResourceTableReader(private val reader: LocalParquetReader colCpuCount resourceCpuCapacity -> colCpuCapacity resourceMemCapacity -> colMemCapacity + resourceNature -> colNature + resourceDeadline -> colDeadline else -> -1 } } override fun isNull(index: Int): Boolean { - require(index in 0..colMemCapacity) { "Invalid column index" } - return false + require(index in 0..colDeadline) { "Invalid column index" } + val record = checkNotNull(record) { "Reader in invalid state" } + + return when (index) { + colNature -> record.nature == null + colDeadline -> record.deadline == -1L + else -> false + } } override fun getBoolean(index: Int): Boolean { @@ -97,6 +109,7 @@ internal class OdcVmResourceTableReader(private val reader: LocalParquetReader record.durationTime + colDeadline -> record.deadline else -> throw IllegalArgumentException("Invalid column") } } @@ -115,11 +128,12 @@ internal class OdcVmResourceTableReader(private val reader: LocalParquetReader record.id + colNature -> record.nature else -> throw IllegalArgumentException("Invalid column") } } diff --git a/opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/formats/opendc/OdcVmResourceTableWriter.kt b/opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/formats/opendc/OdcVmResourceTableWriter.kt index 19409fa7..2b8db7f1 100644 --- a/opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/formats/opendc/OdcVmResourceTableWriter.kt +++ b/opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/formats/opendc/OdcVmResourceTableWriter.kt @@ -26,9 +26,11 @@ import org.apache.parquet.hadoop.ParquetWriter import org.opendc.trace.TableWriter import org.opendc.trace.conv.resourceCpuCapacity import org.opendc.trace.conv.resourceCpuCount +import org.opendc.trace.conv.resourceDeadline import org.opendc.trace.conv.resourceDuration import org.opendc.trace.conv.resourceID import org.opendc.trace.conv.resourceMemCapacity +import org.opendc.trace.conv.resourceNature import org.opendc.trace.conv.resourceSubmissionTime import org.opendc.trace.formats.opendc.parquet.Resource import java.time.Duration @@ -49,6 +51,8 @@ internal class OdcVmResourceTableWriter(private val writer: ParquetWriter colCpuCount resourceCpuCapacity -> colCpuCapacity resourceMemCapacity -> colMemCapacity + resourceNature -> colNature + resourceDeadline -> colDeadline else -> -1 } } @@ -103,6 +122,7 @@ internal class OdcVmResourceTableWriter(private val writer: ParquetWriter localDuration = value + colDeadline -> localDeadline = value else -> throw IllegalArgumentException("Invalid column index $index") } } @@ -133,6 +153,7 @@ internal class OdcVmResourceTableWriter(private val writer: ParquetWriter localId = value + colNature -> localNature = value else -> throw IllegalArgumentException("Invalid column index $index") } } @@ -197,4 +218,6 @@ internal class OdcVmResourceTableWriter(private val writer: ParquetWriter diff --git a/opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/formats/opendc/parquet/Resource.kt b/opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/formats/opendc/parquet/Resource.kt index ea404c65..00922d4f 100644 --- a/opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/formats/opendc/parquet/Resource.kt +++ b/opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/formats/opendc/parquet/Resource.kt @@ -34,4 +34,6 @@ internal data class Resource( val cpuCount: Int, val cpuCapacity: Double, val memCapacity: Double, + val nature: String? = null, + val deadline: Long = -1, ) diff --git a/opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/formats/opendc/parquet/ResourceReadSupport.kt b/opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/formats/opendc/parquet/ResourceReadSupport.kt index e5a28a12..75a2bbb2 100644 --- a/opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/formats/opendc/parquet/ResourceReadSupport.kt +++ b/opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/formats/opendc/parquet/ResourceReadSupport.kt @@ -33,9 +33,11 @@ import org.apache.parquet.schema.Types import org.opendc.trace.TableColumn import org.opendc.trace.conv.resourceCpuCapacity import org.opendc.trace.conv.resourceCpuCount +import org.opendc.trace.conv.resourceDeadline import org.opendc.trace.conv.resourceDuration import org.opendc.trace.conv.resourceID import org.opendc.trace.conv.resourceMemCapacity +import org.opendc.trace.conv.resourceNature import org.opendc.trace.conv.resourceSubmissionTime /** @@ -56,6 +58,8 @@ internal class ResourceReadSupport(private val projection: List?) : Read "cpu_capacity" to resourceCpuCapacity, "requiredMemory" to resourceMemCapacity, "mem_capacity" to resourceMemCapacity, + "nature" to resourceNature, + "deadline" to resourceDeadline, ) override fun init(context: InitContext): ReadContext { @@ -112,6 +116,13 @@ internal class ResourceReadSupport(private val projection: List?) : Read Types .required(PrimitiveType.PrimitiveTypeName.INT64) .named("requiredMemory"), + Types + .optional(PrimitiveType.PrimitiveTypeName.BINARY) + .`as`(LogicalTypeAnnotation.stringType()) + .named("nature"), + Types + .optional(PrimitiveType.PrimitiveTypeName.INT64) + .named("deadline"), ) .named("resource") @@ -142,6 +153,13 @@ internal class ResourceReadSupport(private val projection: List?) : Read Types .required(PrimitiveType.PrimitiveTypeName.INT64) .named("mem_capacity"), + Types + .optional(PrimitiveType.PrimitiveTypeName.BINARY) + .`as`(LogicalTypeAnnotation.stringType()) + .named("nature"), + Types + .optional(PrimitiveType.PrimitiveTypeName.INT64) + .named("deadline"), ) .named("resource") diff --git a/opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/formats/opendc/parquet/ResourceRecordMaterializer.kt b/opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/formats/opendc/parquet/ResourceRecordMaterializer.kt index 5f02ea1e..866b304e 100644 --- a/opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/formats/opendc/parquet/ResourceRecordMaterializer.kt +++ b/opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/formats/opendc/parquet/ResourceRecordMaterializer.kt @@ -43,6 +43,8 @@ internal class ResourceRecordMaterializer(schema: MessageType) : RecordMateriali private var localCpuCount = 0 private var localCpuCapacity = 0.0 private var localMemCapacity = 0.0 + private var localNature: String? = null + private var localDeadline = -1L /** * Root converter for the record. @@ -95,6 +97,18 @@ internal class ResourceRecordMaterializer(schema: MessageType) : RecordMateriali localMemCapacity = value.toDouble() } } + "nature" -> + object : PrimitiveConverter() { + override fun addBinary(value: Binary) { + localNature = value.toStringUsingUTF8() + } + } + "deadline" -> + object : PrimitiveConverter() { + override fun addLong(value: Long) { + localDeadline = value + } + } else -> error("Unknown column $type") } } @@ -106,6 +120,8 @@ internal class ResourceRecordMaterializer(schema: MessageType) : RecordMateriali localCpuCount = 0 localCpuCapacity = 0.0 localMemCapacity = 0.0 + localNature = null + localDeadline = -1 } override fun end() {} @@ -121,6 +137,8 @@ internal class ResourceRecordMaterializer(schema: MessageType) : RecordMateriali localCpuCount, localCpuCapacity, localMemCapacity, + localNature, + localDeadline, ) override fun getRootConverter(): GroupConverter = root diff --git a/opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/formats/opendc/parquet/ResourceWriteSupport.kt b/opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/formats/opendc/parquet/ResourceWriteSupport.kt index e5822b0c..c3e984fb 100644 --- a/opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/formats/opendc/parquet/ResourceWriteSupport.kt +++ b/opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/formats/opendc/parquet/ResourceWriteSupport.kt @@ -83,6 +83,18 @@ internal class ResourceWriteSupport : WriteSupport() { consumer.addLong(record.memCapacity.roundToLong()) consumer.endField("mem_capacity", 5) + record.nature?.let { + consumer.startField("nature", 6) + consumer.addBinary(Binary.fromCharSequence(it)) + consumer.endField("nature", 6) + } + + if (record.deadline != -1L) { + consumer.startField("deadline", 7) + consumer.addLong(record.deadline) + consumer.endField("deadline", 7) + } + consumer.endMessage() } @@ -114,6 +126,13 @@ internal class ResourceWriteSupport : WriteSupport() { Types .required(PrimitiveType.PrimitiveTypeName.INT64) .named("mem_capacity"), + Types + .optional(PrimitiveType.PrimitiveTypeName.BINARY) + .`as`(LogicalTypeAnnotation.stringType()) + .named("nature"), + Types + .optional(PrimitiveType.PrimitiveTypeName.INT64) + .named("deadline"), ) .named("resource") } diff --git a/opendc-trace/opendc-trace-api/src/test/resources/opendc/trace-v2.1/tasks.parquet b/opendc-trace/opendc-trace-api/src/test/resources/opendc/trace-v2.1/tasks.parquet index 5053a192..c73177e2 100644 Binary files a/opendc-trace/opendc-trace-api/src/test/resources/opendc/trace-v2.1/tasks.parquet and b/opendc-trace/opendc-trace-api/src/test/resources/opendc/trace-v2.1/tasks.parquet differ diff --git a/opendc-web/opendc-web-runner/Dockerfile b/opendc-web/opendc-web-runner/Dockerfile index 22c36c65..b3bd0af6 100644 --- a/opendc-web/opendc-web-runner/Dockerfile +++ b/opendc-web/opendc-web-runner/Dockerfile @@ -1,4 +1,4 @@ -FROM openjdk:19-slim +FROM openjdk:21-slim MAINTAINER OpenDC Maintainers # Obtain (cache) Gradle wrapper @@ -11,9 +11,10 @@ RUN ./gradlew --version COPY ./ /app/ RUN ./gradlew --no-daemon :opendc-web:opendc-web-runner:installDist -FROM openjdk:19-slim +FROM openjdk:21-slim COPY --from=0 /app/opendc-web/opendc-web-runner/build/install /opt/ -COPY --from=0 /app/traces /opt/opendc/traces +COPY --from=0 /app/opendc-experiments/opendc-experiments-base/src/test/resources/workloadTraces \ + /opt/opendc/traces WORKDIR /opt/opendc CMD bin/opendc-web-runner diff --git a/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/runner/OpenDCRunner.kt b/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/runner/OpenDCRunner.kt index 7c5c6290..9a1b398e 100644 --- a/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/runner/OpenDCRunner.kt +++ b/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/runner/OpenDCRunner.kt @@ -279,10 +279,10 @@ public class OpenDCRunner( provisioner.runSteps( setupComputeService( serviceDomain, - { createComputeScheduler(scenario.schedulerName, Random(it.seeder.nextLong())) }, + { createComputeScheduler(scenario.schedulerName, Random(it.seeder.nextLong()), timeSource) }, ), registerComputeMonitor(serviceDomain, monitor), - setupHosts(serviceDomain, topology, startTime), + setupHosts(serviceDomain, topology, listOf(), startTime), ) val service = provisioner.registry.resolve(serviceDomain, ComputeService::class.java)!! diff --git a/opendc-web/opendc-web-server/Dockerfile b/opendc-web/opendc-web-server/Dockerfile index bcdb831e..e6cbaff5 100644 --- a/opendc-web/opendc-web-server/Dockerfile +++ b/opendc-web/opendc-web-server/Dockerfile @@ -1,4 +1,4 @@ -FROM openjdk:19-slim +FROM openjdk:21-slim MAINTAINER OpenDC Maintainers # Obtain (cache) Gradle wrapper @@ -19,7 +19,7 @@ ENV OPENDC_AUTH0_DOCS_CLIENT_ID=$OPENDC_AUTH0_DOCS_CLIENT_ID COPY ./ /app/ RUN ./gradlew --no-daemon :opendc-web:opendc-web-server:quarkusBuild -Dquarkus.profile=docker -FROM openjdk:19-slim +FROM openjdk:21-slim COPY --from=0 /app/opendc-web/opendc-web-server/build/quarkus-app /opt/opendc WORKDIR /opt/opendc CMD java -jar quarkus-run.jar diff --git a/site/docs/getting-started/0-installation.md b/site/docs/getting-started/0-installation.md index 281e811f..76ffd015 100644 --- a/site/docs/getting-started/0-installation.md +++ b/site/docs/getting-started/0-installation.md @@ -14,7 +14,7 @@ quicker (The web server is however missing some more complex features). 1. **Supported Platforms** OpenDC is actively tested on Windows, macOS and GNU/Linux. 2. **Required Software** - A Java installation of version 19 or higher is required for OpenDC. You may download the + A Java installation of version 21 or higher is required for OpenDC. You may download the [Java distribution from Oracle](https://www.oracle.com/java/technologies/downloads/) or use the distribution provided by your package manager. diff --git a/site/docs/getting-started/4-start-using-intellij.md b/site/docs/getting-started/4-start-using-intellij.md index eb821088..6aec91f1 100644 --- a/site/docs/getting-started/4-start-using-intellij.md +++ b/site/docs/getting-started/4-start-using-intellij.md @@ -10,7 +10,7 @@ First of all you can download IntelliJ here: https://lp.jetbrains.com/intellij-i git clone git@github.com:atlarge-research/opendc ``` -Check if you have a compatible java version available. Make sure to have one of these versions available: [19, 20, 21] +Check if you have a compatible java version available. Make sure to have one of these versions available: [21] If not install a supported version! diff --git a/traces/bitbrains-small/meta.parquet b/traces/bitbrains-small/meta.parquet deleted file mode 100644 index 9cded35f..00000000 Binary files a/traces/bitbrains-small/meta.parquet and /dev/null differ diff --git a/traces/bitbrains-small/trace.parquet b/traces/bitbrains-small/trace.parquet deleted file mode 100644 index 9d953956..00000000 Binary files a/traces/bitbrains-small/trace.parquet and /dev/null differ -- cgit v1.2.3