summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorSacheendra Talluri <sacheendra.t@gmail.com>2025-03-20 10:16:46 +0100
committerGitHub <noreply@github.com>2025-03-20 10:16:46 +0100
commit1e35c61cd31b8bfb33a6ccbb46b08c0466518e6c (patch)
tree261c84148cd045246bdc2ad7aa3c41524356b699
parent6211b887b68b3ebc9245fada1c0f36725955b052 (diff)
Adds load shifting over time (#319)
* Start time shifting * Existing experiments work with new columns * Remove unused traces dir * Update java to 21 LTS and jacoco to be compatible * Minimal working timeshifting * Timeshift scheduler linked as carbon receiver * Add basic tests for timeshift scheduler * Run spotless apply * Modify tarce format tests to support new fields * Change all mentions of java 19 to 21 * Add a deferAll option to workload to make all tasks deferrable * Run spotless apply * Copy traces from resources in web dockerfile
-rw-r--r--.github/workflows/build.yml4
-rw-r--r--.github/workflows/publish.yml2
-rw-r--r--.github/workflows/release.yml2
-rw-r--r--buildSrc/src/main/kotlin/jacoco-conventions.gradle.kts2
-rw-r--r--gradle/libs.versions.toml2
-rw-r--r--opendc-compute/opendc-compute-simulator/src/main/java/org/opendc/compute/simulator/service/ComputeService.java9
-rw-r--r--opendc-compute/opendc-compute-simulator/src/main/java/org/opendc/compute/simulator/service/ServiceTask.java25
-rw-r--r--opendc-compute/opendc-compute-simulator/src/main/java/org/opendc/compute/simulator/service/TaskNature.java32
-rw-r--r--opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/provisioner/ComputeSteps.kt4
-rw-r--r--opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/provisioner/HostsProvisioningStep.kt5
-rw-r--r--opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/scheduler/ComputeSchedulers.kt14
-rw-r--r--opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/scheduler/FilterScheduler.kt1
-rw-r--r--opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/scheduler/MemorizingScheduler.kt1
-rw-r--r--opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/scheduler/TimeshiftScheduler.kt161
-rw-r--r--opendc-compute/opendc-compute-simulator/src/test/kotlin/org/opendc/compute/simulator/scheduler/TimeshiftSchedulerTest.kt89
-rw-r--r--opendc-compute/opendc-compute-workload/src/main/kotlin/org/opendc/compute/workload/ComputeWorkloadLoader.kt13
-rw-r--r--opendc-compute/opendc-compute-workload/src/main/kotlin/org/opendc/compute/workload/Task.kt4
-rw-r--r--opendc-compute/opendc-compute-workload/src/main/kotlin/org/opendc/compute/workload/WorkloadLoader.kt1
-rw-r--r--opendc-experiments/opendc-experiments-base/build.gradle.kts2
-rw-r--r--opendc-experiments/opendc-experiments-base/src/main/kotlin/org/opendc/experiments/base/experiment/specs/WorkloadSpec.kt3
-rw-r--r--opendc-experiments/opendc-experiments-base/src/main/kotlin/org/opendc/experiments/base/runner/ScenarioReplayer.kt13
-rw-r--r--opendc-experiments/opendc-experiments-base/src/main/kotlin/org/opendc/experiments/base/runner/ScenarioRunner.kt20
-rw-r--r--opendc-experiments/opendc-experiments-base/src/test/kotlin/org/opendc/experiments/base/TestingUtils.kt4
-rw-r--r--opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/conv/ResourceColumns.kt14
-rw-r--r--opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/formats/opendc/OdcVmResourceTableReader.kt20
-rw-r--r--opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/formats/opendc/OdcVmResourceTableWriter.kt25
-rw-r--r--opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/formats/opendc/OdcVmTraceFormat.kt4
-rw-r--r--opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/formats/opendc/parquet/Resource.kt2
-rw-r--r--opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/formats/opendc/parquet/ResourceReadSupport.kt18
-rw-r--r--opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/formats/opendc/parquet/ResourceRecordMaterializer.kt18
-rw-r--r--opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/formats/opendc/parquet/ResourceWriteSupport.kt19
-rw-r--r--opendc-trace/opendc-trace-api/src/test/resources/opendc/trace-v2.1/tasks.parquetbin4597 -> 5407 bytes
-rw-r--r--opendc-web/opendc-web-runner/Dockerfile7
-rw-r--r--opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/runner/OpenDCRunner.kt4
-rw-r--r--opendc-web/opendc-web-server/Dockerfile4
-rw-r--r--site/docs/getting-started/0-installation.md2
-rw-r--r--site/docs/getting-started/4-start-using-intellij.md2
-rw-r--r--traces/bitbrains-small/meta.parquetbin2723 -> 0 bytes
-rw-r--r--traces/bitbrains-small/trace.parquetbin2163354 -> 0 bytes
39 files changed, 521 insertions, 31 deletions
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index e008127f..6d9cca16 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -14,10 +14,10 @@ jobs:
strategy:
matrix:
os: [ ubuntu-22.04 ]
- java: [ 19 ]
+ java: [ 21 ]
include:
- os: windows-2022
- java: 19
+ java: 21
steps:
- name: Checkout repository
uses: actions/checkout@v4
diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml
index 9b83de14..294be1d1 100644
--- a/.github/workflows/publish.yml
+++ b/.github/workflows/publish.yml
@@ -96,7 +96,7 @@ jobs:
uses: actions/setup-java@v4
with:
distribution: 'zulu'
- java-version: 19
+ java-version: 21
- name: Prepare
id: prep
run: |
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index 1efb1f7f..05cf997c 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -19,7 +19,7 @@ jobs:
uses: actions/setup-java@v4
with:
distribution: 'zulu'
- java-version: 19
+ java-version: 21
- name: Publish with Gradle
uses: gradle/actions/setup-gradle@v3
with:
diff --git a/buildSrc/src/main/kotlin/jacoco-conventions.gradle.kts b/buildSrc/src/main/kotlin/jacoco-conventions.gradle.kts
index 57ad1b9d..144fb1aa 100644
--- a/buildSrc/src/main/kotlin/jacoco-conventions.gradle.kts
+++ b/buildSrc/src/main/kotlin/jacoco-conventions.gradle.kts
@@ -26,7 +26,7 @@ plugins {
}
jacoco {
- toolVersion = "0.8.8"
+ toolVersion = "0.8.11"
}
tasks.jacocoTestReport {
diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml
index bec3c35c..44cd4003 100644
--- a/gradle/libs.versions.toml
+++ b/gradle/libs.versions.toml
@@ -8,7 +8,7 @@ hadoop = "3.3.6"
hypersistence-utils = "3.7.3"
jackson = "2.16.1"
jandex-gradle = "1.1.0"
-java = "19"
+java = "21"
jline = "3.25.1"
jmh-gradle = "0.7.0"
jakarta = "3.0.2"
diff --git a/opendc-compute/opendc-compute-simulator/src/main/java/org/opendc/compute/simulator/service/ComputeService.java b/opendc-compute/opendc-compute-simulator/src/main/java/org/opendc/compute/simulator/service/ComputeService.java
index eb8d3377..69625306 100644
--- a/opendc-compute/opendc-compute-simulator/src/main/java/org/opendc/compute/simulator/service/ComputeService.java
+++ b/opendc-compute/opendc-compute-simulator/src/main/java/org/opendc/compute/simulator/service/ComputeService.java
@@ -25,6 +25,7 @@ package org.opendc.compute.simulator.service;
import java.time.Duration;
import java.time.Instant;
import java.time.InstantSource;
+import java.time.temporal.TemporalAmount;
import java.util.ArrayDeque;
import java.util.ArrayList;
import java.util.Collections;
@@ -499,7 +500,6 @@ public final class ComputeService implements AutoCloseable {
SimHost host = hv.getHost();
// Remove request from queue
- taskQueue.remove(req);
tasksPending--;
LOGGER.info("Assigned task {} to host {}", task, host);
@@ -642,6 +642,9 @@ public final class ComputeService implements AutoCloseable {
@NotNull
public ServiceTask newTask(
@NotNull String name,
+ @NotNull TaskNature nature,
+ @NotNull TemporalAmount duration,
+ @NotNull Long deadline,
@NotNull ServiceFlavor flavor,
@NotNull Workload workload,
@NotNull Map<String, ?> meta) {
@@ -652,9 +655,9 @@ public final class ComputeService implements AutoCloseable {
// final ServiceFlavor internalFlavor =
// Objects.requireNonNull(service.flavorById.get(flavor.getUid()), "Unknown flavor");
-
// ServiceTask task = new ServiceTask(service, uid, name, internalFlavor, workload, meta);
- ServiceTask task = new ServiceTask(service, uid, name, flavor, workload, meta);
+
+ ServiceTask task = new ServiceTask(service, uid, name, nature, duration, deadline, flavor, workload, meta);
service.taskById.put(uid, task);
diff --git a/opendc-compute/opendc-compute-simulator/src/main/java/org/opendc/compute/simulator/service/ServiceTask.java b/opendc-compute/opendc-compute-simulator/src/main/java/org/opendc/compute/simulator/service/ServiceTask.java
index dac65d67..4d5611a8 100644
--- a/opendc-compute/opendc-compute-simulator/src/main/java/org/opendc/compute/simulator/service/ServiceTask.java
+++ b/opendc-compute/opendc-compute-simulator/src/main/java/org/opendc/compute/simulator/service/ServiceTask.java
@@ -23,6 +23,7 @@
package org.opendc.compute.simulator.service;
import java.time.Instant;
+import java.time.temporal.TemporalAmount;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
@@ -49,6 +50,9 @@ public class ServiceTask {
private final UUID uid;
private final String name;
+ private final TaskNature nature;
+ private final TemporalAmount duration;
+ private final Long deadline;
private ServiceFlavor flavor;
public Workload workload;
@@ -68,12 +72,18 @@ public class ServiceTask {
ComputeService service,
UUID uid,
String name,
+ TaskNature nature,
+ TemporalAmount duration,
+ Long deadline,
ServiceFlavor flavor,
Workload workload,
Map<String, ?> meta) {
this.service = service;
this.uid = uid;
this.name = name;
+ this.nature = nature;
+ this.duration = duration;
+ this.deadline = deadline;
this.flavor = flavor;
this.workload = workload;
this.meta = meta;
@@ -87,6 +97,21 @@ public class ServiceTask {
}
@NotNull
+ public TaskNature getNature() {
+ return nature;
+ }
+
+ @NotNull
+ public TemporalAmount getDuration() {
+ return duration;
+ }
+
+ @NotNull
+ public Long getDeadline() {
+ return deadline;
+ }
+
+ @NotNull
public String getName() {
return name;
}
diff --git a/opendc-compute/opendc-compute-simulator/src/main/java/org/opendc/compute/simulator/service/TaskNature.java b/opendc-compute/opendc-compute-simulator/src/main/java/org/opendc/compute/simulator/service/TaskNature.java
new file mode 100644
index 00000000..ffb49143
--- /dev/null
+++ b/opendc-compute/opendc-compute-simulator/src/main/java/org/opendc/compute/simulator/service/TaskNature.java
@@ -0,0 +1,32 @@
+/*
+ * Copyright (c) 2025 AtLarge Research
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+package org.opendc.compute.simulator.service;
+
+public class TaskNature {
+
+ public final boolean deferrable;
+
+ public TaskNature(boolean deferrable) {
+ this.deferrable = deferrable;
+ }
+}
diff --git a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/provisioner/ComputeSteps.kt b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/provisioner/ComputeSteps.kt
index a18856f8..1145c15b 100644
--- a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/provisioner/ComputeSteps.kt
+++ b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/provisioner/ComputeSteps.kt
@@ -29,6 +29,7 @@ import org.opendc.compute.simulator.telemetry.ComputeMonitor
import org.opendc.compute.simulator.telemetry.OutputFiles
import org.opendc.compute.topology.specs.ClusterSpec
import org.opendc.compute.topology.specs.HostSpec
+import org.opendc.simulator.compute.power.CarbonReceiver
import java.time.Duration
/**
@@ -84,7 +85,8 @@ public fun registerComputeMonitor(
public fun setupHosts(
serviceDomain: String,
specs: List<ClusterSpec>,
+ carbonReceivers: List<CarbonReceiver>,
startTime: Long = 0L,
): ProvisioningStep {
- return HostsProvisioningStep(serviceDomain, specs, startTime)
+ return HostsProvisioningStep(serviceDomain, specs, carbonReceivers, startTime)
}
diff --git a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/provisioner/HostsProvisioningStep.kt b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/provisioner/HostsProvisioningStep.kt
index 68e263f8..c347e866 100644
--- a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/provisioner/HostsProvisioningStep.kt
+++ b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/provisioner/HostsProvisioningStep.kt
@@ -29,6 +29,7 @@ import org.opendc.compute.topology.specs.ClusterSpec
import org.opendc.compute.topology.specs.HostSpec
import org.opendc.compute.topology.specs.createSimBatteryPolicy
import org.opendc.simulator.compute.power.CarbonModel
+import org.opendc.simulator.compute.power.CarbonReceiver
import org.opendc.simulator.compute.power.SimPowerSource
import org.opendc.simulator.compute.power.batteries.BatteryAggregator
import org.opendc.simulator.compute.power.batteries.SimBattery
@@ -46,6 +47,7 @@ import org.opendc.simulator.engine.graph.FlowEdge
public class HostsProvisioningStep internal constructor(
private val serviceDomain: String,
private val clusterSpecs: List<ClusterSpec>,
+ private val carbonReceivers: List<CarbonReceiver>,
private val startTime: Long = 0L,
) : ProvisioningStep {
override fun apply(ctx: ProvisioningContext): AutoCloseable {
@@ -75,6 +77,9 @@ public class HostsProvisioningStep internal constructor(
if (carbonFragments != null) {
carbonModel = CarbonModel(engine, carbonFragments, startTime)
carbonModel.addReceiver(simPowerSource)
+ for (receiver in carbonReceivers) {
+ carbonModel.addReceiver(receiver)
+ }
}
if (cluster.battery != null) {
diff --git a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/scheduler/ComputeSchedulers.kt b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/scheduler/ComputeSchedulers.kt
index 8f2369dc..b83eafdd 100644
--- a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/scheduler/ComputeSchedulers.kt
+++ b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/scheduler/ComputeSchedulers.kt
@@ -31,6 +31,7 @@ import org.opendc.compute.simulator.scheduler.weights.CoreRamWeigher
import org.opendc.compute.simulator.scheduler.weights.InstanceCountWeigher
import org.opendc.compute.simulator.scheduler.weights.RamWeigher
import org.opendc.compute.simulator.scheduler.weights.VCpuWeigher
+import java.time.InstantSource
import java.util.SplittableRandom
import java.util.random.RandomGenerator
@@ -45,13 +46,15 @@ public enum class ComputeSchedulerEnum {
ProvisionedCoresInv,
Random,
TaskNumMemorizing,
+ Timeshift,
}
public fun createComputeScheduler(
name: String,
seeder: RandomGenerator,
+ clock: InstantSource,
): ComputeScheduler {
- return createComputeScheduler(ComputeSchedulerEnum.valueOf(name.uppercase()), seeder)
+ return createComputeScheduler(ComputeSchedulerEnum.valueOf(name.uppercase()), seeder, clock)
}
/**
@@ -60,6 +63,7 @@ public fun createComputeScheduler(
public fun createComputeScheduler(
name: ComputeSchedulerEnum,
seeder: RandomGenerator,
+ clock: InstantSource,
): ComputeScheduler {
val cpuAllocationRatio = 1.0
val ramAllocationRatio = 1.5
@@ -116,5 +120,13 @@ public fun createComputeScheduler(
filters = listOf(ComputeFilter(), VCpuFilter(cpuAllocationRatio), RamFilter(ramAllocationRatio)),
random = SplittableRandom(seeder.nextLong()),
)
+ ComputeSchedulerEnum.Timeshift ->
+ TimeshiftScheduler(
+ filters = listOf(ComputeFilter(), VCpuFilter(cpuAllocationRatio), RamFilter(ramAllocationRatio)),
+ weighers = listOf(RamWeigher(multiplier = 1.0)),
+ windowSize = 168,
+ clock = clock,
+ random = SplittableRandom(seeder.nextLong()),
+ )
}
}
diff --git a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/scheduler/FilterScheduler.kt b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/scheduler/FilterScheduler.kt
index 832482eb..386e0d63 100644
--- a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/scheduler/FilterScheduler.kt
+++ b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/scheduler/FilterScheduler.kt
@@ -79,7 +79,6 @@ public class FilterScheduler(
}
val task = req.task
- val hosts = hosts
val filteredHosts = hosts.filter { host -> filters.all { filter -> filter.test(host, task) } }
val subset =
diff --git a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/scheduler/MemorizingScheduler.kt b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/scheduler/MemorizingScheduler.kt
index c48e6fb0..faa1be6b 100644
--- a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/scheduler/MemorizingScheduler.kt
+++ b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/scheduler/MemorizingScheduler.kt
@@ -88,6 +88,7 @@ public class MemorizingScheduler(
taskloop@ for (req in iter) {
if (req.isCancelled) {
iter.remove()
+ continue
}
for (chosenListIndex in minAvailableHost until hostsQueue.size) {
diff --git a/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/scheduler/TimeshiftScheduler.kt b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/scheduler/TimeshiftScheduler.kt
new file mode 100644
index 00000000..a856f366
--- /dev/null
+++ b/opendc-compute/opendc-compute-simulator/src/main/kotlin/org/opendc/compute/simulator/scheduler/TimeshiftScheduler.kt
@@ -0,0 +1,161 @@
+/*
+ * Copyright (c) 2025 AtLarge Research
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+package org.opendc.compute.simulator.scheduler
+
+import org.opendc.compute.simulator.scheduler.filters.HostFilter
+import org.opendc.compute.simulator.scheduler.weights.HostWeigher
+import org.opendc.compute.simulator.service.HostView
+import org.opendc.compute.simulator.service.ServiceTask
+import org.opendc.simulator.compute.power.CarbonModel
+import org.opendc.simulator.compute.power.CarbonReceiver
+import java.time.Instant
+import java.time.InstantSource
+import java.util.LinkedList
+import java.util.SplittableRandom
+import java.util.random.RandomGenerator
+import kotlin.math.min
+
+public class TimeshiftScheduler(
+ private val filters: List<HostFilter>,
+ private val weighers: List<HostWeigher>,
+ private val windowSize: Int,
+ private val clock: InstantSource,
+ private val subsetSize: Int = 1,
+ private val random: RandomGenerator = SplittableRandom(0),
+) : ComputeScheduler, CarbonReceiver {
+ /**
+ * The pool of hosts available to the scheduler.
+ */
+ private val hosts = mutableListOf<HostView>()
+
+ init {
+ require(subsetSize >= 1) { "Subset size must be one or greater" }
+ }
+
+ private val pastCarbonIntensities = LinkedList<Double>()
+ private var carbonRunningSum = 0.0
+
+ private var carbonIntensity = 0.0
+
+// private var lowerCarbonIntensity = 0.0
+ private var thresholdCarbonIntensity = 0.0
+
+ override fun addHost(host: HostView) {
+ hosts.add(host)
+ }
+
+ override fun removeHost(host: HostView) {
+ hosts.remove(host)
+ }
+
+ override fun select(iter: MutableIterator<SchedulingRequest>): SchedulingResult {
+ var result: SchedulingResult? = null
+ for (req in iter) {
+ if (req.isCancelled) {
+ iter.remove()
+ continue
+ }
+
+ val task = req.task
+
+ if (carbonIntensity > thresholdCarbonIntensity) {
+ if (task.nature.deferrable) {
+ val currentTime = clock.instant()
+ val estimatedCompletion = currentTime.plus(task.duration)
+ val deadline = Instant.ofEpochMilli(task.deadline)
+ if (estimatedCompletion.isBefore(deadline)) {
+ // No need to schedule this task in a high carbon intensity period
+ continue
+ }
+ }
+ }
+
+ val filteredHosts = hosts.filter { host -> filters.all { filter -> filter.test(host, task) } }
+
+ val subset =
+ if (weighers.isNotEmpty()) {
+ val filterResults = weighers.map { it.getWeights(filteredHosts, task) }
+ val weights = DoubleArray(filteredHosts.size)
+
+ for (fr in filterResults) {
+ val min = fr.min
+ val range = (fr.max - min)
+
+ // Skip result if all weights are the same
+ if (range == 0.0) {
+ continue
+ }
+
+ val multiplier = fr.multiplier
+ val factor = multiplier / range
+
+ for ((i, weight) in fr.weights.withIndex()) {
+ weights[i] += factor * (weight - min)
+ }
+ }
+
+ weights.indices
+ .asSequence()
+ .sortedByDescending { weights[it] }
+ .map { filteredHosts[it] }
+ .take(subsetSize)
+ .toList()
+ } else {
+ filteredHosts
+ }
+
+ val maxSize = min(subsetSize, subset.size)
+ if (maxSize == 0) {
+ result = SchedulingResult(SchedulingResultType.FAILURE, null, req)
+ break
+ } else {
+ iter.remove()
+ result = SchedulingResult(SchedulingResultType.SUCCESS, subset[random.nextInt(maxSize)], req)
+ break
+ }
+ }
+
+ if (result == null) return SchedulingResult(SchedulingResultType.EMPTY)
+
+ return result
+ }
+
+ override fun removeTask(
+ task: ServiceTask,
+ host: HostView?,
+ ) {}
+
+ override fun updateCarbonIntensity(newCarbonIntensity: Double) {
+ this.carbonIntensity = newCarbonIntensity
+ this.pastCarbonIntensities.addLast(newCarbonIntensity)
+ this.carbonRunningSum += newCarbonIntensity
+ if (this.pastCarbonIntensities.size > this.windowSize) {
+ this.carbonRunningSum -= this.pastCarbonIntensities.removeFirst()
+ }
+ this.thresholdCarbonIntensity = this.carbonRunningSum / this.pastCarbonIntensities.size
+ }
+
+ override fun setCarbonModel(carbonModel: CarbonModel?) {}
+
+ override fun removeCarbonModel(carbonModel: CarbonModel?) {}
+}
diff --git a/opendc-compute/opendc-compute-simulator/src/test/kotlin/org/opendc/compute/simulator/scheduler/TimeshiftSchedulerTest.kt b/opendc-compute/opendc-compute-simulator/src/test/kotlin/org/opendc/compute/simulator/scheduler/TimeshiftSchedulerTest.kt
new file mode 100644
index 00000000..b893c1aa
--- /dev/null
+++ b/opendc-compute/opendc-compute-simulator/src/test/kotlin/org/opendc/compute/simulator/scheduler/TimeshiftSchedulerTest.kt
@@ -0,0 +1,89 @@
+/*
+ * Copyright (c) 2025 AtLarge Research
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+package org.opendc.compute.simulator.scheduler
+
+import io.mockk.every
+import io.mockk.mockk
+import org.junit.jupiter.api.Assertions.assertEquals
+import org.junit.jupiter.api.Test
+import org.opendc.compute.simulator.service.TaskNature
+import java.time.Duration
+import java.time.Instant
+import java.time.InstantSource
+
+class TimeshiftSchedulerTest {
+ @Test
+ fun testBasicDeferring() {
+ val clock = mockk<InstantSource>()
+ every { clock.instant() } returns Instant.ofEpochMilli(10)
+
+ val scheduler =
+ TimeshiftScheduler(
+ filters = emptyList(),
+ weighers = emptyList(),
+ windowSize = 2,
+ clock = clock,
+ )
+
+ val req = mockk<SchedulingRequest>()
+ every { req.task.flavor.coreCount } returns 2
+ every { req.task.flavor.memorySize } returns 1024
+ every { req.isCancelled } returns false
+ every { req.task.nature } returns TaskNature(true)
+ every { req.task.duration } returns Duration.ofMillis(10)
+ every { req.task.deadline } returns 50
+
+ scheduler.updateCarbonIntensity(100.0)
+ scheduler.updateCarbonIntensity(200.0)
+
+ assertEquals(SchedulingResultType.EMPTY, scheduler.select(mutableListOf(req).iterator()).resultType)
+ }
+
+ @Test
+ fun testRespectDeadline() {
+ val clock = mockk<InstantSource>()
+ every { clock.instant() } returns Instant.ofEpochMilli(10)
+
+ val scheduler =
+ TimeshiftScheduler(
+ filters = emptyList(),
+ weighers = emptyList(),
+ windowSize = 2,
+ clock = clock,
+ )
+
+ val req = mockk<SchedulingRequest>()
+ every { req.task.flavor.coreCount } returns 2
+ every { req.task.flavor.memorySize } returns 1024
+ every { req.isCancelled } returns false
+ every { req.task.nature } returns TaskNature(true)
+ every { req.task.duration } returns Duration.ofMillis(10)
+ every { req.task.deadline } returns 20
+
+ scheduler.updateCarbonIntensity(100.0)
+ scheduler.updateCarbonIntensity(200.0)
+
+ // The scheduler tries to schedule the task, but fails as there are no hosts.
+ assertEquals(SchedulingResultType.FAILURE, scheduler.select(mutableListOf(req).iterator()).resultType)
+ }
+}
diff --git a/opendc-compute/opendc-compute-workload/src/main/kotlin/org/opendc/compute/workload/ComputeWorkloadLoader.kt b/opendc-compute/opendc-compute-workload/src/main/kotlin/org/opendc/compute/workload/ComputeWorkloadLoader.kt
index cbff5fb9..8e774acb 100644
--- a/opendc-compute/opendc-compute-workload/src/main/kotlin/org/opendc/compute/workload/ComputeWorkloadLoader.kt
+++ b/opendc-compute/opendc-compute-workload/src/main/kotlin/org/opendc/compute/workload/ComputeWorkloadLoader.kt
@@ -31,9 +31,11 @@ import org.opendc.trace.conv.TABLE_RESOURCES
import org.opendc.trace.conv.TABLE_RESOURCE_STATES
import org.opendc.trace.conv.resourceCpuCapacity
import org.opendc.trace.conv.resourceCpuCount
+import org.opendc.trace.conv.resourceDeadline
import org.opendc.trace.conv.resourceDuration
import org.opendc.trace.conv.resourceID
import org.opendc.trace.conv.resourceMemCapacity
+import org.opendc.trace.conv.resourceNature
import org.opendc.trace.conv.resourceStateCpuUsage
import org.opendc.trace.conv.resourceStateDuration
import org.opendc.trace.conv.resourceSubmissionTime
@@ -56,6 +58,7 @@ public class ComputeWorkloadLoader(
private val checkpointDuration: Long = 0L,
private val checkpointIntervalScaling: Double = 1.0,
private val scalingPolicy: ScalingPolicy = NoDelayScaling(),
+ private val deferAll: Boolean = false,
) : WorkloadLoader(subMissionTime) {
/**
* The logger for this instance.
@@ -115,6 +118,8 @@ public class ComputeWorkloadLoader(
val cpuCountCol = reader.resolve(resourceCpuCount)
val cpuCapacityCol = reader.resolve(resourceCpuCapacity)
val memCol = reader.resolve(resourceMemCapacity)
+ val natureCol = reader.resolve(resourceNature)
+ val deadlineCol = reader.resolve(resourceDeadline)
var counter = 0
val entries = mutableListOf<Task>()
@@ -132,6 +137,12 @@ public class ComputeWorkloadLoader(
val cpuCapacity = reader.getDouble(cpuCapacityCol)
val memCapacity = reader.getDouble(memCol) / 1000.0 // Convert from KB to MB
val uid = UUID.nameUUIDFromBytes("$id-${counter++}".toByteArray())
+ var nature = reader.getString(natureCol)
+ var deadline = reader.getLong(deadlineCol)
+ if (deferAll) {
+ nature = "deferrable"
+ deadline = submissionTime + (3 * duration)
+ }
val builder = fragments.getValue(id) // Get all fragments related to this VM
val totalLoad = builder.totalLoad
@@ -146,6 +157,8 @@ public class ComputeWorkloadLoader(
totalLoad,
submissionTime,
duration,
+ nature,
+ deadline,
builder.build(),
),
)
diff --git a/opendc-compute/opendc-compute-workload/src/main/kotlin/org/opendc/compute/workload/Task.kt b/opendc-compute/opendc-compute-workload/src/main/kotlin/org/opendc/compute/workload/Task.kt
index d32c84f6..787f271e 100644
--- a/opendc-compute/opendc-compute-workload/src/main/kotlin/org/opendc/compute/workload/Task.kt
+++ b/opendc-compute/opendc-compute-workload/src/main/kotlin/org/opendc/compute/workload/Task.kt
@@ -46,5 +46,7 @@ public data class Task(
val totalLoad: Double,
var submissionTime: Long,
val duration: Long,
- var trace: TraceWorkload,
+ val nature: String?,
+ var deadline: Long,
+ val trace: TraceWorkload,
)
diff --git a/opendc-compute/opendc-compute-workload/src/main/kotlin/org/opendc/compute/workload/WorkloadLoader.kt b/opendc-compute/opendc-compute-workload/src/main/kotlin/org/opendc/compute/workload/WorkloadLoader.kt
index 6b9c3753..c00bc768 100644
--- a/opendc-compute/opendc-compute-workload/src/main/kotlin/org/opendc/compute/workload/WorkloadLoader.kt
+++ b/opendc-compute/opendc-compute-workload/src/main/kotlin/org/opendc/compute/workload/WorkloadLoader.kt
@@ -40,6 +40,7 @@ public abstract class WorkloadLoader(private val submissionTime: String? = null)
for (task in workload) {
task.submissionTime += timeShift
+ task.deadline = if (task.deadline == -1L) -1L else task.deadline + timeShift
}
}
diff --git a/opendc-experiments/opendc-experiments-base/build.gradle.kts b/opendc-experiments/opendc-experiments-base/build.gradle.kts
index 36073418..8a77f0c4 100644
--- a/opendc-experiments/opendc-experiments-base/build.gradle.kts
+++ b/opendc-experiments/opendc-experiments-base/build.gradle.kts
@@ -55,7 +55,7 @@ val createScenarioApp by tasks.creating(CreateStartScripts::class) {
applicationName = "OpenDCExperimentRunner"
mainClass.set("org.opendc.experiments.base.runner.ExperimentCli")
classpath = tasks.jar.get().outputs.files + configurations["runtimeClasspath"]
- outputDir = project.buildDir.resolve("scripts")
+ outputDir = layout.buildDirectory.dir("scripts").get().asFile
}
// Create custom Scenario distribution
diff --git a/opendc-experiments/opendc-experiments-base/src/main/kotlin/org/opendc/experiments/base/experiment/specs/WorkloadSpec.kt b/opendc-experiments/opendc-experiments-base/src/main/kotlin/org/opendc/experiments/base/experiment/specs/WorkloadSpec.kt
index cf40d88d..4d2e932f 100644
--- a/opendc-experiments/opendc-experiments-base/src/main/kotlin/org/opendc/experiments/base/experiment/specs/WorkloadSpec.kt
+++ b/opendc-experiments/opendc-experiments-base/src/main/kotlin/org/opendc/experiments/base/experiment/specs/WorkloadSpec.kt
@@ -44,6 +44,7 @@ public data class WorkloadSpec(
val type: WorkloadTypes,
val sampleFraction: Double = 1.0,
val submissionTime: String? = null,
+ val deferAll: Boolean = false,
val scalingPolicy: ScalingPolicyEnum = ScalingPolicyEnum.NoDelay,
) {
public val name: String = File(pathToFile).nameWithoutExtension
@@ -74,6 +75,7 @@ public fun getWorkloadLoader(
checkpointDuration: Long,
checkpointIntervalScaling: Double,
scalingPolicy: ScalingPolicy,
+ deferAll: Boolean,
): WorkloadLoader {
return when (type) {
WorkloadTypes.ComputeWorkload ->
@@ -84,6 +86,7 @@ public fun getWorkloadLoader(
checkpointDuration,
checkpointIntervalScaling,
scalingPolicy,
+ deferAll,
)
}
}
diff --git a/opendc-experiments/opendc-experiments-base/src/main/kotlin/org/opendc/experiments/base/runner/ScenarioReplayer.kt b/opendc-experiments/opendc-experiments-base/src/main/kotlin/org/opendc/experiments/base/runner/ScenarioReplayer.kt
index 5664e0cb..d56e4e4b 100644
--- a/opendc-experiments/opendc-experiments-base/src/main/kotlin/org/opendc/experiments/base/runner/ScenarioReplayer.kt
+++ b/opendc-experiments/opendc-experiments-base/src/main/kotlin/org/opendc/experiments/base/runner/ScenarioReplayer.kt
@@ -34,9 +34,11 @@ import org.opendc.compute.failure.models.FailureModel
import org.opendc.compute.simulator.TaskWatcher
import org.opendc.compute.simulator.service.ComputeService
import org.opendc.compute.simulator.service.ServiceTask
+import org.opendc.compute.simulator.service.TaskNature
import org.opendc.compute.workload.Task
import org.opendc.experiments.base.experiment.specs.FailureModelSpec
import org.opendc.experiments.base.experiment.specs.createFailureModel
+import java.time.Duration
import java.time.InstantSource
import java.util.Random
import kotlin.coroutines.coroutineContext
@@ -114,15 +116,26 @@ public suspend fun ComputeService.replay(
// Delay the task based on the startTime given by the trace.
if (!submitImmediately) {
delay(max(0, (start - now - simulationOffset)))
+ entry.deadline -= simulationOffset
}
val workload = entry.trace
val meta = mutableMapOf<String, Any>("workload" to workload)
+ val nature =
+ if (entry.nature == "deferrable") {
+ TaskNature(true)
+ } else {
+ TaskNature(false)
+ }
+
launch {
val task =
client.newTask(
entry.name,
+ nature,
+ Duration.ofMillis(entry.duration),
+ entry.deadline,
client.newFlavor(
entry.name,
entry.cpuCount,
diff --git a/opendc-experiments/opendc-experiments-base/src/main/kotlin/org/opendc/experiments/base/runner/ScenarioRunner.kt b/opendc-experiments/opendc-experiments-base/src/main/kotlin/org/opendc/experiments/base/runner/ScenarioRunner.kt
index 4da035ec..f8cbb4fd 100644
--- a/opendc-experiments/opendc-experiments-base/src/main/kotlin/org/opendc/experiments/base/runner/ScenarioRunner.kt
+++ b/opendc-experiments/opendc-experiments-base/src/main/kotlin/org/opendc/experiments/base/runner/ScenarioRunner.kt
@@ -35,6 +35,7 @@ import org.opendc.compute.topology.clusterTopology
import org.opendc.experiments.base.experiment.Scenario
import org.opendc.experiments.base.experiment.specs.getScalingPolicy
import org.opendc.experiments.base.experiment.specs.getWorkloadLoader
+import org.opendc.simulator.compute.power.CarbonReceiver
import org.opendc.simulator.kotlin.runSimulation
import java.io.File
import java.time.Duration
@@ -92,20 +93,35 @@ public fun runScenario(
checkpointDuration,
checkpointIntervalScaling,
scalingPolicy,
+ scenario.workloadSpec.deferAll,
)
var workload = workloadLoader.sampleByLoad(scenario.workloadSpec.sampleFraction)
val startTimeLong = workload.minOf { it.submissionTime }
val startTime = Duration.ofMillis(startTimeLong)
+ val carbonReceivers = mutableListOf<CarbonReceiver>()
val topology = clusterTopology(scenario.topologySpec.pathToFile)
provisioner.runSteps(
setupComputeService(
serviceDomain,
- { createComputeScheduler(scenario.allocationPolicySpec.policyType, Random(it.seeder.nextLong())) },
+ {
+ val computeScheduler =
+ createComputeScheduler(
+ scenario.allocationPolicySpec.policyType,
+ Random(it.seeder.nextLong()),
+ timeSource,
+ )
+
+ if (computeScheduler is CarbonReceiver) {
+ carbonReceivers.add(computeScheduler)
+ }
+
+ return@setupComputeService computeScheduler
+ },
maxNumFailures = scenario.maxNumFailures,
),
- setupHosts(serviceDomain, topology, startTimeLong),
+ setupHosts(serviceDomain, topology, carbonReceivers, startTimeLong),
)
addExportModel(provisioner, serviceDomain, scenario, seed, startTime, scenario.id)
diff --git a/opendc-experiments/opendc-experiments-base/src/test/kotlin/org/opendc/experiments/base/TestingUtils.kt b/opendc-experiments/opendc-experiments-base/src/test/kotlin/org/opendc/experiments/base/TestingUtils.kt
index 68de1773..b2887de3 100644
--- a/opendc-experiments/opendc-experiments-base/src/test/kotlin/org/opendc/experiments/base/TestingUtils.kt
+++ b/opendc-experiments/opendc-experiments-base/src/test/kotlin/org/opendc/experiments/base/TestingUtils.kt
@@ -82,6 +82,8 @@ fun createTestTask(
1800000.0,
LocalDateTime.parse(submissionTime).toInstant(ZoneOffset.UTC).toEpochMilli(),
duration,
+ "",
+ -1,
TraceWorkload(
fragments,
checkpointInterval,
@@ -114,7 +116,7 @@ fun runTest(
provisioner.runSteps(
setupComputeService(serviceDomain = "compute.opendc.org", { computeScheduler }),
registerComputeMonitor(serviceDomain = "compute.opendc.org", monitor, exportInterval = Duration.ofMinutes(1), startTime),
- setupHosts(serviceDomain = "compute.opendc.org", topology, startTimeLong),
+ setupHosts(serviceDomain = "compute.opendc.org", topology, listOf(), startTimeLong),
)
val service = provisioner.registry.resolve("compute.opendc.org", ComputeService::class.java)!!
diff --git a/opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/conv/ResourceColumns.kt b/opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/conv/ResourceColumns.kt
index 9a826418..d0f56bff 100644
--- a/opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/conv/ResourceColumns.kt
+++ b/opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/conv/ResourceColumns.kt
@@ -43,7 +43,7 @@ public val resourceClusterID: String = "cluster_id"
public val resourceSubmissionTime: String = "submission_time"
/**
- * Start time for the resource.
+ * Carbon intensity of the resource.
*/
@JvmField
public val resourceCarbonIntensity: String = "carbon_intensity"
@@ -71,3 +71,15 @@ public val resourceCpuCapacity: String = "cpu_capacity"
*/
@JvmField
public val resourceMemCapacity: String = "mem_capacity"
+
+/**
+ * Nature of the task. Delayable, interruptible, etc.
+ */
+@JvmField
+public val resourceNature: String = "nature"
+
+/**
+ * Deadline of the task.
+ */
+@JvmField
+public val resourceDeadline: String = "deadline"
diff --git a/opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/formats/opendc/OdcVmResourceTableReader.kt b/opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/formats/opendc/OdcVmResourceTableReader.kt
index 9c489bfd..10f60658 100644
--- a/opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/formats/opendc/OdcVmResourceTableReader.kt
+++ b/opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/formats/opendc/OdcVmResourceTableReader.kt
@@ -25,9 +25,11 @@ package org.opendc.trace.formats.opendc
import org.opendc.trace.TableReader
import org.opendc.trace.conv.resourceCpuCapacity
import org.opendc.trace.conv.resourceCpuCount
+import org.opendc.trace.conv.resourceDeadline
import org.opendc.trace.conv.resourceDuration
import org.opendc.trace.conv.resourceID
import org.opendc.trace.conv.resourceMemCapacity
+import org.opendc.trace.conv.resourceNature
import org.opendc.trace.conv.resourceSubmissionTime
import org.opendc.trace.formats.opendc.parquet.Resource
import org.opendc.trace.util.parquet.LocalParquetReader
@@ -62,6 +64,8 @@ internal class OdcVmResourceTableReader(private val reader: LocalParquetReader<R
private val colCpuCount = 3
private val colCpuCapacity = 4
private val colMemCapacity = 5
+ private val colNature = 6
+ private val colDeadline = 7
override fun resolve(name: String): Int {
return when (name) {
@@ -71,13 +75,21 @@ internal class OdcVmResourceTableReader(private val reader: LocalParquetReader<R
resourceCpuCount -> colCpuCount
resourceCpuCapacity -> colCpuCapacity
resourceMemCapacity -> colMemCapacity
+ resourceNature -> colNature
+ resourceDeadline -> colDeadline
else -> -1
}
}
override fun isNull(index: Int): Boolean {
- require(index in 0..colMemCapacity) { "Invalid column index" }
- return false
+ require(index in 0..colDeadline) { "Invalid column index" }
+ val record = checkNotNull(record) { "Reader in invalid state" }
+
+ return when (index) {
+ colNature -> record.nature == null
+ colDeadline -> record.deadline == -1L
+ else -> false
+ }
}
override fun getBoolean(index: Int): Boolean {
@@ -97,6 +109,7 @@ internal class OdcVmResourceTableReader(private val reader: LocalParquetReader<R
val record = checkNotNull(record) { "Reader in invalid state" }
return when (index) {
colDurationTime -> record.durationTime
+ colDeadline -> record.deadline
else -> throw IllegalArgumentException("Invalid column")
}
}
@@ -115,11 +128,12 @@ internal class OdcVmResourceTableReader(private val reader: LocalParquetReader<R
}
}
- override fun getString(index: Int): String {
+ override fun getString(index: Int): String? {
val record = checkNotNull(record) { "Reader in invalid state" }
return when (index) {
colID -> record.id
+ colNature -> record.nature
else -> throw IllegalArgumentException("Invalid column")
}
}
diff --git a/opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/formats/opendc/OdcVmResourceTableWriter.kt b/opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/formats/opendc/OdcVmResourceTableWriter.kt
index 19409fa7..2b8db7f1 100644
--- a/opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/formats/opendc/OdcVmResourceTableWriter.kt
+++ b/opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/formats/opendc/OdcVmResourceTableWriter.kt
@@ -26,9 +26,11 @@ import org.apache.parquet.hadoop.ParquetWriter
import org.opendc.trace.TableWriter
import org.opendc.trace.conv.resourceCpuCapacity
import org.opendc.trace.conv.resourceCpuCount
+import org.opendc.trace.conv.resourceDeadline
import org.opendc.trace.conv.resourceDuration
import org.opendc.trace.conv.resourceID
import org.opendc.trace.conv.resourceMemCapacity
+import org.opendc.trace.conv.resourceNature
import org.opendc.trace.conv.resourceSubmissionTime
import org.opendc.trace.formats.opendc.parquet.Resource
import java.time.Duration
@@ -49,6 +51,8 @@ internal class OdcVmResourceTableWriter(private val writer: ParquetWriter<Resour
private var localCpuCount: Int = 0
private var localCpuCapacity: Double = Double.NaN
private var localMemCapacity: Double = Double.NaN
+ private var localNature: String? = null
+ private var localDeadline: Long = -1
override fun startRow() {
localIsActive = true
@@ -58,12 +62,25 @@ internal class OdcVmResourceTableWriter(private val writer: ParquetWriter<Resour
localCpuCount = 0
localCpuCapacity = Double.NaN
localMemCapacity = Double.NaN
+ localNature = null
+ localDeadline = -1L
}
override fun endRow() {
check(localIsActive) { "No active row" }
localIsActive = false
- writer.write(Resource(localId, localSubmissionTime, localDuration, localCpuCount, localCpuCapacity, localMemCapacity))
+ writer.write(
+ Resource(
+ localId,
+ localSubmissionTime,
+ localDuration,
+ localCpuCount,
+ localCpuCapacity,
+ localMemCapacity,
+ localNature,
+ localDeadline,
+ ),
+ )
}
override fun resolve(name: String): Int {
@@ -74,6 +91,8 @@ internal class OdcVmResourceTableWriter(private val writer: ParquetWriter<Resour
resourceCpuCount -> colCpuCount
resourceCpuCapacity -> colCpuCapacity
resourceMemCapacity -> colMemCapacity
+ resourceNature -> colNature
+ resourceDeadline -> colDeadline
else -> -1
}
}
@@ -103,6 +122,7 @@ internal class OdcVmResourceTableWriter(private val writer: ParquetWriter<Resour
check(localIsActive) { "No active row" }
when (index) {
colDuration -> localDuration = value
+ colDeadline -> localDeadline = value
else -> throw IllegalArgumentException("Invalid column index $index")
}
}
@@ -133,6 +153,7 @@ internal class OdcVmResourceTableWriter(private val writer: ParquetWriter<Resour
check(localIsActive) { "No active row" }
when (index) {
colID -> localId = value
+ colNature -> localNature = value
else -> throw IllegalArgumentException("Invalid column index $index")
}
}
@@ -197,4 +218,6 @@ internal class OdcVmResourceTableWriter(private val writer: ParquetWriter<Resour
private val colCpuCount = 3
private val colCpuCapacity = 4
private val colMemCapacity = 5
+ private val colNature = 6
+ private val colDeadline = 7
}
diff --git a/opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/formats/opendc/OdcVmTraceFormat.kt b/opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/formats/opendc/OdcVmTraceFormat.kt
index b75cf091..e2013182 100644
--- a/opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/formats/opendc/OdcVmTraceFormat.kt
+++ b/opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/formats/opendc/OdcVmTraceFormat.kt
@@ -39,9 +39,11 @@ import org.opendc.trace.conv.TABLE_RESOURCES
import org.opendc.trace.conv.TABLE_RESOURCE_STATES
import org.opendc.trace.conv.resourceCpuCapacity
import org.opendc.trace.conv.resourceCpuCount
+import org.opendc.trace.conv.resourceDeadline
import org.opendc.trace.conv.resourceDuration
import org.opendc.trace.conv.resourceID
import org.opendc.trace.conv.resourceMemCapacity
+import org.opendc.trace.conv.resourceNature
import org.opendc.trace.conv.resourceStateCpuUsage
import org.opendc.trace.conv.resourceStateDuration
import org.opendc.trace.conv.resourceStateTimestamp
@@ -100,6 +102,8 @@ public class OdcVmTraceFormat : TraceFormat {
TableColumn(resourceCpuCount, TableColumnType.Int),
TableColumn(resourceCpuCapacity, TableColumnType.Double),
TableColumn(resourceMemCapacity, TableColumnType.Double),
+ TableColumn(resourceNature, TableColumnType.String),
+ TableColumn(resourceDeadline, TableColumnType.Long),
),
)
TABLE_RESOURCE_STATES ->
diff --git a/opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/formats/opendc/parquet/Resource.kt b/opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/formats/opendc/parquet/Resource.kt
index ea404c65..00922d4f 100644
--- a/opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/formats/opendc/parquet/Resource.kt
+++ b/opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/formats/opendc/parquet/Resource.kt
@@ -34,4 +34,6 @@ internal data class Resource(
val cpuCount: Int,
val cpuCapacity: Double,
val memCapacity: Double,
+ val nature: String? = null,
+ val deadline: Long = -1,
)
diff --git a/opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/formats/opendc/parquet/ResourceReadSupport.kt b/opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/formats/opendc/parquet/ResourceReadSupport.kt
index e5a28a12..75a2bbb2 100644
--- a/opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/formats/opendc/parquet/ResourceReadSupport.kt
+++ b/opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/formats/opendc/parquet/ResourceReadSupport.kt
@@ -33,9 +33,11 @@ import org.apache.parquet.schema.Types
import org.opendc.trace.TableColumn
import org.opendc.trace.conv.resourceCpuCapacity
import org.opendc.trace.conv.resourceCpuCount
+import org.opendc.trace.conv.resourceDeadline
import org.opendc.trace.conv.resourceDuration
import org.opendc.trace.conv.resourceID
import org.opendc.trace.conv.resourceMemCapacity
+import org.opendc.trace.conv.resourceNature
import org.opendc.trace.conv.resourceSubmissionTime
/**
@@ -56,6 +58,8 @@ internal class ResourceReadSupport(private val projection: List<String>?) : Read
"cpu_capacity" to resourceCpuCapacity,
"requiredMemory" to resourceMemCapacity,
"mem_capacity" to resourceMemCapacity,
+ "nature" to resourceNature,
+ "deadline" to resourceDeadline,
)
override fun init(context: InitContext): ReadContext {
@@ -112,6 +116,13 @@ internal class ResourceReadSupport(private val projection: List<String>?) : Read
Types
.required(PrimitiveType.PrimitiveTypeName.INT64)
.named("requiredMemory"),
+ Types
+ .optional(PrimitiveType.PrimitiveTypeName.BINARY)
+ .`as`(LogicalTypeAnnotation.stringType())
+ .named("nature"),
+ Types
+ .optional(PrimitiveType.PrimitiveTypeName.INT64)
+ .named("deadline"),
)
.named("resource")
@@ -142,6 +153,13 @@ internal class ResourceReadSupport(private val projection: List<String>?) : Read
Types
.required(PrimitiveType.PrimitiveTypeName.INT64)
.named("mem_capacity"),
+ Types
+ .optional(PrimitiveType.PrimitiveTypeName.BINARY)
+ .`as`(LogicalTypeAnnotation.stringType())
+ .named("nature"),
+ Types
+ .optional(PrimitiveType.PrimitiveTypeName.INT64)
+ .named("deadline"),
)
.named("resource")
diff --git a/opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/formats/opendc/parquet/ResourceRecordMaterializer.kt b/opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/formats/opendc/parquet/ResourceRecordMaterializer.kt
index 5f02ea1e..866b304e 100644
--- a/opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/formats/opendc/parquet/ResourceRecordMaterializer.kt
+++ b/opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/formats/opendc/parquet/ResourceRecordMaterializer.kt
@@ -43,6 +43,8 @@ internal class ResourceRecordMaterializer(schema: MessageType) : RecordMateriali
private var localCpuCount = 0
private var localCpuCapacity = 0.0
private var localMemCapacity = 0.0
+ private var localNature: String? = null
+ private var localDeadline = -1L
/**
* Root converter for the record.
@@ -95,6 +97,18 @@ internal class ResourceRecordMaterializer(schema: MessageType) : RecordMateriali
localMemCapacity = value.toDouble()
}
}
+ "nature" ->
+ object : PrimitiveConverter() {
+ override fun addBinary(value: Binary) {
+ localNature = value.toStringUsingUTF8()
+ }
+ }
+ "deadline" ->
+ object : PrimitiveConverter() {
+ override fun addLong(value: Long) {
+ localDeadline = value
+ }
+ }
else -> error("Unknown column $type")
}
}
@@ -106,6 +120,8 @@ internal class ResourceRecordMaterializer(schema: MessageType) : RecordMateriali
localCpuCount = 0
localCpuCapacity = 0.0
localMemCapacity = 0.0
+ localNature = null
+ localDeadline = -1
}
override fun end() {}
@@ -121,6 +137,8 @@ internal class ResourceRecordMaterializer(schema: MessageType) : RecordMateriali
localCpuCount,
localCpuCapacity,
localMemCapacity,
+ localNature,
+ localDeadline,
)
override fun getRootConverter(): GroupConverter = root
diff --git a/opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/formats/opendc/parquet/ResourceWriteSupport.kt b/opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/formats/opendc/parquet/ResourceWriteSupport.kt
index e5822b0c..c3e984fb 100644
--- a/opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/formats/opendc/parquet/ResourceWriteSupport.kt
+++ b/opendc-trace/opendc-trace-api/src/main/kotlin/org/opendc/trace/formats/opendc/parquet/ResourceWriteSupport.kt
@@ -83,6 +83,18 @@ internal class ResourceWriteSupport : WriteSupport<Resource>() {
consumer.addLong(record.memCapacity.roundToLong())
consumer.endField("mem_capacity", 5)
+ record.nature?.let {
+ consumer.startField("nature", 6)
+ consumer.addBinary(Binary.fromCharSequence(it))
+ consumer.endField("nature", 6)
+ }
+
+ if (record.deadline != -1L) {
+ consumer.startField("deadline", 7)
+ consumer.addLong(record.deadline)
+ consumer.endField("deadline", 7)
+ }
+
consumer.endMessage()
}
@@ -114,6 +126,13 @@ internal class ResourceWriteSupport : WriteSupport<Resource>() {
Types
.required(PrimitiveType.PrimitiveTypeName.INT64)
.named("mem_capacity"),
+ Types
+ .optional(PrimitiveType.PrimitiveTypeName.BINARY)
+ .`as`(LogicalTypeAnnotation.stringType())
+ .named("nature"),
+ Types
+ .optional(PrimitiveType.PrimitiveTypeName.INT64)
+ .named("deadline"),
)
.named("resource")
}
diff --git a/opendc-trace/opendc-trace-api/src/test/resources/opendc/trace-v2.1/tasks.parquet b/opendc-trace/opendc-trace-api/src/test/resources/opendc/trace-v2.1/tasks.parquet
index 5053a192..c73177e2 100644
--- a/opendc-trace/opendc-trace-api/src/test/resources/opendc/trace-v2.1/tasks.parquet
+++ b/opendc-trace/opendc-trace-api/src/test/resources/opendc/trace-v2.1/tasks.parquet
Binary files differ
diff --git a/opendc-web/opendc-web-runner/Dockerfile b/opendc-web/opendc-web-runner/Dockerfile
index 22c36c65..b3bd0af6 100644
--- a/opendc-web/opendc-web-runner/Dockerfile
+++ b/opendc-web/opendc-web-runner/Dockerfile
@@ -1,4 +1,4 @@
-FROM openjdk:19-slim
+FROM openjdk:21-slim
MAINTAINER OpenDC Maintainers <opendc@atlarge-research.com>
# Obtain (cache) Gradle wrapper
@@ -11,9 +11,10 @@ RUN ./gradlew --version
COPY ./ /app/
RUN ./gradlew --no-daemon :opendc-web:opendc-web-runner:installDist
-FROM openjdk:19-slim
+FROM openjdk:21-slim
COPY --from=0 /app/opendc-web/opendc-web-runner/build/install /opt/
-COPY --from=0 /app/traces /opt/opendc/traces
+COPY --from=0 /app/opendc-experiments/opendc-experiments-base/src/test/resources/workloadTraces \
+ /opt/opendc/traces
WORKDIR /opt/opendc
CMD bin/opendc-web-runner
diff --git a/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/runner/OpenDCRunner.kt b/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/runner/OpenDCRunner.kt
index 7c5c6290..9a1b398e 100644
--- a/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/runner/OpenDCRunner.kt
+++ b/opendc-web/opendc-web-runner/src/main/kotlin/org/opendc/web/runner/OpenDCRunner.kt
@@ -279,10 +279,10 @@ public class OpenDCRunner(
provisioner.runSteps(
setupComputeService(
serviceDomain,
- { createComputeScheduler(scenario.schedulerName, Random(it.seeder.nextLong())) },
+ { createComputeScheduler(scenario.schedulerName, Random(it.seeder.nextLong()), timeSource) },
),
registerComputeMonitor(serviceDomain, monitor),
- setupHosts(serviceDomain, topology, startTime),
+ setupHosts(serviceDomain, topology, listOf(), startTime),
)
val service = provisioner.registry.resolve(serviceDomain, ComputeService::class.java)!!
diff --git a/opendc-web/opendc-web-server/Dockerfile b/opendc-web/opendc-web-server/Dockerfile
index bcdb831e..e6cbaff5 100644
--- a/opendc-web/opendc-web-server/Dockerfile
+++ b/opendc-web/opendc-web-server/Dockerfile
@@ -1,4 +1,4 @@
-FROM openjdk:19-slim
+FROM openjdk:21-slim
MAINTAINER OpenDC Maintainers <opendc@atlarge-research.com>
# Obtain (cache) Gradle wrapper
@@ -19,7 +19,7 @@ ENV OPENDC_AUTH0_DOCS_CLIENT_ID=$OPENDC_AUTH0_DOCS_CLIENT_ID
COPY ./ /app/
RUN ./gradlew --no-daemon :opendc-web:opendc-web-server:quarkusBuild -Dquarkus.profile=docker
-FROM openjdk:19-slim
+FROM openjdk:21-slim
COPY --from=0 /app/opendc-web/opendc-web-server/build/quarkus-app /opt/opendc
WORKDIR /opt/opendc
CMD java -jar quarkus-run.jar
diff --git a/site/docs/getting-started/0-installation.md b/site/docs/getting-started/0-installation.md
index 281e811f..76ffd015 100644
--- a/site/docs/getting-started/0-installation.md
+++ b/site/docs/getting-started/0-installation.md
@@ -14,7 +14,7 @@ quicker (The web server is however missing some more complex features).
1. **Supported Platforms**
OpenDC is actively tested on Windows, macOS and GNU/Linux.
2. **Required Software**
- A Java installation of version 19 or higher is required for OpenDC. You may download the
+ A Java installation of version 21 or higher is required for OpenDC. You may download the
[Java distribution from Oracle](https://www.oracle.com/java/technologies/downloads/) or use the distribution provided
by your package manager.
diff --git a/site/docs/getting-started/4-start-using-intellij.md b/site/docs/getting-started/4-start-using-intellij.md
index eb821088..6aec91f1 100644
--- a/site/docs/getting-started/4-start-using-intellij.md
+++ b/site/docs/getting-started/4-start-using-intellij.md
@@ -10,7 +10,7 @@ First of all you can download IntelliJ here: https://lp.jetbrains.com/intellij-i
git clone git@github.com:atlarge-research/opendc
```
-Check if you have a compatible java version available. Make sure to have one of these versions available: [19, 20, 21]
+Check if you have a compatible java version available. Make sure to have one of these versions available: [21]
If not install a supported version!
diff --git a/traces/bitbrains-small/meta.parquet b/traces/bitbrains-small/meta.parquet
deleted file mode 100644
index 9cded35f..00000000
--- a/traces/bitbrains-small/meta.parquet
+++ /dev/null
Binary files differ
diff --git a/traces/bitbrains-small/trace.parquet b/traces/bitbrains-small/trace.parquet
deleted file mode 100644
index 9d953956..00000000
--- a/traces/bitbrains-small/trace.parquet
+++ /dev/null
Binary files differ