diff options
Diffstat (limited to 'opendc-experiments/opendc-experiments-base/src/jmh/kotlin/org')
8 files changed, 745 insertions, 93 deletions
diff --git a/opendc-experiments/opendc-experiments-base/src/jmh/kotlin/org/opendc/experiments/CIBenchmark.kt b/opendc-experiments/opendc-experiments-base/src/jmh/kotlin/org/opendc/experiments/CIBenchmark.kt new file mode 100644 index 00000000..a719b9f3 --- /dev/null +++ b/opendc-experiments/opendc-experiments-base/src/jmh/kotlin/org/opendc/experiments/CIBenchmark.kt @@ -0,0 +1,56 @@ +/* + * Copyright (c) 2026 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.experiments + +import org.opendc.experiments.base.runner.ExperimentCommand +import org.openjdk.jmh.annotations.Benchmark +import org.openjdk.jmh.annotations.BenchmarkMode +import org.openjdk.jmh.annotations.Fork +import org.openjdk.jmh.annotations.Measurement +import org.openjdk.jmh.annotations.Mode +import org.openjdk.jmh.annotations.OutputTimeUnit +import org.openjdk.jmh.annotations.Scope +import org.openjdk.jmh.annotations.State +import org.openjdk.jmh.annotations.Warmup +import java.io.File +import java.util.concurrent.TimeUnit + +@State(Scope.Thread) +@Fork(1) +@Warmup(iterations = 1, batchSize = 1) +@Measurement(iterations = 5, batchSize = 1) +@BenchmarkMode(Mode.SingleShotTime) +@OutputTimeUnit(TimeUnit.MILLISECONDS) +open class CIBenchmark : OpenDCBenchmark() { + @Benchmark + fun benchmark1() { + ExperimentCommand().main(arrayOf("--experiment-path", "src/jmh/resources/experiments/experiment_100.json")) + File("output").deleteRecursively() + } + + @Benchmark + fun benchmark2() { + ExperimentCommand().main(arrayOf("--experiment-path", "src/jmh/resources/experiments/experiment_2.json")) + File("output").deleteRecursively() + } +} diff --git a/opendc-experiments/opendc-experiments-base/src/jmh/kotlin/org/opendc/experiments/FragmentScalingBenchmark.kt b/opendc-experiments/opendc-experiments-base/src/jmh/kotlin/org/opendc/experiments/FragmentScalingBenchmark.kt new file mode 100644 index 00000000..3da4946a --- /dev/null +++ b/opendc-experiments/opendc-experiments-base/src/jmh/kotlin/org/opendc/experiments/FragmentScalingBenchmark.kt @@ -0,0 +1,97 @@ +/* + * Copyright (c) 2026 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.experiments + +import org.opendc.experiments.base.runner.ExperimentCommand +import org.openjdk.jmh.annotations.Benchmark +import org.openjdk.jmh.annotations.BenchmarkMode +import org.openjdk.jmh.annotations.Fork +import org.openjdk.jmh.annotations.Measurement +import org.openjdk.jmh.annotations.Mode +import org.openjdk.jmh.annotations.OutputTimeUnit +import org.openjdk.jmh.annotations.Scope +import org.openjdk.jmh.annotations.State +import org.openjdk.jmh.annotations.Warmup +import java.io.File +import java.util.concurrent.TimeUnit + +/** + * benchmark that measures simulation performance for workloads with + * increasing number of fragments from 1 to 10k. + * + * Each benchmark runs a full workload and measures runtime and [HeapStats] + * (average and peak heap). + * + * Every benchmark simulates a workload consisting of 1000 tasks. + * However, the number of fragments increases from 1 to 10k. + * + * The following variables are all fixed: + * - **Topology**: 100-host cluster (`topologies/taskScaling/topology_100.json`) + * - **Allocation policy**: memory-based (`Mem`) + * - **Export interval**: set to 999999 to make the simulator export only at the end of the simulation, + * to minimize the impact of telemetry export on the results. + * + * JFR profiling and heap statistics collection are provided by [OpenDCBenchmark]. + */ +@State(Scope.Thread) +@Fork(1) +@Warmup(iterations = 1, batchSize = 1) +@Measurement(iterations = 5, batchSize = 1) +@BenchmarkMode(Mode.SingleShotTime) +@OutputTimeUnit(TimeUnit.MILLISECONDS) +open class FragmentScalingBenchmark : OpenDCBenchmark() { + /** Runs the simulation with a workload containing 1 fragment. */ + @Benchmark + fun fragmentScalingBenchmark1() { + ExperimentCommand().main(arrayOf("--experiment-path", "src/jmh/resources/experiments/fragmentScaling/experiment_1.json")) + File("output").deleteRecursively() + } + + /** Runs the simulation with a workload containing 10 fragments. */ + @Benchmark + fun fragmentScalingBenchmark10() { + ExperimentCommand().main(arrayOf("--experiment-path", "src/jmh/resources/experiments/fragmentScaling/experiment_10.json")) + File("output").deleteRecursively() + } + + /** Runs the simulation with a workload containing 100 fragments. */ + @Benchmark + fun fragmentScalingBenchmark100() { + ExperimentCommand().main(arrayOf("--experiment-path", "src/jmh/resources/experiments/fragmentScaling/experiment_100.json")) + File("output").deleteRecursively() + } + + /** Runs the simulation with a workload containing 1 000 fragments. */ + @Benchmark + fun fragmentScalingBenchmark1K() { + ExperimentCommand().main(arrayOf("--experiment-path", "src/jmh/resources/experiments/fragmentScaling/experiment_1K.json")) + File("output").deleteRecursively() + } + + /** Runs the simulation with a workload containing 10 000 fragments. */ + @Benchmark + fun fragmentScalingBenchmark10K() { + ExperimentCommand().main(arrayOf("--experiment-path", "src/jmh/resources/experiments/fragmentScaling/experiment_10K.json")) + File("output").deleteRecursively() + } +} diff --git a/opendc-experiments/opendc-experiments-base/src/jmh/kotlin/org/opendc/experiments/JfrHeapAnalyzer.kt b/opendc-experiments/opendc-experiments-base/src/jmh/kotlin/org/opendc/experiments/JfrHeapAnalyzer.kt new file mode 100644 index 00000000..ab57b736 --- /dev/null +++ b/opendc-experiments/opendc-experiments-base/src/jmh/kotlin/org/opendc/experiments/JfrHeapAnalyzer.kt @@ -0,0 +1,100 @@ +/* + * Copyright (c) 2026 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.experiments + +import jdk.jfr.consumer.RecordingFile +import java.nio.file.Path +import kotlin.math.sqrt + +/** + * Heap usage statistics derived from a single JFR recording. + * + * All memory values are in megabytes and are computed from the `heapUsed` field + * of `jdk.GCHeapSummary` events captured during one JMH iteration. + * + * @property avgMb Mean heap usage across all GC samples in the recording (MB). + * @property maxMb Peak heap usage observed in any single GC sample (MB). + * @property stdMb Population standard deviation of heap usage across all samples (MB). + * @property sampleCount Number of `jdk.GCHeapSummary` events found in the recording. + */ +data class HeapStats( + val avgMb: Double, + val maxMb: Double, + val stdMb: Double, + val sampleCount: Int, +) + +/** + * Parses a JFR recording file and computes heap usage statistics from its + * `jdk.GCHeapSummary` events. + * + * Each `jdk.GCHeapSummary` event is emitted by the JVM after a GC cycle and + * carries the `heapUsed` value (bytes). This function collects all such values, + * then computes mean, peak, and standard deviation, converting the results from + * bytes to megabytes. + * + * @param jfrPath Path to the `.jfr` recording file to analyze. + * @return A [HeapStats] instance if at least one `jdk.GCHeapSummary` event was + * found, or `null` if the recording contained no such events (e.g., no GC + * occurred during the iteration). + */ +fun analyzeHeap(jfrPath: Path): HeapStats? { + val samples = mutableListOf<Long>() + + RecordingFile(jfrPath).use { recording -> + while (recording.hasMoreEvents()) { + val event = recording.readEvent() + if (event.eventType.name == "jdk.GCHeapSummary") { + samples.add(event.getLong("heapUsed")) + } + } + } + + if (samples.isEmpty()) return null + + val avg = samples.average() + val max = samples.max().toDouble() + val std = sqrt(samples.sumOf { (it - avg).let { d -> d * d } } / samples.size) + val toMb = 1.0 / (1024.0 * 1024.0) + + return HeapStats(avg * toMb, max * toMb, std * toMb, samples.size) +} + +/** + * Standalone entry point for inspecting the heap statistics of an existing JFR file. + * + * Reads `build/bench.jfr` (the default output path used by [OpenDCBenchmark]) and + * prints a human-readable summary to stdout. Useful for quickly inspecting the + * recording from the most recently completed benchmark iteration without re-running + * the full benchmark. + * + * Exits with an error if the file contains no `jdk.GCHeapSummary` events. + */ +fun main() { + val path = Path.of("build/bench.jfr") + val stats = analyzeHeap(path) ?: error("No jdk.GCHeapSummary events found in $path") + println("Heap statistics from $path (${stats.sampleCount} GC samples):") + println(" Avg: ${"%.2f".format(stats.avgMb)} MB") + println(" Max: ${"%.2f".format(stats.maxMb)} MB") + println(" Std: ${"%.2f".format(stats.stdMb)} MB") +} diff --git a/opendc-experiments/opendc-experiments-base/src/jmh/kotlin/org/opendc/experiments/OpenDCBenchmark.kt b/opendc-experiments/opendc-experiments-base/src/jmh/kotlin/org/opendc/experiments/OpenDCBenchmark.kt new file mode 100644 index 00000000..d8b5c887 --- /dev/null +++ b/opendc-experiments/opendc-experiments-base/src/jmh/kotlin/org/opendc/experiments/OpenDCBenchmark.kt @@ -0,0 +1,145 @@ +/* + * Copyright (c) 2026 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.experiments + +import jdk.jfr.Configuration +import jdk.jfr.Recording +import org.openjdk.jmh.annotations.Level +import org.openjdk.jmh.annotations.Setup +import org.openjdk.jmh.annotations.TearDown +import org.openjdk.jmh.infra.BenchmarkParams +import org.openjdk.jmh.infra.IterationParams +import org.openjdk.jmh.runner.IterationType +import java.io.File +import java.nio.file.Path +import kotlin.math.sqrt + +/** Computes the population standard deviation of the receiver list. */ +private fun List<Double>.std(): Double { + val avg = average() + return sqrt(sumOf { (it - avg) * (it - avg) } / size) +} + +/** + * Abstract base class for JMH benchmarks in OpenDC. + * + * This base class provides: + * + * **JFR profiling** — a Java Flight Recorder session using the built-in `profile` + * configuration is started before every JMH iteration and stopped afterwards. + * The recording is written to `build/bench.jfr` and overwritten each iteration, + * so only the most-recent recording is kept on disk. + * + * **Heap statistics** — after each *measurement* iteration the JFR file is processed + * to get the average and peak heap usage. At the end of the trial, per-iteration [HeapStats] + * are aggregated (mean ± std-dev). The jmh task configuration in [build.gradle.kts](../build.gradle.kts) + * is set up to merge these heap stats into the final JSON report as a + * `heapMetric` field for each benchmark entry. + * + * ### Subclassing + * Concrete benchmark classes should: + * 1. Annotate the class with the desired JMH mode/time-unit annotations. See [CIBenchmark] as example. + * 2. Implement one or more `@Benchmark` methods. + * 3. Add any extra `@Setup` / `@TearDown` methods as needed; the lifecycle callbacks + * defined here run at [Level.Iteration] and [Level.Trial] respectively. + * + * @see analyzeHeap + * @see HeapStats + */ +abstract class OpenDCBenchmark { + /** Active JFR recording for the current iteration, or `null` between iterations. */ + private var recording: Recording? = null + + /** Destination path for the JFR file; overwritten on every iteration. */ + private val jfrPath = Path.of("build/bench.jfr") + + /** Accumulated heap statistics, one entry per completed measurement iteration. */ + private val heapResults = mutableListOf<HeapStats>() + + /** + * Starts a JFR recording before each iteration. + * + * Uses the JDK's built-in `profile` configuration, which captures GC heap + * summaries, CPU load, thread activity, and other standard events needed for + * heap analysis. + */ + @Setup(Level.Iteration) + fun setupIteration() { + recording = Recording(Configuration.getConfiguration("profile")) + recording!!.setDestination(jfrPath) + recording!!.start() + } + + /** + * Stops the JFR recording after each iteration and, for measurement iterations, + * parses the resulting file to extract [HeapStats]. + * + * Warmup iterations are skipped so that only steady-state heap behaviour is + * included in the final CSV report. + * + * @param params JMH-injected iteration metadata; used to distinguish warmup + * from measurement iterations via [IterationParams.type]. + */ + @TearDown(Level.Iteration) + fun tearDownIteration(params: IterationParams) { + recording?.stop() + recording = null + if (params.type == IterationType.MEASUREMENT) { + analyzeHeap(jfrPath)?.let { heapResults.add(it) } + } + } + + /** + * Aggregates heap statistics across all measurement iterations and appends a + * summary row to `build/heap-stats.csv`. + * + * The CSV columns are: + * `benchmark, avg_heap_mb, std_avg_heap_mb, max_heap_mb, std_max_heap_mb` + * + * If no measurement iterations produced heap data (e.g., the JFR file was + * empty), the method returns without writing anything. + * + * @param params JMH-injected trial metadata; provides the fully-qualified + * benchmark name used as the first CSV column. + */ + @TearDown(Level.Trial) + fun tearDownTrial(params: BenchmarkParams) { + if (heapResults.isEmpty()) return + + val avgs = heapResults.map { it.avgMb } + val maxes = heapResults.map { it.maxMb } + + val line = + "\"${params.benchmark}\"," + + "${"%.4f".format(avgs.average())}," + + "${"%.4f".format(avgs.std())}," + + "${"%.4f".format(maxes.average())}," + + "${"%.4f".format(maxes.std())}\n" + + println(line) + + val heapFile = File("build/heap-stats.csv") + heapFile.parentFile.mkdirs() + heapFile.appendText(line) + } +} diff --git a/opendc-experiments/opendc-experiments-base/src/jmh/kotlin/org/opendc/experiments/SamplingScalingBenchmark.kt b/opendc-experiments/opendc-experiments-base/src/jmh/kotlin/org/opendc/experiments/SamplingScalingBenchmark.kt new file mode 100644 index 00000000..4c2b7e9e --- /dev/null +++ b/opendc-experiments/opendc-experiments-base/src/jmh/kotlin/org/opendc/experiments/SamplingScalingBenchmark.kt @@ -0,0 +1,178 @@ +/* + * Copyright (c) 2026 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.experiments + +import org.opendc.experiments.base.runner.ExperimentCommand +import org.openjdk.jmh.annotations.Benchmark +import org.openjdk.jmh.annotations.BenchmarkMode +import org.openjdk.jmh.annotations.Fork +import org.openjdk.jmh.annotations.Measurement +import org.openjdk.jmh.annotations.Mode +import org.openjdk.jmh.annotations.OutputTimeUnit +import org.openjdk.jmh.annotations.Scope +import org.openjdk.jmh.annotations.State +import org.openjdk.jmh.annotations.Warmup +import java.io.File +import java.util.concurrent.TimeUnit + +/** + * JMH benchmark that measures the cost of telemetry export as the number of + * export samples scales from 1 to 360 000. + * + * The number suffix in each benchmark method name is the number of export samples + * produced during the simulation. This is controlled via the `exportInterval` in + * the experiment JSON: a smaller interval means more frequent sampling and therefore + * more data written. The workload and topology are fixed across all tiers so that + * only the export overhead changes: + * - **Topology**: 100-host cluster (`topologies/taskScaling/topology_100.json`) + * - **Workload**: 10 000-task trace (`workloadTraces/taskScaling/10K_tasks`) + * - **Allocation policy**: memory-based (`Mem`) + * + * JFR profiling and heap statistics collection are provided by [OpenDCBenchmark]. + */ +@State(Scope.Thread) +@Fork(1) +@Warmup(iterations = 1, batchSize = 1) +@Measurement(iterations = 5, batchSize = 1) +@BenchmarkMode(Mode.SingleShotTime) +@OutputTimeUnit(TimeUnit.MILLISECONDS) +open class SamplingScalingBenchmark : OpenDCBenchmark() { + /** Runs the simulation producing 1 export sample. */ + @Benchmark + fun samplingScalingBenchmark1() { + ExperimentCommand().main(arrayOf("--experiment-path", "src/jmh/resources/experiments/samplingScaling/experiment_1.json")) + File("output").deleteRecursively() + } + + /** Runs the simulation producing 10 export samples. */ + @Benchmark + fun samplingScalingBenchmark10() { + ExperimentCommand().main(arrayOf("--experiment-path", "src/jmh/resources/experiments/samplingScaling/experiment_10.json")) + File("output").deleteRecursively() + } + + /** Runs the simulation producing 100 export samples. */ + @Benchmark + fun samplingScalingBenchmark100() { + ExperimentCommand().main(arrayOf("--experiment-path", "src/jmh/resources/experiments/samplingScaling/experiment_100.json")) + File("output").deleteRecursively() + } + + /** Runs the simulation producing 1 000 export samples. */ + @Benchmark + fun samplingScalingBenchmark1K() { + ExperimentCommand().main(arrayOf("--experiment-path", "src/jmh/resources/experiments/samplingScaling/experiment_1K.json")) + File("output").deleteRecursively() + } + + /** Runs the simulation producing 5 000 export samples. */ + @Benchmark + fun samplingScalingBenchmark5K() { + ExperimentCommand().main(arrayOf("--experiment-path", "src/jmh/resources/experiments/samplingScaling/experiment_5K.json")) + File("output").deleteRecursively() + } + + /** Runs the simulation producing 10 000 export samples. */ + @Benchmark + fun samplingScalingBenchmark10K() { + ExperimentCommand().main(arrayOf("--experiment-path", "src/jmh/resources/experiments/samplingScaling/experiment_10K.json")) + File("output").deleteRecursively() + } + + /** Runs the simulation producing 20 000 export samples. */ + @Benchmark + fun samplingScalingBenchmark20K() { + ExperimentCommand().main(arrayOf("--experiment-path", "src/jmh/resources/experiments/samplingScaling/experiment_20K.json")) + File("output").deleteRecursively() + } + + /** Runs the simulation producing 30 000 export samples. */ + @Benchmark + fun samplingScalingBenchmark30K() { + ExperimentCommand().main(arrayOf("--experiment-path", "src/jmh/resources/experiments/samplingScaling/experiment_30K.json")) + File("output").deleteRecursively() + } + + /** Runs the simulation producing 40 000 export samples. */ + @Benchmark + fun samplingScalingBenchmark40K() { + ExperimentCommand().main(arrayOf("--experiment-path", "src/jmh/resources/experiments/samplingScaling/experiment_40K.json")) + File("output").deleteRecursively() + } + + /** Runs the simulation producing 50 000 export samples. */ + @Benchmark + fun samplingScalingBenchmark50K() { + ExperimentCommand().main(arrayOf("--experiment-path", "src/jmh/resources/experiments/samplingScaling/experiment_50K.json")) + File("output").deleteRecursively() + } + + /** Runs the simulation producing 60 000 export samples. */ + @Benchmark + fun samplingScalingBenchmark60K() { + ExperimentCommand().main(arrayOf("--experiment-path", "src/jmh/resources/experiments/samplingScaling/experiment_60K.json")) + File("output").deleteRecursively() + } + + /** Runs the simulation producing 70 000 export samples. */ + @Benchmark + fun samplingScalingBenchmark70K() { + ExperimentCommand().main(arrayOf("--experiment-path", "src/jmh/resources/experiments/samplingScaling/experiment_70K.json")) + File("output").deleteRecursively() + } + + /** Runs the simulation producing 80 000 export samples. */ + @Benchmark + fun samplingScalingBenchmark80K() { + ExperimentCommand().main(arrayOf("--experiment-path", "src/jmh/resources/experiments/samplingScaling/experiment_80K.json")) + File("output").deleteRecursively() + } + + /** Runs the simulation producing 90 000 export samples. */ + @Benchmark + fun samplingScalingBenchmark90K() { + ExperimentCommand().main(arrayOf("--experiment-path", "src/jmh/resources/experiments/samplingScaling/experiment_90K.json")) + File("output").deleteRecursively() + } + + /** Runs the simulation producing 100 000 export samples. */ + @Benchmark + fun samplingScalingBenchmark100K() { + ExperimentCommand().main(arrayOf("--experiment-path", "src/jmh/resources/experiments/samplingScaling/experiment_100K.json")) + File("output").deleteRecursively() + } + + /** Runs the simulation producing 200 000 export samples. */ + @Benchmark + fun samplingScalingBenchmark200K() { + ExperimentCommand().main(arrayOf("--experiment-path", "src/jmh/resources/experiments/samplingScaling/experiment_200K.json")) + File("output").deleteRecursively() + } + + /** Runs the simulation producing 360 000 export samples. */ + @Benchmark + fun samplingScalingBenchmark360K() { + ExperimentCommand().main(arrayOf("--experiment-path", "src/jmh/resources/experiments/samplingScaling/experiment_360K.json")) + File("output").deleteRecursively() + } +} diff --git a/opendc-experiments/opendc-experiments-base/src/jmh/kotlin/org/opendc/experiments/TaskScalingBenchmark.kt b/opendc-experiments/opendc-experiments-base/src/jmh/kotlin/org/opendc/experiments/TaskScalingBenchmark.kt new file mode 100644 index 00000000..4cacd09c --- /dev/null +++ b/opendc-experiments/opendc-experiments-base/src/jmh/kotlin/org/opendc/experiments/TaskScalingBenchmark.kt @@ -0,0 +1,98 @@ +/* + * Copyright (c) 2026 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.experiments + +import org.opendc.experiments.base.runner.ExperimentCommand +import org.openjdk.jmh.annotations.Benchmark +import org.openjdk.jmh.annotations.BenchmarkMode +import org.openjdk.jmh.annotations.Fork +import org.openjdk.jmh.annotations.Measurement +import org.openjdk.jmh.annotations.Mode +import org.openjdk.jmh.annotations.OutputTimeUnit +import org.openjdk.jmh.annotations.Scope +import org.openjdk.jmh.annotations.State +import org.openjdk.jmh.annotations.Warmup +import java.io.File +import java.util.concurrent.TimeUnit + +/** + * JMH benchmark that measures simulation performance as the number of scheduled + * tasks scales from 100 to 5 000 000. All tasks have a single fragment. + * + * Each benchmark method runs a full simulation with an increasing task count while + * keeping all other variables fixed: + * - **Topology**: 100-host cluster (`topologies/taskScaling/topology_100.json`) + * - **Allocation policy**: memory-based (`Mem`) + * - **Export interval**: set to 999999 to make the simulator export only at the end of the simulation, + * + * JFR profiling and heap statistics collection are provided by [OpenDCBenchmark]. + */ +@State(Scope.Thread) +@Fork(1) +@Warmup(iterations = 1, batchSize = 1) +@Measurement(iterations = 5, batchSize = 1) +@BenchmarkMode(Mode.SingleShotTime) +@OutputTimeUnit(TimeUnit.MILLISECONDS) +open class TaskScalingBenchmark : OpenDCBenchmark() { + /** Runs the simulation with a workload of 100 tasks. */ + @Benchmark + fun taskScalingBenchmark100() { + ExperimentCommand().main(arrayOf("--experiment-path", "src/jmh/resources/experiments/taskScaling/experiment_100.json")) + File("output").deleteRecursively() + } + + /** Runs the simulation with a workload of 1 000 tasks. */ + @Benchmark + fun taskScalingBenchmark1K() { + ExperimentCommand().main(arrayOf("--experiment-path", "src/jmh/resources/experiments/taskScaling/experiment_1K.json")) + File("output").deleteRecursively() + } + + /** Runs the simulation with a workload of 10 000 tasks. */ + @Benchmark + fun taskScalingBenchmark10K() { + ExperimentCommand().main(arrayOf("--experiment-path", "src/jmh/resources/experiments/taskScaling/experiment_10K.json")) + File("output").deleteRecursively() + } + + /** Runs the simulation with a workload of 100 000 tasks. */ + @Benchmark + fun taskScalingBenchmark100K() { + ExperimentCommand().main(arrayOf("--experiment-path", "src/jmh/resources/experiments/taskScaling/experiment_100K.json")) + File("output").deleteRecursively() + } + + /** Runs the simulation with a workload of 1 000 000 tasks. */ + @Benchmark + fun taskScalingBenchmark1M() { + ExperimentCommand().main(arrayOf("--experiment-path", "src/jmh/resources/experiments/taskScaling/experiment_1M.json")) + File("output").deleteRecursively() + } + + /** Runs the simulation with a workload of 2 500 000 tasks. */ + @Benchmark + fun taskScalingBenchmark2M500K() { + ExperimentCommand().main(arrayOf("--experiment-path", "src/jmh/resources/experiments/taskScaling/experiment_2_5M.json")) + File("output").deleteRecursively() + } +} diff --git a/opendc-experiments/opendc-experiments-base/src/jmh/kotlin/org/opendc/experiments/WorkloadBenchmark.kt b/opendc-experiments/opendc-experiments-base/src/jmh/kotlin/org/opendc/experiments/WorkloadBenchmark.kt new file mode 100644 index 00000000..98423cd3 --- /dev/null +++ b/opendc-experiments/opendc-experiments-base/src/jmh/kotlin/org/opendc/experiments/WorkloadBenchmark.kt @@ -0,0 +1,71 @@ +/* + * Copyright (c) 2026 AtLarge Research + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package org.opendc.experiments + +import org.opendc.experiments.base.runner.ExperimentCommand +import org.openjdk.jmh.annotations.Benchmark +import org.openjdk.jmh.annotations.BenchmarkMode +import org.openjdk.jmh.annotations.Fork +import org.openjdk.jmh.annotations.Measurement +import org.openjdk.jmh.annotations.Mode +import org.openjdk.jmh.annotations.OutputTimeUnit +import org.openjdk.jmh.annotations.Scope +import org.openjdk.jmh.annotations.State +import org.openjdk.jmh.annotations.Warmup +import java.io.File +import java.util.concurrent.TimeUnit + +/** + * JMH benchmark that measures simulation performance across real-world workload + * datasets of varying size and duration, with telemetry export enabled. + * + * JFR profiling and heap statistics collection are provided by [OpenDCBenchmark]. + */ +@State(Scope.Thread) +@Fork(1) +@Warmup(iterations = 1, batchSize = 1) +@Measurement(iterations = 5, batchSize = 1) +@BenchmarkMode(Mode.SingleShotTime) +@OutputTimeUnit(TimeUnit.MILLISECONDS) +open class WorkloadBenchmark : OpenDCBenchmark() { + /** Runs the simulation with the SURF one-week workload trace. */ + @Benchmark + fun surfWeekBenchmark() { + ExperimentCommand().main(arrayOf("--experiment-path", "src/jmh/resources/experiments/workloadScaling/surf_week.json")) + File("output").deleteRecursively() + } + + /** Runs the simulation with the SURF one-month workload trace. */ + @Benchmark + fun surfMonthBenchmark() { + ExperimentCommand().main(arrayOf("--experiment-path", "src/jmh/resources/experiments/workloadScaling/surf_month.json")) + File("output").deleteRecursively() + } + + /** Runs the simulation with the SURF half-year workload trace. */ + @Benchmark + fun surfHalfYearBenchmark() { + ExperimentCommand().main(arrayOf("--experiment-path", "src/jmh/resources/experiments/workloadScaling/surf_halfyear.json")) + File("output").deleteRecursively() + } +} diff --git a/opendc-experiments/opendc-experiments-base/src/jmh/kotlin/org/opendc/experiments/capelin/CapelinBenchmarks.kt b/opendc-experiments/opendc-experiments-base/src/jmh/kotlin/org/opendc/experiments/capelin/CapelinBenchmarks.kt deleted file mode 100644 index c3408226..00000000 --- a/opendc-experiments/opendc-experiments-base/src/jmh/kotlin/org/opendc/experiments/capelin/CapelinBenchmarks.kt +++ /dev/null @@ -1,93 +0,0 @@ -/* - * Copyright (c) 2021 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.experiments.capelin - -import org.opendc.compute.service.ComputeService -import org.opendc.compute.service.scheduler.FilterScheduler -import org.opendc.compute.service.scheduler.filters.ComputeFilter -import org.opendc.compute.service.scheduler.filters.RamFilter -import org.opendc.compute.service.scheduler.filters.VCpuFilter -import org.opendc.compute.service.scheduler.weights.CoreRamWeigher -import org.opendc.experiments.capelin.topology.clusterTopology -import org.opendc.experiments.compute.ComputeWorkloadLoader -import org.opendc.experiments.compute.VirtualMachine -import org.opendc.experiments.compute.replay -import org.opendc.experiments.compute.setupComputeService -import org.opendc.experiments.compute.setupHosts -import org.opendc.experiments.compute.topology.HostSpec -import org.opendc.experiments.compute.trace -import org.opendc.common.experiments.provisioner.Provisioner -import org.opendc.simulator.kotlin.runSimulation -import org.openjdk.jmh.annotations.Benchmark -import org.openjdk.jmh.annotations.Fork -import org.openjdk.jmh.annotations.Measurement -import org.openjdk.jmh.annotations.Param -import org.openjdk.jmh.annotations.Scope -import org.openjdk.jmh.annotations.Setup -import org.openjdk.jmh.annotations.State -import org.openjdk.jmh.annotations.Warmup -import java.io.File -import java.util.Random -import java.util.concurrent.TimeUnit - -/** - * Benchmark suite for the Capelin experiments. - */ -@State(Scope.Thread) -@Fork(1) -@Warmup(iterations = 2, time = 5, timeUnit = TimeUnit.SECONDS) -@Measurement(iterations = 5, time = 5, timeUnit = TimeUnit.SECONDS) -class CapelinBenchmarks { - private lateinit var vms: List<VirtualMachine> - private lateinit var topology: List<HostSpec> - - @Param("true", "false") - private var isOptimized: Boolean = false - - @Setup - fun setUp() { - val loader = ComputeWorkloadLoader(File("src/test/resources/trace")) - vms = trace("bitbrains-small").resolve(loader, Random(1L)) - topology = checkNotNull(object {}.javaClass.getResourceAsStream("/topology.txt")).use { clusterTopology(it) } - } - - @Benchmark - fun benchmarkCapelin() = runSimulation { - val serviceDomain = "compute.opendc.org" - - Provisioner(dispatcher, seed = 0).use { provisioner -> - val computeScheduler = FilterScheduler( - filters = listOf(ComputeFilter(), VCpuFilter(16.0), RamFilter(1.0)), - weighers = listOf(CoreRamWeigher(multiplier = 1.0)) - ) - - provisioner.runSteps( - setupComputeService(serviceDomain, { computeScheduler }), - setupHosts(serviceDomain, topology, optimize = isOptimized) - ) - - val service = provisioner.registry.resolve(serviceDomain, ComputeService::class.java)!! - service.replay(timeSource, vms, 0L, interference = true) - } - } -} |