summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorFabian Mastenbroek <mail.fabianm@gmail.com>2021-08-31 14:56:08 +0200
committerFabian Mastenbroek <mail.fabianm@gmail.com>2021-09-02 09:26:41 +0200
commit23c1502c2668305fd5f4c38c6c794c985d2037e3 (patch)
treeef97d7332eb64ceae13b09acdcbdd1aad03697a9
parentb0806dcf21ab811c46b715cfdff8a6307e117810 (diff)
refactor(trace): Move GWF trace reader into separate module
This change starts the process of moving the different trace formats into separate modules. This change in particular moves the GWF trace format into a new module, opendc-trace-gwf. Furthermore, this change also implements the trace API for the GWF module.
-rw-r--r--opendc-format/build.gradle.kts2
-rw-r--r--opendc-format/src/main/kotlin/org/opendc/format/trace/gwf/GwfTraceReader.kt176
-rw-r--r--opendc-trace/build.gradle.kts21
-rw-r--r--opendc-trace/opendc-trace-gwf/build.gradle.kts37
-rw-r--r--opendc-trace/opendc-trace-gwf/src/main/kotlin/org/opendc/trace/gwf/GwfTaskTable.kt59
-rw-r--r--opendc-trace/opendc-trace-gwf/src/main/kotlin/org/opendc/trace/gwf/GwfTaskTableReader.kt211
-rw-r--r--opendc-trace/opendc-trace-gwf/src/main/kotlin/org/opendc/trace/gwf/GwfTrace.kt46
-rw-r--r--opendc-trace/opendc-trace-gwf/src/main/kotlin/org/opendc/trace/gwf/GwfTraceFormat.kt56
-rw-r--r--opendc-trace/opendc-trace-gwf/src/main/resources/META-INF/services/org.opendc.trace.spi.TraceFormat1
-rw-r--r--opendc-trace/opendc-trace-gwf/src/test/kotlin/org/opendc/trace/gwf/GwfTraceFormatTest.kt109
-rw-r--r--opendc-trace/opendc-trace-gwf/src/test/resources/trace.gwf71
-rw-r--r--opendc-workflow/opendc-workflow-service/build.gradle.kts6
-rw-r--r--opendc-workflow/opendc-workflow-service/src/test/kotlin/org/opendc/workflow/service/TraceReplayer.kt127
-rw-r--r--opendc-workflow/opendc-workflow-service/src/test/kotlin/org/opendc/workflow/service/WorkflowServiceTest.kt (renamed from opendc-workflow/opendc-workflow-service/src/test/kotlin/org/opendc/workflow/service/WorkflowServiceIntegrationTest.kt)75
-rw-r--r--settings.gradle.kts1
15 files changed, 778 insertions, 220 deletions
diff --git a/opendc-format/build.gradle.kts b/opendc-format/build.gradle.kts
index c8e30846..c1258428 100644
--- a/opendc-format/build.gradle.kts
+++ b/opendc-format/build.gradle.kts
@@ -39,7 +39,7 @@ dependencies {
exclude(group = "org.jetbrains.kotlin", module = "kotlin-reflect")
}
implementation(libs.jackson.dataformat.csv)
- implementation(kotlin("reflect"))
+ implementation("org.jetbrains.kotlin:kotlin-reflect:1.5.30")
/* This configuration is necessary for a slim dependency on Apache Parquet */
implementation(libs.parquet) {
diff --git a/opendc-format/src/main/kotlin/org/opendc/format/trace/gwf/GwfTraceReader.kt b/opendc-format/src/main/kotlin/org/opendc/format/trace/gwf/GwfTraceReader.kt
deleted file mode 100644
index e68afeb7..00000000
--- a/opendc-format/src/main/kotlin/org/opendc/format/trace/gwf/GwfTraceReader.kt
+++ /dev/null
@@ -1,176 +0,0 @@
-/*
- * Copyright (c) 2020 AtLarge Research
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to deal
- * in the Software without restriction, including without limitation the rights
- * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- * copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in all
- * copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-
-package org.opendc.format.trace.gwf
-
-import org.opendc.format.trace.TraceEntry
-import org.opendc.format.trace.TraceReader
-import org.opendc.simulator.compute.workload.SimFlopsWorkload
-import org.opendc.workflow.api.Job
-import org.opendc.workflow.api.Task
-import org.opendc.workflow.api.WORKFLOW_TASK_CORES
-import org.opendc.workflow.api.WORKFLOW_TASK_DEADLINE
-import java.io.BufferedReader
-import java.io.File
-import java.io.InputStream
-import java.util.*
-import kotlin.collections.HashSet
-import kotlin.collections.Iterator
-import kotlin.collections.List
-import kotlin.collections.MutableSet
-import kotlin.collections.component1
-import kotlin.collections.component2
-import kotlin.collections.filter
-import kotlin.collections.forEach
-import kotlin.collections.getOrPut
-import kotlin.collections.map
-import kotlin.collections.mapIndexed
-import kotlin.collections.mapOf
-import kotlin.collections.mutableMapOf
-import kotlin.collections.set
-import kotlin.collections.sortedBy
-import kotlin.collections.toMap
-import kotlin.math.max
-import kotlin.math.min
-
-/**
- * A [TraceReader] for the Grid Workload Format. See the Grid Workloads Archive (http://gwa.ewi.tudelft.nl/) for more
- * information about the format.
- *
- * Be aware that in the Grid Workload Format, workflows are not required to be ordered by submission time and therefore
- * this reader needs to read the whole trace into memory before an entry can be read. Consider converting the trace to a
- * different format for better performance.
- *
- * @param reader The buffered reader to read the trace with.
- */
-public class GwfTraceReader(reader: BufferedReader) : TraceReader<Job> {
- /**
- * The internal iterator to use for this reader.
- */
- private val iterator: Iterator<TraceEntry<Job>>
-
- /**
- * Create a [GwfTraceReader] instance from the specified [File].
- *
- * @param file The file to read from.
- */
- public constructor(file: File) : this(file.bufferedReader())
-
- /**
- * Create a [GwfTraceReader] instance from the specified [InputStream].
- *
- * @param input The input stream to read from.
- */
- public constructor(input: InputStream) : this(input.bufferedReader())
-
- /**
- * Initialize the reader.
- */
- init {
- val workflows = mutableMapOf<Long, Job>()
- val starts = mutableMapOf<Long, Long>()
- val tasks = mutableMapOf<Long, Task>()
- val taskDependencies = mutableMapOf<Task, List<Long>>()
-
- var workflowIdCol = 0
- var taskIdCol = 0
- var submitTimeCol = 0
- var runtimeCol = 0
- var coreCol = 0
- var dependencyCol = 0
-
- try {
- reader.lineSequence()
- .filter { line ->
- // Ignore comments in the trace
- !line.startsWith("#") && line.isNotBlank()
- }
- .forEachIndexed { idx, line ->
- val values = line.split(",")
-
- // Parse GWF header
- if (idx == 0) {
- val header = values.mapIndexed { col, name -> Pair(name.trim(), col) }.toMap()
- workflowIdCol = header["WorkflowID"]!!
- taskIdCol = header["JobID"]!!
- submitTimeCol = header["SubmitTime"]!!
- runtimeCol = header["RunTime"]!!
- coreCol = header["NProcs"]!!
- dependencyCol = header["Dependencies"]!!
- return@forEachIndexed
- }
-
- val workflowId = values[workflowIdCol].trim().toLong()
- val taskId = values[taskIdCol].trim().toLong()
- val submitTime = values[submitTimeCol].trim().toLong() * 1000 // ms
- val runtime = max(0, values[runtimeCol].trim().toLong()) // s
- val cores = values[coreCol].trim().toInt()
- val dependencies = values[dependencyCol].split(" ")
- .filter { it.isNotEmpty() }
- .map { it.trim().toLong() }
-
- val flops: Long = 4000 * runtime * cores
-
- val workflow = workflows.getOrPut(workflowId) {
- Job(UUID(0L, workflowId), "<unnamed>", HashSet())
- }
- val workload = SimFlopsWorkload(flops)
- val task = Task(
- UUID(0L, taskId),
- "<unnamed>",
- HashSet(),
- mapOf(
- "workload" to workload,
- WORKFLOW_TASK_CORES to cores,
- WORKFLOW_TASK_DEADLINE to (runtime * 1000)
- ),
- )
- starts.merge(workflowId, submitTime, ::min)
- (workflow.tasks as MutableSet<Task>).add(task)
- tasks[taskId] = task
- taskDependencies[task] = dependencies
- }
- } finally {
- reader.close()
- }
-
- // Fix dependencies and dependents for all tasks
- taskDependencies.forEach { (task, dependencies) ->
- (task.dependencies as MutableSet<Task>).addAll(
- dependencies.map { taskId ->
- tasks[taskId] ?: throw IllegalArgumentException("Dependency task with id $taskId not found")
- }
- )
- }
-
- // Create the entry iterator
- iterator = workflows.map { (id, job) -> TraceEntry(job.uid, job.name, starts.getValue(id), job, job.metadata) }
- .sortedBy { it.start }
- .iterator()
- }
-
- override fun hasNext(): Boolean = iterator.hasNext()
-
- override fun next(): TraceEntry<Job> = iterator.next()
-
- override fun close() {}
-}
diff --git a/opendc-trace/build.gradle.kts b/opendc-trace/build.gradle.kts
new file mode 100644
index 00000000..7edfd134
--- /dev/null
+++ b/opendc-trace/build.gradle.kts
@@ -0,0 +1,21 @@
+/*
+ * Copyright (c) 2021 AtLarge Research
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
diff --git a/opendc-trace/opendc-trace-gwf/build.gradle.kts b/opendc-trace/opendc-trace-gwf/build.gradle.kts
new file mode 100644
index 00000000..f3dfd6ef
--- /dev/null
+++ b/opendc-trace/opendc-trace-gwf/build.gradle.kts
@@ -0,0 +1,37 @@
+/*
+ * Copyright (c) 2021 AtLarge Research
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+description = "Support for GWF traces in OpenDC"
+
+/* Build configuration */
+plugins {
+ `kotlin-library-conventions`
+ `testing-conventions`
+ `jacoco-conventions`
+}
+
+dependencies {
+ api(platform(projects.opendcPlatform))
+ api(projects.opendcTrace.opendcTraceApi)
+
+ implementation(libs.jackson.dataformat.csv)
+}
diff --git a/opendc-trace/opendc-trace-gwf/src/main/kotlin/org/opendc/trace/gwf/GwfTaskTable.kt b/opendc-trace/opendc-trace-gwf/src/main/kotlin/org/opendc/trace/gwf/GwfTaskTable.kt
new file mode 100644
index 00000000..80a99d10
--- /dev/null
+++ b/opendc-trace/opendc-trace-gwf/src/main/kotlin/org/opendc/trace/gwf/GwfTaskTable.kt
@@ -0,0 +1,59 @@
+/*
+ * Copyright (c) 2021 AtLarge Research
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+package org.opendc.trace.gwf
+
+import com.fasterxml.jackson.dataformat.csv.CsvFactory
+import org.opendc.trace.*
+import java.net.URL
+
+/**
+ * A [Table] containing the tasks in a GWF trace.
+ */
+internal class GwfTaskTable(private val factory: CsvFactory, private val url: URL) : Table {
+ override val name: String = TABLE_TASKS
+
+ override val isSynthetic: Boolean = false
+
+ override fun isSupported(column: TableColumn<*>): Boolean {
+ return when (column) {
+ TASK_WORKFLOW_ID -> true
+ TASK_ID -> true
+ TASK_SUBMIT_TIME -> true
+ TASK_RUNTIME -> true
+ TASK_REQ_NCPUS -> true
+ TASK_ALLOC_NCPUS -> true
+ TASK_PARENTS -> true
+ else -> false
+ }
+ }
+
+ override fun newReader(): TableReader {
+ return GwfTaskTableReader(factory.createParser(url))
+ }
+
+ override fun newReader(partition: String): TableReader {
+ throw IllegalArgumentException("Invalid partition $partition")
+ }
+
+ override fun toString(): String = "GwfTaskTable"
+}
diff --git a/opendc-trace/opendc-trace-gwf/src/main/kotlin/org/opendc/trace/gwf/GwfTaskTableReader.kt b/opendc-trace/opendc-trace-gwf/src/main/kotlin/org/opendc/trace/gwf/GwfTaskTableReader.kt
new file mode 100644
index 00000000..64b7d465
--- /dev/null
+++ b/opendc-trace/opendc-trace-gwf/src/main/kotlin/org/opendc/trace/gwf/GwfTaskTableReader.kt
@@ -0,0 +1,211 @@
+/*
+ * Copyright (c) 2021 AtLarge Research
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+package org.opendc.trace.gwf
+
+import com.fasterxml.jackson.core.JsonToken
+import com.fasterxml.jackson.dataformat.csv.CsvParser
+import com.fasterxml.jackson.dataformat.csv.CsvSchema
+import org.opendc.trace.*
+import java.util.regex.Pattern
+
+/**
+ * A [TableReader] implementation for the GWF format.
+ */
+internal class GwfTaskTableReader(private val parser: CsvParser) : TableReader {
+ /**
+ * The current parser state.
+ */
+ private val state = RowState()
+
+ init {
+ parser.schema = schema
+ }
+
+ override fun nextRow(): Boolean {
+ // Reset the row state
+ state.reset()
+
+ if (!nextStart()) {
+ return false
+ }
+
+ while (true) {
+ val token = parser.nextValue()
+
+ if (token == null || token == JsonToken.END_OBJECT) {
+ break
+ }
+
+ when (parser.currentName) {
+ "WorkflowID" -> state.workflowId = parser.longValue
+ "JobID" -> state.jobId = parser.longValue
+ "SubmitTime" -> state.submitTime = parser.longValue
+ "RunTime" -> state.runtime = parser.longValue
+ "NProcs" -> state.nProcs = parser.intValue
+ "ReqNProcs" -> state.reqNProcs = parser.intValue
+ "Dependencies" -> parseParents(parser.valueAsString)
+ }
+ }
+
+ return true
+ }
+
+ override fun hasColumn(column: TableColumn<*>): Boolean {
+ return when (column) {
+ TASK_WORKFLOW_ID -> true
+ TASK_ID -> true
+ TASK_SUBMIT_TIME -> true
+ TASK_RUNTIME -> true
+ TASK_REQ_NCPUS -> true
+ TASK_ALLOC_NCPUS -> true
+ TASK_PARENTS -> true
+ else -> false
+ }
+ }
+
+ override fun <T> get(column: TableColumn<T>): T {
+ val res: Any = when (column) {
+ TASK_WORKFLOW_ID -> state.workflowId
+ TASK_ID -> state.jobId
+ TASK_SUBMIT_TIME -> state.submitTime
+ TASK_RUNTIME -> state.runtime
+ TASK_REQ_NCPUS -> state.nProcs
+ TASK_ALLOC_NCPUS -> state.reqNProcs
+ TASK_PARENTS -> state.dependencies
+ else -> throw IllegalArgumentException("Invalid column")
+ }
+
+ @Suppress("UNCHECKED_CAST")
+ return res as T
+ }
+
+ override fun getBoolean(column: TableColumn<Boolean>): Boolean {
+ throw IllegalArgumentException("Invalid column")
+ }
+
+ override fun getInt(column: TableColumn<Int>): Int {
+ return when (column) {
+ TASK_REQ_NCPUS -> state.nProcs
+ TASK_ALLOC_NCPUS -> state.reqNProcs
+ else -> throw IllegalArgumentException("Invalid column")
+ }
+ }
+
+ override fun getLong(column: TableColumn<Long>): Long {
+ return when (column) {
+ TASK_WORKFLOW_ID -> state.workflowId
+ TASK_ID -> state.jobId
+ TASK_SUBMIT_TIME -> state.submitTime
+ TASK_RUNTIME -> state.runtime
+ else -> throw IllegalArgumentException("Invalid column")
+ }
+ }
+
+ override fun getDouble(column: TableColumn<Double>): Double {
+ throw IllegalArgumentException("Invalid column")
+ }
+
+ override fun close() {
+ parser.close()
+ }
+
+ /**
+ * The pattern used to parse the parents.
+ */
+ private val pattern = Pattern.compile("\\s+")
+
+ /**
+ * Parse the parents into a set of longs.
+ */
+ private fun parseParents(value: String): Set<Long> {
+ val result = mutableSetOf<Long>()
+ val deps = value.split(pattern)
+
+ for (dep in deps) {
+ if (dep.isBlank()) {
+ continue
+ }
+
+ result.add(dep.toLong(10))
+ }
+
+ return result
+ }
+
+ /**
+ * Advance the parser until the next object start.
+ */
+ private fun nextStart(): Boolean {
+ var token = parser.nextValue()
+
+ while (token != null && token != JsonToken.START_OBJECT) {
+ token = parser.nextValue()
+ }
+
+ return token != null
+ }
+
+ /**
+ * The current row state.
+ */
+ private class RowState {
+ var workflowId = -1L
+ var jobId = -1L
+ var submitTime = -1L
+ var runtime = -1L
+ var nProcs = -1
+ var reqNProcs = -1
+ var dependencies = emptySet<Long>()
+
+ /**
+ * Reset the state.
+ */
+ fun reset() {
+ workflowId = -1
+ jobId = -1
+ submitTime = -1
+ runtime = -1
+ nProcs = -1
+ reqNProcs = -1
+ dependencies = emptySet()
+ }
+ }
+
+ companion object {
+ /**
+ * The [CsvSchema] that is used to parse the trace.
+ */
+ private val schema = CsvSchema.builder()
+ .addColumn("WorkflowID", CsvSchema.ColumnType.NUMBER)
+ .addColumn("JobID", CsvSchema.ColumnType.NUMBER)
+ .addColumn("SubmitTime", CsvSchema.ColumnType.NUMBER)
+ .addColumn("RunTime", CsvSchema.ColumnType.NUMBER)
+ .addColumn("NProcs", CsvSchema.ColumnType.NUMBER)
+ .addColumn("ReqNProcs", CsvSchema.ColumnType.NUMBER)
+ .addColumn("Dependencies", CsvSchema.ColumnType.STRING)
+ .setAllowComments(true)
+ .setUseHeader(true)
+ .setColumnSeparator(',')
+ .build()
+ }
+}
diff --git a/opendc-trace/opendc-trace-gwf/src/main/kotlin/org/opendc/trace/gwf/GwfTrace.kt b/opendc-trace/opendc-trace-gwf/src/main/kotlin/org/opendc/trace/gwf/GwfTrace.kt
new file mode 100644
index 00000000..166c1e56
--- /dev/null
+++ b/opendc-trace/opendc-trace-gwf/src/main/kotlin/org/opendc/trace/gwf/GwfTrace.kt
@@ -0,0 +1,46 @@
+/*
+ * Copyright (c) 2021 AtLarge Research
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+package org.opendc.trace.gwf
+
+import com.fasterxml.jackson.dataformat.csv.CsvFactory
+import org.opendc.trace.*
+import java.net.URL
+
+/**
+ * [Trace] implementation for the GWF format.
+ */
+public class GwfTrace internal constructor(private val factory: CsvFactory, private val url: URL) : Trace {
+ override val tables: List<String> = listOf(TABLE_TASKS)
+
+ override fun containsTable(name: String): Boolean = TABLE_TASKS == name
+
+ override fun getTable(name: String): Table? {
+ if (!containsTable(name)) {
+ return null
+ }
+
+ return GwfTaskTable(factory, url)
+ }
+
+ override fun toString(): String = "GwfTrace[$url]"
+}
diff --git a/opendc-trace/opendc-trace-gwf/src/main/kotlin/org/opendc/trace/gwf/GwfTraceFormat.kt b/opendc-trace/opendc-trace-gwf/src/main/kotlin/org/opendc/trace/gwf/GwfTraceFormat.kt
new file mode 100644
index 00000000..6d542503
--- /dev/null
+++ b/opendc-trace/opendc-trace-gwf/src/main/kotlin/org/opendc/trace/gwf/GwfTraceFormat.kt
@@ -0,0 +1,56 @@
+/*
+ * Copyright (c) 2021 AtLarge Research
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+package org.opendc.trace.gwf
+
+import com.fasterxml.jackson.dataformat.csv.CsvFactory
+import com.fasterxml.jackson.dataformat.csv.CsvParser
+import org.opendc.trace.spi.TraceFormat
+import java.net.URL
+import java.nio.file.Paths
+import kotlin.io.path.exists
+
+/**
+ * A [TraceFormat] implementation for the GWF trace format.
+ */
+public class GwfTraceFormat : TraceFormat {
+ /**
+ * The name of this trace format.
+ */
+ override val name: String = "gwf"
+
+ /**
+ * The [CsvFactory] used to create the parser.
+ */
+ private val factory = CsvFactory()
+ .enable(CsvParser.Feature.ALLOW_COMMENTS)
+ .enable(CsvParser.Feature.TRIM_SPACES)
+
+ /**
+ * Read the tasks in the GWF trace.
+ */
+ public override fun open(url: URL): GwfTrace {
+ val path = Paths.get(url.toURI())
+ require(path.exists()) { "URL $url does not exist" }
+ return GwfTrace(factory, url)
+ }
+}
diff --git a/opendc-trace/opendc-trace-gwf/src/main/resources/META-INF/services/org.opendc.trace.spi.TraceFormat b/opendc-trace/opendc-trace-gwf/src/main/resources/META-INF/services/org.opendc.trace.spi.TraceFormat
new file mode 100644
index 00000000..99a874c8
--- /dev/null
+++ b/opendc-trace/opendc-trace-gwf/src/main/resources/META-INF/services/org.opendc.trace.spi.TraceFormat
@@ -0,0 +1 @@
+org.opendc.trace.gwf.GwfTraceFormat
diff --git a/opendc-trace/opendc-trace-gwf/src/test/kotlin/org/opendc/trace/gwf/GwfTraceFormatTest.kt b/opendc-trace/opendc-trace-gwf/src/test/kotlin/org/opendc/trace/gwf/GwfTraceFormatTest.kt
new file mode 100644
index 00000000..6b0568fe
--- /dev/null
+++ b/opendc-trace/opendc-trace-gwf/src/test/kotlin/org/opendc/trace/gwf/GwfTraceFormatTest.kt
@@ -0,0 +1,109 @@
+/*
+ * Copyright (c) 2021 AtLarge Research
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+package org.opendc.trace.gwf
+
+import org.junit.jupiter.api.Assertions.*
+import org.junit.jupiter.api.Test
+import org.junit.jupiter.api.assertAll
+import org.junit.jupiter.api.assertDoesNotThrow
+import org.junit.jupiter.api.assertThrows
+import org.opendc.trace.*
+import java.net.URL
+
+/**
+ * Test suite for the [GwfTraceFormat] class.
+ */
+internal class GwfTraceFormatTest {
+ @Test
+ fun testTraceExists() {
+ val input = checkNotNull(GwfTraceFormatTest::class.java.getResource("/trace.gwf"))
+ val format = GwfTraceFormat()
+ assertDoesNotThrow {
+ format.open(input)
+ }
+ }
+
+ @Test
+ fun testTraceDoesNotExists() {
+ val input = checkNotNull(GwfTraceFormatTest::class.java.getResource("/trace.gwf"))
+ val format = GwfTraceFormat()
+ assertThrows<IllegalArgumentException> {
+ format.open(URL(input.toString() + "help"))
+ }
+ }
+
+ @Test
+ fun testTables() {
+ val input = checkNotNull(GwfTraceFormatTest::class.java.getResource("/trace.gwf"))
+ val format = GwfTraceFormat()
+ val trace = format.open(input)
+
+ assertEquals(listOf(TABLE_TASKS), trace.tables)
+ }
+
+ @Test
+ fun testTableExists() {
+ val input = checkNotNull(GwfTraceFormatTest::class.java.getResource("/trace.gwf"))
+ val format = GwfTraceFormat()
+ val table = format.open(input).getTable(TABLE_TASKS)
+
+ assertNotNull(table)
+ assertDoesNotThrow { table!!.newReader() }
+ }
+
+ @Test
+ fun testTableDoesNotExist() {
+ val input = checkNotNull(GwfTraceFormatTest::class.java.getResource("/trace.gwf"))
+ val format = GwfTraceFormat()
+ val trace = format.open(input)
+
+ assertFalse(trace.containsTable("test"))
+ assertNull(trace.getTable("test"))
+ }
+
+ @Test
+ fun testTableReader() {
+ val input = checkNotNull(GwfTraceFormatTest::class.java.getResource("/trace.gwf"))
+ val format = GwfTraceFormat()
+ val table = format.open(input).getTable(TABLE_TASKS)!!
+ val reader = table.newReader()
+
+ assertAll(
+ { assertTrue(reader.nextRow()) },
+ { assertEquals(0L, reader.getLong(TASK_WORKFLOW_ID)) },
+ { assertEquals(1L, reader.getLong(TASK_ID)) },
+ { assertEquals(16, reader.getLong(TASK_SUBMIT_TIME)) },
+ { assertEquals(11, reader.getLong(TASK_RUNTIME)) },
+ { assertEquals(setOf<Long>(), reader.get(TASK_PARENTS)) },
+ )
+ }
+
+ @Test
+ fun testTableReaderPartition() {
+ val input = checkNotNull(GwfTraceFormatTest::class.java.getResource("/trace.gwf"))
+ val format = GwfTraceFormat()
+ val table = format.open(input).getTable(TABLE_TASKS)!!
+
+ assertThrows<IllegalArgumentException> { table.newReader("test") }
+ }
+}
diff --git a/opendc-trace/opendc-trace-gwf/src/test/resources/trace.gwf b/opendc-trace/opendc-trace-gwf/src/test/resources/trace.gwf
new file mode 100644
index 00000000..2f99616d
--- /dev/null
+++ b/opendc-trace/opendc-trace-gwf/src/test/resources/trace.gwf
@@ -0,0 +1,71 @@
+WorkflowID, JobID , SubmitTime, RunTime , NProcs , ReqNProcs , Dependencies
+0 , 1 , 16 , 11 , 1 , 1 ,
+0 , 2 , 40 , 11 , 1 , 1 , 1
+0 , 3 , 40 , 11 , 1 , 1 , 1
+0 , 4 , 64 , 11 , 1 , 1 , 2
+0 , 5 , 63 , 11 , 1 , 1 , 3
+0 , 6 , 64 , 11 , 1 , 1 , 3
+0 , 7 , 87 , 11 , 1 , 1 , 4 5 6
+1 , 8 , 4 , 11 , 1 , 1 ,
+1 , 9 , 15 , 11 , 1 , 1 , 8
+1 , 10 , 15 , 11 , 1 , 1 , 8
+1 , 11 , 27 , 11 , 1 , 1 , 9
+1 , 12 , 27 , 11 , 1 , 1 , 10
+1 , 13 , 27 , 11 , 1 , 1 , 10
+1 , 14 , 38 , 11 , 1 , 1 , 12 11 13
+2 , 15 , 3 , 11 , 1 , 1 ,
+2 , 16 , 27 , 11 , 1 , 1 , 15
+2 , 17 , 27 , 11 , 1 , 1 , 15
+2 , 18 , 52 , 11 , 1 , 1 , 16
+2 , 19 , 51 , 11 , 1 , 1 , 17
+2 , 20 , 51 , 11 , 1 , 1 , 17
+2 , 21 , 75 , 11 , 1 , 1 , 20 18 19
+3 , 22 , 3 , 11 , 1 , 1 ,
+3 , 23 , 27 , 11 , 1 , 1 , 22
+3 , 24 , 27 , 11 , 1 , 1 , 22
+3 , 25 , 51 , 11 , 1 , 1 , 23
+3 , 26 , 50 , 11 , 1 , 1 , 24
+3 , 27 , 51 , 11 , 1 , 1 , 24
+3 , 28 , 75 , 11 , 1 , 1 , 25 27 26
+4 , 29 , 3 , 11 , 1 , 1 ,
+4 , 30 , 27 , 11 , 1 , 1 , 29
+4 , 31 , 27 , 11 , 1 , 1 , 29
+4 , 32 , 50 , 11 , 1 , 1 , 30
+4 , 33 , 50 , 11 , 1 , 1 , 31
+4 , 34 , 51 , 11 , 1 , 1 , 31
+4 , 35 , 74 , 11 , 1 , 1 , 33 32 34
+5 , 36 , 3 , 11 , 1 , 1 ,
+5 , 37 , 27 , 11 , 1 , 1 , 36
+5 , 38 , 26 , 11 , 1 , 1 , 36
+5 , 39 , 51 , 11 , 1 , 1 , 37
+5 , 40 , 50 , 11 , 1 , 1 , 38
+5 , 41 , 50 , 11 , 1 , 1 , 38
+5 , 42 , 74 , 11 , 1 , 1 , 39 40 41
+6 , 43 , 4 , 11 , 1 , 1 ,
+6 , 44 , 27 , 11 , 1 , 1 , 43
+6 , 45 , 27 , 11 , 1 , 1 , 43
+6 , 46 , 51 , 11 , 1 , 1 , 44
+6 , 47 , 51 , 11 , 1 , 1 , 45
+6 , 48 , 51 , 11 , 1 , 1 , 45
+6 , 49 , 75 , 11 , 1 , 1 , 46 47 48
+7 , 50 , 3 , 0 , 1 , 1 ,
+7 , 51 , 17 , 0 , 1 , 1 , 50
+7 , 52 , 17 , 0 , 1 , 1 , 50
+7 , 53 , 30 , 0 , 1 , 1 , 51
+7 , 54 , 30 , 0 , 1 , 1 , 52
+7 , 55 , 31 , 0 , 1 , 1 , 52
+7 , 56 , 44 , 0 , 1 , 1 , 55 54 53
+8 , 57 , 3 , 11 , 1 , 1 ,
+8 , 58 , 26 , 11 , 1 , 1 , 57
+8 , 59 , 27 , 11 , 1 , 1 , 57
+8 , 60 , 50 , 11 , 1 , 1 , 58
+8 , 61 , 51 , 11 , 1 , 1 , 59
+8 , 62 , 50 , 11 , 1 , 1 , 59
+8 , 63 , 74 , 11 , 1 , 1 , 62 61 60
+9 , 64 , 3 , 11 , 1 , 1 ,
+9 , 65 , 27 , 11 , 1 , 1 , 64
+9 , 66 , 27 , 11 , 1 , 1 , 64
+9 , 67 , 51 , 11 , 1 , 1 , 65
+9 , 68 , 50 , 11 , 1 , 1 , 66
+9 , 69 , 51 , 11 , 1 , 1 , 66
+9 , 70 , 74 , 11 , 1 , 1 , 68 69 67
diff --git a/opendc-workflow/opendc-workflow-service/build.gradle.kts b/opendc-workflow/opendc-workflow-service/build.gradle.kts
index bc082dbc..941202d2 100644
--- a/opendc-workflow/opendc-workflow-service/build.gradle.kts
+++ b/opendc-workflow/opendc-workflow-service/build.gradle.kts
@@ -39,11 +39,7 @@ dependencies {
testImplementation(projects.opendcSimulator.opendcSimulatorCore)
testImplementation(projects.opendcCompute.opendcComputeSimulator)
- testImplementation(projects.opendcFormat)
+ testImplementation(projects.opendcTrace.opendcTraceGwf)
testImplementation(projects.opendcTelemetry.opendcTelemetrySdk)
- testImplementation(libs.jackson.module.kotlin) {
- exclude(group = "org.jetbrains.kotlin", module = "kotlin-reflect")
- }
- testImplementation(kotlin("reflect"))
testRuntimeOnly(libs.log4j.slf4j)
}
diff --git a/opendc-workflow/opendc-workflow-service/src/test/kotlin/org/opendc/workflow/service/TraceReplayer.kt b/opendc-workflow/opendc-workflow-service/src/test/kotlin/org/opendc/workflow/service/TraceReplayer.kt
new file mode 100644
index 00000000..a390fe08
--- /dev/null
+++ b/opendc-workflow/opendc-workflow-service/src/test/kotlin/org/opendc/workflow/service/TraceReplayer.kt
@@ -0,0 +1,127 @@
+/*
+ * Copyright (c) 2021 AtLarge Research
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+package org.opendc.workflow.service
+
+import kotlinx.coroutines.coroutineScope
+import kotlinx.coroutines.delay
+import kotlinx.coroutines.launch
+import org.opendc.simulator.compute.workload.SimFlopsWorkload
+import org.opendc.trace.*
+import org.opendc.workflow.api.Job
+import org.opendc.workflow.api.Task
+import org.opendc.workflow.api.WORKFLOW_TASK_CORES
+import org.opendc.workflow.api.WORKFLOW_TASK_DEADLINE
+import java.time.Clock
+import java.util.*
+import kotlin.collections.HashMap
+import kotlin.collections.HashSet
+import kotlin.math.max
+import kotlin.math.min
+
+/**
+ * Helper tool to replay workflow trace.
+ */
+internal class TraceReplayer(private val trace: Trace) {
+ /**
+ * Replay the workload.
+ */
+ public suspend fun replay(clock: Clock, service: WorkflowService) {
+ val jobs = parseTrace(trace)
+
+ // Sort jobs by their arrival time
+ (jobs as MutableList<Job>).sortBy { it.metadata["WORKFLOW_SUBMIT_TIME"] as Long }
+
+ // Wait until the trace is started
+ val startTime = jobs[0].metadata["WORKFLOW_SUBMIT_TIME"] as Long
+ delay(min(0L, startTime - clock.millis()))
+
+ val offset = startTime - clock.millis()
+
+ coroutineScope {
+ for (job in jobs) {
+ val submitTime = job.metadata["WORKFLOW_SUBMIT_TIME"] as Long
+ delay(max(0, (submitTime - offset) - clock.millis()))
+
+ launch { service.run(job) }
+ }
+ }
+ }
+
+ /**
+ * Convert [trace] into a list of [Job]s that can be submitted to the workflow service.
+ */
+ public fun parseTrace(trace: Trace): List<Job> {
+ val table = checkNotNull(trace.getTable(TABLE_TASKS))
+ val reader = table.newReader()
+
+ val jobs = mutableMapOf<Long, Job>()
+ val tasks = mutableMapOf<Long, Task>()
+ val taskDependencies = mutableMapOf<Task, Set<Long>>()
+
+ try {
+ while (reader.nextRow()) {
+ // Bag of tasks without workflow ID all share the same workflow
+ val workflowId = if (reader.hasColumn(TASK_WORKFLOW_ID)) reader.getLong(TASK_WORKFLOW_ID) else 0L
+ val workflow = jobs.computeIfAbsent(workflowId) { id -> Job(UUID(0L, id), "<unnamed>", HashSet(), HashMap()) }
+
+ val id = reader.getLong(TASK_ID)
+ val grantedCpus = if (reader.hasColumn(TASK_ALLOC_NCPUS))
+ reader.getInt(TASK_ALLOC_NCPUS)
+ else
+ reader.getInt(TASK_REQ_NCPUS)
+ val submitTime = reader.getLong(TASK_SUBMIT_TIME)
+ val runtime = reader.getLong(TASK_RUNTIME)
+ val flops: Long = 4000 * runtime * grantedCpus
+ val workload = SimFlopsWorkload(flops)
+ val task = Task(
+ UUID(0L, id),
+ "<unnamed>",
+ HashSet(),
+ mapOf(
+ "workload" to workload,
+ WORKFLOW_TASK_CORES to grantedCpus,
+ WORKFLOW_TASK_DEADLINE to (runtime * 1000)
+ ),
+ )
+
+ tasks[id] = task
+ taskDependencies[task] = reader.get(TASK_PARENTS)
+
+ (workflow.metadata as MutableMap<String, Any>).merge("WORKFLOW_SUBMIT_TIME", submitTime) { a, b -> min(a as Long, b as Long) }
+ (workflow.tasks as MutableSet<Task>).add(task)
+ }
+
+ // Resolve dependencies for all tasks
+ for ((task, deps) in taskDependencies) {
+ for (dep in deps) {
+ val parent = requireNotNull(tasks[dep]) { "Dependency task with id $dep not found" }
+ (task.dependencies as MutableSet<Task>).add(parent)
+ }
+ }
+ } finally {
+ reader.close()
+ }
+
+ return jobs.values.toList()
+ }
+}
diff --git a/opendc-workflow/opendc-workflow-service/src/test/kotlin/org/opendc/workflow/service/WorkflowServiceIntegrationTest.kt b/opendc-workflow/opendc-workflow-service/src/test/kotlin/org/opendc/workflow/service/WorkflowServiceTest.kt
index d82959e7..07433d1f 100644
--- a/opendc-workflow/opendc-workflow-service/src/test/kotlin/org/opendc/workflow/service/WorkflowServiceIntegrationTest.kt
+++ b/opendc-workflow/opendc-workflow-service/src/test/kotlin/org/opendc/workflow/service/WorkflowServiceTest.kt
@@ -25,9 +25,6 @@ package org.opendc.workflow.service
import io.opentelemetry.api.metrics.MeterProvider
import io.opentelemetry.sdk.metrics.SdkMeterProvider
import io.opentelemetry.sdk.metrics.export.MetricProducer
-import kotlinx.coroutines.coroutineScope
-import kotlinx.coroutines.delay
-import kotlinx.coroutines.launch
import org.junit.jupiter.api.Assertions.assertEquals
import org.junit.jupiter.api.DisplayName
import org.junit.jupiter.api.Test
@@ -39,25 +36,28 @@ import org.opendc.compute.service.scheduler.filters.RamFilter
import org.opendc.compute.service.scheduler.filters.VCpuFilter
import org.opendc.compute.service.scheduler.weights.VCpuWeigher
import org.opendc.compute.simulator.SimHost
-import org.opendc.format.environment.sc18.Sc18EnvironmentReader
-import org.opendc.format.trace.gwf.GwfTraceReader
import org.opendc.simulator.compute.kernel.SimSpaceSharedHypervisorProvider
+import org.opendc.simulator.compute.model.MachineModel
+import org.opendc.simulator.compute.model.MemoryUnit
+import org.opendc.simulator.compute.model.ProcessingNode
+import org.opendc.simulator.compute.model.ProcessingUnit
import org.opendc.simulator.core.runBlockingSimulation
import org.opendc.simulator.resources.SimResourceInterpreter
import org.opendc.telemetry.sdk.toOtelClock
+import org.opendc.trace.gwf.GwfTraceFormat
import org.opendc.workflow.service.internal.WorkflowServiceImpl
import org.opendc.workflow.service.scheduler.WorkflowSchedulerMode
import org.opendc.workflow.service.scheduler.job.NullJobAdmissionPolicy
import org.opendc.workflow.service.scheduler.job.SubmissionTimeJobOrderPolicy
import org.opendc.workflow.service.scheduler.task.NullTaskEligibilityPolicy
import org.opendc.workflow.service.scheduler.task.SubmissionTimeTaskOrderPolicy
-import kotlin.math.max
+import java.util.*
/**
* Integration test suite for the [WorkflowServiceImpl].
*/
@DisplayName("WorkflowService")
-internal class WorkflowServiceIntegrationTest {
+internal class WorkflowServiceTest {
/**
* A large integration test where we check whether all tasks in some trace are executed correctly.
*/
@@ -69,20 +69,20 @@ internal class WorkflowServiceIntegrationTest {
.build()
val interpreter = SimResourceInterpreter(coroutineContext, clock)
- val hosts = Sc18EnvironmentReader(checkNotNull(object {}.javaClass.getResourceAsStream("/environment.json")))
- .use { it.read() }
- .map { def ->
- SimHost(
- def.uid,
- def.name,
- def.model,
- def.meta,
- coroutineContext,
- interpreter,
- MeterProvider.noop().get("opendc-compute-simulator"),
- SimSpaceSharedHypervisorProvider()
- )
- }
+ val machineModel = createMachineModel()
+ val hvProvider = SimSpaceSharedHypervisorProvider()
+ val hosts = List(4) { id ->
+ SimHost(
+ UUID(0, id.toLong()),
+ "node-$id",
+ machineModel,
+ emptyMap(),
+ coroutineContext,
+ interpreter,
+ meterProvider.get("opendc-compute-simulator"),
+ hvProvider,
+ )
+ }
val meter = MeterProvider.noop().get("opendc-compute")
val computeScheduler = FilterScheduler(
@@ -105,23 +105,10 @@ internal class WorkflowServiceIntegrationTest {
taskOrderPolicy = SubmissionTimeTaskOrderPolicy(),
)
- val reader = GwfTraceReader(checkNotNull(object {}.javaClass.getResourceAsStream("/trace.gwf")))
- var offset = Long.MIN_VALUE
-
- coroutineScope {
- while (reader.hasNext()) {
- val entry = reader.next()
+ val trace = GwfTraceFormat().open(checkNotNull(WorkflowServiceTest::class.java.getResource("/trace.gwf")))
+ val replayer = TraceReplayer(trace)
- if (offset < 0) {
- offset = entry.start - clock.millis()
- }
-
- delay(max(0, (entry.start - offset) - clock.millis()))
- launch {
- scheduler.run(entry.workload)
- }
- }
- }
+ replayer.replay(clock, scheduler)
hosts.forEach(SimHost::close)
scheduler.close()
@@ -134,10 +121,22 @@ internal class WorkflowServiceIntegrationTest {
{ assertEquals(0, metrics.jobsActive, "Not all submitted jobs started") },
{ assertEquals(metrics.jobsSubmitted, metrics.jobsFinished, "Not all started jobs finished") },
{ assertEquals(0, metrics.tasksActive, "Not all started tasks finished") },
- { assertEquals(metrics.tasksSubmitted, metrics.tasksFinished, "Not all started tasks finished") }
+ { assertEquals(metrics.tasksSubmitted, metrics.tasksFinished, "Not all started tasks finished") },
+ { assertEquals(33213237L, clock.millis()) }
)
}
+ /**
+ * The machine model based on: https://www.spec.org/power_ssj2008/results/res2020q1/power_ssj2008-20191125-01012.html
+ */
+ private fun createMachineModel(): MachineModel {
+ val node = ProcessingNode("AMD", "am64", "EPYC 7742", 32)
+ val cpus = List(node.coreCount) { id -> ProcessingUnit(node, id, 3400.0) }
+ val memory = List(8) { MemoryUnit("Samsung", "Unknown", 2933.0, 16_000) }
+
+ return MachineModel(cpus, memory)
+ }
+
class WorkflowMetrics {
var jobsSubmitted = 0L
var jobsActive = 0L
diff --git a/settings.gradle.kts b/settings.gradle.kts
index 36c1d9e0..e941356e 100644
--- a/settings.gradle.kts
+++ b/settings.gradle.kts
@@ -47,6 +47,7 @@ include(":opendc-simulator:opendc-simulator-failures")
include(":opendc-telemetry:opendc-telemetry-api")
include(":opendc-telemetry:opendc-telemetry-sdk")
include(":opendc-trace:opendc-trace-api")
+include(":opendc-trace:opendc-trace-gwf")
include(":opendc-harness:opendc-harness-api")
include(":opendc-harness:opendc-harness-engine")
include(":opendc-harness:opendc-harness-cli")