summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorFabian Mastenbroek <mail.fabianm@gmail.com>2021-09-01 11:29:27 +0200
committerFabian Mastenbroek <mail.fabianm@gmail.com>2021-09-02 11:30:15 +0200
commitb2308e1077dc60ec6a4dc646613a4be5b59695a6 (patch)
treea39aa4b5a41bc4616dbe5cade8522221fa6c10af
parent5c6bf9739aa0ffd9651df4fcb4cd46a8545144f0 (diff)
refactor(trace): Implement trace API for WTF reader
This change updates the WTF trace reader to support the new streaming trace API.
-rw-r--r--opendc-experiments/opendc-experiments-capelin/build.gradle.kts4
-rw-r--r--opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/ExperimentHelpers.kt2
-rw-r--r--opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/ParquetTraceReader.kt2
-rw-r--r--opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/RawParquetTraceReader.kt2
-rw-r--r--opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/StreamingParquetTraceReader.kt2
-rw-r--r--opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/TraceEntry.kt (renamed from opendc-format/src/main/kotlin/org/opendc/format/trace/TraceEntry.kt)2
-rw-r--r--opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/TraceReader.kt (renamed from opendc-format/src/main/kotlin/org/opendc/format/trace/TraceReader.kt)4
-rw-r--r--opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/WorkloadSampler.kt1
-rw-r--r--opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/CapelinIntegrationTest.kt2
-rw-r--r--opendc-experiments/opendc-experiments-energy21/build.gradle.kts1
-rw-r--r--opendc-format/src/main/kotlin/org/opendc/format/trace/wtf/WtfTraceReader.kt126
-rw-r--r--opendc-trace/opendc-trace-wtf/build.gradle.kts37
-rw-r--r--opendc-trace/opendc-trace-wtf/src/main/kotlin/org/opendc/trace/wtf/WtfTaskTable.kt64
-rw-r--r--opendc-trace/opendc-trace-wtf/src/main/kotlin/org/opendc/trace/wtf/WtfTaskTableReader.kt116
-rw-r--r--opendc-trace/opendc-trace-wtf/src/main/kotlin/org/opendc/trace/wtf/WtfTrace.kt47
-rw-r--r--opendc-trace/opendc-trace-wtf/src/main/kotlin/org/opendc/trace/wtf/WtfTraceFormat.kt (renamed from opendc-format/src/test/kotlin/org/opendc/format/trace/wtf/WtfTraceReaderTest.kt)32
-rw-r--r--opendc-trace/opendc-trace-wtf/src/main/resources/META-INF/services/org.opendc.trace.spi.TraceFormat1
-rw-r--r--opendc-trace/opendc-trace-wtf/src/test/kotlin/org/opendc/trace/wtf/WtfTraceFormatTest.kt121
-rw-r--r--opendc-trace/opendc-trace-wtf/src/test/resources/wtf-trace/tasks/schema-1.0/part.0.parquet (renamed from opendc-format/src/test/resources/wtf-trace/tasks/schema-1.0/part.0.parquet)bin87475 -> 87475 bytes
-rw-r--r--opendc-web/opendc-web-runner/build.gradle.kts1
-rw-r--r--settings.gradle.kts2
21 files changed, 408 insertions, 161 deletions
diff --git a/opendc-experiments/opendc-experiments-capelin/build.gradle.kts b/opendc-experiments/opendc-experiments-capelin/build.gradle.kts
index 97ca97ec..65cebe1b 100644
--- a/opendc-experiments/opendc-experiments-capelin/build.gradle.kts
+++ b/opendc-experiments/opendc-experiments-capelin/build.gradle.kts
@@ -31,7 +31,6 @@ plugins {
dependencies {
api(platform(projects.opendcPlatform))
api(projects.opendcHarness.opendcHarnessApi)
- implementation(projects.opendcFormat)
implementation(projects.opendcTrace.opendcTraceParquet)
implementation(projects.opendcTrace.opendcTraceBitbrains)
implementation(projects.opendcSimulator.opendcSimulatorCore)
@@ -45,6 +44,9 @@ dependencies {
implementation(libs.config)
implementation(libs.progressbar)
implementation(libs.clikt)
+ implementation(libs.jackson.module.kotlin) {
+ exclude(group = "org.jetbrains.kotlin", module = "kotlin-reflect")
+ }
implementation(libs.parquet)
testImplementation(libs.log4j.slf4j)
diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/ExperimentHelpers.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/ExperimentHelpers.kt
index 20dd603f..46e11056 100644
--- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/ExperimentHelpers.kt
+++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/ExperimentHelpers.kt
@@ -48,7 +48,7 @@ import org.opendc.compute.simulator.SimHost
import org.opendc.experiments.capelin.env.EnvironmentReader
import org.opendc.experiments.capelin.monitor.ExperimentMetricExporter
import org.opendc.experiments.capelin.monitor.ExperimentMonitor
-import org.opendc.format.trace.TraceReader
+import org.opendc.experiments.capelin.trace.TraceReader
import org.opendc.simulator.compute.kernel.SimFairShareHypervisorProvider
import org.opendc.simulator.compute.kernel.interference.VmInterferenceModel
import org.opendc.simulator.compute.power.SimplePowerDriver
diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/ParquetTraceReader.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/ParquetTraceReader.kt
index 0f49ecd2..0bf4ada6 100644
--- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/ParquetTraceReader.kt
+++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/ParquetTraceReader.kt
@@ -24,8 +24,6 @@ package org.opendc.experiments.capelin.trace
import org.opendc.experiments.capelin.model.CompositeWorkload
import org.opendc.experiments.capelin.model.Workload
-import org.opendc.format.trace.TraceEntry
-import org.opendc.format.trace.TraceReader
import org.opendc.simulator.compute.workload.SimWorkload
/**
diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/RawParquetTraceReader.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/RawParquetTraceReader.kt
index 2630784b..24ff0ba1 100644
--- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/RawParquetTraceReader.kt
+++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/RawParquetTraceReader.kt
@@ -23,8 +23,6 @@
package org.opendc.experiments.capelin.trace
import org.apache.avro.generic.GenericData
-import org.opendc.format.trace.TraceEntry
-import org.opendc.format.trace.TraceReader
import org.opendc.simulator.compute.workload.SimTraceWorkload
import org.opendc.simulator.compute.workload.SimWorkload
import org.opendc.trace.util.parquet.LocalParquetReader
diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/StreamingParquetTraceReader.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/StreamingParquetTraceReader.kt
index 9bcbdc75..61e4cab5 100644
--- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/StreamingParquetTraceReader.kt
+++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/StreamingParquetTraceReader.kt
@@ -30,8 +30,6 @@ import org.apache.parquet.filter2.predicate.FilterApi
import org.apache.parquet.filter2.predicate.Statistics
import org.apache.parquet.filter2.predicate.UserDefinedPredicate
import org.apache.parquet.io.api.Binary
-import org.opendc.format.trace.TraceEntry
-import org.opendc.format.trace.TraceReader
import org.opendc.simulator.compute.workload.SimTraceWorkload
import org.opendc.simulator.compute.workload.SimWorkload
import org.opendc.trace.util.parquet.LocalInputFile
diff --git a/opendc-format/src/main/kotlin/org/opendc/format/trace/TraceEntry.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/TraceEntry.kt
index 3ce79d69..303a6a8c 100644
--- a/opendc-format/src/main/kotlin/org/opendc/format/trace/TraceEntry.kt
+++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/TraceEntry.kt
@@ -22,7 +22,7 @@
* SOFTWARE.
*/
-package org.opendc.format.trace
+package org.opendc.experiments.capelin.trace
import java.util.UUID
diff --git a/opendc-format/src/main/kotlin/org/opendc/format/trace/TraceReader.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/TraceReader.kt
index 797a88d5..08304edc 100644
--- a/opendc-format/src/main/kotlin/org/opendc/format/trace/TraceReader.kt
+++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/TraceReader.kt
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2020 AtLarge Research
+ * Copyright (c) 2021 AtLarge Research
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
@@ -20,7 +20,7 @@
* SOFTWARE.
*/
-package org.opendc.format.trace
+package org.opendc.experiments.capelin.trace
/**
* An interface for reading workloads into memory.
diff --git a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/WorkloadSampler.kt b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/WorkloadSampler.kt
index 6de3f265..cb32ce88 100644
--- a/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/WorkloadSampler.kt
+++ b/opendc-experiments/opendc-experiments-capelin/src/main/kotlin/org/opendc/experiments/capelin/trace/WorkloadSampler.kt
@@ -26,7 +26,6 @@ import mu.KotlinLogging
import org.opendc.experiments.capelin.model.CompositeWorkload
import org.opendc.experiments.capelin.model.SamplingStrategy
import org.opendc.experiments.capelin.model.Workload
-import org.opendc.format.trace.TraceEntry
import org.opendc.simulator.compute.workload.SimWorkload
import java.util.*
import kotlin.random.Random
diff --git a/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/CapelinIntegrationTest.kt b/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/CapelinIntegrationTest.kt
index 9d6329d1..2934bbe6 100644
--- a/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/CapelinIntegrationTest.kt
+++ b/opendc-experiments/opendc-experiments-capelin/src/test/kotlin/org/opendc/experiments/capelin/CapelinIntegrationTest.kt
@@ -42,7 +42,7 @@ import org.opendc.experiments.capelin.monitor.ExperimentMonitor
import org.opendc.experiments.capelin.trace.ParquetTraceReader
import org.opendc.experiments.capelin.trace.PerformanceInterferenceReader
import org.opendc.experiments.capelin.trace.RawParquetTraceReader
-import org.opendc.format.trace.TraceReader
+import org.opendc.experiments.capelin.trace.TraceReader
import org.opendc.simulator.compute.kernel.interference.VmInterferenceModel
import org.opendc.simulator.compute.workload.SimWorkload
import org.opendc.simulator.core.runBlockingSimulation
diff --git a/opendc-experiments/opendc-experiments-energy21/build.gradle.kts b/opendc-experiments/opendc-experiments-energy21/build.gradle.kts
index bc05f09b..7d34d098 100644
--- a/opendc-experiments/opendc-experiments-energy21/build.gradle.kts
+++ b/opendc-experiments/opendc-experiments-energy21/build.gradle.kts
@@ -31,7 +31,6 @@ plugins {
dependencies {
api(platform(projects.opendcPlatform))
api(projects.opendcHarness.opendcHarnessApi)
- implementation(projects.opendcFormat)
implementation(projects.opendcSimulator.opendcSimulatorCore)
implementation(projects.opendcSimulator.opendcSimulatorCompute)
implementation(projects.opendcSimulator.opendcSimulatorFailures)
diff --git a/opendc-format/src/main/kotlin/org/opendc/format/trace/wtf/WtfTraceReader.kt b/opendc-format/src/main/kotlin/org/opendc/format/trace/wtf/WtfTraceReader.kt
deleted file mode 100644
index e8e72f0e..00000000
--- a/opendc-format/src/main/kotlin/org/opendc/format/trace/wtf/WtfTraceReader.kt
+++ /dev/null
@@ -1,126 +0,0 @@
-/*
- * Copyright (c) 2020 AtLarge Research
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to deal
- * in the Software without restriction, including without limitation the rights
- * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- * copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in all
- * copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-
-package org.opendc.format.trace.wtf
-
-import org.apache.avro.generic.GenericRecord
-import org.opendc.format.trace.TraceEntry
-import org.opendc.format.trace.TraceReader
-import org.opendc.simulator.compute.workload.SimFlopsWorkload
-import org.opendc.trace.util.parquet.LocalParquetReader
-import org.opendc.workflow.api.Job
-import org.opendc.workflow.api.Task
-import org.opendc.workflow.api.WORKFLOW_TASK_CORES
-import org.opendc.workflow.api.WORKFLOW_TASK_DEADLINE
-import java.io.File
-import java.nio.file.Path
-import java.util.UUID
-import kotlin.math.min
-
-/**
- * A [TraceReader] for the Workflow Trace Format (WTF). See the Workflow Trace Archive
- * (https://wta.atlarge-research.com/) for more information about the format.
- *
- * @param path The path to the trace.
- */
-public class WtfTraceReader(path: Path) : TraceReader<Job> {
- /**
- * The internal iterator to use for this reader.
- */
- private val iterator: Iterator<TraceEntry<Job>>
-
- /**
- * Construct a [TraceReader] from the specified [path].
- *
- * @param path The path to the trace.
- */
- public constructor(path: File) : this(path.toPath())
-
- /**
- * Initialize the reader.
- */
- init {
- val workflows = mutableMapOf<Long, Job>()
- val starts = mutableMapOf<Long, Long>()
- val tasks = mutableMapOf<Long, Task>()
- val taskDependencies = mutableMapOf<Task, List<Long>>()
-
- LocalParquetReader<GenericRecord>(path.resolve("tasks/schema-1.0")).use { reader ->
- while (true) {
- val nextRecord = reader.read() ?: break
-
- val workflowId = nextRecord.get("workflow_id") as Long
- val taskId = nextRecord.get("id") as Long
- val submitTime = nextRecord.get("ts_submit") as Long
- val runtime = nextRecord.get("runtime") as Long
- val cores = (nextRecord.get("resource_amount_requested") as Double).toInt()
-
- @Suppress("UNCHECKED_CAST")
- val dependencies = (nextRecord.get("parents") as ArrayList<GenericRecord>).map {
- it.get("item") as Long
- }
-
- val flops: Long = 4100 * (runtime / 1000) * cores
-
- val workflow = workflows.getOrPut(workflowId) {
- Job(UUID(0L, workflowId), "<unnamed>", HashSet())
- }
- val workload = SimFlopsWorkload(flops)
- val task = Task(
- UUID(0L, taskId),
- "<unnamed>",
- HashSet(),
- mapOf(
- "workload" to workload,
- WORKFLOW_TASK_CORES to cores,
- WORKFLOW_TASK_DEADLINE to runtime
- )
- )
-
- starts.merge(workflowId, submitTime, ::min)
- (workflow.tasks as MutableSet<Task>).add(task)
- tasks[taskId] = task
- taskDependencies[task] = dependencies
- }
- }
-
- // Fix dependencies and dependents for all tasks
- taskDependencies.forEach { (task, dependencies) ->
- (task.dependencies as MutableSet<Task>).addAll(
- dependencies.map { taskId ->
- tasks[taskId] ?: throw IllegalArgumentException("Dependency task with id $taskId not found")
- }
- )
- }
-
- // Create the entry iterator
- iterator = workflows.map { (id, job) -> TraceEntry(job.uid, job.name, starts.getValue(id), job, job.metadata) }
- .sortedBy { it.start }
- .iterator()
- }
-
- override fun hasNext(): Boolean = iterator.hasNext()
-
- override fun next(): TraceEntry<Job> = iterator.next()
-
- override fun close() {}
-}
diff --git a/opendc-trace/opendc-trace-wtf/build.gradle.kts b/opendc-trace/opendc-trace-wtf/build.gradle.kts
new file mode 100644
index 00000000..5051c7b0
--- /dev/null
+++ b/opendc-trace/opendc-trace-wtf/build.gradle.kts
@@ -0,0 +1,37 @@
+/*
+ * Copyright (c) 2021 AtLarge Research
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+description = "Support for Workflow Trace Format (WTF) traces in OpenDC"
+
+/* Build configuration */
+plugins {
+ `kotlin-library-conventions`
+ `testing-conventions`
+ `jacoco-conventions`
+}
+
+dependencies {
+ api(platform(projects.opendcPlatform))
+ api(projects.opendcTrace.opendcTraceApi)
+
+ implementation(projects.opendcTrace.opendcTraceParquet)
+}
diff --git a/opendc-trace/opendc-trace-wtf/src/main/kotlin/org/opendc/trace/wtf/WtfTaskTable.kt b/opendc-trace/opendc-trace-wtf/src/main/kotlin/org/opendc/trace/wtf/WtfTaskTable.kt
new file mode 100644
index 00000000..be26f540
--- /dev/null
+++ b/opendc-trace/opendc-trace-wtf/src/main/kotlin/org/opendc/trace/wtf/WtfTaskTable.kt
@@ -0,0 +1,64 @@
+/*
+ * Copyright (c) 2021 AtLarge Research
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+package org.opendc.trace.wtf
+
+import org.apache.avro.generic.GenericRecord
+import org.opendc.trace.*
+import org.opendc.trace.util.parquet.LocalParquetReader
+import java.nio.file.Path
+
+/**
+ * A [Table] containing the tasks in a GWF trace.
+ */
+internal class WtfTaskTable(private val path: Path) : Table {
+ override val name: String = TABLE_TASKS
+
+ override val isSynthetic: Boolean = false
+
+ override fun isSupported(column: TableColumn<*>): Boolean {
+ return when (column) {
+ TASK_ID -> true
+ TASK_WORKFLOW_ID -> true
+ TASK_SUBMIT_TIME -> true
+ TASK_WAIT_TIME -> true
+ TASK_RUNTIME -> true
+ TASK_REQ_NCPUS -> true
+ TASK_PARENTS -> true
+ TASK_CHILDREN -> true
+ TASK_GROUP_ID -> true
+ TASK_USER_ID -> true
+ else -> false
+ }
+ }
+
+ override fun newReader(): TableReader {
+ val reader = LocalParquetReader<GenericRecord>(path.resolve("tasks/schema-1.0"))
+ return WtfTaskTableReader(reader)
+ }
+
+ override fun newReader(partition: String): TableReader {
+ throw IllegalArgumentException("Invalid partition $partition")
+ }
+
+ override fun toString(): String = "WtfTaskTable"
+}
diff --git a/opendc-trace/opendc-trace-wtf/src/main/kotlin/org/opendc/trace/wtf/WtfTaskTableReader.kt b/opendc-trace/opendc-trace-wtf/src/main/kotlin/org/opendc/trace/wtf/WtfTaskTableReader.kt
new file mode 100644
index 00000000..b6789542
--- /dev/null
+++ b/opendc-trace/opendc-trace-wtf/src/main/kotlin/org/opendc/trace/wtf/WtfTaskTableReader.kt
@@ -0,0 +1,116 @@
+/*
+ * Copyright (c) 2021 AtLarge Research
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+package org.opendc.trace.wtf
+
+import org.apache.avro.generic.GenericRecord
+import org.opendc.trace.*
+import org.opendc.trace.util.parquet.LocalParquetReader
+
+/**
+ * A [TableReader] implementation for the WTF format.
+ */
+internal class WtfTaskTableReader(private val reader: LocalParquetReader<GenericRecord>) : TableReader {
+ /**
+ * The current record.
+ */
+ private var record: GenericRecord? = null
+
+ override fun nextRow(): Boolean {
+ record = reader.read()
+ return record != null
+ }
+
+ override fun hasColumn(column: TableColumn<*>): Boolean {
+ return when (column) {
+ TASK_ID -> true
+ TASK_WORKFLOW_ID -> true
+ TASK_SUBMIT_TIME -> true
+ TASK_WAIT_TIME -> true
+ TASK_RUNTIME -> true
+ TASK_REQ_NCPUS -> true
+ TASK_PARENTS -> true
+ TASK_CHILDREN -> true
+ TASK_GROUP_ID -> true
+ TASK_USER_ID -> true
+ else -> false
+ }
+ }
+
+ override fun <T> get(column: TableColumn<T>): T {
+ val record = checkNotNull(record) { "Reader in invalid state" }
+
+ @Suppress("UNCHECKED_CAST")
+ val res: Any = when (column) {
+ TASK_ID -> record["id"]
+ TASK_WORKFLOW_ID -> record["workflow_id"]
+ TASK_SUBMIT_TIME -> record["ts_submit"]
+ TASK_WAIT_TIME -> record["wait_time"]
+ TASK_RUNTIME -> record["runtime"]
+ TASK_REQ_NCPUS -> (record["resource_amount_requested"] as Double).toInt()
+ TASK_PARENTS -> (record["parents"] as ArrayList<GenericRecord>).map { it["item"] as Long }.toSet()
+ TASK_CHILDREN -> (record["children"] as ArrayList<GenericRecord>).map { it["item"] as Long }.toSet()
+ TASK_GROUP_ID -> record["group_id"]
+ TASK_USER_ID -> record["user_id"]
+ else -> throw IllegalArgumentException("Invalid column")
+ }
+
+ @Suppress("UNCHECKED_CAST")
+ return res as T
+ }
+
+ override fun getBoolean(column: TableColumn<Boolean>): Boolean {
+ throw IllegalArgumentException("Invalid column")
+ }
+
+ override fun getInt(column: TableColumn<Int>): Int {
+ val record = checkNotNull(record) { "Reader in invalid state" }
+
+ return when (column) {
+ TASK_REQ_NCPUS -> (record["resource_amount_requested"] as Double).toInt()
+ TASK_GROUP_ID -> record["group_id"] as Int
+ TASK_USER_ID -> record["user_id"] as Int
+ else -> throw IllegalArgumentException("Invalid column")
+ }
+ }
+
+ override fun getLong(column: TableColumn<Long>): Long {
+ val record = checkNotNull(record) { "Reader in invalid state" }
+
+ return when (column) {
+ TASK_ID -> record["id"] as Long
+ TASK_WORKFLOW_ID -> record["workflow_id"] as Long
+ TASK_SUBMIT_TIME -> record["ts_submit"] as Long
+ TASK_WAIT_TIME -> record["wait_time"] as Long
+ TASK_RUNTIME -> record["runtime"] as Long
+ else -> throw IllegalArgumentException("Invalid column")
+ }
+ }
+
+ override fun getDouble(column: TableColumn<Double>): Double {
+ throw IllegalArgumentException("Invalid column")
+ }
+
+ override fun close() {
+ reader.close()
+ }
+}
diff --git a/opendc-trace/opendc-trace-wtf/src/main/kotlin/org/opendc/trace/wtf/WtfTrace.kt b/opendc-trace/opendc-trace-wtf/src/main/kotlin/org/opendc/trace/wtf/WtfTrace.kt
new file mode 100644
index 00000000..7eff0f5a
--- /dev/null
+++ b/opendc-trace/opendc-trace-wtf/src/main/kotlin/org/opendc/trace/wtf/WtfTrace.kt
@@ -0,0 +1,47 @@
+/*
+ * Copyright (c) 2021 AtLarge Research
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+package org.opendc.trace.wtf
+
+import org.opendc.trace.TABLE_TASKS
+import org.opendc.trace.Table
+import org.opendc.trace.Trace
+import java.nio.file.Path
+
+/**
+ * [Trace] implementation for the WTF format.
+ */
+public class WtfTrace internal constructor(private val path: Path) : Trace {
+ override val tables: List<String> = listOf(TABLE_TASKS)
+
+ override fun containsTable(name: String): Boolean = TABLE_TASKS == name
+
+ override fun getTable(name: String): Table? {
+ if (!containsTable(name)) {
+ return null
+ }
+
+ return WtfTaskTable(path)
+ }
+
+ override fun toString(): String = "SwfTrace[$path]"
+}
diff --git a/opendc-format/src/test/kotlin/org/opendc/format/trace/wtf/WtfTraceReaderTest.kt b/opendc-trace/opendc-trace-wtf/src/main/kotlin/org/opendc/trace/wtf/WtfTraceFormat.kt
index 31ae03e0..781cb335 100644
--- a/opendc-format/src/test/kotlin/org/opendc/format/trace/wtf/WtfTraceReaderTest.kt
+++ b/opendc-trace/opendc-trace-wtf/src/main/kotlin/org/opendc/trace/wtf/WtfTraceFormat.kt
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2020 AtLarge Research
+ * Copyright (c) 2021 AtLarge Research
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
@@ -20,28 +20,22 @@
* SOFTWARE.
*/
-package org.opendc.format.trace.wtf
+package org.opendc.trace.wtf
-import org.junit.jupiter.api.Assertions.*
-import org.junit.jupiter.api.Test
-import java.io.File
+import org.opendc.trace.spi.TraceFormat
+import java.net.URL
+import java.nio.file.Paths
+import kotlin.io.path.exists
/**
- * Test suite for the [WtfTraceReader] class.
+ * A [TraceFormat] implementation for the Workflow Trace Format (WTF).
*/
-class WtfTraceReaderTest {
- /**
- * Smoke test for parsing WTF traces.
- */
- @Test
- fun testParseWtf() {
- val reader = WtfTraceReader(File("src/test/resources/wtf-trace"))
- var entry = reader.next()
- assertEquals(0, entry.start)
- assertEquals(23, entry.workload.tasks.size)
+public class WtfTraceFormat : TraceFormat {
+ override val name: String = "wtf"
- entry = reader.next()
- assertEquals(333387, entry.start)
- assertEquals(23, entry.workload.tasks.size)
+ override fun open(url: URL): WtfTrace {
+ val path = Paths.get(url.toURI())
+ require(path.exists()) { "URL $url does not exist" }
+ return WtfTrace(path)
}
}
diff --git a/opendc-trace/opendc-trace-wtf/src/main/resources/META-INF/services/org.opendc.trace.spi.TraceFormat b/opendc-trace/opendc-trace-wtf/src/main/resources/META-INF/services/org.opendc.trace.spi.TraceFormat
new file mode 100644
index 00000000..32da52ff
--- /dev/null
+++ b/opendc-trace/opendc-trace-wtf/src/main/resources/META-INF/services/org.opendc.trace.spi.TraceFormat
@@ -0,0 +1 @@
+org.opendc.trace.wtf.WtfTraceFormat
diff --git a/opendc-trace/opendc-trace-wtf/src/test/kotlin/org/opendc/trace/wtf/WtfTraceFormatTest.kt b/opendc-trace/opendc-trace-wtf/src/test/kotlin/org/opendc/trace/wtf/WtfTraceFormatTest.kt
new file mode 100644
index 00000000..a05a523e
--- /dev/null
+++ b/opendc-trace/opendc-trace-wtf/src/test/kotlin/org/opendc/trace/wtf/WtfTraceFormatTest.kt
@@ -0,0 +1,121 @@
+/*
+ * Copyright (c) 2021 AtLarge Research
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+package org.opendc.trace.wtf
+
+import org.junit.jupiter.api.Assertions.*
+import org.junit.jupiter.api.Test
+import org.junit.jupiter.api.assertThrows
+import org.opendc.trace.*
+import java.io.File
+import java.net.URL
+
+/**
+ * Test suite for the [WtfTraceFormat] class.
+ */
+class WtfTraceFormatTest {
+ @Test
+ fun testTraceExists() {
+ val input = File("src/test/resources/wtf-trace").toURI().toURL()
+ val format = WtfTraceFormat()
+ org.junit.jupiter.api.assertDoesNotThrow {
+ format.open(input)
+ }
+ }
+
+ @Test
+ fun testTraceDoesNotExists() {
+ val input = File("src/test/resources/wtf-trace").toURI().toURL()
+ val format = WtfTraceFormat()
+ assertThrows<IllegalArgumentException> {
+ format.open(URL(input.toString() + "help"))
+ }
+ }
+
+ @Test
+ fun testTables() {
+ val input = File("src/test/resources/wtf-trace").toURI().toURL()
+ val format = WtfTraceFormat()
+ val trace = format.open(input)
+
+ assertEquals(listOf(TABLE_TASKS), trace.tables)
+ }
+
+ @Test
+ fun testTableExists() {
+ val input = File("src/test/resources/wtf-trace").toURI().toURL()
+ val format = WtfTraceFormat()
+ val table = format.open(input).getTable(TABLE_TASKS)
+
+ assertNotNull(table)
+ org.junit.jupiter.api.assertDoesNotThrow { table!!.newReader() }
+ }
+
+ @Test
+ fun testTableDoesNotExist() {
+ val input = File("src/test/resources/wtf-trace").toURI().toURL()
+ val format = WtfTraceFormat()
+ val trace = format.open(input)
+
+ assertFalse(trace.containsTable("test"))
+ assertNull(trace.getTable("test"))
+ }
+
+ /**
+ * Smoke test for parsing WTF traces.
+ */
+ @Test
+ fun testTableReader() {
+ val input = File("src/test/resources/wtf-trace")
+ val trace = WtfTraceFormat().open(input.toURI().toURL())
+ val reader = trace.getTable(TABLE_TASKS)!!.newReader()
+
+ assertAll(
+ { assertTrue(reader.nextRow()) },
+ { assertEquals(362334516345962206, reader.getLong(TASK_ID)) },
+ { assertEquals(1078341553348591493, reader.getLong(TASK_WORKFLOW_ID)) },
+ { assertEquals(245604, reader.getLong(TASK_SUBMIT_TIME)) },
+ { assertEquals(8163, reader.getLong(TASK_RUNTIME)) },
+ { assertEquals(setOf(584055316413447529, 133113685133695608, 1008582348422865408), reader.get(TASK_PARENTS)) },
+ )
+
+ assertAll(
+ { assertTrue(reader.nextRow()) },
+ { assertEquals(502010169100446658, reader.getLong(TASK_ID)) },
+ { assertEquals(1078341553348591493, reader.getLong(TASK_WORKFLOW_ID)) },
+ { assertEquals(251325, reader.getLong(TASK_SUBMIT_TIME)) },
+ { assertEquals(8216, reader.getLong(TASK_RUNTIME)) },
+ { assertEquals(setOf(584055316413447529, 133113685133695608, 1008582348422865408), reader.get(TASK_PARENTS)) },
+ )
+
+ reader.close()
+ }
+
+ @Test
+ fun testTableReaderPartition() {
+ val input = File("src/test/resources/wtf-trace").toURI().toURL()
+ val format = WtfTraceFormat()
+ val table = format.open(input).getTable(TABLE_TASKS)!!
+
+ assertThrows<IllegalArgumentException> { table.newReader("test") }
+ }
+}
diff --git a/opendc-format/src/test/resources/wtf-trace/tasks/schema-1.0/part.0.parquet b/opendc-trace/opendc-trace-wtf/src/test/resources/wtf-trace/tasks/schema-1.0/part.0.parquet
index d2044038..d2044038 100644
--- a/opendc-format/src/test/resources/wtf-trace/tasks/schema-1.0/part.0.parquet
+++ b/opendc-trace/opendc-trace-wtf/src/test/resources/wtf-trace/tasks/schema-1.0/part.0.parquet
Binary files differ
diff --git a/opendc-web/opendc-web-runner/build.gradle.kts b/opendc-web/opendc-web-runner/build.gradle.kts
index 1f705b79..ec4a4673 100644
--- a/opendc-web/opendc-web-runner/build.gradle.kts
+++ b/opendc-web/opendc-web-runner/build.gradle.kts
@@ -36,7 +36,6 @@ application {
dependencies {
api(platform(projects.opendcPlatform))
implementation(projects.opendcCompute.opendcComputeSimulator)
- implementation(projects.opendcFormat)
implementation(projects.opendcExperiments.opendcExperimentsCapelin)
implementation(projects.opendcSimulator.opendcSimulatorCore)
implementation(projects.opendcTelemetry.opendcTelemetrySdk)
diff --git a/settings.gradle.kts b/settings.gradle.kts
index 73d9a151..5cae0a31 100644
--- a/settings.gradle.kts
+++ b/settings.gradle.kts
@@ -30,7 +30,6 @@ include(":opendc-workflow:opendc-workflow-service")
include(":opendc-faas:opendc-faas-api")
include(":opendc-faas:opendc-faas-service")
include(":opendc-faas:opendc-faas-simulator")
-include(":opendc-format")
include(":opendc-experiments:opendc-experiments-capelin")
include(":opendc-experiments:opendc-experiments-energy21")
include(":opendc-experiments:opendc-experiments-serverless20")
@@ -49,6 +48,7 @@ include(":opendc-telemetry:opendc-telemetry-sdk")
include(":opendc-trace:opendc-trace-api")
include(":opendc-trace:opendc-trace-gwf")
include(":opendc-trace:opendc-trace-swf")
+include(":opendc-trace:opendc-trace-wtf")
include(":opendc-trace:opendc-trace-bitbrains")
include(":opendc-trace:opendc-trace-parquet")
include(":opendc-harness:opendc-harness-api")