refactor(trace): Implement trace API for WTF reader

This change updates the WTF trace reader to support the new streaming
trace API.

8 files changed, 427 insertions, 0 deletions

diff --git a/opendc-trace/opendc-trace-wtf/build.gradle.kts b/opendc-trace/opendc-trace-wtf/build.gradle.kts
new file mode 100644
index 00000000..5051c7b0
--- /dev/null
+++ b/opendc-trace/opendc-trace-wtf/build.gradle.kts
@@ -0,0 +1,37 @@
+/*
+ * Copyright (c) 2021 AtLarge Research
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+description = "Support for Workflow Trace Format (WTF) traces in OpenDC"
+
+/* Build configuration */
+plugins {
+    `kotlin-library-conventions`
+    `testing-conventions`
+    `jacoco-conventions`
+}
+
+dependencies {
+    api(platform(projects.opendcPlatform))
+    api(projects.opendcTrace.opendcTraceApi)
+
+    implementation(projects.opendcTrace.opendcTraceParquet)
+}
diff --git a/opendc-trace/opendc-trace-wtf/src/main/kotlin/org/opendc/trace/wtf/WtfTaskTable.kt b/opendc-trace/opendc-trace-wtf/src/main/kotlin/org/opendc/trace/wtf/WtfTaskTable.kt
new file mode 100644
index 00000000..be26f540
--- /dev/null
+++ b/opendc-trace/opendc-trace-wtf/src/main/kotlin/org/opendc/trace/wtf/WtfTaskTable.kt
@@ -0,0 +1,64 @@
+/*
+ * Copyright (c) 2021 AtLarge Research
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+package org.opendc.trace.wtf
+
+import org.apache.avro.generic.GenericRecord
+import org.opendc.trace.*
+import org.opendc.trace.util.parquet.LocalParquetReader
+import java.nio.file.Path
+
+/**
+ * A [Table] containing the tasks in a GWF trace.
+ */
+internal class WtfTaskTable(private val path: Path) : Table {
+    override val name: String = TABLE_TASKS
+
+    override val isSynthetic: Boolean = false
+
+    override fun isSupported(column: TableColumn<*>): Boolean {
+        return when (column) {
+            TASK_ID -> true
+            TASK_WORKFLOW_ID -> true
+            TASK_SUBMIT_TIME -> true
+            TASK_WAIT_TIME -> true
+            TASK_RUNTIME -> true
+            TASK_REQ_NCPUS -> true
+            TASK_PARENTS -> true
+            TASK_CHILDREN -> true
+            TASK_GROUP_ID -> true
+            TASK_USER_ID -> true
+            else -> false
+        }
+    }
+
+    override fun newReader(): TableReader {
+        val reader = LocalParquetReader<GenericRecord>(path.resolve("tasks/schema-1.0"))
+        return WtfTaskTableReader(reader)
+    }
+
+    override fun newReader(partition: String): TableReader {
+        throw IllegalArgumentException("Invalid partition $partition")
+    }
+
+    override fun toString(): String = "WtfTaskTable"
+}
diff --git a/opendc-trace/opendc-trace-wtf/src/main/kotlin/org/opendc/trace/wtf/WtfTaskTableReader.kt b/opendc-trace/opendc-trace-wtf/src/main/kotlin/org/opendc/trace/wtf/WtfTaskTableReader.kt
new file mode 100644
index 00000000..b6789542
--- /dev/null
+++ b/opendc-trace/opendc-trace-wtf/src/main/kotlin/org/opendc/trace/wtf/WtfTaskTableReader.kt
@@ -0,0 +1,116 @@
+/*
+ * Copyright (c) 2021 AtLarge Research
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+package org.opendc.trace.wtf
+
+import org.apache.avro.generic.GenericRecord
+import org.opendc.trace.*
+import org.opendc.trace.util.parquet.LocalParquetReader
+
+/**
+ * A [TableReader] implementation for the WTF format.
+ */
+internal class WtfTaskTableReader(private val reader: LocalParquetReader<GenericRecord>) : TableReader {
+    /**
+     * The current record.
+     */
+    private var record: GenericRecord? = null
+
+    override fun nextRow(): Boolean {
+        record = reader.read()
+        return record != null
+    }
+
+    override fun hasColumn(column: TableColumn<*>): Boolean {
+        return when (column) {
+            TASK_ID -> true
+            TASK_WORKFLOW_ID -> true
+            TASK_SUBMIT_TIME -> true
+            TASK_WAIT_TIME -> true
+            TASK_RUNTIME -> true
+            TASK_REQ_NCPUS -> true
+            TASK_PARENTS -> true
+            TASK_CHILDREN -> true
+            TASK_GROUP_ID -> true
+            TASK_USER_ID -> true
+            else -> false
+        }
+    }
+
+    override fun <T> get(column: TableColumn<T>): T {
+        val record = checkNotNull(record) { "Reader in invalid state" }
+
+        @Suppress("UNCHECKED_CAST")
+        val res: Any = when (column) {
+            TASK_ID -> record["id"]
+            TASK_WORKFLOW_ID -> record["workflow_id"]
+            TASK_SUBMIT_TIME -> record["ts_submit"]
+            TASK_WAIT_TIME -> record["wait_time"]
+            TASK_RUNTIME -> record["runtime"]
+            TASK_REQ_NCPUS -> (record["resource_amount_requested"] as Double).toInt()
+            TASK_PARENTS -> (record["parents"] as ArrayList<GenericRecord>).map { it["item"] as Long }.toSet()
+            TASK_CHILDREN -> (record["children"] as ArrayList<GenericRecord>).map { it["item"] as Long }.toSet()
+            TASK_GROUP_ID -> record["group_id"]
+            TASK_USER_ID -> record["user_id"]
+            else -> throw IllegalArgumentException("Invalid column")
+        }
+
+        @Suppress("UNCHECKED_CAST")
+        return res as T
+    }
+
+    override fun getBoolean(column: TableColumn<Boolean>): Boolean {
+        throw IllegalArgumentException("Invalid column")
+    }
+
+    override fun getInt(column: TableColumn<Int>): Int {
+        val record = checkNotNull(record) { "Reader in invalid state" }
+
+        return when (column) {
+            TASK_REQ_NCPUS -> (record["resource_amount_requested"] as Double).toInt()
+            TASK_GROUP_ID -> record["group_id"] as Int
+            TASK_USER_ID -> record["user_id"] as Int
+            else -> throw IllegalArgumentException("Invalid column")
+        }
+    }
+
+    override fun getLong(column: TableColumn<Long>): Long {
+        val record = checkNotNull(record) { "Reader in invalid state" }
+
+        return when (column) {
+            TASK_ID -> record["id"] as Long
+            TASK_WORKFLOW_ID -> record["workflow_id"] as Long
+            TASK_SUBMIT_TIME -> record["ts_submit"] as Long
+            TASK_WAIT_TIME -> record["wait_time"] as Long
+            TASK_RUNTIME -> record["runtime"] as Long
+            else -> throw IllegalArgumentException("Invalid column")
+        }
+    }
+
+    override fun getDouble(column: TableColumn<Double>): Double {
+        throw IllegalArgumentException("Invalid column")
+    }
+
+    override fun close() {
+        reader.close()
+    }
+}
diff --git a/opendc-trace/opendc-trace-wtf/src/main/kotlin/org/opendc/trace/wtf/WtfTrace.kt b/opendc-trace/opendc-trace-wtf/src/main/kotlin/org/opendc/trace/wtf/WtfTrace.kt
new file mode 100644
index 00000000..7eff0f5a
--- /dev/null
+++ b/opendc-trace/opendc-trace-wtf/src/main/kotlin/org/opendc/trace/wtf/WtfTrace.kt
@@ -0,0 +1,47 @@
+/*
+ * Copyright (c) 2021 AtLarge Research
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+package org.opendc.trace.wtf
+
+import org.opendc.trace.TABLE_TASKS
+import org.opendc.trace.Table
+import org.opendc.trace.Trace
+import java.nio.file.Path
+
+/**
+ * [Trace] implementation for the WTF format.
+ */
+public class WtfTrace internal constructor(private val path: Path) : Trace {
+    override val tables: List<String> = listOf(TABLE_TASKS)
+
+    override fun containsTable(name: String): Boolean = TABLE_TASKS == name
+
+    override fun getTable(name: String): Table? {
+        if (!containsTable(name)) {
+            return null
+        }
+
+        return WtfTaskTable(path)
+    }
+
+    override fun toString(): String = "SwfTrace[$path]"
+}
diff --git a/opendc-trace/opendc-trace-wtf/src/main/kotlin/org/opendc/trace/wtf/WtfTraceFormat.kt b/opendc-trace/opendc-trace-wtf/src/main/kotlin/org/opendc/trace/wtf/WtfTraceFormat.kt
new file mode 100644
index 00000000..781cb335
--- /dev/null
+++ b/opendc-trace/opendc-trace-wtf/src/main/kotlin/org/opendc/trace/wtf/WtfTraceFormat.kt
@@ -0,0 +1,41 @@
+/*
+ * Copyright (c) 2021 AtLarge Research
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+package org.opendc.trace.wtf
+
+import org.opendc.trace.spi.TraceFormat
+import java.net.URL
+import java.nio.file.Paths
+import kotlin.io.path.exists
+
+/**
+ * A [TraceFormat] implementation for the Workflow Trace Format (WTF).
+ */
+public class WtfTraceFormat : TraceFormat {
+    override val name: String = "wtf"
+
+    override fun open(url: URL): WtfTrace {
+        val path = Paths.get(url.toURI())
+        require(path.exists()) { "URL $url does not exist" }
+        return WtfTrace(path)
+    }
+}
diff --git a/opendc-trace/opendc-trace-wtf/src/main/resources/META-INF/services/org.opendc.trace.spi.TraceFormat b/opendc-trace/opendc-trace-wtf/src/main/resources/META-INF/services/org.opendc.trace.spi.TraceFormat
new file mode 100644
index 00000000..32da52ff
--- /dev/null
+++ b/opendc-trace/opendc-trace-wtf/src/main/resources/META-INF/services/org.opendc.trace.spi.TraceFormat
@@ -0,0 +1 @@
+org.opendc.trace.wtf.WtfTraceFormat
diff --git a/opendc-trace/opendc-trace-wtf/src/test/kotlin/org/opendc/trace/wtf/WtfTraceFormatTest.kt b/opendc-trace/opendc-trace-wtf/src/test/kotlin/org/opendc/trace/wtf/WtfTraceFormatTest.kt
new file mode 100644
index 00000000..a05a523e
--- /dev/null
+++ b/opendc-trace/opendc-trace-wtf/src/test/kotlin/org/opendc/trace/wtf/WtfTraceFormatTest.kt
@@ -0,0 +1,121 @@
+/*
+ * Copyright (c) 2021 AtLarge Research
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+package org.opendc.trace.wtf
+
+import org.junit.jupiter.api.Assertions.*
+import org.junit.jupiter.api.Test
+import org.junit.jupiter.api.assertThrows
+import org.opendc.trace.*
+import java.io.File
+import java.net.URL
+
+/**
+ * Test suite for the [WtfTraceFormat] class.
+ */
+class WtfTraceFormatTest {
+    @Test
+    fun testTraceExists() {
+        val input = File("src/test/resources/wtf-trace").toURI().toURL()
+        val format = WtfTraceFormat()
+        org.junit.jupiter.api.assertDoesNotThrow {
+            format.open(input)
+        }
+    }
+
+    @Test
+    fun testTraceDoesNotExists() {
+        val input = File("src/test/resources/wtf-trace").toURI().toURL()
+        val format = WtfTraceFormat()
+        assertThrows<IllegalArgumentException> {
+            format.open(URL(input.toString() + "help"))
+        }
+    }
+
+    @Test
+    fun testTables() {
+        val input = File("src/test/resources/wtf-trace").toURI().toURL()
+        val format = WtfTraceFormat()
+        val trace = format.open(input)
+
+        assertEquals(listOf(TABLE_TASKS), trace.tables)
+    }
+
+    @Test
+    fun testTableExists() {
+        val input = File("src/test/resources/wtf-trace").toURI().toURL()
+        val format = WtfTraceFormat()
+        val table = format.open(input).getTable(TABLE_TASKS)
+
+        assertNotNull(table)
+        org.junit.jupiter.api.assertDoesNotThrow { table!!.newReader() }
+    }
+
+    @Test
+    fun testTableDoesNotExist() {
+        val input = File("src/test/resources/wtf-trace").toURI().toURL()
+        val format = WtfTraceFormat()
+        val trace = format.open(input)
+
+        assertFalse(trace.containsTable("test"))
+        assertNull(trace.getTable("test"))
+    }
+
+    /**
+     * Smoke test for parsing WTF traces.
+     */
+    @Test
+    fun testTableReader() {
+        val input = File("src/test/resources/wtf-trace")
+        val trace = WtfTraceFormat().open(input.toURI().toURL())
+        val reader = trace.getTable(TABLE_TASKS)!!.newReader()
+
+        assertAll(
+            { assertTrue(reader.nextRow()) },
+            { assertEquals(362334516345962206, reader.getLong(TASK_ID)) },
+            { assertEquals(1078341553348591493, reader.getLong(TASK_WORKFLOW_ID)) },
+            { assertEquals(245604, reader.getLong(TASK_SUBMIT_TIME)) },
+            { assertEquals(8163, reader.getLong(TASK_RUNTIME)) },
+            { assertEquals(setOf(584055316413447529, 133113685133695608, 1008582348422865408), reader.get(TASK_PARENTS)) },
+        )
+
+        assertAll(
+            { assertTrue(reader.nextRow()) },
+            { assertEquals(502010169100446658, reader.getLong(TASK_ID)) },
+            { assertEquals(1078341553348591493, reader.getLong(TASK_WORKFLOW_ID)) },
+            { assertEquals(251325, reader.getLong(TASK_SUBMIT_TIME)) },
+            { assertEquals(8216, reader.getLong(TASK_RUNTIME)) },
+            { assertEquals(setOf(584055316413447529, 133113685133695608, 1008582348422865408), reader.get(TASK_PARENTS)) },
+        )
+
+        reader.close()
+    }
+
+    @Test
+    fun testTableReaderPartition() {
+        val input = File("src/test/resources/wtf-trace").toURI().toURL()
+        val format = WtfTraceFormat()
+        val table = format.open(input).getTable(TABLE_TASKS)!!
+
+        assertThrows<IllegalArgumentException> { table.newReader("test") }
+    }
+}
diff --git a/opendc-trace/opendc-trace-wtf/src/test/resources/wtf-trace/tasks/schema-1.0/part.0.parquet b/opendc-trace/opendc-trace-wtf/src/test/resources/wtf-trace/tasks/schema-1.0/part.0.parquet
new file mode 100644
index 00000000..d2044038
--- /dev/null
+++ b/opendc-trace/opendc-trace-wtf/src/test/resources/wtf-trace/tasks/schema-1.0/part.0.parquet