merge: Add support for trace writing

This pull request extends the trace API to support writing new traces.

- Unify columns of different tables 
- Support column lookup via index
- Use index lookup in trace loader 
- Add property for describing partition keys
- Simplify TraceFormat SPI interface
- Add support for writing traces

**Breaking API Changes**

- `TraceFormat` SPI interface has been redesigned.

5 files changed, 86 insertions, 192 deletions

diff --git a/opendc-trace/opendc-trace-gwf/src/main/kotlin/org/opendc/trace/gwf/GwfTaskTable.kt b/opendc-trace/opendc-trace-gwf/src/main/kotlin/org/opendc/trace/gwf/GwfTaskTable.kt
deleted file mode 100644
index fd7bd068..00000000
--- a/opendc-trace/opendc-trace-gwf/src/main/kotlin/org/opendc/trace/gwf/GwfTaskTable.kt
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Copyright (c) 2021 AtLarge Research
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to deal
- * in the Software without restriction, including without limitation the rights
- * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- * copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in all
- * copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-
-package org.opendc.trace.gwf
-
-import com.fasterxml.jackson.dataformat.csv.CsvFactory
-import org.opendc.trace.*
-import java.net.URL
-
-/**
- * A [Table] containing the tasks in a GWF trace.
- */
-internal class GwfTaskTable(private val factory: CsvFactory, private val url: URL) : Table {
-    override val name: String = TABLE_TASKS
-
-    override val isSynthetic: Boolean = false
-
-    override val columns: List<TableColumn<*>> = listOf(
-        TASK_WORKFLOW_ID,
-        TASK_ID,
-        TASK_SUBMIT_TIME,
-        TASK_RUNTIME,
-        TASK_REQ_NCPUS,
-        TASK_ALLOC_NCPUS,
-        TASK_PARENTS
-    )
-
-    override fun newReader(): TableReader {
-        return GwfTaskTableReader(factory.createParser(url))
-    }
-
-    override fun newReader(partition: String): TableReader {
-        throw IllegalArgumentException("Invalid partition $partition")
-    }
-
-    override fun toString(): String = "GwfTaskTable"
-}
diff --git a/opendc-trace/opendc-trace-gwf/src/main/kotlin/org/opendc/trace/gwf/GwfTaskTableReader.kt b/opendc-trace/opendc-trace-gwf/src/main/kotlin/org/opendc/trace/gwf/GwfTaskTableReader.kt
index 39eb5520..aa4c543b 100644
--- a/opendc-trace/opendc-trace-gwf/src/main/kotlin/org/opendc/trace/gwf/GwfTaskTableReader.kt
+++ b/opendc-trace/opendc-trace-gwf/src/main/kotlin/org/opendc/trace/gwf/GwfTaskTableReader.kt
@@ -67,52 +67,43 @@ internal class GwfTaskTableReader(private val parser: CsvParser) : TableReader {
         return true
     }
 
-    override fun hasColumn(column: TableColumn<*>): Boolean {
-        return when (column) {
-            TASK_WORKFLOW_ID -> true
-            TASK_ID -> true
-            TASK_SUBMIT_TIME -> true
-            TASK_RUNTIME -> true
-            TASK_REQ_NCPUS -> true
-            TASK_ALLOC_NCPUS -> true
-            TASK_PARENTS -> true
-            else -> false
-        }
+    override fun resolve(column: TableColumn<*>): Int = columns[column] ?: -1
+
+    override fun isNull(index: Int): Boolean {
+        check(index in 0..columns.size) { "Invalid column" }
+        return false
     }
 
-    override fun <T> get(column: TableColumn<T>): T {
-        val res: Any? = when (column) {
-            TASK_WORKFLOW_ID -> workflowId
-            TASK_ID -> jobId
-            TASK_SUBMIT_TIME -> submitTime
-            TASK_RUNTIME -> runtime
-            TASK_REQ_NCPUS -> nProcs
-            TASK_ALLOC_NCPUS -> reqNProcs
-            TASK_PARENTS -> dependencies
+    override fun get(index: Int): Any? {
+        return when (index) {
+            COL_JOB_ID -> jobId
+            COL_WORKFLOW_ID -> workflowId
+            COL_SUBMIT_TIME -> submitTime
+            COL_RUNTIME -> runtime
+            COL_REQ_NPROC -> getInt(index)
+            COL_NPROC -> getInt(index)
+            COL_DEPS -> dependencies
             else -> throw IllegalArgumentException("Invalid column")
         }
-
-        @Suppress("UNCHECKED_CAST")
-        return res as T
     }
 
-    override fun getBoolean(column: TableColumn<Boolean>): Boolean {
+    override fun getBoolean(index: Int): Boolean {
         throw IllegalArgumentException("Invalid column")
     }
 
-    override fun getInt(column: TableColumn<Int>): Int {
-        return when (column) {
-            TASK_REQ_NCPUS -> nProcs
-            TASK_ALLOC_NCPUS -> reqNProcs
+    override fun getInt(index: Int): Int {
+        return when (index) {
+            COL_REQ_NPROC -> reqNProcs
+            COL_NPROC -> nProcs
             else -> throw IllegalArgumentException("Invalid column")
         }
     }
 
-    override fun getLong(column: TableColumn<Long>): Long {
+    override fun getLong(index: Int): Long {
         throw IllegalArgumentException("Invalid column")
     }
 
-    override fun getDouble(column: TableColumn<Double>): Double {
+    override fun getDouble(index: Int): Double {
         throw IllegalArgumentException("Invalid column")
     }
 
@@ -180,6 +171,24 @@ internal class GwfTaskTableReader(private val parser: CsvParser) : TableReader {
         dependencies = emptySet()
     }
 
+    private val COL_WORKFLOW_ID = 0
+    private val COL_JOB_ID = 1
+    private val COL_SUBMIT_TIME = 2
+    private val COL_RUNTIME = 3
+    private val COL_NPROC = 4
+    private val COL_REQ_NPROC = 5
+    private val COL_DEPS = 6
+
+    private val columns = mapOf(
+        TASK_ID to COL_JOB_ID,
+        TASK_WORKFLOW_ID to COL_WORKFLOW_ID,
+        TASK_SUBMIT_TIME to COL_SUBMIT_TIME,
+        TASK_RUNTIME to COL_RUNTIME,
+        TASK_ALLOC_NCPUS to COL_NPROC,
+        TASK_REQ_NCPUS to COL_REQ_NPROC,
+        TASK_PARENTS to COL_DEPS
+    )
+
     companion object {
         /**
          * The [CsvSchema] that is used to parse the trace.
diff --git a/opendc-trace/opendc-trace-gwf/src/main/kotlin/org/opendc/trace/gwf/GwfTrace.kt b/opendc-trace/opendc-trace-gwf/src/main/kotlin/org/opendc/trace/gwf/GwfTrace.kt
deleted file mode 100644
index 166c1e56..00000000
--- a/opendc-trace/opendc-trace-gwf/src/main/kotlin/org/opendc/trace/gwf/GwfTrace.kt
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- * Copyright (c) 2021 AtLarge Research
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to deal
- * in the Software without restriction, including without limitation the rights
- * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- * copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in all
- * copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-
-package org.opendc.trace.gwf
-
-import com.fasterxml.jackson.dataformat.csv.CsvFactory
-import org.opendc.trace.*
-import java.net.URL
-
-/**
- * [Trace] implementation for the GWF format.
- */
-public class GwfTrace internal constructor(private val factory: CsvFactory, private val url: URL) : Trace {
-    override val tables: List<String> = listOf(TABLE_TASKS)
-
-    override fun containsTable(name: String): Boolean = TABLE_TASKS == name
-
-    override fun getTable(name: String): Table? {
-        if (!containsTable(name)) {
-            return null
-        }
-
-        return GwfTaskTable(factory, url)
-    }
-
-    override fun toString(): String = "GwfTrace[$url]"
-}
diff --git a/opendc-trace/opendc-trace-gwf/src/main/kotlin/org/opendc/trace/gwf/GwfTraceFormat.kt b/opendc-trace/opendc-trace-gwf/src/main/kotlin/org/opendc/trace/gwf/GwfTraceFormat.kt
index 6d542503..d4287420 100644
--- a/opendc-trace/opendc-trace-gwf/src/main/kotlin/org/opendc/trace/gwf/GwfTraceFormat.kt
+++ b/opendc-trace/opendc-trace-gwf/src/main/kotlin/org/opendc/trace/gwf/GwfTraceFormat.kt
@@ -24,10 +24,10 @@ package org.opendc.trace.gwf
 
 import com.fasterxml.jackson.dataformat.csv.CsvFactory
 import com.fasterxml.jackson.dataformat.csv.CsvParser
+import org.opendc.trace.*
+import org.opendc.trace.spi.TableDetails
 import org.opendc.trace.spi.TraceFormat
-import java.net.URL
-import java.nio.file.Paths
-import kotlin.io.path.exists
+import java.nio.file.Path
 
 /**
  * A [TraceFormat] implementation for the GWF trace format.
@@ -45,12 +45,38 @@ public class GwfTraceFormat : TraceFormat {
         .enable(CsvParser.Feature.ALLOW_COMMENTS)
         .enable(CsvParser.Feature.TRIM_SPACES)
 
-    /**
-     * Read the tasks in the GWF trace.
-     */
-    public override fun open(url: URL): GwfTrace {
-        val path = Paths.get(url.toURI())
-        require(path.exists()) { "URL $url does not exist" }
-        return GwfTrace(factory, url)
+    override fun create(path: Path) {
+        throw UnsupportedOperationException("Writing not supported for this format")
+    }
+
+    override fun getTables(path: Path): List<String> = listOf(TABLE_TASKS)
+
+    override fun getDetails(path: Path, table: String): TableDetails {
+        return when (table) {
+            TABLE_TASKS -> TableDetails(
+                listOf(
+                    TASK_WORKFLOW_ID,
+                    TASK_ID,
+                    TASK_SUBMIT_TIME,
+                    TASK_RUNTIME,
+                    TASK_REQ_NCPUS,
+                    TASK_ALLOC_NCPUS,
+                    TASK_PARENTS,
+                ),
+                listOf(TASK_WORKFLOW_ID)
+            )
+            else -> throw IllegalArgumentException("Table $table not supported")
+        }
+    }
+
+    override fun newReader(path: Path, table: String): TableReader {
+        return when (table) {
+            TABLE_TASKS -> GwfTaskTableReader(factory.createParser(path.toFile()))
+            else -> throw IllegalArgumentException("Table $table not supported")
+        }
+    }
+
+    override fun newWriter(path: Path, table: String): TableWriter {
+        throw UnsupportedOperationException("Writing not supported for this format")
     }
 }
diff --git a/opendc-trace/opendc-trace-gwf/src/test/kotlin/org/opendc/trace/gwf/GwfTraceFormatTest.kt b/opendc-trace/opendc-trace-gwf/src/test/kotlin/org/opendc/trace/gwf/GwfTraceFormatTest.kt
index b209b979..7fe403b2 100644
--- a/opendc-trace/opendc-trace-gwf/src/test/kotlin/org/opendc/trace/gwf/GwfTraceFormatTest.kt
+++ b/opendc-trace/opendc-trace-gwf/src/test/kotlin/org/opendc/trace/gwf/GwfTraceFormatTest.kt
@@ -22,13 +22,10 @@
 
 package org.opendc.trace.gwf
 
+import org.junit.jupiter.api.*
 import org.junit.jupiter.api.Assertions.*
-import org.junit.jupiter.api.Test
-import org.junit.jupiter.api.assertAll
-import org.junit.jupiter.api.assertDoesNotThrow
-import org.junit.jupiter.api.assertThrows
 import org.opendc.trace.*
-import java.net.URL
+import java.nio.file.Paths
 import java.time.Duration
 import java.time.Instant
 
@@ -36,59 +33,32 @@ import java.time.Instant
  * Test suite for the [GwfTraceFormat] class.
  */
 internal class GwfTraceFormatTest {
-    @Test
-    fun testTraceExists() {
-        val input = checkNotNull(GwfTraceFormatTest::class.java.getResource("/trace.gwf"))
-        val format = GwfTraceFormat()
-        assertDoesNotThrow {
-            format.open(input)
-        }
-    }
-
-    @Test
-    fun testTraceDoesNotExists() {
-        val input = checkNotNull(GwfTraceFormatTest::class.java.getResource("/trace.gwf"))
-        val format = GwfTraceFormat()
-        assertThrows<IllegalArgumentException> {
-            format.open(URL(input.toString() + "help"))
-        }
-    }
+    private val format = GwfTraceFormat()
 
     @Test
     fun testTables() {
-        val input = checkNotNull(GwfTraceFormatTest::class.java.getResource("/trace.gwf"))
-        val format = GwfTraceFormat()
-        val trace = format.open(input)
+        val path = Paths.get(checkNotNull(GwfTraceFormatTest::class.java.getResource("/trace.gwf")).toURI())
 
-        assertEquals(listOf(TABLE_TASKS), trace.tables)
+        assertEquals(listOf(TABLE_TASKS), format.getTables(path))
     }
 
     @Test
     fun testTableExists() {
-        val input = checkNotNull(GwfTraceFormatTest::class.java.getResource("/trace.gwf"))
-        val format = GwfTraceFormat()
-        val table = format.open(input).getTable(TABLE_TASKS)
-
-        assertNotNull(table)
-        assertDoesNotThrow { table!!.newReader() }
+        val path = Paths.get(checkNotNull(GwfTraceFormatTest::class.java.getResource("/trace.gwf")).toURI())
+        assertDoesNotThrow { format.getDetails(path, TABLE_TASKS) }
     }
 
     @Test
     fun testTableDoesNotExist() {
-        val input = checkNotNull(GwfTraceFormatTest::class.java.getResource("/trace.gwf"))
-        val format = GwfTraceFormat()
-        val trace = format.open(input)
+        val path = Paths.get(checkNotNull(GwfTraceFormatTest::class.java.getResource("/trace.gwf")).toURI())
 
-        assertFalse(trace.containsTable("test"))
-        assertNull(trace.getTable("test"))
+        assertThrows<IllegalArgumentException> { format.getDetails(path, "test") }
     }
 
     @Test
     fun testTableReader() {
-        val input = checkNotNull(GwfTraceFormatTest::class.java.getResource("/trace.gwf"))
-        val format = GwfTraceFormat()
-        val table = format.open(input).getTable(TABLE_TASKS)!!
-        val reader = table.newReader()
+        val path = Paths.get(checkNotNull(GwfTraceFormatTest::class.java.getResource("/trace.gwf")).toURI())
+        val reader = format.newReader(path, TABLE_TASKS)
 
         assertAll(
             { assertTrue(reader.nextRow()) },
@@ -99,13 +69,4 @@ internal class GwfTraceFormatTest {
             { assertEquals(emptySet<String>(), reader.get(TASK_PARENTS)) },
         )
     }
-
-    @Test
-    fun testTableReaderPartition() {
-        val input = checkNotNull(GwfTraceFormatTest::class.java.getResource("/trace.gwf"))
-        val format = GwfTraceFormat()
-        val table = format.open(input).getTable(TABLE_TASKS)!!
-
-        assertThrows<IllegalArgumentException> { table.newReader("test") }
-    }
 }