feat(trace): Add support for WfCommons (WorkflowHub) traces

This change adds support for reading WfCommons workflow traces in
OpenDC. This functionality is available in the new
`opendc-trace-wfformat` module.

2 files changed, 478 insertions, 0 deletions

diff --git a/opendc-trace/opendc-trace-wfformat/src/test/kotlin/org/opendc/trace/wfformat/WfFormatTaskTableReaderTest.kt b/opendc-trace/opendc-trace-wfformat/src/test/kotlin/org/opendc/trace/wfformat/WfFormatTaskTableReaderTest.kt
new file mode 100644
index 00000000..b07f27ed
--- /dev/null
+++ b/opendc-trace/opendc-trace-wfformat/src/test/kotlin/org/opendc/trace/wfformat/WfFormatTaskTableReaderTest.kt
@@ -0,0 +1,345 @@
+/*
+ * Copyright (c) 2021 AtLarge Research
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+package org.opendc.trace.wfformat
+
+import com.fasterxml.jackson.core.JsonFactory
+import com.fasterxml.jackson.core.JsonParseException
+import org.junit.jupiter.api.Assertions.*
+import org.junit.jupiter.api.Test
+import org.junit.jupiter.api.assertDoesNotThrow
+import org.junit.jupiter.api.assertThrows
+import org.opendc.trace.TASK_ID
+import org.opendc.trace.TASK_PARENTS
+
+/**
+ * Test suite for the [WfFormatTaskTableReader] class.
+ */
+internal class WfFormatTaskTableReaderTest {
+    /**
+     * The [JsonFactory] used to construct the parser.
+     */
+    private val factory = JsonFactory()
+
+    @Test
+    fun testEmptyInput() {
+        val content = ""
+        val parser = factory.createParser(content)
+        val reader = WfFormatTaskTableReader(parser)
+
+        assertFalse(reader.nextRow())
+        reader.close()
+    }
+
+    @Test
+    fun testTopLevelArrayInput() {
+        val content = "[]"
+        val parser = factory.createParser(content)
+        val reader = WfFormatTaskTableReader(parser)
+
+        assertThrows<JsonParseException> {
+            while (reader.nextRow()) {
+                continue
+            }
+        }
+
+        reader.close()
+    }
+
+    @Test
+    fun testNoWorkflow() {
+        val content = """
+        {
+            "name": "eager-nextflow-chameleon"
+        }
+        """.trimIndent()
+        val parser = factory.createParser(content)
+        val reader = WfFormatTaskTableReader(parser)
+
+        assertDoesNotThrow {
+            while (reader.nextRow()) {
+                continue
+            }
+        }
+
+        reader.close()
+    }
+
+    @Test
+    fun testWorkflowArrayType() {
+        val content = """
+        {
+            "name": "eager-nextflow-chameleon",
+            "workflow": []
+        }
+        """.trimIndent()
+        val parser = factory.createParser(content)
+        val reader = WfFormatTaskTableReader(parser)
+
+        assertThrows<JsonParseException> {
+            while (reader.nextRow()) {
+                continue
+            }
+        }
+
+        reader.close()
+    }
+
+    @Test
+    fun testWorkflowNullType() {
+        val content = """
+        {
+            "name": "eager-nextflow-chameleon",
+            "workflow": null
+        }
+        """.trimIndent()
+        val parser = factory.createParser(content)
+        val reader = WfFormatTaskTableReader(parser)
+
+        assertThrows<JsonParseException> {
+            while (reader.nextRow()) {
+                continue
+            }
+        }
+
+        reader.close()
+    }
+
+    @Test
+    fun testNoJobs() {
+        val content = """
+        {
+            "name": "eager-nextflow-chameleon",
+            "workflow": {
+
+            }
+        }
+        """.trimIndent()
+        val parser = factory.createParser(content)
+        val reader = WfFormatTaskTableReader(parser)
+
+        assertDoesNotThrow { reader.nextRow() }
+
+        reader.close()
+    }
+
+    @Test
+    fun testJobsObjectType() {
+        val content = """
+        {
+            "name": "eager-nextflow-chameleon",
+            "workflow": { "jobs": {} }
+        }
+        """.trimIndent()
+        val parser = factory.createParser(content)
+        val reader = WfFormatTaskTableReader(parser)
+
+        assertThrows<JsonParseException> { reader.nextRow() }
+
+        reader.close()
+    }
+
+    @Test
+    fun testJobsNullType() {
+        val content = """
+        {
+            "name": "eager-nextflow-chameleon",
+            "workflow": { "jobs": null }
+        }
+        """.trimIndent()
+        val parser = factory.createParser(content)
+        val reader = WfFormatTaskTableReader(parser)
+
+        assertThrows<JsonParseException> { reader.nextRow() }
+
+        reader.close()
+    }
+
+    @Test
+    fun testJobsInvalidChildType() {
+        val content = """
+        {
+            "name": "eager-nextflow-chameleon",
+            "workflow": {
+                "jobs": [1]
+            }
+        }
+        """.trimIndent()
+        val parser = factory.createParser(content)
+        val reader = WfFormatTaskTableReader(parser)
+
+        assertThrows<JsonParseException> { reader.nextRow() }
+
+        reader.close()
+    }
+
+    @Test
+    fun testJobsValidChildType() {
+        val content = """
+        {
+            "name": "eager-nextflow-chameleon",
+            "workflow": {
+                "jobs": [
+                    {
+                        "name": "test"
+                    }
+                ]
+            }
+        }
+        """.trimIndent()
+        val parser = factory.createParser(content)
+        val reader = WfFormatTaskTableReader(parser)
+
+        assertTrue(reader.nextRow())
+        assertEquals("test", reader.get(TASK_ID))
+        assertFalse(reader.nextRow())
+
+        reader.close()
+    }
+
+    @Test
+    fun testJobsInvalidParents() {
+        val content = """
+        {
+            "name": "eager-nextflow-chameleon",
+            "workflow": {
+                "jobs": [
+                    {
+                        "name": "test",
+                        "parents": 1,
+                    }
+                ]
+            }
+        }
+        """.trimIndent()
+        val parser = factory.createParser(content)
+        val reader = WfFormatTaskTableReader(parser)
+
+        assertThrows<JsonParseException> { reader.nextRow() }
+
+        reader.close()
+    }
+
+    @Test
+    fun testJobsInvalidParentsItem() {
+        val content = """
+        {
+            "name": "eager-nextflow-chameleon",
+            "workflow": {
+                "jobs": [
+                    {
+                        "name": "test",
+                        "parents": [1],
+                    }
+                ]
+            }
+        }
+        """.trimIndent()
+        val parser = factory.createParser(content)
+        val reader = WfFormatTaskTableReader(parser)
+
+        assertThrows<JsonParseException> { reader.nextRow() }
+
+        reader.close()
+    }
+
+    @Test
+    fun testJobsValidParents() {
+        val content = """
+        {
+            "name": "eager-nextflow-chameleon",
+            "workflow": {
+                "jobs": [
+                    {
+                        "name": "test",
+                        "parents": ["1"]
+                    }
+                ]
+            }
+        }
+        """.trimIndent()
+        val parser = factory.createParser(content)
+        val reader = WfFormatTaskTableReader(parser)
+
+        assertTrue(reader.nextRow())
+        assertEquals(setOf("1"), reader.get(TASK_PARENTS))
+        assertFalse(reader.nextRow())
+
+        reader.close()
+    }
+
+    @Test
+    fun testJobsInvalidSecondEntry() {
+        val content = """
+        {
+            "workflow": {
+                "jobs": [
+                    {
+                        "name": "test",
+                        "parents": ["1"]
+                    },
+                    "test"
+                ]
+            }
+        }
+        """.trimIndent()
+        val parser = factory.createParser(content)
+        val reader = WfFormatTaskTableReader(parser)
+
+        assertDoesNotThrow { reader.nextRow() }
+        assertThrows<JsonParseException> { reader.nextRow() }
+
+        reader.close()
+    }
+
+    @Test
+    fun testDuplicateJobsArray() {
+        val content = """
+        {
+            "name": "eager-nextflow-chameleon",
+            "workflow": {
+                "jobs": [
+                    {
+                        "name": "test",
+                        "parents": ["1"]
+                    }
+                ],
+                "jobs": [
+                    {
+                        "name": "test2",
+                        "parents": ["test"]
+                    }
+                ]
+            }
+        }
+        """.trimIndent()
+        val parser = factory.createParser(content)
+        val reader = WfFormatTaskTableReader(parser)
+
+        assertTrue(reader.nextRow())
+        assertTrue(reader.nextRow())
+        assertEquals("test2", reader.get(TASK_ID))
+        assertFalse(reader.nextRow())
+
+        reader.close()
+    }
+}
diff --git a/opendc-trace/opendc-trace-wfformat/src/test/kotlin/org/opendc/trace/wfformat/WfFormatTraceFormatTest.kt b/opendc-trace/opendc-trace-wfformat/src/test/kotlin/org/opendc/trace/wfformat/WfFormatTraceFormatTest.kt
new file mode 100644
index 00000000..0bfc8840
--- /dev/null
+++ b/opendc-trace/opendc-trace-wfformat/src/test/kotlin/org/opendc/trace/wfformat/WfFormatTraceFormatTest.kt
@@ -0,0 +1,133 @@
+/*
+ * Copyright (c) 2021 AtLarge Research
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+package org.opendc.trace.wfformat
+
+import org.junit.jupiter.api.Assertions.*
+import org.junit.jupiter.api.Test
+import org.junit.jupiter.api.assertDoesNotThrow
+import org.junit.jupiter.api.assertThrows
+import org.opendc.trace.*
+import java.io.File
+import java.net.URL
+
+/**
+ * Test suite for the [WfFormatTraceFormat] class.
+ */
+class WfFormatTraceFormatTest {
+    @Test
+    fun testTraceExists() {
+        val input = File("src/test/resources/trace.json").toURI().toURL()
+        val format = WfFormatTraceFormat()
+        assertDoesNotThrow { format.open(input) }
+    }
+
+    @Test
+    fun testTraceDoesNotExists() {
+        val input = File("src/test/resources/trace.json").toURI().toURL()
+        val format = WfFormatTraceFormat()
+        assertThrows<IllegalArgumentException> { format.open(URL(input.toString() + "help")) }
+    }
+
+    @Test
+    fun testTables() {
+        val input = File("src/test/resources/trace.json").toURI().toURL()
+        val format = WfFormatTraceFormat()
+        val trace = format.open(input)
+
+        assertEquals(listOf(TABLE_TASKS), trace.tables)
+    }
+
+    @Test
+    fun testTableExists() {
+        val input = File("src/test/resources/trace.json").toURI().toURL()
+        val format = WfFormatTraceFormat()
+        val table = format.open(input).getTable(TABLE_TASKS)
+
+        assertNotNull(table)
+        assertDoesNotThrow { table!!.newReader() }
+    }
+
+    @Test
+    fun testTableDoesNotExist() {
+        val input = File("src/test/resources/trace.json").toURI().toURL()
+        val format = WfFormatTraceFormat()
+        val trace = format.open(input)
+
+        assertFalse(trace.containsTable("test"))
+        assertNull(trace.getTable("test"))
+    }
+
+    /**
+     * Smoke test for parsing WfCommons traces.
+     */
+    @Test
+    fun testTableReader() {
+        val input = File("src/test/resources/trace.json").toURI().toURL()
+        val trace = WfFormatTraceFormat().open(input)
+        val reader = trace.getTable(TABLE_TASKS)!!.newReader()
+
+        assertAll(
+            { assertTrue(reader.nextRow()) },
+            { assertEquals("makebwaindex_mammoth_mt_krause.fasta", reader.get(TASK_ID)) },
+            { assertEquals("eager-nextflow-chameleon", reader.get(TASK_WORKFLOW_ID)) },
+            { assertEquals(172000, reader.get(TASK_RUNTIME).toMillis()) },
+            { assertEquals(emptySet<String>(), reader.get(TASK_PARENTS)) },
+        )
+
+        assertAll(
+            { assertTrue(reader.nextRow()) },
+            { assertEquals("makeseqdict_mammoth_mt_krause.fasta", reader.get(TASK_ID)) },
+            { assertEquals("eager-nextflow-chameleon", reader.get(TASK_WORKFLOW_ID)) },
+            { assertEquals(175000, reader.get(TASK_RUNTIME).toMillis()) },
+            { assertEquals(setOf("makebwaindex_mammoth_mt_krause.fasta"), reader.get(TASK_PARENTS)) },
+        )
+
+        reader.close()
+    }
+
+    /**
+     * Test full iteration of the table.
+     */
+    @Test
+    fun testTableReaderFull() {
+        val input = File("src/test/resources/trace.json").toURI().toURL()
+        val trace = WfFormatTraceFormat().open(input)
+        val reader = trace.getTable(TABLE_TASKS)!!.newReader()
+
+        assertDoesNotThrow {
+            while (reader.nextRow()) {
+                // reader.get(TASK_ID)
+            }
+            reader.close()
+        }
+    }
+
+    @Test
+    fun testTableReaderPartition() {
+        val input = File("src/test/resources/trace.json").toURI().toURL()
+        val format = WfFormatTraceFormat()
+        val table = format.open(input).getTable(TABLE_TASKS)!!
+
+        assertThrows<IllegalArgumentException> { table.newReader("test") }
+    }
+}