merge: Add generic trace reading library

This pull request adds a generic trace reading library to OpenDC. 
The library has been designed to support a wide range of trace formats 
and uses a streaming approach to improve performance of reading large traces.

* Add trace reading API
* Implement API for GWF format
* Implement API for SWF format
* Implement API for WTF format
* Implement API for Bitbrains format
* Implement API for Bitbrains Parquet format

**Breaking API Changes**

* `opendc-format` has been removed in favour of `opendc-trace-*`

5 files changed, 460 insertions, 0 deletions

diff --git a/opendc-trace/opendc-trace-bitbrains/src/main/kotlin/org/opendc/trace/bitbrains/BitbrainsResourceStateTable.kt b/opendc-trace/opendc-trace-bitbrains/src/main/kotlin/org/opendc/trace/bitbrains/BitbrainsResourceStateTable.kt
new file mode 100644
index 00000000..767ef919
--- /dev/null
+++ b/opendc-trace/opendc-trace-bitbrains/src/main/kotlin/org/opendc/trace/bitbrains/BitbrainsResourceStateTable.kt
@@ -0,0 +1,139 @@
+/*
+ * Copyright (c) 2021 AtLarge Research
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+package org.opendc.trace.bitbrains
+
+import com.fasterxml.jackson.dataformat.csv.CsvFactory
+import org.opendc.trace.*
+import java.nio.file.Files
+import java.nio.file.Path
+import java.util.stream.Collectors
+import kotlin.io.path.extension
+import kotlin.io.path.nameWithoutExtension
+
+/**
+ * The resource state [Table] in the Bitbrains format.
+ */
+internal class BitbrainsResourceStateTable(private val factory: CsvFactory, private val path: Path) : Table {
+    /**
+     * The partitions that belong to the table.
+     */
+    private val partitions =
+        Files.walk(path, 1)
+            .filter { !Files.isDirectory(it) && it.extension == "csv" }
+            .collect(Collectors.toMap({ it.nameWithoutExtension }, { it }))
+
+    override val name: String = TABLE_RESOURCE_STATES
+
+    override val isSynthetic: Boolean = false
+
+    override fun isSupported(column: TableColumn<*>): Boolean {
+        return when (column) {
+            RESOURCE_STATE_ID -> true
+            RESOURCE_STATE_TIMESTAMP -> true
+            RESOURCE_STATE_NCPUS -> true
+            RESOURCE_STATE_CPU_CAPACITY -> true
+            RESOURCE_STATE_CPU_USAGE -> true
+            RESOURCE_STATE_CPU_USAGE_PCT -> true
+            RESOURCE_STATE_MEM_CAPACITY -> true
+            RESOURCE_STATE_MEM_USAGE -> true
+            RESOURCE_STATE_DISK_READ -> true
+            RESOURCE_STATE_DISK_WRITE -> true
+            RESOURCE_STATE_NET_RX -> true
+            RESOURCE_STATE_NET_TX -> true
+            else -> false
+        }
+    }
+
+    override fun newReader(): TableReader {
+        val it = partitions.iterator()
+
+        return object : TableReader {
+            var delegate: TableReader? = nextDelegate()
+
+            override fun nextRow(): Boolean {
+                var delegate = delegate
+
+                while (delegate != null) {
+                    if (delegate.nextRow()) {
+                        break
+                    }
+
+                    delegate.close()
+                    delegate = nextDelegate()
+                }
+
+                this.delegate = delegate
+                return delegate != null
+            }
+
+            override fun hasColumn(column: TableColumn<*>): Boolean = delegate?.hasColumn(column) ?: false
+
+            override fun <T> get(column: TableColumn<T>): T {
+                val delegate = checkNotNull(delegate) { "Invalid reader state" }
+                return delegate.get(column)
+            }
+
+            override fun getBoolean(column: TableColumn<Boolean>): Boolean {
+                val delegate = checkNotNull(delegate) { "Invalid reader state" }
+                return delegate.getBoolean(column)
+            }
+
+            override fun getInt(column: TableColumn<Int>): Int {
+                val delegate = checkNotNull(delegate) { "Invalid reader state" }
+                return delegate.getInt(column)
+            }
+
+            override fun getLong(column: TableColumn<Long>): Long {
+                val delegate = checkNotNull(delegate) { "Invalid reader state" }
+                return delegate.getLong(column)
+            }
+
+            override fun getDouble(column: TableColumn<Double>): Double {
+                val delegate = checkNotNull(delegate) { "Invalid reader state" }
+                return delegate.getDouble(column)
+            }
+
+            override fun close() {
+                delegate?.close()
+            }
+
+            private fun nextDelegate(): TableReader? {
+                return if (it.hasNext()) {
+                    val (partition, path) = it.next()
+                    return BitbrainsResourceStateTableReader(partition, factory.createParser(path.toFile()))
+                } else {
+                    null
+                }
+            }
+
+            override fun toString(): String = "BitbrainsCompositeTableReader"
+        }
+    }
+
+    override fun newReader(partition: String): TableReader {
+        val path = requireNotNull(partitions[partition]) { "Invalid partition $partition" }
+        return BitbrainsResourceStateTableReader(partition, factory.createParser(path.toFile()))
+    }
+
+    override fun toString(): String = "BitbrainsResourceStateTable"
+}
diff --git a/opendc-trace/opendc-trace-bitbrains/src/main/kotlin/org/opendc/trace/bitbrains/BitbrainsResourceStateTableReader.kt b/opendc-trace/opendc-trace-bitbrains/src/main/kotlin/org/opendc/trace/bitbrains/BitbrainsResourceStateTableReader.kt
new file mode 100644
index 00000000..5687ac7f
--- /dev/null
+++ b/opendc-trace/opendc-trace-bitbrains/src/main/kotlin/org/opendc/trace/bitbrains/BitbrainsResourceStateTableReader.kt
@@ -0,0 +1,218 @@
+/*
+ * Copyright (c) 2021 AtLarge Research
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+package org.opendc.trace.bitbrains
+
+import com.fasterxml.jackson.core.JsonToken
+import com.fasterxml.jackson.dataformat.csv.CsvParser
+import com.fasterxml.jackson.dataformat.csv.CsvSchema
+import org.opendc.trace.*
+import java.time.Instant
+
+/**
+ * A [TableReader] for the Bitbrains resource state table.
+ */
+internal class BitbrainsResourceStateTableReader(private val partition: String, private val parser: CsvParser) : TableReader {
+    /**
+     * The current parser state.
+     */
+    private val state = RowState()
+
+    init {
+        parser.schema = schema
+    }
+
+    override fun nextRow(): Boolean {
+        // Reset the row state
+        state.reset()
+
+        if (!nextStart()) {
+            return false
+        }
+
+        while (true) {
+            val token = parser.nextValue()
+
+            if (token == null || token == JsonToken.END_OBJECT) {
+                break
+            }
+
+            when (parser.currentName) {
+                "Timestamp [ms]" -> state.timestamp = Instant.ofEpochSecond(parser.longValue)
+                "CPU cores" -> state.cpuCores = parser.intValue
+                "CPU capacity provisioned [MHZ]" -> state.cpuCapacity = parser.doubleValue
+                "CPU usage [MHZ]" -> state.cpuUsage = parser.doubleValue
+                "CPU usage [%]" -> state.cpuUsagePct = parser.doubleValue
+                "Memory capacity provisioned [KB]" -> state.memCapacity = parser.doubleValue
+                "Memory usage [KB]" -> state.memUsage = parser.doubleValue
+                "Disk read throughput [KB/s]" -> state.diskRead = parser.doubleValue
+                "Disk write throughput [KB/s]" -> state.diskWrite = parser.doubleValue
+                "Network received throughput [KB/s]" -> state.netReceived = parser.doubleValue
+                "Network transmitted throughput [KB/s]" -> state.netTransmitted = parser.doubleValue
+            }
+        }
+
+        return true
+    }
+
+    override fun hasColumn(column: TableColumn<*>): Boolean {
+        return when (column) {
+            RESOURCE_STATE_ID -> true
+            RESOURCE_STATE_TIMESTAMP -> true
+            RESOURCE_STATE_NCPUS -> true
+            RESOURCE_STATE_CPU_CAPACITY -> true
+            RESOURCE_STATE_CPU_USAGE -> true
+            RESOURCE_STATE_CPU_USAGE_PCT -> true
+            RESOURCE_STATE_MEM_CAPACITY -> true
+            RESOURCE_STATE_MEM_USAGE -> true
+            RESOURCE_STATE_DISK_READ -> true
+            RESOURCE_STATE_DISK_WRITE -> true
+            RESOURCE_STATE_NET_RX -> true
+            RESOURCE_STATE_NET_TX -> true
+            else -> false
+        }
+    }
+
+    override fun <T> get(column: TableColumn<T>): T {
+        val res: Any? = when (column) {
+            RESOURCE_STATE_ID -> partition
+            RESOURCE_STATE_TIMESTAMP -> state.timestamp
+            RESOURCE_STATE_NCPUS -> state.cpuCores
+            RESOURCE_STATE_CPU_CAPACITY -> state.cpuCapacity
+            RESOURCE_STATE_CPU_USAGE -> state.cpuUsage
+            RESOURCE_STATE_CPU_USAGE_PCT -> state.cpuUsagePct
+            RESOURCE_STATE_MEM_CAPACITY -> state.memCapacity
+            RESOURCE_STATE_MEM_USAGE -> state.memUsage
+            RESOURCE_STATE_DISK_READ -> state.diskRead
+            RESOURCE_STATE_DISK_WRITE -> state.diskWrite
+            RESOURCE_STATE_NET_RX -> state.netReceived
+            RESOURCE_STATE_NET_TX -> state.netTransmitted
+            else -> throw IllegalArgumentException("Invalid column")
+        }
+
+        @Suppress("UNCHECKED_CAST")
+        return res as T
+    }
+
+    override fun getBoolean(column: TableColumn<Boolean>): Boolean {
+        throw IllegalArgumentException("Invalid column")
+    }
+
+    override fun getInt(column: TableColumn<Int>): Int {
+        return when (column) {
+            RESOURCE_STATE_NCPUS -> state.cpuCores
+            else -> throw IllegalArgumentException("Invalid column")
+        }
+    }
+
+    override fun getLong(column: TableColumn<Long>): Long {
+        throw IllegalArgumentException("Invalid column")
+    }
+
+    override fun getDouble(column: TableColumn<Double>): Double {
+        return when (column) {
+            RESOURCE_STATE_CPU_CAPACITY -> state.cpuCapacity
+            RESOURCE_STATE_CPU_USAGE -> state.cpuUsage
+            RESOURCE_STATE_CPU_USAGE_PCT -> state.cpuUsagePct
+            RESOURCE_STATE_MEM_CAPACITY -> state.memCapacity
+            RESOURCE_STATE_MEM_USAGE -> state.memUsage
+            RESOURCE_STATE_DISK_READ -> state.diskRead
+            RESOURCE_STATE_DISK_WRITE -> state.diskWrite
+            RESOURCE_STATE_NET_RX -> state.netReceived
+            RESOURCE_STATE_NET_TX -> state.netTransmitted
+            else -> throw IllegalArgumentException("Invalid column")
+        }
+    }
+
+    override fun close() {
+        parser.close()
+    }
+
+    /**
+     * Advance the parser until the next object start.
+     */
+    private fun nextStart(): Boolean {
+        var token = parser.nextValue()
+
+        while (token != null && token != JsonToken.START_OBJECT) {
+            token = parser.nextValue()
+        }
+
+        return token != null
+    }
+
+    /**
+     * The current row state.
+     */
+    private class RowState {
+        var timestamp: Instant? = null
+        var cpuCores = -1
+        var cpuCapacity = Double.NaN
+        var cpuUsage = Double.NaN
+        var cpuUsagePct = Double.NaN
+        var memCapacity = Double.NaN
+        var memUsage = Double.NaN
+        var diskRead = Double.NaN
+        var diskWrite = Double.NaN
+        var netReceived = Double.NaN
+        var netTransmitted = Double.NaN
+
+        /**
+         * Reset the state.
+         */
+        fun reset() {
+            timestamp = null
+            cpuCores = -1
+            cpuCapacity = Double.NaN
+            cpuUsage = Double.NaN
+            cpuUsagePct = Double.NaN
+            memCapacity = Double.NaN
+            memUsage = Double.NaN
+            diskRead = Double.NaN
+            diskWrite = Double.NaN
+            netReceived = Double.NaN
+            netTransmitted = Double.NaN
+        }
+    }
+
+    companion object {
+        /**
+         * The [CsvSchema] that is used to parse the trace.
+         */
+        private val schema = CsvSchema.builder()
+            .addColumn("Timestamp [ms]", CsvSchema.ColumnType.NUMBER)
+            .addColumn("CPU cores", CsvSchema.ColumnType.NUMBER)
+            .addColumn("CPU capacity provisioned [MHZ]", CsvSchema.ColumnType.NUMBER)
+            .addColumn("CPU usage [MHZ]", CsvSchema.ColumnType.NUMBER)
+            .addColumn("CPU usage [%]", CsvSchema.ColumnType.NUMBER)
+            .addColumn("Memory capacity provisioned [KB]", CsvSchema.ColumnType.NUMBER)
+            .addColumn("Memory usage [KB]", CsvSchema.ColumnType.NUMBER)
+            .addColumn("Disk read throughput [KB/s]", CsvSchema.ColumnType.NUMBER)
+            .addColumn("Disk write throughput [KB/s]", CsvSchema.ColumnType.NUMBER)
+            .addColumn("Network received throughput [KB/s]", CsvSchema.ColumnType.NUMBER)
+            .addColumn("Network transmitted throughput [KB/s]", CsvSchema.ColumnType.NUMBER)
+            .setAllowComments(true)
+            .setUseHeader(true)
+            .setColumnSeparator(';')
+            .build()
+    }
+}
diff --git a/opendc-trace/opendc-trace-bitbrains/src/main/kotlin/org/opendc/trace/bitbrains/BitbrainsTrace.kt b/opendc-trace/opendc-trace-bitbrains/src/main/kotlin/org/opendc/trace/bitbrains/BitbrainsTrace.kt
new file mode 100644
index 00000000..5a2d4243
--- /dev/null
+++ b/opendc-trace/opendc-trace-bitbrains/src/main/kotlin/org/opendc/trace/bitbrains/BitbrainsTrace.kt
@@ -0,0 +1,46 @@
+/*
+ * Copyright (c) 2021 AtLarge Research
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+package org.opendc.trace.bitbrains
+
+import com.fasterxml.jackson.dataformat.csv.CsvFactory
+import org.opendc.trace.*
+import java.nio.file.Path
+
+/**
+ * [Trace] implementation for the Bitbrains format.
+ */
+public class BitbrainsTrace internal constructor(private val factory: CsvFactory, private val path: Path) : Trace {
+    override val tables: List<String> = listOf(TABLE_RESOURCE_STATES)
+
+    override fun containsTable(name: String): Boolean = TABLE_RESOURCE_STATES == name
+
+    override fun getTable(name: String): Table? {
+        if (!containsTable(name)) {
+            return null
+        }
+
+        return BitbrainsResourceStateTable(factory, path)
+    }
+
+    override fun toString(): String = "BitbrainsTrace[$path]"
+}
diff --git a/opendc-trace/opendc-trace-bitbrains/src/main/kotlin/org/opendc/trace/bitbrains/BitbrainsTraceFormat.kt b/opendc-trace/opendc-trace-bitbrains/src/main/kotlin/org/opendc/trace/bitbrains/BitbrainsTraceFormat.kt
new file mode 100644
index 00000000..55b11fe3
--- /dev/null
+++ b/opendc-trace/opendc-trace-bitbrains/src/main/kotlin/org/opendc/trace/bitbrains/BitbrainsTraceFormat.kt
@@ -0,0 +1,56 @@
+/*
+ * Copyright (c) 2021 AtLarge Research
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+package org.opendc.trace.bitbrains
+
+import com.fasterxml.jackson.dataformat.csv.CsvFactory
+import com.fasterxml.jackson.dataformat.csv.CsvParser
+import org.opendc.trace.spi.TraceFormat
+import java.net.URL
+import java.nio.file.Paths
+import kotlin.io.path.exists
+
+/**
+ * A format implementation for the GWF trace format.
+ */
+public class BitbrainsTraceFormat : TraceFormat {
+    /**
+     * The name of this trace format.
+     */
+    override val name: String = "bitbrains"
+
+    /**
+     * The [CsvFactory] used to create the parser.
+     */
+    private val factory = CsvFactory()
+        .enable(CsvParser.Feature.ALLOW_COMMENTS)
+        .enable(CsvParser.Feature.TRIM_SPACES)
+
+    /**
+     * Open a Bitbrains trace.
+     */
+    override fun open(url: URL): BitbrainsTrace {
+        val path = Paths.get(url.toURI())
+        require(path.exists()) { "URL $url does not exist" }
+        return BitbrainsTrace(factory, path)
+    }
+}
diff --git a/opendc-trace/opendc-trace-bitbrains/src/main/resources/META-INF/services/org.opendc.trace.spi.TraceFormat b/opendc-trace/opendc-trace-bitbrains/src/main/resources/META-INF/services/org.opendc.trace.spi.TraceFormat
new file mode 100644
index 00000000..f18135d0
--- /dev/null
+++ b/opendc-trace/opendc-trace-bitbrains/src/main/resources/META-INF/services/org.opendc.trace.spi.TraceFormat
@@ -0,0 +1 @@
+org.opendc.trace.bitbrains.BitbrainsTraceFormat