merge: Improve discovery of interference models (#76)

This pull request intends to improve discovery of interference models. Previously, interference models were not tied to the workload trace, meaning they had to be resolved separately from the workload trace. In reality, the interference model is always tied to the particular workload trace.

With this pull request, we integrate the interference model into the `odcvm` trace format and make it available through the `opendc-trace` library. This has as additional benefit that we can support different interference formats in the future using the same API.

Furthermore, this change allows us to ship the interference model with the workload traces and resolve them automatically in the future using some form of package manager.

## Implementation Notes :hammer_and_pick:

* Incorporate interference model in trace format 
* Load interference model via trace library
* Move conventions into separate package 

## External Dependencies :four_leaf_clover:

* N/A

## Breaking API Changes :warning:

* `VmInterferenceModelReader` has been removed from `opendc-compute-workload`
* Table and column conventions have been moved in `org.opendc.trace.conv` package

9 files changed, 381 insertions, 3 deletions

diff --git a/opendc-trace/opendc-trace-opendc/src/main/kotlin/org/opendc/trace/opendc/OdcVmInterferenceJsonTableReader.kt b/opendc-trace/opendc-trace-opendc/src/main/kotlin/org/opendc/trace/opendc/OdcVmInterferenceJsonTableReader.kt
new file mode 100644
index 00000000..eb91e305
--- /dev/null
+++ b/opendc-trace/opendc-trace-opendc/src/main/kotlin/org/opendc/trace/opendc/OdcVmInterferenceJsonTableReader.kt
@@ -0,0 +1,169 @@
+/*
+ * Copyright (c) 2022 AtLarge Research
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+package org.opendc.trace.opendc
+
+import org.opendc.trace.*
+import org.opendc.trace.conv.INTERFERENCE_GROUP_MEMBERS
+import org.opendc.trace.conv.INTERFERENCE_GROUP_SCORE
+import org.opendc.trace.conv.INTERFERENCE_GROUP_TARGET
+import shaded.parquet.com.fasterxml.jackson.core.JsonParseException
+import shaded.parquet.com.fasterxml.jackson.core.JsonParser
+import shaded.parquet.com.fasterxml.jackson.core.JsonToken
+
+/**
+ * A [TableReader] implementation for the OpenDC VM interference JSON format.
+ */
+internal class OdcVmInterferenceJsonTableReader(private val parser: JsonParser) : TableReader {
+    /**
+     * A flag to indicate whether a single row has been read already.
+     */
+    private var isStarted = false
+
+    override fun nextRow(): Boolean {
+        if (!isStarted) {
+            isStarted = true
+
+            parser.nextToken()
+
+            if (!parser.isExpectedStartArrayToken) {
+                throw JsonParseException(parser, "Expected array at start, but got ${parser.currentToken()}")
+            }
+        }
+
+        return if (parser.nextToken() != JsonToken.END_ARRAY) {
+            parseGroup(parser)
+            true
+        } else {
+            reset()
+            false
+        }
+    }
+
+    override fun resolve(column: TableColumn<*>): Int {
+        return when (column) {
+            INTERFERENCE_GROUP_MEMBERS -> COL_MEMBERS
+            INTERFERENCE_GROUP_TARGET -> COL_TARGET
+            INTERFERENCE_GROUP_SCORE -> COL_SCORE
+            else -> -1
+        }
+    }
+
+    override fun isNull(index: Int): Boolean {
+        return when (index) {
+            COL_MEMBERS, COL_TARGET, COL_SCORE -> false
+            else -> throw IllegalArgumentException("Invalid column index $index")
+        }
+    }
+
+    override fun get(index: Int): Any {
+        return when (index) {
+            COL_MEMBERS -> members
+            COL_TARGET -> targetLoad
+            COL_SCORE -> score
+            else -> throw IllegalArgumentException("Invalid column $index")
+        }
+    }
+
+    override fun getBoolean(index: Int): Boolean {
+        throw IllegalArgumentException("Invalid column $index")
+    }
+
+    override fun getInt(index: Int): Int {
+        throw IllegalArgumentException("Invalid column $index")
+    }
+
+    override fun getLong(index: Int): Long {
+        throw IllegalArgumentException("Invalid column $index")
+    }
+
+    override fun getDouble(index: Int): Double {
+        return when (index) {
+            COL_TARGET -> targetLoad
+            COL_SCORE -> score
+            else -> throw IllegalArgumentException("Invalid column $index")
+        }
+    }
+
+    override fun close() {
+        parser.close()
+    }
+
+    private val COL_MEMBERS = 0
+    private val COL_TARGET = 1
+    private val COL_SCORE = 2
+
+    private var members = emptySet<String>()
+    private var targetLoad = Double.POSITIVE_INFINITY
+    private var score = 1.0
+
+    /**
+     * Reset the state.
+     */
+    private fun reset() {
+        members = emptySet()
+        targetLoad = Double.POSITIVE_INFINITY
+        score = 1.0
+    }
+
+    /**
+     * Parse a group an interference JSON file.
+     */
+    private fun parseGroup(parser: JsonParser) {
+        var targetLoad = Double.POSITIVE_INFINITY
+        var score = 1.0
+        val members = mutableSetOf<String>()
+
+        if (!parser.isExpectedStartObjectToken) {
+            throw JsonParseException(parser, "Expected object, but got ${parser.currentToken()}")
+        }
+
+        while (parser.nextValue() != JsonToken.END_OBJECT) {
+            when (parser.currentName) {
+                "vms" -> parseGroupMembers(parser, members)
+                "minServerLoad" -> targetLoad = parser.doubleValue
+                "performanceScore" -> score = parser.doubleValue
+            }
+        }
+
+        this.members = members
+        this.targetLoad = targetLoad
+        this.score = score
+    }
+
+    /**
+     * Parse the members of a group.
+     */
+    private fun parseGroupMembers(parser: JsonParser, members: MutableSet<String>) {
+        if (!parser.isExpectedStartArrayToken) {
+            throw JsonParseException(parser, "Expected array for group members, but got ${parser.currentToken()}")
+        }
+
+        while (parser.nextValue() != JsonToken.END_ARRAY) {
+            if (parser.currentToken() != JsonToken.VALUE_STRING) {
+                throw JsonParseException(parser, "Expected string value for group member")
+            }
+
+            members.add(parser.text)
+        }
+    }
+}
diff --git a/opendc-trace/opendc-trace-opendc/src/main/kotlin/org/opendc/trace/opendc/OdcVmInterferenceJsonTableWriter.kt b/opendc-trace/opendc-trace-opendc/src/main/kotlin/org/opendc/trace/opendc/OdcVmInterferenceJsonTableWriter.kt
new file mode 100644
index 00000000..64bc4356
--- /dev/null
+++ b/opendc-trace/opendc-trace-opendc/src/main/kotlin/org/opendc/trace/opendc/OdcVmInterferenceJsonTableWriter.kt
@@ -0,0 +1,127 @@
+/*
+ * Copyright (c) 2022 AtLarge Research
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+package org.opendc.trace.opendc
+
+import org.opendc.trace.*
+import org.opendc.trace.conv.INTERFERENCE_GROUP_MEMBERS
+import org.opendc.trace.conv.INTERFERENCE_GROUP_SCORE
+import org.opendc.trace.conv.INTERFERENCE_GROUP_TARGET
+import shaded.parquet.com.fasterxml.jackson.core.JsonGenerator
+
+/**
+ * A [TableWriter] implementation for the OpenDC VM interference JSON format.
+ */
+internal class OdcVmInterferenceJsonTableWriter(private val generator: JsonGenerator) : TableWriter {
+    /**
+     * A flag to indicate whether a row has been started.
+     */
+    private var isRowActive = false
+
+    init {
+        generator.writeStartArray()
+    }
+
+    override fun startRow() {
+        // Reset state
+        members = emptySet()
+        targetLoad = Double.POSITIVE_INFINITY
+        score = 1.0
+
+        // Mark row as active
+        isRowActive = true
+    }
+
+    override fun endRow() {
+        check(isRowActive) { "No active row" }
+
+        generator.writeStartObject()
+        generator.writeArrayFieldStart("vms")
+        for (member in members) {
+            generator.writeString(member)
+        }
+        generator.writeEndArray()
+        generator.writeNumberField("minServerLoad", targetLoad)
+        generator.writeNumberField("performanceScore", score)
+        generator.writeEndObject()
+    }
+
+    override fun resolve(column: TableColumn<*>): Int {
+        return when (column) {
+            INTERFERENCE_GROUP_MEMBERS -> COL_MEMBERS
+            INTERFERENCE_GROUP_TARGET -> COL_TARGET
+            INTERFERENCE_GROUP_SCORE -> COL_SCORE
+            else -> -1
+        }
+    }
+
+    override fun set(index: Int, value: Any) {
+        check(isRowActive) { "No active row" }
+
+        @Suppress("UNCHECKED_CAST")
+        when (index) {
+            COL_MEMBERS -> members = value as Set<String>
+            COL_TARGET -> targetLoad = (value as Number).toDouble()
+            COL_SCORE -> score = (value as Number).toDouble()
+            else -> throw IllegalArgumentException("Invalid column index $index")
+        }
+    }
+
+    override fun setBoolean(index: Int, value: Boolean) {
+        throw IllegalArgumentException("Invalid column $index")
+    }
+
+    override fun setInt(index: Int, value: Int) {
+        throw IllegalArgumentException("Invalid column $index")
+    }
+
+    override fun setLong(index: Int, value: Long) {
+        throw IllegalArgumentException("Invalid column $index")
+    }
+
+    override fun setDouble(index: Int, value: Double) {
+        check(isRowActive) { "No active row" }
+
+        when (index) {
+            COL_TARGET -> targetLoad = (value as Number).toDouble()
+            COL_SCORE -> score = (value as Number).toDouble()
+            else -> throw IllegalArgumentException("Invalid column $index")
+        }
+    }
+
+    override fun flush() {
+        generator.flush()
+    }
+
+    override fun close() {
+        generator.writeEndArray()
+        generator.close()
+    }
+
+    private val COL_MEMBERS = 0
+    private val COL_TARGET = 1
+    private val COL_SCORE = 2
+
+    private var members = emptySet<String>()
+    private var targetLoad = Double.POSITIVE_INFINITY
+    private var score = 1.0
+}
diff --git a/opendc-trace/opendc-trace-opendc/src/main/kotlin/org/opendc/trace/opendc/OdcVmResourceStateTableReader.kt b/opendc-trace/opendc-trace-opendc/src/main/kotlin/org/opendc/trace/opendc/OdcVmResourceStateTableReader.kt
index b5043f82..b82da888 100644
--- a/opendc-trace/opendc-trace-opendc/src/main/kotlin/org/opendc/trace/opendc/OdcVmResourceStateTableReader.kt
+++ b/opendc-trace/opendc-trace-opendc/src/main/kotlin/org/opendc/trace/opendc/OdcVmResourceStateTableReader.kt
@@ -25,6 +25,7 @@ package org.opendc.trace.opendc
 import org.apache.avro.Schema
 import org.apache.avro.generic.GenericRecord
 import org.opendc.trace.*
+import org.opendc.trace.conv.*
 import org.opendc.trace.util.parquet.LocalParquetReader
 import java.time.Duration
 import java.time.Instant
diff --git a/opendc-trace/opendc-trace-opendc/src/main/kotlin/org/opendc/trace/opendc/OdcVmResourceStateTableWriter.kt b/opendc-trace/opendc-trace-opendc/src/main/kotlin/org/opendc/trace/opendc/OdcVmResourceStateTableWriter.kt
index 15a8cb85..01b9750c 100644
--- a/opendc-trace/opendc-trace-opendc/src/main/kotlin/org/opendc/trace/opendc/OdcVmResourceStateTableWriter.kt
+++ b/opendc-trace/opendc-trace-opendc/src/main/kotlin/org/opendc/trace/opendc/OdcVmResourceStateTableWriter.kt
@@ -27,6 +27,7 @@ import org.apache.avro.generic.GenericRecord
 import org.apache.avro.generic.GenericRecordBuilder
 import org.apache.parquet.hadoop.ParquetWriter
 import org.opendc.trace.*
+import org.opendc.trace.conv.*
 import java.time.Duration
 import java.time.Instant
 
diff --git a/opendc-trace/opendc-trace-opendc/src/main/kotlin/org/opendc/trace/opendc/OdcVmResourceTableReader.kt b/opendc-trace/opendc-trace-opendc/src/main/kotlin/org/opendc/trace/opendc/OdcVmResourceTableReader.kt
index ffbdc440..4909e70e 100644
--- a/opendc-trace/opendc-trace-opendc/src/main/kotlin/org/opendc/trace/opendc/OdcVmResourceTableReader.kt
+++ b/opendc-trace/opendc-trace-opendc/src/main/kotlin/org/opendc/trace/opendc/OdcVmResourceTableReader.kt
@@ -25,6 +25,7 @@ package org.opendc.trace.opendc
 import org.apache.avro.Schema
 import org.apache.avro.generic.GenericRecord
 import org.opendc.trace.*
+import org.opendc.trace.conv.*
 import org.opendc.trace.util.parquet.LocalParquetReader
 import java.time.Instant
 
diff --git a/opendc-trace/opendc-trace-opendc/src/main/kotlin/org/opendc/trace/opendc/OdcVmResourceTableWriter.kt b/opendc-trace/opendc-trace-opendc/src/main/kotlin/org/opendc/trace/opendc/OdcVmResourceTableWriter.kt
index 4b66a86f..edc89ee6 100644
--- a/opendc-trace/opendc-trace-opendc/src/main/kotlin/org/opendc/trace/opendc/OdcVmResourceTableWriter.kt
+++ b/opendc-trace/opendc-trace-opendc/src/main/kotlin/org/opendc/trace/opendc/OdcVmResourceTableWriter.kt
@@ -27,6 +27,7 @@ import org.apache.avro.generic.GenericRecord
 import org.apache.avro.generic.GenericRecordBuilder
 import org.apache.parquet.hadoop.ParquetWriter
 import org.opendc.trace.*
+import org.opendc.trace.conv.*
 import java.time.Instant
 import kotlin.math.roundToLong
 
diff --git a/opendc-trace/opendc-trace-opendc/src/main/kotlin/org/opendc/trace/opendc/OdcVmTraceFormat.kt b/opendc-trace/opendc-trace-opendc/src/main/kotlin/org/opendc/trace/opendc/OdcVmTraceFormat.kt
index 886f3d54..36a1b4a0 100644
--- a/opendc-trace/opendc-trace-opendc/src/main/kotlin/org/opendc/trace/opendc/OdcVmTraceFormat.kt
+++ b/opendc-trace/opendc-trace-opendc/src/main/kotlin/org/opendc/trace/opendc/OdcVmTraceFormat.kt
@@ -29,19 +29,28 @@ import org.apache.parquet.avro.AvroParquetWriter
 import org.apache.parquet.hadoop.ParquetFileWriter
 import org.apache.parquet.hadoop.metadata.CompressionCodecName
 import org.opendc.trace.*
+import org.opendc.trace.conv.*
 import org.opendc.trace.spi.TableDetails
 import org.opendc.trace.spi.TraceFormat
 import org.opendc.trace.util.parquet.LocalOutputFile
 import org.opendc.trace.util.parquet.LocalParquetReader
 import org.opendc.trace.util.parquet.TIMESTAMP_SCHEMA
+import shaded.parquet.com.fasterxml.jackson.core.JsonEncoding
+import shaded.parquet.com.fasterxml.jackson.core.JsonFactory
 import java.nio.file.Files
 import java.nio.file.Path
+import kotlin.io.path.exists
 
 /**
  * A [TraceFormat] implementation of the OpenDC virtual machine trace format.
  */
 public class OdcVmTraceFormat : TraceFormat {
     /**
+     * A [JsonFactory] that is used to parse the JSON-based interference model.
+     */
+    private val jsonFactory = JsonFactory()
+
+    /**
      * The name of this trace format.
      */
     override val name: String = "opendc-vm"
@@ -58,7 +67,7 @@ public class OdcVmTraceFormat : TraceFormat {
         }
     }
 
-    override fun getTables(path: Path): List<String> = listOf(TABLE_RESOURCES, TABLE_RESOURCE_STATES)
+    override fun getTables(path: Path): List<String> = listOf(TABLE_RESOURCES, TABLE_RESOURCE_STATES, TABLE_INTERFERENCE_GROUPS)
 
     override fun getDetails(path: Path, table: String): TableDetails {
         return when (table) {
@@ -82,6 +91,13 @@ public class OdcVmTraceFormat : TraceFormat {
                 ),
                 listOf(RESOURCE_ID, RESOURCE_STATE_TIMESTAMP)
             )
+            TABLE_INTERFERENCE_GROUPS -> TableDetails(
+                listOf(
+                    INTERFERENCE_GROUP_MEMBERS,
+                    INTERFERENCE_GROUP_TARGET,
+                    INTERFERENCE_GROUP_SCORE,
+                )
+            )
             else -> throw IllegalArgumentException("Table $table not supported")
         }
     }
@@ -96,6 +112,15 @@ public class OdcVmTraceFormat : TraceFormat {
                 val reader = LocalParquetReader<GenericRecord>(path.resolve("trace.parquet"))
                 OdcVmResourceStateTableReader(reader)
             }
+            TABLE_INTERFERENCE_GROUPS -> {
+                val modelPath = path.resolve("interference-model.json")
+                val parser = if (modelPath.exists())
+                    jsonFactory.createParser(modelPath.toFile())
+                else
+                    jsonFactory.createParser("[]") // If model does not exist, return empty model
+
+                OdcVmInterferenceJsonTableReader(parser)
+            }
             else -> throw IllegalArgumentException("Table $table not supported")
         }
     }
@@ -122,6 +147,10 @@ public class OdcVmTraceFormat : TraceFormat {
                     .build()
                 OdcVmResourceStateTableWriter(writer, schema)
             }
+            TABLE_INTERFERENCE_GROUPS -> {
+                val generator = jsonFactory.createGenerator(path.resolve("interference-model.json").toFile(), JsonEncoding.UTF8)
+                OdcVmInterferenceJsonTableWriter(generator)
+            }
             else -> throw IllegalArgumentException("Table $table not supported")
         }
     }
diff --git a/opendc-trace/opendc-trace-opendc/src/test/kotlin/org/opendc/trace/opendc/OdcVmTraceFormatTest.kt b/opendc-trace/opendc-trace-opendc/src/test/kotlin/org/opendc/trace/opendc/OdcVmTraceFormatTest.kt
index bfe0f881..c8742624 100644
--- a/opendc-trace/opendc-trace-opendc/src/test/kotlin/org/opendc/trace/opendc/OdcVmTraceFormatTest.kt
+++ b/opendc-trace/opendc-trace-opendc/src/test/kotlin/org/opendc/trace/opendc/OdcVmTraceFormatTest.kt
@@ -28,7 +28,7 @@ import org.junit.jupiter.api.assertDoesNotThrow
 import org.junit.jupiter.api.assertThrows
 import org.junit.jupiter.params.ParameterizedTest
 import org.junit.jupiter.params.provider.ValueSource
-import org.opendc.trace.*
+import org.opendc.trace.conv.*
 import java.nio.file.Paths
 
 /**
@@ -41,7 +41,7 @@ internal class OdcVmTraceFormatTest {
     fun testTables() {
         val path = Paths.get("src/test/resources/trace-v2.1")
 
-        assertEquals(listOf(TABLE_RESOURCES, TABLE_RESOURCE_STATES), format.getTables(path))
+        assertEquals(listOf(TABLE_RESOURCES, TABLE_RESOURCE_STATES, TABLE_INTERFERENCE_GROUPS), format.getTables(path))
     }
 
     @Test
@@ -93,4 +93,33 @@ internal class OdcVmTraceFormatTest {
 
         reader.close()
     }
+
+    @Test
+    fun testInterferenceGroups() {
+        val path = Paths.get("src/test/resources/trace-v2.1")
+        val reader = format.newReader(path, TABLE_INTERFERENCE_GROUPS)
+
+        assertAll(
+            { assertTrue(reader.nextRow()) },
+            { assertEquals(setOf("1019", "1023", "1052"), reader.get(INTERFERENCE_GROUP_MEMBERS)) },
+            { assertEquals(0.0, reader.get(INTERFERENCE_GROUP_TARGET)) },
+            { assertEquals(0.8830158730158756, reader.get(INTERFERENCE_GROUP_SCORE)) },
+            { assertTrue(reader.nextRow()) },
+            { assertEquals(setOf("1023", "1052", "1073"), reader.get(INTERFERENCE_GROUP_MEMBERS)) },
+            { assertEquals(0.0, reader.get(INTERFERENCE_GROUP_TARGET)) },
+            { assertEquals(0.7133055555552751, reader.get(INTERFERENCE_GROUP_SCORE)) },
+            { assertFalse(reader.nextRow()) }
+        )
+
+        reader.close()
+    }
+
+    @Test
+    fun testInterferenceGroupsEmpty() {
+        val path = Paths.get("src/test/resources/trace-v2.0")
+        val reader = format.newReader(path, TABLE_INTERFERENCE_GROUPS)
+
+        assertFalse(reader.nextRow())
+        reader.close()
+    }
 }
diff --git a/opendc-trace/opendc-trace-opendc/src/test/resources/trace-v2.1/interference-model.json b/opendc-trace/opendc-trace-opendc/src/test/resources/trace-v2.1/interference-model.json
new file mode 100644
index 00000000..6a0616d9
--- /dev/null
+++ b/opendc-trace/opendc-trace-opendc/src/test/resources/trace-v2.1/interference-model.json
@@ -0,0 +1,20 @@
+[
+    {
+        "vms": [
+            "1019",
+            "1023",
+            "1052"
+        ],
+        "minServerLoad": 0.0,
+        "performanceScore": 0.8830158730158756
+    },
+    {
+        "vms": [
+            "1023",
+            "1052",
+            "1073"
+        ],
+        "minServerLoad": 0.0,
+        "performanceScore": 0.7133055555552751
+    }
+]