From fff89d25bd3c7b874e68261d21695c473c30ed7d Mon Sep 17 00:00:00 2001 From: Dante Niewenhuis Date: Tue, 16 Apr 2024 09:29:53 +0200 Subject: =?UTF-8?q?Revamped=20the=20trace=20system.=20All=20TraceFormat=20?= =?UTF-8?q?files=20are=20now=20in=20the=20api=20m=E2=80=A6=20(#216)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Revamped the trace system. All TraceFormat files are now in the api module. This fixes some problems with not being able to use types of traces * applied spotless --- .../org/opendc/trace/wtf/WtfTraceFormatTest.kt | 141 --------------------- .../shell/tasks/schema-1.0/part.0.parquet | Bin 259381 -> 0 bytes .../shell/workflows/schema-1.0/part.0.parquet | Bin 63858 -> 0 bytes .../workload/schema-1.0/generic_information.json | 1 - .../wtf-trace/tasks/schema-1.0/part.0.parquet | Bin 87475 -> 0 bytes 5 files changed, 142 deletions(-) delete mode 100644 opendc-trace/opendc-trace-wtf/src/test/kotlin/org/opendc/trace/wtf/WtfTraceFormatTest.kt delete mode 100755 opendc-trace/opendc-trace-wtf/src/test/resources/shell/tasks/schema-1.0/part.0.parquet delete mode 100755 opendc-trace/opendc-trace-wtf/src/test/resources/shell/workflows/schema-1.0/part.0.parquet delete mode 100755 opendc-trace/opendc-trace-wtf/src/test/resources/shell/workload/schema-1.0/generic_information.json delete mode 100644 opendc-trace/opendc-trace-wtf/src/test/resources/wtf-trace/tasks/schema-1.0/part.0.parquet (limited to 'opendc-trace/opendc-trace-wtf/src/test') diff --git a/opendc-trace/opendc-trace-wtf/src/test/kotlin/org/opendc/trace/wtf/WtfTraceFormatTest.kt b/opendc-trace/opendc-trace-wtf/src/test/kotlin/org/opendc/trace/wtf/WtfTraceFormatTest.kt deleted file mode 100644 index ad49cce0..00000000 --- a/opendc-trace/opendc-trace-wtf/src/test/kotlin/org/opendc/trace/wtf/WtfTraceFormatTest.kt +++ /dev/null @@ -1,141 +0,0 @@ -/* - * Copyright (c) 2021 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.trace.wtf - -import org.junit.jupiter.api.Assertions.assertAll -import org.junit.jupiter.api.Assertions.assertDoesNotThrow -import org.junit.jupiter.api.Assertions.assertEquals -import org.junit.jupiter.api.Assertions.assertTrue -import org.junit.jupiter.api.BeforeEach -import org.junit.jupiter.api.DisplayName -import org.junit.jupiter.api.Nested -import org.junit.jupiter.api.Test -import org.junit.jupiter.api.assertThrows -import org.opendc.trace.TableColumn -import org.opendc.trace.TableReader -import org.opendc.trace.conv.TABLE_TASKS -import org.opendc.trace.conv.TASK_ID -import org.opendc.trace.conv.TASK_PARENTS -import org.opendc.trace.conv.TASK_RUNTIME -import org.opendc.trace.conv.TASK_SUBMIT_TIME -import org.opendc.trace.conv.TASK_WORKFLOW_ID -import org.opendc.trace.testkit.TableReaderTestKit -import java.nio.file.Paths -import java.time.Duration -import java.time.Instant - -/** - * Test suite for the [WtfTraceFormat] class. - */ -@DisplayName("WTF TraceFormat") -class WtfTraceFormatTest { - private val format = WtfTraceFormat() - - @Test - fun testTables() { - val path = Paths.get("src/test/resources/wtf-trace") - assertEquals(listOf(TABLE_TASKS), format.getTables(path)) - } - - @Test - fun testTableExists() { - val path = Paths.get("src/test/resources/wtf-trace") - assertDoesNotThrow { format.getDetails(path, TABLE_TASKS) } - } - - @Test - fun testTableDoesNotExist() { - val path = Paths.get("src/test/resources/wtf-trace") - - assertThrows { format.getDetails(path, "test") } - } - - /** - * Smoke test for parsing WTF traces. - */ - @Test - fun testTableReader() { - val path = Paths.get("src/test/resources/wtf-trace") - val reader = format.newReader(path, TABLE_TASKS, listOf(TASK_ID, TASK_WORKFLOW_ID, TASK_SUBMIT_TIME, TASK_RUNTIME, TASK_PARENTS)) - - assertAll( - { assertTrue(reader.nextRow()) }, - { assertEquals("362334516345962206", reader.getString(TASK_ID)) }, - { assertEquals("1078341553348591493", reader.getString(TASK_WORKFLOW_ID)) }, - { assertEquals(Instant.ofEpochMilli(245604), reader.getInstant(TASK_SUBMIT_TIME)) }, - { assertEquals(Duration.ofMillis(8163), reader.getDuration(TASK_RUNTIME)) }, - { - assertEquals( - setOf("584055316413447529", "133113685133695608", "1008582348422865408"), - reader.getSet(TASK_PARENTS, String::class.java), - ) - }, - ) - - assertAll( - { assertTrue(reader.nextRow()) }, - { assertEquals("502010169100446658", reader.getString(TASK_ID)) }, - { assertEquals("1078341553348591493", reader.getString(TASK_WORKFLOW_ID)) }, - { assertEquals(Instant.ofEpochMilli(251325), reader.getInstant(TASK_SUBMIT_TIME)) }, - { assertEquals(Duration.ofMillis(8216), reader.getDuration(TASK_RUNTIME)) }, - { - assertEquals( - setOf("584055316413447529", "133113685133695608", "1008582348422865408"), - reader.getSet(TASK_PARENTS, String::class.java), - ) - }, - ) - - reader.close() - } - - @DisplayName("TableReader for Tasks") - @Nested - inner class TasksTableReaderTest : TableReaderTestKit() { - override lateinit var reader: TableReader - override lateinit var columns: List - - @BeforeEach - fun setUp() { - val path = Paths.get("src/test/resources/wtf-trace") - - columns = format.getDetails(path, TABLE_TASKS).columns - reader = format.newReader(path, TABLE_TASKS, null) - } - } - - @DisplayName("TableReader for Tasks (Shell trace)") - @Nested - inner class ShellTasksTableReaderTest : TableReaderTestKit() { - override lateinit var reader: TableReader - override lateinit var columns: List - - @BeforeEach - fun setUp() { - val path = Paths.get("src/test/resources/shell") - - columns = format.getDetails(path, TABLE_TASKS).columns - reader = format.newReader(path, TABLE_TASKS, null) - } - } -} diff --git a/opendc-trace/opendc-trace-wtf/src/test/resources/shell/tasks/schema-1.0/part.0.parquet b/opendc-trace/opendc-trace-wtf/src/test/resources/shell/tasks/schema-1.0/part.0.parquet deleted file mode 100755 index 31256990..00000000 Binary files a/opendc-trace/opendc-trace-wtf/src/test/resources/shell/tasks/schema-1.0/part.0.parquet and /dev/null differ diff --git a/opendc-trace/opendc-trace-wtf/src/test/resources/shell/workflows/schema-1.0/part.0.parquet b/opendc-trace/opendc-trace-wtf/src/test/resources/shell/workflows/schema-1.0/part.0.parquet deleted file mode 100755 index 872469d5..00000000 Binary files a/opendc-trace/opendc-trace-wtf/src/test/resources/shell/workflows/schema-1.0/part.0.parquet and /dev/null differ diff --git a/opendc-trace/opendc-trace-wtf/src/test/resources/shell/workload/schema-1.0/generic_information.json b/opendc-trace/opendc-trace-wtf/src/test/resources/shell/workload/schema-1.0/generic_information.json deleted file mode 100755 index 5949ab59..00000000 --- a/opendc-trace/opendc-trace-wtf/src/test/resources/shell/workload/schema-1.0/generic_information.json +++ /dev/null @@ -1 +0,0 @@ -{"total_workflows": 3403, "total_tasks": 10208, "domain": "Industrial", "date_start": null, "date_end": null, "num_sites": 3403, "num_resources": 10208.0, "num_users": 1, "num_groups": 1, "total_resource_seconds": 89229.863, "authors": ["Shenjun Ma", "Alexey Ilyushkin", "Alexander Stegehuis", "Alexandru Iosup"], "min_resource_task": 1.0, "max_resource_task": 1.0, "std_resource_task": 0.0, "mean_resource_task": 1.0, "median_resource_task": 1.0, "first_quartile_resource_task": 1.0, "third_quartile_resource_task": 1.0, "cov_resource_task": 0.0, "min_memory": -1, "max_memory": -1, "std_memory": 0.0, "mean_memory": -1.0, "median_memory": -1, "first_quartile_memory": -1, "third_quartile_memory": -1, "cov_memory": -0.0, "min_network_usage": -1, "max_network_usage": -1, "std_network_usage": 0.0, "mean_network_usage": -1.0, "median_network_usage": -1, "first_quartile_network_usage": -1, "third_quartile_network_usage": -1, "cov_network_usage": -0.0, "min_disk_space_usage": -1, "max_disk_space_usage": -1, "std_disk_space_usage": 0.0, "mean_disk_space_usage": -1.0, "median_disk_space_usage": -1, "first_quartile_disk_space_usage": -1, "third_quartile_disk_space_usage": -1, "cov_disk_space_usage": -0.0, "min_energy": -1, "max_energy": -1, "std_energy": 0.0, "mean_energy": -1.0, "median_energy": -1, "first_quartile_energy": -1, "third_quartile_energy": -1, "cov_energy": -0.0, "workload_description": "Chronos is a trace from Shell's Chronos IoT production system. It contains pipelines where sensor data is obtained, checked if values are within range (e.g. temperature, operational status, etc.), and the outcomes are written to persistent storage."} \ No newline at end of file diff --git a/opendc-trace/opendc-trace-wtf/src/test/resources/wtf-trace/tasks/schema-1.0/part.0.parquet b/opendc-trace/opendc-trace-wtf/src/test/resources/wtf-trace/tasks/schema-1.0/part.0.parquet deleted file mode 100644 index d2044038..00000000 Binary files a/opendc-trace/opendc-trace-wtf/src/test/resources/wtf-trace/tasks/schema-1.0/part.0.parquet and /dev/null differ -- cgit v1.2.3