From e8cdfbcec3f75b3f303ce52bac5f5595a94555e4 Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Tue, 31 Aug 2021 15:14:46 +0200 Subject: refactor(trace): Extract Parquet helpers into separate module This change extracts the Parquet helpers outside format module into a new module, in order to improve re-usability of these helpers. --- .../kotlin/org/opendc/format/util/ParquetTest.kt | 125 --------------------- 1 file changed, 125 deletions(-) delete mode 100644 opendc-format/src/test/kotlin/org/opendc/format/util/ParquetTest.kt (limited to 'opendc-format/src/test/kotlin') diff --git a/opendc-format/src/test/kotlin/org/opendc/format/util/ParquetTest.kt b/opendc-format/src/test/kotlin/org/opendc/format/util/ParquetTest.kt deleted file mode 100644 index e496dd96..00000000 --- a/opendc-format/src/test/kotlin/org/opendc/format/util/ParquetTest.kt +++ /dev/null @@ -1,125 +0,0 @@ -/* - * Copyright (c) 2021 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.format.util - -import org.apache.avro.SchemaBuilder -import org.apache.avro.generic.GenericData -import org.apache.parquet.avro.AvroParquetReader -import org.apache.parquet.avro.AvroParquetWriter -import org.apache.parquet.hadoop.ParquetFileWriter -import org.junit.jupiter.api.* -import org.junit.jupiter.api.Assertions.assertEquals -import java.io.File -import java.nio.file.FileAlreadyExistsException -import java.nio.file.NoSuchFileException - -/** - * Test suite for the Parquet helper classes. - */ -internal class ParquetTest { - private val schema = SchemaBuilder - .record("test") - .namespace("org.opendc.format.util") - .fields() - .name("field").type().intType().noDefault() - .endRecord() - - private lateinit var file: File - - /** - * Setup the test - */ - @BeforeEach - fun setUp() { - file = File.createTempFile("opendc", "parquet") - } - - /** - * Tear down the test. - */ - @AfterEach - fun tearDown() { - file.delete() - } - - /** - * Initial test to verify whether the Parquet writer works. - */ - @Test - fun testSmoke() { - val n = 4 - val writer = AvroParquetWriter.builder(LocalOutputFile(file)) - .withSchema(schema) - .withWriteMode(ParquetFileWriter.Mode.OVERWRITE) - .build() - - try { - repeat(n) { i -> - val record = GenericData.Record(schema) - record.put("field", i) - writer.write(record) - } - } finally { - writer.close() - } - - val reader = AvroParquetReader.builder(LocalInputFile(file)) - .build() - - var counter = 0 - try { - while (true) { - val record = reader.read() ?: break - assertEquals(counter++, record.get("field")) - } - } finally { - reader.close() - } - - assertEquals(n, counter) - } - - /** - * Test if overwriting fails if not specified. - */ - @Test - fun testOverwrite() { - assertThrows { - AvroParquetWriter.builder(LocalOutputFile(file)) - .withSchema(schema) - .build() - } - } - - /** - * Test non-existent file. - */ - @Test - fun testNonExistent() { - file.delete() - assertThrows { - AvroParquetReader.builder(LocalInputFile(file)) - .build() - } - } -} -- cgit v1.2.3 From 214480d154771f0b783829b6e5ec82b837304ad2 Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Tue, 31 Aug 2021 16:18:56 +0200 Subject: refactor(trace): Move Bitbrains format into separate module This change moves Bitbrains trace support into a separate module and adds support for the new trace api. --- .../trace/bitbrains/BitbrainsTraceReaderTest.kt | 45 ---------------------- 1 file changed, 45 deletions(-) delete mode 100644 opendc-format/src/test/kotlin/org/opendc/format/trace/bitbrains/BitbrainsTraceReaderTest.kt (limited to 'opendc-format/src/test/kotlin') diff --git a/opendc-format/src/test/kotlin/org/opendc/format/trace/bitbrains/BitbrainsTraceReaderTest.kt b/opendc-format/src/test/kotlin/org/opendc/format/trace/bitbrains/BitbrainsTraceReaderTest.kt deleted file mode 100644 index 48b4a2e3..00000000 --- a/opendc-format/src/test/kotlin/org/opendc/format/trace/bitbrains/BitbrainsTraceReaderTest.kt +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Copyright (c) 2021 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.format.trace.bitbrains - -import org.junit.jupiter.api.Assertions.assertAll -import org.junit.jupiter.api.Assertions.assertEquals -import org.junit.jupiter.api.Test - -/** - * Test suite for the [BitbrainsTraceReader] class. - */ -class BitbrainsTraceReaderTest { - @Test - fun testSmoke() { - val file = BitbrainsTraceReaderTest::class.java.getResourceAsStream("/bitbrains.csv")!! - BitbrainsRawTraceReader(file).use { reader -> - val entry = reader.next() - - assertAll( - { assertEquals(1376314846, entry.timestamp) }, - { assertEquals(19.066, entry.cpuUsage, 0.01) } - ) - } - } -} -- cgit v1.2.3 From 5c6bf9739aa0ffd9651df4fcb4cd46a8545144f0 Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Tue, 31 Aug 2021 18:08:14 +0200 Subject: refactor(trace): Implement trace API for SWF reader This change updates the SWF trace reader to support the new streaming trace API. --- .../opendc/format/trace/swf/SwfTraceReaderTest.kt | 45 ---------------------- 1 file changed, 45 deletions(-) delete mode 100644 opendc-format/src/test/kotlin/org/opendc/format/trace/swf/SwfTraceReaderTest.kt (limited to 'opendc-format/src/test/kotlin') diff --git a/opendc-format/src/test/kotlin/org/opendc/format/trace/swf/SwfTraceReaderTest.kt b/opendc-format/src/test/kotlin/org/opendc/format/trace/swf/SwfTraceReaderTest.kt deleted file mode 100644 index e0e049cf..00000000 --- a/opendc-format/src/test/kotlin/org/opendc/format/trace/swf/SwfTraceReaderTest.kt +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Copyright (c) 2020 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.format.trace.swf - -import org.junit.jupiter.api.Assertions.assertEquals -import org.junit.jupiter.api.Test -import org.opendc.simulator.compute.workload.SimTraceWorkload -import java.io.File - -class SwfTraceReaderTest { - @Test - internal fun testParseSwf() { - val reader = SwfTraceReader(File(SwfTraceReaderTest::class.java.getResource("/swf_trace.txt").toURI())) - var entry = reader.next() - assertEquals(0, entry.start) - // 1961 slices for waiting, 3 full and 1 partial running slices - assertEquals(1965, (entry.workload as SimTraceWorkload).trace.toList().size) - - entry = reader.next() - assertEquals(164472, entry.start) - // 1188 slices for waiting, 0 full and 1 partial running slices - assertEquals(1189, (entry.workload as SimTraceWorkload).trace.toList().size) - assertEquals(0.25, (entry.workload as SimTraceWorkload).trace.toList().last().usage) - } -} -- cgit v1.2.3 From b2308e1077dc60ec6a4dc646613a4be5b59695a6 Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Wed, 1 Sep 2021 11:29:27 +0200 Subject: refactor(trace): Implement trace API for WTF reader This change updates the WTF trace reader to support the new streaming trace API. --- .../opendc/format/trace/wtf/WtfTraceReaderTest.kt | 47 ---------------------- 1 file changed, 47 deletions(-) delete mode 100644 opendc-format/src/test/kotlin/org/opendc/format/trace/wtf/WtfTraceReaderTest.kt (limited to 'opendc-format/src/test/kotlin') diff --git a/opendc-format/src/test/kotlin/org/opendc/format/trace/wtf/WtfTraceReaderTest.kt b/opendc-format/src/test/kotlin/org/opendc/format/trace/wtf/WtfTraceReaderTest.kt deleted file mode 100644 index 31ae03e0..00000000 --- a/opendc-format/src/test/kotlin/org/opendc/format/trace/wtf/WtfTraceReaderTest.kt +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Copyright (c) 2020 AtLarge Research - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package org.opendc.format.trace.wtf - -import org.junit.jupiter.api.Assertions.* -import org.junit.jupiter.api.Test -import java.io.File - -/** - * Test suite for the [WtfTraceReader] class. - */ -class WtfTraceReaderTest { - /** - * Smoke test for parsing WTF traces. - */ - @Test - fun testParseWtf() { - val reader = WtfTraceReader(File("src/test/resources/wtf-trace")) - var entry = reader.next() - assertEquals(0, entry.start) - assertEquals(23, entry.workload.tasks.size) - - entry = reader.next() - assertEquals(333387, entry.start) - assertEquals(23, entry.workload.tasks.size) - } -} -- cgit v1.2.3