diff options
| author | Fabian Mastenbroek <mail.fabianm@gmail.com> | 2022-12-13 23:59:47 +0000 |
|---|---|---|
| committer | Fabian Mastenbroek <mail.fabianm@gmail.com> | 2022-12-14 00:24:24 +0000 |
| commit | 3542350909b1213240e5097a1793a7c0733f6196 (patch) | |
| tree | e602b041f6c054e40e01548ae918cc7faf21e2a7 /opendc-trace/opendc-trace-parquet/src/main | |
| parent | de739998fde6b856e40f8a98f78efddc0c57f167 (diff) | |
fix(trace/wtf): Disable Parquet strict typing
This change fixes an issue where some of the traces from the Workflow
Trace Archive would fail to load with the trace format in OpenDC. This
was caused by one of the fields being stored as a double, while the
formats expects it to be a long.
Parquet does not support unioning primitive types. Therefore, we have to
disable strict type checking when reading the file. Furthermore, we need
to support double entries for storing the workflow ids.
Diffstat (limited to 'opendc-trace/opendc-trace-parquet/src/main')
| -rw-r--r-- | opendc-trace/opendc-trace-parquet/src/main/kotlin/org/opendc/trace/util/parquet/LocalParquetReader.kt | 8 |
1 files changed, 6 insertions, 2 deletions
diff --git a/opendc-trace/opendc-trace-parquet/src/main/kotlin/org/opendc/trace/util/parquet/LocalParquetReader.kt b/opendc-trace/opendc-trace-parquet/src/main/kotlin/org/opendc/trace/util/parquet/LocalParquetReader.kt index 031bad60..de8a56d0 100644 --- a/opendc-trace/opendc-trace-parquet/src/main/kotlin/org/opendc/trace/util/parquet/LocalParquetReader.kt +++ b/opendc-trace/opendc-trace-parquet/src/main/kotlin/org/opendc/trace/util/parquet/LocalParquetReader.kt @@ -38,10 +38,12 @@ import kotlin.io.path.isDirectory * * @param path The path to the Parquet file or directory to read. * @param readSupport Helper class to perform conversion from Parquet to [T]. + * @param strictTyping A flag to disable strict typing of primitive types. */ public class LocalParquetReader<out T>( path: Path, - private val readSupport: ReadSupport<T> + private val readSupport: ReadSupport<T>, + private val strictTyping: Boolean = true ) : AutoCloseable { /** * The input files to process. @@ -119,6 +121,8 @@ public class LocalParquetReader<out T>( private fun createReader(input: InputFile): ParquetReader<T> { return object : ParquetReader.Builder<T>(input) { override fun getReadSupport(): ReadSupport<@UnsafeVariance T> = this@LocalParquetReader.readSupport - }.build() + } + .set("parquet.strict.typing", strictTyping.toString()) + .build() } } |
