From d6190d20d0de9b5e8258dfe626fbe0d6bf48ba15 Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Tue, 8 Jun 2021 22:52:32 +0200 Subject: test: Fix logging warning for tests This change fixes the SLF4J logging warnings that occur during the tests. --- opendc-format/build.gradle.kts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'opendc-format/build.gradle.kts') diff --git a/opendc-format/build.gradle.kts b/opendc-format/build.gradle.kts index e95cb666..6c87dd99 100644 --- a/opendc-format/build.gradle.kts +++ b/opendc-format/build.gradle.kts @@ -39,10 +39,11 @@ dependencies { exclude(group = "org.jetbrains.kotlin", module = "kotlin-reflect") } implementation(kotlin("reflect")) - implementation(libs.parquet) implementation(libs.hadoop.client) { exclude(group = "org.slf4j", module = "slf4j-log4j12") exclude(group = "log4j") } + + testRuntimeOnly(libs.slf4j.simple) } -- cgit v1.2.3 From 9097811e0ac6872de3e4ff5f521d8859870b1000 Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Tue, 8 Jun 2021 23:42:48 +0200 Subject: format: Add implementation of local Parquet InputFile This change adds an implementation of Parquet's local InputFile in order to eliminate the dependency on the entire Hadoop system. This implementation allows users to read Parquet files locally without needing a Parquet filesystem implementation. --- opendc-format/build.gradle.kts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'opendc-format/build.gradle.kts') diff --git a/opendc-format/build.gradle.kts b/opendc-format/build.gradle.kts index 6c87dd99..c0ffeb3e 100644 --- a/opendc-format/build.gradle.kts +++ b/opendc-format/build.gradle.kts @@ -44,6 +44,6 @@ dependencies { exclude(group = "org.slf4j", module = "slf4j-log4j12") exclude(group = "log4j") } - + testRuntimeOnly(libs.slf4j.simple) } -- cgit v1.2.3 From 1b52a443e508bc4130071e67a1a8e17a6714c6b8 Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Tue, 8 Jun 2021 23:46:07 +0200 Subject: format: Use LocalInputFile for Parquet reader This change updates the format implementations that use Parquet by switching to our InputFile implementation for local files, which eliminates the need for Hadoop's filesystem support. --- opendc-format/build.gradle.kts | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) (limited to 'opendc-format/build.gradle.kts') diff --git a/opendc-format/build.gradle.kts b/opendc-format/build.gradle.kts index c0ffeb3e..d3c1a59a 100644 --- a/opendc-format/build.gradle.kts +++ b/opendc-format/build.gradle.kts @@ -39,8 +39,15 @@ dependencies { exclude(group = "org.jetbrains.kotlin", module = "kotlin-reflect") } implementation(kotlin("reflect")) - implementation(libs.parquet) - implementation(libs.hadoop.client) { + + implementation(libs.parquet) { + exclude(group = "org.apache.hadoop") + } + implementation(libs.hadoop.common) { + exclude(group = "org.slf4j", module = "slf4j-log4j12") + exclude(group = "log4j") + } + implementation(libs.hadoop.mapreduce.client.core) { exclude(group = "org.slf4j", module = "slf4j-log4j12") exclude(group = "log4j") } -- cgit v1.2.3 From 0eb4fa604efe4e0b84d69749f688a79c2249c8b3 Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Wed, 9 Jun 2021 13:38:56 +0200 Subject: build: Eliminate most Hadoop dependencies This change eliminates all Hadoop dependencies that are not necessary for Parquet to work correctly. As a result, the number of dependencies should now be greatly reduced, which in turn leads to less artifacts that need to be retrieved at build time. --- opendc-format/build.gradle.kts | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) (limited to 'opendc-format/build.gradle.kts') diff --git a/opendc-format/build.gradle.kts b/opendc-format/build.gradle.kts index d3c1a59a..e19e0ec8 100644 --- a/opendc-format/build.gradle.kts +++ b/opendc-format/build.gradle.kts @@ -40,16 +40,28 @@ dependencies { } implementation(kotlin("reflect")) + /* This configuration is necessary for a slim dependency on Apache Parquet */ implementation(libs.parquet) { exclude(group = "org.apache.hadoop") } - implementation(libs.hadoop.common) { + runtimeOnly(libs.hadoop.common) { exclude(group = "org.slf4j", module = "slf4j-log4j12") exclude(group = "log4j") + exclude(group = "org.apache.hadoop") + exclude(group = "org.apache.curator") + exclude(group = "org.apache.zookeeper") + exclude(group = "org.apache.kerby") + exclude(group = "org.apache.httpcomponents") + exclude(group = "org.apache.htrace") + exclude(group = "commons-cli") + exclude(group = "javax.servlet") + exclude(group = "org.eclipse.jetty") + exclude(group = "com.sun.jersey") + exclude(group = "com.jcraft") + exclude(group = "dnsjava") } - implementation(libs.hadoop.mapreduce.client.core) { - exclude(group = "org.slf4j", module = "slf4j-log4j12") - exclude(group = "log4j") + runtimeOnly(libs.hadoop.mapreduce.client.core) { + isTransitive = false } testRuntimeOnly(libs.slf4j.simple) -- cgit v1.2.3