From 1b52a443e508bc4130071e67a1a8e17a6714c6b8 Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Tue, 8 Jun 2021 23:46:07 +0200 Subject: format: Use LocalInputFile for Parquet reader This change updates the format implementations that use Parquet by switching to our InputFile implementation for local files, which eliminates the need for Hadoop's filesystem support. --- gradle/libs.versions.toml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'gradle/libs.versions.toml') diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 1d7fdd3e..4faa0476 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -5,6 +5,7 @@ slf4j = "1.7.30" log4j = "2.14.1" opentelemetry-main = "1.2.0" opentelemetry-metrics = "1.2.0-alpha" +hadoop = "3.3.0" [libraries] kotlinx-coroutines = { module = "org.jetbrains.kotlinx:kotlinx-coroutines-core", version = "1.5.0" } @@ -45,4 +46,6 @@ kotlinx-benchmark-runtime-jvm = { module = "org.jetbrains.kotlinx:kotlinx-benchm # Other mongodb = { module = "org.mongodb:mongodb-driver-sync", version = "4.2.3" } classgraph = { module = "io.github.classgraph:classgraph", version = "4.8.105" } -hadoop-client = { module = "org.apache.hadoop:hadoop-client", version = "3.3.0" } +hadoop-client = { module = "org.apache.hadoop:hadoop-client", version.ref = "hadoop" } +hadoop-common = { module = "org.apache.hadoop:hadoop-common", version.ref = "hadoop" } +hadoop-mapreduce-client-core = { module = "org.apache.hadoop:hadoop-mapreduce-client-core", version.ref = "hadoop" } -- cgit v1.2.3 From 0eb4fa604efe4e0b84d69749f688a79c2249c8b3 Mon Sep 17 00:00:00 2001 From: Fabian Mastenbroek Date: Wed, 9 Jun 2021 13:38:56 +0200 Subject: build: Eliminate most Hadoop dependencies This change eliminates all Hadoop dependencies that are not necessary for Parquet to work correctly. As a result, the number of dependencies should now be greatly reduced, which in turn leads to less artifacts that need to be retrieved at build time. --- gradle/libs.versions.toml | 1 - 1 file changed, 1 deletion(-) (limited to 'gradle/libs.versions.toml') diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 4faa0476..22f713f6 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -46,6 +46,5 @@ kotlinx-benchmark-runtime-jvm = { module = "org.jetbrains.kotlinx:kotlinx-benchm # Other mongodb = { module = "org.mongodb:mongodb-driver-sync", version = "4.2.3" } classgraph = { module = "io.github.classgraph:classgraph", version = "4.8.105" } -hadoop-client = { module = "org.apache.hadoop:hadoop-client", version.ref = "hadoop" } hadoop-common = { module = "org.apache.hadoop:hadoop-common", version.ref = "hadoop" } hadoop-mapreduce-client-core = { module = "org.apache.hadoop:hadoop-mapreduce-client-core", version.ref = "hadoop" } -- cgit v1.2.3