summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorFabian Mastenbroek <mail.fabianm@gmail.com>2021-06-09 13:38:56 +0200
committerFabian Mastenbroek <mail.fabianm@gmail.com>2021-06-09 14:03:54 +0200
commit0eb4fa604efe4e0b84d69749f688a79c2249c8b3 (patch)
tree96efef3c0aa511093c793dae6ea648448510bb68
parent77075ff4680667da70bdee00be7fa83539a50439 (diff)
build: Eliminate most Hadoop dependencies
This change eliminates all Hadoop dependencies that are not necessary for Parquet to work correctly. As a result, the number of dependencies should now be greatly reduced, which in turn leads to less artifacts that need to be retrieved at build time.
-rw-r--r--gradle/libs.versions.toml1
-rw-r--r--opendc-experiments/opendc-experiments-capelin/build.gradle.kts5
-rw-r--r--opendc-experiments/opendc-experiments-serverless20/build.gradle.kts6
-rw-r--r--opendc-experiments/opendc-experiments-tf20/build.gradle.kts5
-rw-r--r--opendc-format/build.gradle.kts20
5 files changed, 16 insertions, 21 deletions
diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml
index 4faa0476..22f713f6 100644
--- a/gradle/libs.versions.toml
+++ b/gradle/libs.versions.toml
@@ -46,6 +46,5 @@ kotlinx-benchmark-runtime-jvm = { module = "org.jetbrains.kotlinx:kotlinx-benchm
# Other
mongodb = { module = "org.mongodb:mongodb-driver-sync", version = "4.2.3" }
classgraph = { module = "io.github.classgraph:classgraph", version = "4.8.105" }
-hadoop-client = { module = "org.apache.hadoop:hadoop-client", version.ref = "hadoop" }
hadoop-common = { module = "org.apache.hadoop:hadoop-common", version.ref = "hadoop" }
hadoop-mapreduce-client-core = { module = "org.apache.hadoop:hadoop-mapreduce-client-core", version.ref = "hadoop" }
diff --git a/opendc-experiments/opendc-experiments-capelin/build.gradle.kts b/opendc-experiments/opendc-experiments-capelin/build.gradle.kts
index 0dade513..324cae3e 100644
--- a/opendc-experiments/opendc-experiments-capelin/build.gradle.kts
+++ b/opendc-experiments/opendc-experiments-capelin/build.gradle.kts
@@ -44,10 +44,5 @@ dependencies {
implementation(libs.clikt)
implementation(libs.parquet)
- implementation(libs.hadoop.client) {
- exclude(group = "org.slf4j", module = "slf4j-log4j12")
- exclude(group = "log4j")
- }
-
testImplementation(libs.log4j.slf4j)
}
diff --git a/opendc-experiments/opendc-experiments-serverless20/build.gradle.kts b/opendc-experiments/opendc-experiments-serverless20/build.gradle.kts
index 88479765..7d68cb3a 100644
--- a/opendc-experiments/opendc-experiments-serverless20/build.gradle.kts
+++ b/opendc-experiments/opendc-experiments-serverless20/build.gradle.kts
@@ -37,10 +37,4 @@ dependencies {
implementation(projects.opendcTelemetry.opendcTelemetrySdk)
implementation(libs.kotlin.logging)
implementation(libs.config)
-
- implementation(libs.parquet)
- implementation(libs.hadoop.client) {
- exclude(group = "org.slf4j", module = "slf4j-log4j12")
- exclude(group = "log4j")
- }
}
diff --git a/opendc-experiments/opendc-experiments-tf20/build.gradle.kts b/opendc-experiments/opendc-experiments-tf20/build.gradle.kts
index 64483bd4..b088045b 100644
--- a/opendc-experiments/opendc-experiments-tf20/build.gradle.kts
+++ b/opendc-experiments/opendc-experiments-tf20/build.gradle.kts
@@ -38,9 +38,4 @@ dependencies {
implementation(projects.opendcUtils)
implementation(libs.kotlin.logging)
- implementation(libs.parquet)
- implementation(libs.hadoop.client) {
- exclude(group = "org.slf4j", module = "slf4j-log4j12")
- exclude(group = "log4j")
- }
}
diff --git a/opendc-format/build.gradle.kts b/opendc-format/build.gradle.kts
index d3c1a59a..e19e0ec8 100644
--- a/opendc-format/build.gradle.kts
+++ b/opendc-format/build.gradle.kts
@@ -40,16 +40,28 @@ dependencies {
}
implementation(kotlin("reflect"))
+ /* This configuration is necessary for a slim dependency on Apache Parquet */
implementation(libs.parquet) {
exclude(group = "org.apache.hadoop")
}
- implementation(libs.hadoop.common) {
+ runtimeOnly(libs.hadoop.common) {
exclude(group = "org.slf4j", module = "slf4j-log4j12")
exclude(group = "log4j")
+ exclude(group = "org.apache.hadoop")
+ exclude(group = "org.apache.curator")
+ exclude(group = "org.apache.zookeeper")
+ exclude(group = "org.apache.kerby")
+ exclude(group = "org.apache.httpcomponents")
+ exclude(group = "org.apache.htrace")
+ exclude(group = "commons-cli")
+ exclude(group = "javax.servlet")
+ exclude(group = "org.eclipse.jetty")
+ exclude(group = "com.sun.jersey")
+ exclude(group = "com.jcraft")
+ exclude(group = "dnsjava")
}
- implementation(libs.hadoop.mapreduce.client.core) {
- exclude(group = "org.slf4j", module = "slf4j-log4j12")
- exclude(group = "log4j")
+ runtimeOnly(libs.hadoop.mapreduce.client.core) {
+ isTransitive = false
}
testRuntimeOnly(libs.slf4j.simple)