Skip to content

Commit 2b3cfd6

Browse files
jbaierajbaiera
committed
Fix project artifact sharing. (#1442)
Embed the MR project into dependent projects by depending on the project and embedding its contents into the jar using the configuration instead of the project's jar task.
1 parent de8c04a commit 2b3cfd6

File tree

5 files changed

+106
-77
lines changed

5 files changed

+106
-77
lines changed

‎hive/build.gradle

Lines changed: 25 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -5,16 +5,38 @@ description = "Elasticsearch Hadoop Hive"
55

66
evaluationDependsOn(':elasticsearch-hadoop-mr')
77

8+
configurations {
9+
embedded {
10+
transitive = false
11+
canBeResolved = true
12+
}
13+
implementation {
14+
extendsFrom project.configurations.embedded
15+
}
16+
}
17+
818
dependencies {
9-
provided(project(":elasticsearch-hadoop-mr"))
10-
provided(project(path: ":elasticsearch-hadoop-mr", configuration:"compile"))
19+
embedded(project(":elasticsearch-hadoop-mr"))
20+
21+
provided("org.apache.hive:hive-service:$hiveVersion") {
22+
exclude module: "log4j-slf4j-impl"
23+
}
24+
provided("org.apache.hive:hive-exec:$hiveVersion")
25+
provided("org.apache.hive:hive-metastore:$hiveVersion")
1126

1227
testImplementation(project(":test:shared"))
28+
1329
itestImplementation(project(":test:shared"))
30+
itestImplementation("org.apache.hive:hive-service:$hiveVersion") {
31+
exclude module: "log4j-slf4j-impl"
32+
}
33+
itestImplementation("org.apache.hive:hive-jdbc:$hiveVersion") {
34+
exclude module: "log4j-slf4j-impl"
35+
}
1436
}
1537

1638
jar {
17-
from(zipTree(project(":elasticsearch-hadoop-mr").jar.archivePath)) {
39+
from(project.configurations.embedded.collect { it.isDirectory() ? it : zipTree(it)}) {
1840
include "org/elasticsearch/hadoop/**"
1941
include "esh-build.properties"
2042
include "META-INF/services/*"
@@ -29,18 +51,3 @@ javadoc {
2951
sourcesJar {
3052
from project(":elasticsearch-hadoop-mr").sourceSets.main.allJava.srcDirs
3153
}
32-
33-
dependencies {
34-
provided("org.apache.hive:hive-service:$hiveVersion") {
35-
exclude module: "log4j-slf4j-impl"
36-
}
37-
provided("org.apache.hive:hive-exec:$hiveVersion")
38-
provided("org.apache.hive:hive-metastore:$hiveVersion")
39-
40-
itestImplementation("org.apache.hive:hive-service:$hiveVersion") {
41-
exclude module: "log4j-slf4j-impl"
42-
}
43-
itestImplementation("org.apache.hive:hive-jdbc:$hiveVersion") {
44-
exclude module: "log4j-slf4j-impl"
45-
}
46-
}

‎pig/build.gradle

Lines changed: 23 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -5,16 +5,36 @@ description = "Elasticsearch Hadoop Pig"
55

66
evaluationDependsOn(':elasticsearch-hadoop-mr')
77

8+
configurations {
9+
embedded {
10+
transitive = false
11+
canBeResolved = true
12+
}
13+
implementation {
14+
extendsFrom project.configurations.embedded
15+
}
16+
}
17+
18+
ext.pigClassifier = "h2"
19+
820
dependencies {
9-
provided(project(":elasticsearch-hadoop-mr"))
10-
provided(project(path: ":elasticsearch-hadoop-mr", configuration:"compile"))
21+
embedded(project(":elasticsearch-hadoop-mr"))
22+
23+
provided("org.apache.pig:pig:$pigVersion:$pigClassifier")
24+
provided("joda-time:joda-time:$jodaVersion")
1125

1226
testImplementation(project(":test:shared"))
27+
testImplementation("org.apache.pig:pig:$pigVersion:$pigClassifier")
28+
testImplementation("joda-time:joda-time:$jodaVersion")
29+
testImplementation("com.google.guava:guava:11.0")
30+
testImplementation("jline:jline:0.9.94")
31+
1332
itestImplementation(project(":test:shared"))
33+
itestImplementation("dk.brics.automaton:automaton:1.11-8")
1434
}
1535

1636
jar {
17-
from(zipTree(project(":elasticsearch-hadoop-mr").jar.archivePath)) {
37+
from(project.configurations.embedded.collect { it.isDirectory() ? it : zipTree(it)}) {
1838
include "org/elasticsearch/hadoop/**"
1939
include "esh-build.properties"
2040
include "META-INF/services/*"
@@ -29,18 +49,3 @@ javadoc {
2949
sourcesJar {
3050
from project(":elasticsearch-hadoop-mr").sourceSets.main.allJava.srcDirs
3151
}
32-
33-
ext.pigClassifier = "h2"
34-
35-
dependencies {
36-
provided("org.apache.pig:pig:$pigVersion:$pigClassifier")
37-
provided("joda-time:joda-time:$jodaVersion")
38-
39-
testImplementation("org.apache.pig:pig:$pigVersion:$pigClassifier")
40-
testImplementation("joda-time:joda-time:$jodaVersion")
41-
42-
testImplementation("com.google.guava:guava:11.0")
43-
testImplementation("jline:jline:0.9.94")
44-
45-
itestImplementation("dk.brics.automaton:automaton:1.11-8")
46-
}

‎spark/sql-13/build.gradle

Lines changed: 30 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -13,18 +13,44 @@ variants {
1313
targetVersions '2.11.12'
1414
}
1515

16+
println "Compiled using Scala ${project.ext.scalaMajorVersion} [${project.ext.scalaVersion}]"
17+
String sparkVersion = spark13Version
18+
1619
configurations {
20+
embedded {
21+
transitive = false
22+
canBeResolved = true
23+
}
24+
implementation {
25+
extendsFrom project.configurations.embedded
26+
}
1727
scalaCompilerPlugin {
1828
defaultDependencies { dependencies ->
1929
if (project.ext.scalaMajorVersion != '2.10') {
2030
dependencies.add(project.dependencies.create("com.typesafe.genjavadoc:genjavadoc-plugin_${scalaVersion}:0.13"))
2131
}
2232
}
2333
}
34+
testImplementation {
35+
exclude group: "org.mortbay.jetty"
36+
}
37+
itestImplementation {
38+
exclude group: "org.mortbay.jetty"
39+
}
2440
}
2541

26-
println "Compiled using Scala ${project.ext.scalaMajorVersion} [${project.ext.scalaVersion}]"
27-
String sparkVersion = spark13Version
42+
// deal with the messy conflicts out there
43+
configurations.all { Configuration conf ->
44+
conf.resolutionStrategy {
45+
eachDependency { details ->
46+
// in a similar vein, change all javax.servlet artifacts to the one used by Spark
47+
// otherwise these will lead to SecurityException (signer information wrong)
48+
if (details.requested.name.contains("servlet") && !details.requested.name.contains("guice")) {
49+
details.useTarget group: "org.eclipse.jetty.orbit", name: "javax.servlet", version: "3.0.0.v201112011016"
50+
}
51+
}
52+
}
53+
}
2854

2955
tasks.withType(ScalaCompile) { ScalaCompile task ->
3056
task.sourceCompatibility = project.ext.minimumRuntimeVersion
@@ -90,7 +116,7 @@ eclipse {
90116
}
91117

92118
dependencies {
93-
provided(project(":elasticsearch-hadoop-mr"))
119+
embedded(project(":elasticsearch-hadoop-mr"))
94120

95121
api("org.scala-lang:scala-library:${project.ext.scalaVersion}")
96122
api("org.scala-lang:scala-reflect:${project.ext.scalaVersion}")
@@ -140,7 +166,7 @@ dependencies {
140166
}
141167

142168
jar {
143-
from(zipTree(project(":elasticsearch-hadoop-mr").jar.archivePath)) {
169+
from(project.configurations.embedded.collect { it.isDirectory() ? it : zipTree(it)}) {
144170
include "org/elasticsearch/hadoop/**"
145171
include "esh-build.properties"
146172
include "META-INF/services/*"
@@ -164,27 +190,6 @@ scaladoc {
164190
title = "${rootProject.description} ${version} API"
165191
}
166192

167-
// deal with the messy conflicts out there
168-
configurations.all { Configuration conf ->
169-
conf.resolutionStrategy {
170-
eachDependency { details ->
171-
// in a similar vein, change all javax.servlet artifacts to the one used by Spark
172-
// otherwise these will lead to SecurityException (signer information wrong)
173-
if (details.requested.name.contains("servlet") && !details.requested.name.contains("guice")) {
174-
details.useTarget group: "org.eclipse.jetty.orbit", name: "javax.servlet", version: "3.0.0.v201112011016"
175-
}
176-
}
177-
}
178-
}
179-
configurations {
180-
testImplementation {
181-
exclude group: "org.mortbay.jetty"
182-
}
183-
itestImplementation {
184-
exclude group: "org.mortbay.jetty"
185-
}
186-
}
187-
188193
tasks.withType(ScalaCompile) {
189194
if (project.ext.scalaMajorVersion != '2.10') {
190195
scalaCompileOptions.with {

‎spark/sql-20/build.gradle

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,13 @@ variants {
1414
}
1515

1616
configurations {
17+
embedded {
18+
transitive = false
19+
canBeResolved = true
20+
}
21+
implementation {
22+
extendsFrom project.configurations.embedded
23+
}
1724
scalaCompilerPlugin {
1825
defaultDependencies { dependencies ->
1926
if (project.ext.scalaMajorVersion != '2.10') {
@@ -95,7 +102,7 @@ eclipse {
95102
}
96103

97104
dependencies {
98-
provided(project(":elasticsearch-hadoop-mr"))
105+
embedded(project(":elasticsearch-hadoop-mr"))
99106

100107
api("org.scala-lang:scala-library:$scalaVersion")
101108
api("org.scala-lang:scala-reflect:$scalaVersion")
@@ -154,7 +161,7 @@ dependencies {
154161
}
155162

156163
jar {
157-
from(zipTree(project(":elasticsearch-hadoop-mr").jar.archivePath)) {
164+
from(project.configurations.embedded.collect { it.isDirectory() ? it : zipTree(it)}) {
158165
include "org/elasticsearch/hadoop/**"
159166
include "esh-build.properties"
160167
include "META-INF/services/*"

‎storm/build.gradle

Lines changed: 19 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -5,15 +5,32 @@ description = "Elasticsearch Storm"
55

66
evaluationDependsOn(':elasticsearch-hadoop-mr')
77

8+
configurations {
9+
embedded {
10+
transitive = false
11+
canBeResolved = true
12+
}
13+
implementation {
14+
extendsFrom project.configurations.embedded
15+
}
16+
}
17+
818
dependencies {
9-
provided(project(":elasticsearch-hadoop-mr"))
19+
embedded(project(":elasticsearch-hadoop-mr"))
20+
21+
provided("org.apache.storm:storm-core:$stormVersion") {
22+
exclude module: "log4j-slf4j-impl"
23+
}
1024

1125
testImplementation(project(":test:shared"))
26+
1227
itestImplementation(project(":test:shared"))
28+
itestImplementation("com.google.guava:guava:16.0.1")
29+
itestImplementation("com.twitter:carbonite:1.4.0")
1330
}
1431

1532
jar {
16-
from(zipTree(project(":elasticsearch-hadoop-mr").jar.archivePath)) {
33+
from(project.configurations.embedded.collect { it.isDirectory() ? it : zipTree(it)}) {
1734
include "org/elasticsearch/hadoop/**"
1835
include "esh-build.properties"
1936
include "META-INF/services/*"
@@ -29,16 +46,4 @@ sourcesJar {
2946
from project(":elasticsearch-hadoop-mr").sourceSets.main.allJava.srcDirs
3047
}
3148

32-
dependencies {
33-
provided("org.apache.storm:storm-core:$stormVersion") {
34-
exclude module: "log4j-slf4j-impl"
35-
}
36-
37-
itestImplementation("com.google.guava:guava:16.0.1")
38-
itestImplementation("com.twitter:carbonite:1.4.0")
39-
}
40-
41-
// add itest to Eclipse
42-
//eclipse.classpath.plusConfigurations += [configurations.itestCompile]
43-
4449
tasks.getByName('integrationTest').enabled = false

0 commit comments

Comments
 (0)