Skip to content

Commit b34e112

Browse files
jbaierajbaiera
authored
Fix project artifact sharing. (#1442)
Embed the MR project into dependent projects by depending on the project and embedding its contents into the jar using the configuration instead of the project's jar task.
1 parent 7f36061 commit b34e112

File tree

5 files changed

+106
-77
lines changed

5 files changed

+106
-77
lines changed

‎hive/build.gradle

Lines changed: 25 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -5,16 +5,38 @@ description = "Elasticsearch Hadoop Hive"
55

66
evaluationDependsOn(':elasticsearch-hadoop-mr')
77

8+
configurations {
9+
embedded {
10+
transitive = false
11+
canBeResolved = true
12+
}
13+
implementation {
14+
extendsFrom project.configurations.embedded
15+
}
16+
}
17+
818
dependencies {
9-
provided(project(":elasticsearch-hadoop-mr"))
10-
provided(project(path: ":elasticsearch-hadoop-mr", configuration:"compile"))
19+
embedded(project(":elasticsearch-hadoop-mr"))
20+
21+
provided("org.apache.hive:hive-service:$hiveVersion") {
22+
exclude module: "log4j-slf4j-impl"
23+
}
24+
provided("org.apache.hive:hive-exec:$hiveVersion")
25+
provided("org.apache.hive:hive-metastore:$hiveVersion")
1126

1227
testImplementation(project(":test:shared"))
28+
1329
itestImplementation(project(":test:shared"))
30+
itestImplementation("org.apache.hive:hive-service:$hiveVersion") {
31+
exclude module: "log4j-slf4j-impl"
32+
}
33+
itestImplementation("org.apache.hive:hive-jdbc:$hiveVersion") {
34+
exclude module: "log4j-slf4j-impl"
35+
}
1436
}
1537

1638
jar {
17-
from(zipTree(project(":elasticsearch-hadoop-mr").jar.archivePath)) {
39+
from(project.configurations.embedded.collect { it.isDirectory() ? it : zipTree(it)}) {
1840
include "org/elasticsearch/hadoop/**"
1941
include "esh-build.properties"
2042
include "META-INF/services/*"
@@ -29,18 +51,3 @@ javadoc {
2951
sourcesJar {
3052
from project(":elasticsearch-hadoop-mr").sourceSets.main.allJava.srcDirs
3153
}
32-
33-
dependencies {
34-
provided("org.apache.hive:hive-service:$hiveVersion") {
35-
exclude module: "log4j-slf4j-impl"
36-
}
37-
provided("org.apache.hive:hive-exec:$hiveVersion")
38-
provided("org.apache.hive:hive-metastore:$hiveVersion")
39-
40-
itestImplementation("org.apache.hive:hive-service:$hiveVersion") {
41-
exclude module: "log4j-slf4j-impl"
42-
}
43-
itestImplementation("org.apache.hive:hive-jdbc:$hiveVersion") {
44-
exclude module: "log4j-slf4j-impl"
45-
}
46-
}

‎pig/build.gradle

Lines changed: 23 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -5,16 +5,36 @@ description = "Elasticsearch Hadoop Pig"
55

66
evaluationDependsOn(':elasticsearch-hadoop-mr')
77

8+
configurations {
9+
embedded {
10+
transitive = false
11+
canBeResolved = true
12+
}
13+
implementation {
14+
extendsFrom project.configurations.embedded
15+
}
16+
}
17+
18+
ext.pigClassifier = "h2"
19+
820
dependencies {
9-
provided(project(":elasticsearch-hadoop-mr"))
10-
provided(project(path: ":elasticsearch-hadoop-mr", configuration:"compile"))
21+
embedded(project(":elasticsearch-hadoop-mr"))
22+
23+
provided("org.apache.pig:pig:$pigVersion:$pigClassifier")
24+
provided("joda-time:joda-time:$jodaVersion")
1125

1226
testImplementation(project(":test:shared"))
27+
testImplementation("org.apache.pig:pig:$pigVersion:$pigClassifier")
28+
testImplementation("joda-time:joda-time:$jodaVersion")
29+
testImplementation("com.google.guava:guava:11.0")
30+
testImplementation("jline:jline:0.9.94")
31+
1332
itestImplementation(project(":test:shared"))
33+
itestImplementation("dk.brics.automaton:automaton:1.11-8")
1434
}
1535

1636
jar {
17-
from(zipTree(project(":elasticsearch-hadoop-mr").jar.archivePath)) {
37+
from(project.configurations.embedded.collect { it.isDirectory() ? it : zipTree(it)}) {
1838
include "org/elasticsearch/hadoop/**"
1939
include "esh-build.properties"
2040
include "META-INF/services/*"
@@ -29,18 +49,3 @@ javadoc {
2949
sourcesJar {
3050
from project(":elasticsearch-hadoop-mr").sourceSets.main.allJava.srcDirs
3151
}
32-
33-
ext.pigClassifier = "h2"
34-
35-
dependencies {
36-
provided("org.apache.pig:pig:$pigVersion:$pigClassifier")
37-
provided("joda-time:joda-time:$jodaVersion")
38-
39-
testImplementation("org.apache.pig:pig:$pigVersion:$pigClassifier")
40-
testImplementation("joda-time:joda-time:$jodaVersion")
41-
42-
testImplementation("com.google.guava:guava:11.0")
43-
testImplementation("jline:jline:0.9.94")
44-
45-
itestImplementation("dk.brics.automaton:automaton:1.11-8")
46-
}

‎spark/sql-13/build.gradle

Lines changed: 30 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -13,16 +13,42 @@ variants {
1313
targetVersions '2.11.12'
1414
}
1515

16+
println "Compiled using Scala ${project.ext.scalaMajorVersion} [${project.ext.scalaVersion}]"
17+
String sparkVersion = spark13Version
18+
1619
configurations {
20+
embedded {
21+
transitive = false
22+
canBeResolved = true
23+
}
24+
implementation {
25+
extendsFrom project.configurations.embedded
26+
}
1727
scalaCompilerPlugin {
1828
defaultDependencies { dependencies ->
1929
dependencies.add(project.dependencies.create( "com.typesafe.genjavadoc:genjavadoc-plugin_${scalaVersion}:0.13"))
2030
}
2131
}
32+
testImplementation {
33+
exclude group: "org.mortbay.jetty"
34+
}
35+
itestImplementation {
36+
exclude group: "org.mortbay.jetty"
37+
}
2238
}
2339

24-
println "Compiled using Scala ${project.ext.scalaMajorVersion} [${project.ext.scalaVersion}]"
25-
String sparkVersion = spark13Version
40+
// deal with the messy conflicts out there
41+
configurations.all { Configuration conf ->
42+
conf.resolutionStrategy {
43+
eachDependency { details ->
44+
// in a similar vein, change all javax.servlet artifacts to the one used by Spark
45+
// otherwise these will lead to SecurityException (signer information wrong)
46+
if (details.requested.name.contains("servlet") && !details.requested.name.contains("guice")) {
47+
details.useTarget group: "org.eclipse.jetty.orbit", name: "javax.servlet", version: "3.0.0.v201112011016"
48+
}
49+
}
50+
}
51+
}
2652

2753
tasks.withType(ScalaCompile) { ScalaCompile task ->
2854
task.sourceCompatibility = project.ext.minimumRuntimeVersion
@@ -88,7 +114,7 @@ eclipse {
88114
}
89115

90116
dependencies {
91-
provided(project(":elasticsearch-hadoop-mr"))
117+
embedded(project(":elasticsearch-hadoop-mr"))
92118

93119
api("org.scala-lang:scala-library:${project.ext.scalaVersion}")
94120
api("org.scala-lang:scala-reflect:${project.ext.scalaVersion}")
@@ -138,7 +164,7 @@ dependencies {
138164
}
139165

140166
jar {
141-
from(zipTree(project(":elasticsearch-hadoop-mr").jar.archivePath)) {
167+
from(project.configurations.embedded.collect { it.isDirectory() ? it : zipTree(it)}) {
142168
include "org/elasticsearch/hadoop/**"
143169
include "esh-build.properties"
144170
include "META-INF/services/*"
@@ -160,27 +186,6 @@ scaladoc {
160186
title = "${rootProject.description} ${version} API"
161187
}
162188

163-
// deal with the messy conflicts out there
164-
configurations.all { Configuration conf ->
165-
conf.resolutionStrategy {
166-
eachDependency { details ->
167-
// in a similar vein, change all javax.servlet artifacts to the one used by Spark
168-
// otherwise these will lead to SecurityException (signer information wrong)
169-
if (details.requested.name.contains("servlet") && !details.requested.name.contains("guice")) {
170-
details.useTarget group: "org.eclipse.jetty.orbit", name: "javax.servlet", version: "3.0.0.v201112011016"
171-
}
172-
}
173-
}
174-
}
175-
configurations {
176-
testImplementation {
177-
exclude group: "org.mortbay.jetty"
178-
}
179-
itestImplementation {
180-
exclude group: "org.mortbay.jetty"
181-
}
182-
}
183-
184189
tasks.withType(ScalaCompile) {
185190
scalaCompileOptions.with {
186191
additionalParameters = [

‎spark/sql-20/build.gradle

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,13 @@ variants {
1414
}
1515

1616
configurations {
17+
embedded {
18+
transitive = false
19+
canBeResolved = true
20+
}
21+
implementation {
22+
extendsFrom project.configurations.embedded
23+
}
1724
scalaCompilerPlugin {
1825
defaultDependencies { dependencies ->
1926
dependencies.add(project.dependencies.create( "com.typesafe.genjavadoc:genjavadoc-plugin_${scalaVersion}:0.13"))
@@ -93,7 +100,7 @@ eclipse {
93100
}
94101

95102
dependencies {
96-
provided(project(":elasticsearch-hadoop-mr"))
103+
embedded(project(":elasticsearch-hadoop-mr"))
97104

98105
api("org.scala-lang:scala-library:$scalaVersion")
99106
api("org.scala-lang:scala-reflect:$scalaVersion")
@@ -152,7 +159,7 @@ dependencies {
152159
}
153160

154161
jar {
155-
from(zipTree(project(":elasticsearch-hadoop-mr").jar.archivePath)) {
162+
from(project.configurations.embedded.collect { it.isDirectory() ? it : zipTree(it)}) {
156163
include "org/elasticsearch/hadoop/**"
157164
include "esh-build.properties"
158165
include "META-INF/services/*"

‎storm/build.gradle

Lines changed: 19 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -5,15 +5,32 @@ description = "Elasticsearch Storm"
55

66
evaluationDependsOn(':elasticsearch-hadoop-mr')
77

8+
configurations {
9+
embedded {
10+
transitive = false
11+
canBeResolved = true
12+
}
13+
implementation {
14+
extendsFrom project.configurations.embedded
15+
}
16+
}
17+
818
dependencies {
9-
provided(project(":elasticsearch-hadoop-mr"))
19+
embedded(project(":elasticsearch-hadoop-mr"))
20+
21+
provided("org.apache.storm:storm-core:$stormVersion") {
22+
exclude module: "log4j-slf4j-impl"
23+
}
1024

1125
testImplementation(project(":test:shared"))
26+
1227
itestImplementation(project(":test:shared"))
28+
itestImplementation("com.google.guava:guava:16.0.1")
29+
itestImplementation("com.twitter:carbonite:1.4.0")
1330
}
1431

1532
jar {
16-
from(zipTree(project(":elasticsearch-hadoop-mr").jar.archivePath)) {
33+
from(project.configurations.embedded.collect { it.isDirectory() ? it : zipTree(it)}) {
1734
include "org/elasticsearch/hadoop/**"
1835
include "esh-build.properties"
1936
include "META-INF/services/*"
@@ -29,16 +46,4 @@ sourcesJar {
2946
from project(":elasticsearch-hadoop-mr").sourceSets.main.allJava.srcDirs
3047
}
3148

32-
dependencies {
33-
provided("org.apache.storm:storm-core:$stormVersion") {
34-
exclude module: "log4j-slf4j-impl"
35-
}
36-
37-
itestImplementation("com.google.guava:guava:16.0.1")
38-
itestImplementation("com.twitter:carbonite:1.4.0")
39-
}
40-
41-
// add itest to Eclipse
42-
//eclipse.classpath.plusConfigurations += [configurations.itestCompile]
43-
4449
tasks.getByName('integrationTest').enabled = false

0 commit comments

Comments
 (0)