Replace apache download tasks with custom ivy repository. (#1438)

Hadoop ecosystem tarballs are hosted on Apache's download mirrors only instead of in maven repositories. Previously we were downloading these artifacts from a round robin of mirror endpoints, using securely obtained hashes to validate the downloaded files even from non HTTPS sources.

This PR removes those download tasks, replacing it with a custom Ivy repository backed by an HTTPS enabled apache mirror. This allows us to ensure the artifacts are securely downloaded and cached by Gradle instead of relying on task outputs like the old mirror download task.

Author: jbaiera

buildSrc/src/main/groovy/org/elasticsearch/hadoop/gradle/fixture/hadoop/HadoopClusterFormationTasks.groovy

@@ -28,13 +28,12 @@ import org.elasticsearch.hadoop.gradle.fixture.hadoop.conf.HadoopClusterConfigur
 import org.elasticsearch.hadoop.gradle.fixture.hadoop.conf.InstanceConfiguration
 import org.elasticsearch.hadoop.gradle.fixture.hadoop.conf.RoleConfiguration
 import org.elasticsearch.hadoop.gradle.fixture.hadoop.conf.ServiceConfiguration
-import org.elasticsearch.hadoop.gradle.tasks.ApacheMirrorDownload
-import org.elasticsearch.hadoop.gradle.tasks.VerifyChecksums
 import org.gradle.api.AntBuilder
 import org.gradle.api.DefaultTask
 import org.gradle.api.GradleException
 import org.gradle.api.Project
 import org.gradle.api.Task
+import org.gradle.api.artifacts.Configuration
 import org.gradle.api.logging.Logger
 import org.gradle.api.tasks.Copy
 import org.gradle.api.tasks.Delete
@@ -59,14 +58,6 @@ class HadoopClusterFormationTasks {
         Task stopTask
     }
 
-    /**
-     * Pairing of download and verification tasks for a distribution
-     */
-    static class DistributionTasks {
-        ApacheMirrorDownload download
-        VerifyChecksums verify
-    }
-
     /**
      * Adds dependent tasks to the given task to start and stop a cluster with the given configuration.
      * <p>
@@ -104,7 +95,7 @@ class HadoopClusterFormationTasks {
         for (ServiceConfiguration serviceConfiguration : clusterConfiguration.getServices()) {
 
             // Get the download task for this service's package and add it to the service's dependency tasks
-            DistributionTasks distributionTasks = getOrConfigureDistributionDownload(project, serviceConfiguration)
+            Configuration distributionConfiguration = getOrConfigureDistributionDownload(project, serviceConfiguration)
 
             // Keep track of the start tasks in this service
             List<TaskPair> serviceTaskPairs = []
@@ -140,7 +131,7 @@ class HadoopClusterFormationTasks {
                     TaskPair instanceTasks
                     try {
                         instanceTasks = configureNode(project, prefix, instanceDependencies, instanceInfo,
-                                distributionTasks)
+                                distributionConfiguration)
                     } catch (Exception e) {
                         throw new GradleException(
                                 "Exception occurred while initializing instance [${instanceInfo.toString()}]", e)
@@ -207,36 +198,21 @@ class HadoopClusterFormationTasks {
      * either an already created one from the root project, or a newly created download task. These also contain the
      * verify task to ensure the download has been securely captured.
      */
-    static DistributionTasks getOrConfigureDistributionDownload(Project project, ServiceConfiguration serviceConfiguration) {
+    static Configuration getOrConfigureDistributionDownload(Project project, ServiceConfiguration serviceConfiguration) {
         Version serviceVersion = serviceConfiguration.getVersion()
 
-        String downloadTaskName = "download${serviceConfiguration.serviceDescriptor.packageName().capitalize()}#${serviceVersion}"
-        String verifyTaskName = "verify${serviceConfiguration.serviceDescriptor.packageName().capitalize()}#${serviceVersion}"
-
-        ApacheMirrorDownload downloadTask = project.rootProject.tasks.findByName(downloadTaskName) as ApacheMirrorDownload
-        if (downloadTask == null) {
-            downloadTask = project.rootProject.tasks.create(name: downloadTaskName, type: ApacheMirrorDownload) as ApacheMirrorDownload
-            serviceConfiguration.getServiceDescriptor().configureDownload(downloadTask, serviceConfiguration)
-            downloadTask.group = 'downloads'
-            downloadTask.onlyIf { !downloadTask.outputFile().exists() }
-        }
-
-        VerifyChecksums verifyTask = project.rootProject.tasks.findByName(verifyTaskName) as VerifyChecksums
-        if (verifyTask == null) {
-            verifyTask = project.rootProject.tasks.create(name: verifyTaskName, type: VerifyChecksums) as VerifyChecksums
-            verifyTask.group = 'downloads'
-            verifyTask.dependsOn downloadTask
-            verifyTask.inputFile downloadTask.outputFile()
-            for (Map.Entry<String, String> hash : serviceConfiguration.serviceDescriptor.packageHashVerification(serviceVersion)) {
-                verifyTask.checksum hash.key, hash.value
-            }
+        String configurationName = "download${serviceConfiguration.serviceDescriptor.packageName().capitalize()}#${serviceVersion}"
+        Configuration configuration = project.configurations.findByName(configurationName)
+        if (configuration == null) {
+            configuration = project.configurations.create(configurationName)
+            project.dependencies.add(configurationName, serviceConfiguration.getServiceDescriptor().getDependencyCoordinates(serviceConfiguration))
         }
 
-        return new DistributionTasks(download: downloadTask, verify: verifyTask)
+        return configuration
     }
 
     static TaskPair configureNode(Project project, String prefix, Object dependsOn, InstanceInfo node,
-                                  DistributionTasks distribution) {
+                                  Configuration distributionConfiguration) {
         Task setup = project.tasks.create(name: taskName(prefix, node, 'clean'), type: Delete, dependsOn: dependsOn) {
             delete node.homeDir
             delete node.cwd
@@ -257,7 +233,7 @@ class HadoopClusterFormationTasks {
         }
 
         // Always extract the package contents, and configure the files
-        setup = configureExtractTask(taskName(prefix, node, 'extract'), project, setup, node, distribution)
+        setup = configureExtractTask(taskName(prefix, node, 'extract'), project, setup, node, distributionConfiguration)
         setup = configureWriteConfigTask(taskName(prefix, node, 'configure'), project, setup, node)
         setup = configureExtraConfigFilesTask(taskName(prefix, node, 'extraConfig'), project, setup, node)
 
@@ -329,13 +305,13 @@ class HadoopClusterFormationTasks {
         return setup
     }
 
-    static Task configureExtractTask(String name, Project project, Task setup, InstanceInfo node, DistributionTasks distribution) {
-        List extractDependsOn = [distribution.verify, setup]
+    static Task configureExtractTask(String name, Project project, Task setup, InstanceInfo node, Configuration distributionConfiguration) {
+        List extractDependsOn = [distributionConfiguration, setup]
         return project.tasks.create(name: name, type: Copy, dependsOn: extractDependsOn) {
             group = 'hadoopFixture'
             // TODO: Switch logic if a service is ever not a tar distribution
             from {
-                project.tarTree(project.resources.gzip(distribution.download.outputFile()))
+                project.tarTree(project.resources.gzip(distributionConfiguration.files.first()))
             }
             into node.baseDir
         }

buildSrc/src/main/groovy/org/elasticsearch/hadoop/gradle/fixture/hadoop/HadoopFixturePlugin.groovy

@@ -0,0 +1,87 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.hadoop.gradle.fixture.hadoop
+
+import org.elasticsearch.hadoop.gradle.fixture.hadoop.conf.HadoopClusterConfiguration
+import org.gradle.api.Action
+import org.gradle.api.NamedDomainObjectContainer
+import org.gradle.api.NamedDomainObjectFactory
+import org.gradle.api.Plugin
+import org.gradle.api.Project
+import org.gradle.api.artifacts.dsl.RepositoryHandler
+import org.gradle.api.artifacts.repositories.IvyArtifactRepository
+import org.gradle.api.artifacts.repositories.IvyPatternRepositoryLayout
+import org.gradle.api.publish.ivy.IvyArtifact
+
+class HadoopFixturePlugin implements Plugin<Project> {
+
+    private static final String APACHE_MIRROR = "https://apache.osuosl.org/"
+
+    static class HadoopFixturePluginExtension {
+        private NamedDomainObjectContainer<HadoopClusterConfiguration> clusters
+
+        HadoopFixturePluginExtension(final Project project) {
+            this.clusters = project.container(HadoopClusterConfiguration.class, new NamedDomainObjectFactory<HadoopClusterConfiguration>() {
+                @Override
+                HadoopClusterConfiguration create(String name) {
+                    return new HadoopClusterConfiguration(project, name)
+                }
+            })
+        }
+
+        HadoopClusterConfiguration cluster(String name, Closure config) {
+            clusters.maybeCreate(name)
+            return clusters.getByName(name, config)
+        }
+
+        NamedDomainObjectContainer<HadoopClusterConfiguration> getClusters() {
+            return clusters
+        }
+    }
+
+    @Override
+    void apply(Project project) {
+        HadoopFixturePluginExtension extension = project.getExtensions().create("hadoop", HadoopFixturePluginExtension.class, project)
+        configureApacheMirrorRepository(project)
+        project.afterEvaluate {
+            extension.getClusters().forEach { config ->
+                // Finish cluster setup
+                HadoopClusterFormationTasks.setup(project, config)
+            }
+        }
+    }
+
+    private static configureApacheMirrorRepository(Project project) {
+        RepositoryHandler repositoryHandler = project.getRepositories()
+        repositoryHandler.add(repositoryHandler.ivy({IvyArtifactRepository ivyArtifactRepository ->
+            ivyArtifactRepository.setUrl(APACHE_MIRROR)
+            ivyArtifactRepository.patternLayout({IvyPatternRepositoryLayout ivyPatternRepositoryLayout ->
+                // We use this pattern normally and break the regular tradition of a strictly numerical version
+                // because Hive does not provide a reasonable artifact name that makes a more robust pattern
+                // reasonable (it has a very unorthodox layout)
+                ivyPatternRepositoryLayout.artifact("[organization]/[module]/[revision].[ext]")
+                ivyPatternRepositoryLayout.setM2compatible(true)
+            })
+            ivyArtifactRepository.metadataSources({IvyArtifactRepository.MetadataSources metadataSources ->
+                metadataSources.artifact()
+            })
+        }))
+    }
+}

buildSrc/src/main/groovy/org/elasticsearch/hadoop/gradle/fixture/hadoop/ServiceDescriptor.groovy

@@ -22,7 +22,6 @@ package org.elasticsearch.hadoop.gradle.fixture.hadoop
 import org.elasticsearch.gradle.Version
 import org.elasticsearch.hadoop.gradle.fixture.hadoop.conf.InstanceConfiguration
 import org.elasticsearch.hadoop.gradle.fixture.hadoop.conf.ServiceConfiguration
-import org.elasticsearch.hadoop.gradle.tasks.ApacheMirrorDownload
 
 import static org.elasticsearch.hadoop.gradle.fixture.hadoop.conf.SettingsContainer.FileSettings
 
@@ -60,9 +59,10 @@ interface ServiceDescriptor {
     Version defaultVersion()
 
     /**
-     * Callback to configure a download task to perform the package download.
+     * The coordinates for this dependency that will be used with a custom Ivy Repository to download the artifact from
+     * an Apache mirror.
      */
-    void configureDownload(ApacheMirrorDownload task, ServiceConfiguration configuration)
+    String getDependencyCoordinates(ServiceConfiguration configuration)
 
     /**
      * The official apache package name for the artifact.
@@ -74,11 +74,6 @@ interface ServiceDescriptor {
      */
     String artifactName(ServiceConfiguration configuration)
 
-    /**
-     * Return a mapping of hash algorithm id to hash value for an artifact of the given version.
-     */
-    Map<String, String> packageHashVerification(Version version)
-
     /**
      * The name of the directory under the base dir that contains the package contents.
      */

buildSrc/src/main/groovy/org/elasticsearch/hadoop/gradle/fixture/hadoop/services/HadoopServiceDescriptor.groovy

@@ -27,19 +27,11 @@ import org.elasticsearch.hadoop.gradle.fixture.hadoop.ServiceDescriptor
 import org.elasticsearch.hadoop.gradle.fixture.hadoop.conf.InstanceConfiguration
 import org.elasticsearch.hadoop.gradle.fixture.hadoop.conf.ServiceConfiguration
 import org.elasticsearch.hadoop.gradle.fixture.hadoop.conf.SettingsContainer
-import org.elasticsearch.hadoop.gradle.tasks.ApacheMirrorDownload
-import org.gradle.api.GradleException
 
 import static org.elasticsearch.hadoop.gradle.fixture.hadoop.conf.SettingsContainer.FileSettings
 
 class HadoopServiceDescriptor implements ServiceDescriptor {
 
-    static final Map<Version, Map<String, String>> VERSION_MAP = [:]
-    static {
-        VERSION_MAP.put(new Version(2, 7, 7),
-                ['SHA-512': '17c8917211dd4c25f78bf60130a390f9e273b0149737094e45f4ae5c917b1174b97eb90818c5df068e607835120126281bcc07514f38bd7fd3cb8e9d3db1bdde'])
-    }
-
     static final RoleDescriptor NAMENODE = RoleDescriptor.requiredProcess('namenode')
     static final RoleDescriptor DATANODE = RoleDescriptor.requiredProcess('datanode', [NAMENODE])
     static final RoleDescriptor RESOURCEMANAGER = RoleDescriptor.requiredProcess('resourcemanager')
@@ -73,12 +65,8 @@ class HadoopServiceDescriptor implements ServiceDescriptor {
     }
 
     @Override
-    void configureDownload(ApacheMirrorDownload task, ServiceConfiguration configuration) {
-        Version version = configuration.getVersion()
-        task.packagePath = 'hadoop/common'
-        task.packageName = 'hadoop'
-        task.artifactFileName = "hadoop-${version}.tar.gz"
-        task.version = "${version}"
+    String getDependencyCoordinates(ServiceConfiguration configuration) {
+        return "hadoop.common:hadoop-${configuration.getVersion()}:${artifactName(configuration)}@tar.gz"
     }
 
     @Override
@@ -92,15 +80,6 @@ class HadoopServiceDescriptor implements ServiceDescriptor {
         return "hadoop-${version}"
     }
 
-    @Override
-    Map<String, String> packageHashVerification(Version version) {
-        Map<String, String> hashVerifications = VERSION_MAP.get(version)
-        if (hashVerifications == null) {
-            throw new GradleException("Unsupported version [$version] - No download hash configured")
-        }
-        return hashVerifications
-    }
-
     @Override
     String homeDirName(InstanceConfiguration configuration) {
         return artifactName(configuration.getServiceConf())

buildSrc/src/main/groovy/org/elasticsearch/hadoop/gradle/fixture/hadoop/services/HiveServiceDescriptor.groovy

@@ -26,19 +26,11 @@ import org.elasticsearch.hadoop.gradle.fixture.hadoop.RoleDescriptor
 import org.elasticsearch.hadoop.gradle.fixture.hadoop.ServiceDescriptor
 import org.elasticsearch.hadoop.gradle.fixture.hadoop.conf.InstanceConfiguration
 import org.elasticsearch.hadoop.gradle.fixture.hadoop.conf.ServiceConfiguration
-import org.elasticsearch.hadoop.gradle.tasks.ApacheMirrorDownload
-import org.gradle.api.GradleException
 
 import static org.elasticsearch.hadoop.gradle.fixture.hadoop.conf.SettingsContainer.FileSettings
 
 class HiveServiceDescriptor implements ServiceDescriptor {
 
-    static final Map<Version, Map<String, String>> VERSION_MAP = [:]
-    static {
-        VERSION_MAP.put(new Version(1, 2, 2),
-                ['SHA-256' : '763b246a1a1ceeb815493d1e5e1d71836b0c5b9be1c4cd9c8d685565113771d1'])
-    }
-
     static RoleDescriptor HIVESERVER = RoleDescriptor.requiredProcess('hiveserver')
 
     @Override
@@ -67,12 +59,8 @@ class HiveServiceDescriptor implements ServiceDescriptor {
     }
 
     @Override
-    void configureDownload(ApacheMirrorDownload task, ServiceConfiguration configuration) {
-        Version version = configuration.getVersion()
-        task.packagePath = 'hive'
-        task.packageName = 'hive'
-        task.artifactFileName = "apache-hive-${version}-bin.tar.gz"
-        task.version = "${version}"
+    String getDependencyCoordinates(ServiceConfiguration configuration) {
+        return "hive:hive-${configuration.getVersion()}:${artifactName(configuration)}@tar.gz"
     }
 
     @Override
@@ -86,15 +74,6 @@ class HiveServiceDescriptor implements ServiceDescriptor {
         return "apache-hive-${version}-bin"
     }
 
-    @Override
-    Map<String, String> packageHashVerification(Version version) {
-        Map<String, String> hashVerifications = VERSION_MAP.get(version)
-        if (hashVerifications == null) {
-            throw new GradleException("Unsupported version [$version] - No download hash configured")
-        }
-        return hashVerifications
-    }
-
     @Override
     String homeDirName(InstanceConfiguration configuration) {
         return artifactName(configuration.getServiceConf())

buildSrc/src/main/groovy/org/elasticsearch/hadoop/gradle/fixture/hadoop/services/PigServiceDescriptor.groovy

@@ -26,19 +26,11 @@ import org.elasticsearch.hadoop.gradle.fixture.hadoop.ServiceDescriptor
 import org.elasticsearch.hadoop.gradle.fixture.hadoop.conf.HadoopClusterConfiguration
 import org.elasticsearch.hadoop.gradle.fixture.hadoop.conf.InstanceConfiguration
 import org.elasticsearch.hadoop.gradle.fixture.hadoop.conf.ServiceConfiguration
-import org.elasticsearch.hadoop.gradle.tasks.ApacheMirrorDownload
-import org.gradle.api.GradleException
 
 import static org.elasticsearch.hadoop.gradle.fixture.hadoop.conf.SettingsContainer.FileSettings
 
 class PigServiceDescriptor implements ServiceDescriptor {
 
-    static final Map<Version, Map<String, String>> VERSION_MAP = [:]
-    static {
-        VERSION_MAP.put(new Version(0, 17, 0),
-                ['MD5': 'da76998409fe88717b970b45678e00d4'])
-    }
-
     static RoleDescriptor GATEWAY = RoleDescriptor.requiredGateway('pig', [])
 
     @Override
@@ -67,11 +59,8 @@ class PigServiceDescriptor implements ServiceDescriptor {
     }
 
     @Override
-    void configureDownload(ApacheMirrorDownload task, ServiceConfiguration configuration) {
-        task.setPackagePath('pig')
-        task.setPackageName('pig')
-        task.setVersion(configuration.getVersion().toString())
-        task.setArtifactFileName("${artifactName(configuration)}.tar.gz")
+    String getDependencyCoordinates(ServiceConfiguration configuration) {
+        return "pig:pig-${configuration.getVersion()}:${artifactName(configuration)}@tar.gz"
     }
 
     @Override
@@ -84,15 +73,6 @@ class PigServiceDescriptor implements ServiceDescriptor {
         return "pig-${configuration.getVersion()}"
     }
 
-    @Override
-    Map<String, String> packageHashVerification(Version version) {
-        Map<String, String> hashVerifications = VERSION_MAP.get(version)
-        if (hashVerifications == null) {
-            throw new GradleException("Unsupported version [$version] - No download hash configured")
-        }
-        return hashVerifications
-    }
-
     @Override
     String homeDirName(InstanceConfiguration configuration) {
         return artifactName(configuration.getServiceConf())