Replace apache download tasks with custom ivy repository.

buildSrc/src/main/groovy/org/elasticsearch/hadoop/gradle/fixture/hadoop/HadoopClusterFormationTasks.groovy

@@ -28,13 +28,12 @@ import org.elasticsearch.hadoop.gradle.fixture.hadoop.conf.HadoopClusterConfigur
 import org.elasticsearch.hadoop.gradle.fixture.hadoop.conf.InstanceConfiguration
 import org.elasticsearch.hadoop.gradle.fixture.hadoop.conf.RoleConfiguration
 import org.elasticsearch.hadoop.gradle.fixture.hadoop.conf.ServiceConfiguration
-import org.elasticsearch.hadoop.gradle.tasks.ApacheMirrorDownload
-import org.elasticsearch.hadoop.gradle.tasks.VerifyChecksums
 import org.gradle.api.AntBuilder
 import org.gradle.api.DefaultTask
 import org.gradle.api.GradleException
 import org.gradle.api.Project
 import org.gradle.api.Task
+import org.gradle.api.artifacts.Configuration
 import org.gradle.api.logging.Logger
 import org.gradle.api.tasks.Copy
 import org.gradle.api.tasks.Delete
@@ -59,14 +58,6 @@ class HadoopClusterFormationTasks {
         Task stopTask
     }
 
-    /**
-     * Pairing of download and verification tasks for a distribution
-     */
-    static class DistributionTasks {
-        ApacheMirrorDownload download
-        VerifyChecksums verify
-    }
-
     /**
      * Adds dependent tasks to the given task to start and stop a cluster with the given configuration.
      * <p>
@@ -104,7 +95,7 @@ class HadoopClusterFormationTasks {
         for (ServiceConfiguration serviceConfiguration : clusterConfiguration.getServices()) {
 
             // Get the download task for this service's package and add it to the service's dependency tasks
-            DistributionTasks distributionTasks = getOrConfigureDistributionDownload(project, serviceConfiguration)
+            Configuration distributionConfiguration = getOrConfigureDistributionDownload(project, serviceConfiguration)
 
             // Keep track of the start tasks in this service
             List<TaskPair> serviceTaskPairs = []
@@ -140,7 +131,7 @@ class HadoopClusterFormationTasks {
                     TaskPair instanceTasks
                     try {
                         instanceTasks = configureNode(project, prefix, instanceDependencies, instanceInfo,
-                                distributionTasks)
+                                distributionConfiguration)
                     } catch (Exception e) {
                         throw new GradleException(
                                 "Exception occurred while initializing instance [${instanceInfo.toString()}]", e)
@@ -207,36 +198,21 @@ class HadoopClusterFormationTasks {
      * either an already created one from the root project, or a newly created download task. These also contain the
      * verify task to ensure the download has been securely captured.
      */
-    static DistributionTasks getOrConfigureDistributionDownload(Project project, ServiceConfiguration serviceConfiguration) {
+    static Configuration getOrConfigureDistributionDownload(Project project, ServiceConfiguration serviceConfiguration) {
         Version serviceVersion = serviceConfiguration.getVersion()
 
-        String downloadTaskName = "download${serviceConfiguration.serviceDescriptor.packageName().capitalize()}#${serviceVersion}"
-        String verifyTaskName = "verify${serviceConfiguration.serviceDescriptor.packageName().capitalize()}#${serviceVersion}"
-
-        ApacheMirrorDownload downloadTask = project.rootProject.tasks.findByName(downloadTaskName) as ApacheMirrorDownload
-        if (downloadTask == null) {
-            downloadTask = project.rootProject.tasks.create(name: downloadTaskName, type: ApacheMirrorDownload) as ApacheMirrorDownload
-            serviceConfiguration.getServiceDescriptor().configureDownload(downloadTask, serviceConfiguration)
-            downloadTask.group = 'downloads'
-            downloadTask.onlyIf { !downloadTask.outputFile().exists() }
-        }
-
-        VerifyChecksums verifyTask = project.rootProject.tasks.findByName(verifyTaskName) as VerifyChecksums
-        if (verifyTask == null) {
-            verifyTask = project.rootProject.tasks.create(name: verifyTaskName, type: VerifyChecksums) as VerifyChecksums
-            verifyTask.group = 'downloads'
-            verifyTask.dependsOn downloadTask
-            verifyTask.inputFile downloadTask.outputFile()
-            for (Map.Entry<String, String> hash : serviceConfiguration.serviceDescriptor.packageHashVerification(serviceVersion)) {
-                verifyTask.checksum hash.key, hash.value
-            }
+        String configurationName = "download${serviceConfiguration.serviceDescriptor.packageName().capitalize()}#${serviceVersion}"
+        Configuration configuration = project.configurations.findByName(configurationName)
+        if (configuration == null) {
+            configuration = project.configurations.create(configurationName)
+            project.dependencies.add(configurationName, serviceConfiguration.getServiceDescriptor().getDependencyCoordinates(serviceConfiguration))
         }
 
-        return new DistributionTasks(download: downloadTask, verify: verifyTask)
+        return configuration
     }
 
     static TaskPair configureNode(Project project, String prefix, Object dependsOn, InstanceInfo node,
-                                  DistributionTasks distribution) {
+                                  Configuration distributionConfiguration) {
         Task setup = project.tasks.create(name: taskName(prefix, node, 'clean'), type: Delete, dependsOn: dependsOn) {
             delete node.homeDir
             delete node.cwd
@@ -257,7 +233,7 @@ class HadoopClusterFormationTasks {
         }
 
         // Always extract the package contents, and configure the files
-        setup = configureExtractTask(taskName(prefix, node, 'extract'), project, setup, node, distribution)
+        setup = configureExtractTask(taskName(prefix, node, 'extract'), project, setup, node, distributionConfiguration)
         setup = configureWriteConfigTask(taskName(prefix, node, 'configure'), project, setup, node)
         setup = configureExtraConfigFilesTask(taskName(prefix, node, 'extraConfig'), project, setup, node)
 
@@ -329,13 +305,13 @@ class HadoopClusterFormationTasks {
         return setup
     }
 
-    static Task configureExtractTask(String name, Project project, Task setup, InstanceInfo node, DistributionTasks distribution) {
-        List extractDependsOn = [distribution.verify, setup]
+    static Task configureExtractTask(String name, Project project, Task setup, InstanceInfo node, Configuration distributionConfiguration) {
+        List extractDependsOn = [distributionConfiguration, setup]
         return project.tasks.create(name: name, type: Copy, dependsOn: extractDependsOn) {
             group = 'hadoopFixture'
             // TODO: Switch logic if a service is ever not a tar distribution
             from {
-                project.tarTree(project.resources.gzip(distribution.download.outputFile()))
+                project.tarTree(project.resources.gzip(distributionConfiguration.files.first()))
             }
             into node.baseDir
         }

buildSrc/src/main/groovy/org/elasticsearch/hadoop/gradle/fixture/hadoop/HadoopFixturePlugin.groovy

@@ -0,0 +1,87 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.hadoop.gradle.fixture.hadoop
+
+import org.elasticsearch.hadoop.gradle.fixture.hadoop.conf.HadoopClusterConfiguration
+import org.gradle.api.Action
+import org.gradle.api.NamedDomainObjectContainer
+import org.gradle.api.NamedDomainObjectFactory
+import org.gradle.api.Plugin
+import org.gradle.api.Project
+import org.gradle.api.artifacts.dsl.RepositoryHandler
+import org.gradle.api.artifacts.repositories.IvyArtifactRepository
+import org.gradle.api.artifacts.repositories.IvyPatternRepositoryLayout
+import org.gradle.api.publish.ivy.IvyArtifact
+
+class HadoopFixturePlugin implements Plugin<Project> {
+
+    private static final String APACHE_MIRROR = "https://apache.osuosl.org/"
+
+    static class HadoopFixturePluginExtension {
+        private NamedDomainObjectContainer<HadoopClusterConfiguration> clusters
+
+        HadoopFixturePluginExtension(final Project project) {
+            this.clusters = project.container(HadoopClusterConfiguration.class, new NamedDomainObjectFactory<HadoopClusterConfiguration>() {
+                @Override
+                HadoopClusterConfiguration create(String name) {
+                    return new HadoopClusterConfiguration(project, name)
+                }
+            })
+        }
+
+        HadoopClusterConfiguration cluster(String name, Closure config) {
+            clusters.maybeCreate(name)
+            return clusters.getByName(name, config)
+        }
+
+        NamedDomainObjectContainer<HadoopClusterConfiguration> getClusters() {
+            return clusters
+        }
+    }
+
+    @Override
+    void apply(Project project) {
+        HadoopFixturePluginExtension extension = project.getExtensions().create("hadoop", HadoopFixturePluginExtension.class, project)
+        configureApacheMirrorRepository(project)
+        project.afterEvaluate {
+            extension.getClusters().forEach { config ->
+                // Finish cluster setup
+                HadoopClusterFormationTasks.setup(project, config)
+            }
+        }
+    }
+
+    private static configureApacheMirrorRepository(Project project) {
+        RepositoryHandler repositoryHandler = project.getRepositories()
+        repositoryHandler.add(repositoryHandler.ivy({IvyArtifactRepository ivyArtifactRepository ->
+            ivyArtifactRepository.setUrl(APACHE_MIRROR)
+            ivyArtifactRepository.patternLayout({IvyPatternRepositoryLayout ivyPatternRepositoryLayout ->
+                // We use this pattern normally and break the regular tradition of a strictly numerical version
+                // because Hive does not provide a reasonable artifact name that makes a more robust pattern
+                // reasonable (it has a very unorthodox layout)
+                ivyPatternRepositoryLayout.artifact("[organization]/[module]/[revision].[ext]")
+                ivyPatternRepositoryLayout.setM2compatible(true)
+            })
+            ivyArtifactRepository.metadataSources({IvyArtifactRepository.MetadataSources metadataSources ->
+                metadataSources.artifact()
+            })
+        }))
+    }
+}

buildSrc/src/main/groovy/org/elasticsearch/hadoop/gradle/fixture/hadoop/ServiceDescriptor.groovy

@@ -22,7 +22,6 @@ package org.elasticsearch.hadoop.gradle.fixture.hadoop
 import org.elasticsearch.gradle.Version
 import org.elasticsearch.hadoop.gradle.fixture.hadoop.conf.InstanceConfiguration
 import org.elasticsearch.hadoop.gradle.fixture.hadoop.conf.ServiceConfiguration
-import org.elasticsearch.hadoop.gradle.tasks.ApacheMirrorDownload
 
 import static org.elasticsearch.hadoop.gradle.fixture.hadoop.conf.SettingsContainer.FileSettings
 
@@ -60,9 +59,10 @@ interface ServiceDescriptor {
     Version defaultVersion()
 
     /**
-     * Callback to configure a download task to perform the package download.
+     * The coordinates for this dependency that will be used with a custom Ivy Repository to download the artifact from
+     * an Apache mirror.
      */
-    void configureDownload(ApacheMirrorDownload task, ServiceConfiguration configuration)
+    String getDependencyCoordinates(ServiceConfiguration configuration)
 
     /**
      * The official apache package name for the artifact.
@@ -74,11 +74,6 @@ interface ServiceDescriptor {
      */
     String artifactName(ServiceConfiguration configuration)
 
-    /**
-     * Return a mapping of hash algorithm id to hash value for an artifact of the given version.
-     */
-    Map<String, String> packageHashVerification(Version version)
-
     /**
      * The name of the directory under the base dir that contains the package contents.
      */

buildSrc/src/main/groovy/org/elasticsearch/hadoop/gradle/fixture/hadoop/services/HadoopServiceDescriptor.groovy

@@ -27,19 +27,11 @@ import org.elasticsearch.hadoop.gradle.fixture.hadoop.ServiceDescriptor
 import org.elasticsearch.hadoop.gradle.fixture.hadoop.conf.InstanceConfiguration
 import org.elasticsearch.hadoop.gradle.fixture.hadoop.conf.ServiceConfiguration
 import org.elasticsearch.hadoop.gradle.fixture.hadoop.conf.SettingsContainer
-import org.elasticsearch.hadoop.gradle.tasks.ApacheMirrorDownload
-import org.gradle.api.GradleException
 
 import static org.elasticsearch.hadoop.gradle.fixture.hadoop.conf.SettingsContainer.FileSettings
 
 class HadoopServiceDescriptor implements ServiceDescriptor {
 
-    static final Map<Version, Map<String, String>> VERSION_MAP = [:]
-    static {
-        VERSION_MAP.put(new Version(2, 7, 7),
-                ['SHA-512': '17c8917211dd4c25f78bf60130a390f9e273b0149737094e45f4ae5c917b1174b97eb90818c5df068e607835120126281bcc07514f38bd7fd3cb8e9d3db1bdde'])
-    }
-
     static final RoleDescriptor NAMENODE = RoleDescriptor.requiredProcess('namenode')
     static final RoleDescriptor DATANODE = RoleDescriptor.requiredProcess('datanode', [NAMENODE])
     static final RoleDescriptor RESOURCEMANAGER = RoleDescriptor.requiredProcess('resourcemanager')
@@ -73,12 +65,8 @@ class HadoopServiceDescriptor implements ServiceDescriptor {
     }
 
     @Override
-    void configureDownload(ApacheMirrorDownload task, ServiceConfiguration configuration) {
-        Version version = configuration.getVersion()
-        task.packagePath = 'hadoop/common'
-        task.packageName = 'hadoop'
-        task.artifactFileName = "hadoop-${version}.tar.gz"
-        task.version = "${version}"
+    String getDependencyCoordinates(ServiceConfiguration configuration) {
+        return "hadoop.common:hadoop-${configuration.getVersion()}:${artifactName(configuration)}@tar.gz"
     }
 
     @Override
@@ -92,15 +80,6 @@ class HadoopServiceDescriptor implements ServiceDescriptor {
         return "hadoop-${version}"
     }
 
-    @Override
-    Map<String, String> packageHashVerification(Version version) {
-        Map<String, String> hashVerifications = VERSION_MAP.get(version)
-        if (hashVerifications == null) {
-            throw new GradleException("Unsupported version [$version] - No download hash configured")
-        }
-        return hashVerifications
-    }
-
     @Override
     String homeDirName(InstanceConfiguration configuration) {
         return artifactName(configuration.getServiceConf())

buildSrc/src/main/groovy/org/elasticsearch/hadoop/gradle/fixture/hadoop/services/HiveServiceDescriptor.groovy

@@ -26,19 +26,11 @@ import org.elasticsearch.hadoop.gradle.fixture.hadoop.RoleDescriptor
 import org.elasticsearch.hadoop.gradle.fixture.hadoop.ServiceDescriptor
 import org.elasticsearch.hadoop.gradle.fixture.hadoop.conf.InstanceConfiguration
 import org.elasticsearch.hadoop.gradle.fixture.hadoop.conf.ServiceConfiguration
-import org.elasticsearch.hadoop.gradle.tasks.ApacheMirrorDownload
-import org.gradle.api.GradleException
 
 import static org.elasticsearch.hadoop.gradle.fixture.hadoop.conf.SettingsContainer.FileSettings
 
 class HiveServiceDescriptor implements ServiceDescriptor {
 
-    static final Map<Version, Map<String, String>> VERSION_MAP = [:]
-    static {
-        VERSION_MAP.put(new Version(1, 2, 2),
-                ['SHA-256' : '763b246a1a1ceeb815493d1e5e1d71836b0c5b9be1c4cd9c8d685565113771d1'])
-    }
-
     static RoleDescriptor HIVESERVER = RoleDescriptor.requiredProcess('hiveserver')
 
     @Override
@@ -67,12 +59,8 @@ class HiveServiceDescriptor implements ServiceDescriptor {
     }
 
     @Override
-    void configureDownload(ApacheMirrorDownload task, ServiceConfiguration configuration) {
-        Version version = configuration.getVersion()
-        task.packagePath = 'hive'
-        task.packageName = 'hive'
-        task.artifactFileName = "apache-hive-${version}-bin.tar.gz"
-        task.version = "${version}"
+    String getDependencyCoordinates(ServiceConfiguration configuration) {
+        return "hive:hive-${configuration.getVersion()}:${artifactName(configuration)}@tar.gz"
     }
 
     @Override
@@ -86,15 +74,6 @@ class HiveServiceDescriptor implements ServiceDescriptor {
         return "apache-hive-${version}-bin"
     }
 
-    @Override
-    Map<String, String> packageHashVerification(Version version) {
-        Map<String, String> hashVerifications = VERSION_MAP.get(version)
-        if (hashVerifications == null) {
-            throw new GradleException("Unsupported version [$version] - No download hash configured")
-        }
-        return hashVerifications
-    }
-
     @Override
     String homeDirName(InstanceConfiguration configuration) {
         return artifactName(configuration.getServiceConf())

buildSrc/src/main/groovy/org/elasticsearch/hadoop/gradle/fixture/hadoop/services/PigServiceDescriptor.groovy

@@ -26,19 +26,11 @@ import org.elasticsearch.hadoop.gradle.fixture.hadoop.ServiceDescriptor
 import org.elasticsearch.hadoop.gradle.fixture.hadoop.conf.HadoopClusterConfiguration
 import org.elasticsearch.hadoop.gradle.fixture.hadoop.conf.InstanceConfiguration
 import org.elasticsearch.hadoop.gradle.fixture.hadoop.conf.ServiceConfiguration
-import org.elasticsearch.hadoop.gradle.tasks.ApacheMirrorDownload
-import org.gradle.api.GradleException
 
 import static org.elasticsearch.hadoop.gradle.fixture.hadoop.conf.SettingsContainer.FileSettings
 
 class PigServiceDescriptor implements ServiceDescriptor {
 
-    static final Map<Version, Map<String, String>> VERSION_MAP = [:]
-    static {
-        VERSION_MAP.put(new Version(0, 17, 0),
-                ['MD5': 'da76998409fe88717b970b45678e00d4'])
-    }
-
     static RoleDescriptor GATEWAY = RoleDescriptor.requiredGateway('pig', [])
 
     @Override
@@ -67,11 +59,8 @@ class PigServiceDescriptor implements ServiceDescriptor {
     }
 
     @Override
-    void configureDownload(ApacheMirrorDownload task, ServiceConfiguration configuration) {
-        task.setPackagePath('pig')
-        task.setPackageName('pig')
-        task.setVersion(configuration.getVersion().toString())
-        task.setArtifactFileName("${artifactName(configuration)}.tar.gz")
+    String getDependencyCoordinates(ServiceConfiguration configuration) {
+        return "pig:pig-${configuration.getVersion()}:${artifactName(configuration)}@tar.gz"
     }
 
     @Override
@@ -84,15 +73,6 @@ class PigServiceDescriptor implements ServiceDescriptor {
         return "pig-${configuration.getVersion()}"
     }
 
-    @Override
-    Map<String, String> packageHashVerification(Version version) {
-        Map<String, String> hashVerifications = VERSION_MAP.get(version)
-        if (hashVerifications == null) {
-            throw new GradleException("Unsupported version [$version] - No download hash configured")
-        }
-        return hashVerifications
-    }
-
     @Override
     String homeDirName(InstanceConfiguration configuration) {
         return artifactName(configuration.getServiceConf())