Support creating Spark 2.0 cluster using Azure HDInsight PowerShell

src/ResourceManager/HDInsight/Commands.HDInsight.Test/Commands.HDInsight.Test.csproj

@@ -52,7 +52,7 @@
     </Reference>
     <Reference Include="Microsoft.Azure.Management.HDInsight, Version=1.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
       <SpecificVersion>False</SpecificVersion>
-      <HintPath>..\..\..\packages\Microsoft.Azure.Management.HDInsight.1.0.14-preview\lib\net40\Microsoft.Azure.Management.HDInsight.dll</HintPath>
+      <HintPath>..\..\..\packages\Microsoft.Azure.Management.HDInsight.1.3.0\lib\net40\Microsoft.Azure.Management.HDInsight.dll</HintPath>
     </Reference>
     <Reference Include="Microsoft.Azure.Management.HDInsight.Job">
       <SpecificVersion>False</SpecificVersion>

src/ResourceManager/HDInsight/Commands.HDInsight.Test/UnitTests/NewClusterTests.cs

@@ -221,5 +221,109 @@ public void CanCreateNewHDInsightCluster_Linux()
                     clusterout.OperatingSystemType == OSType.Linux)),
                     Times.Once);
         }
+
+        [Fact]
+        [Trait(Category.AcceptanceType, Category.CheckIn)]
+        public void CanCreateNewHDInsightCluster_LinuxComponentVersion()
+        {
+            string sparkClusterType = "Spark";
+            var componentVersion = new Dictionary<string, string>
+            {
+                {"Spark", "2.0"}
+            };
+            var componentVersionResponse = "[Spark, 2.0]";
+
+            cmdlet.ClusterName = ClusterName;
+            cmdlet.ResourceGroupName = ResourceGroupName;
+            cmdlet.ClusterSizeInNodes = ClusterSize;
+            cmdlet.Location = Location;
+            cmdlet.HttpCredential = _httpCred;
+            cmdlet.DefaultStorageAccountName = StorageName;
+            cmdlet.DefaultStorageAccountKey = StorageKey;
+            cmdlet.ClusterType = "Spark";
+            cmdlet.SshCredential = _sshCred;
+            cmdlet.OSType = OSType.Linux;
+            cmdlet.ComponentVersion = componentVersion;
+
+            var cluster = new Cluster
+            {
+                Id = "id",
+                Name = ClusterName,
+                Location = Location,
+                Properties = new ClusterGetProperties
+                {
+                    ClusterVersion = "3.5",
+                    ClusterState = "Running",
+                    ClusterDefinition = new ClusterDefinition
+                    {
+                        ClusterType = sparkClusterType
+                    },
+                    QuotaInfo = new QuotaInfo
+                    {
+                        CoresUsed = 24
+                    },
+                    OperatingSystemType = OSType.Linux
+                }
+            };
+
+
+            cluster.Properties.ClusterDefinition.ComponentVersion = componentVersion;
+
+            var coreConfigs = new Dictionary<string, string>
+            {
+                {"fs.defaultFS", "wasb://giyertestcsmv2@" + StorageName},
+                {
+                    "fs.azure.account.key." + StorageName,
+                    StorageKey
+                }
+            };
+            var gatewayConfigs = new Dictionary<string, string>
+            {
+                {"restAuthCredential.isEnabled", "true"},
+                {"restAuthCredential.username", _httpCred.UserName},
+                {"restAuthCredential.password", _httpCred.Password.ConvertToString()}
+            };
+
+            var configurations = new Dictionary<string, Dictionary<string, string>>
+            {
+                {"core-site", coreConfigs},
+                {"gateway", gatewayConfigs}
+            };
+            var serializedConfig = JsonConvert.SerializeObject(configurations);
+            cluster.Properties.ClusterDefinition.Configurations = serializedConfig;
+
+            var getresponse = new ClusterGetResponse { Cluster = cluster };
+
+            hdinsightManagementMock.Setup(c => c.CreateNewCluster(ResourceGroupName, ClusterName, It.Is<ClusterCreateParameters>(
+                parameters =>
+                    parameters.ClusterSizeInNodes == ClusterSize &&
+                    parameters.DefaultStorageAccountName == StorageName &&
+                    parameters.DefaultStorageAccountKey == StorageKey &&
+                    parameters.Location == Location &&
+                    parameters.UserName == _httpCred.UserName &&
+                    parameters.Password == _httpCred.Password.ConvertToString() &&
+                    parameters.ClusterType == sparkClusterType &&
+                    parameters.OSType == OSType.Linux &&
+                    parameters.SshUserName == _sshCred.UserName &&
+                    parameters.SshPassword == _sshCred.Password.ConvertToString() &&
+                    parameters.ComponentVersion["Spark"] == componentVersion["Spark"])))
+            .Returns(getresponse)
+            .Verifiable();
+
+            cmdlet.ExecuteCmdlet();
+
+            commandRuntimeMock.VerifyAll();
+            commandRuntimeMock.Verify(f => f.WriteObject(It.Is<AzureHDInsightCluster>(
+                clusterout =>
+                    clusterout.ClusterState == "Running" &&
+                    clusterout.ClusterType == sparkClusterType &&
+                    clusterout.ClusterVersion == "3.5" &&
+                    clusterout.CoresUsed == 24 &&
+                    clusterout.Location == Location &&
+                    clusterout.Name == ClusterName &&
+                    clusterout.OperatingSystemType == OSType.Linux &&
+                    clusterout.ComponentVersion[0] == componentVersionResponse)),
+                    Times.Once);
+        }
     }
 }

src/ResourceManager/HDInsight/Commands.HDInsight/Commands.HDInsight.csproj

@@ -50,6 +50,7 @@
     <Compile Include="JobCommands\UseAzureHDInsightClusterCommand.cs" />
     <Compile Include="JobCommands\StopAzureHDInsightJobCommand.cs" />
     <Compile Include="JobCommands\StartAzureHDInsightJobCommand.cs" />
+    <Compile Include="ManagementCommands\AddAzureHDInsightComponentVersionCommand.cs" />
     <Compile Include="ManagementCommands\SetAzureHDInsightDefaultStorageCommand.cs" />
     <Compile Include="ManagementCommands\AddAzureHDInsightClusterIdentity.cs" />
     <Compile Include="ManagementCommands\AddAzureHDInsightStorageCommand.cs" />
@@ -124,7 +125,7 @@
     </Reference>
     <Reference Include="Microsoft.Azure.Management.HDInsight, Version=1.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
       <SpecificVersion>False</SpecificVersion>
-      <HintPath>..\..\..\packages\Microsoft.Azure.Management.HDInsight.1.0.14-preview\lib\net40\Microsoft.Azure.Management.HDInsight.dll</HintPath>
+      <HintPath>..\..\..\packages\Microsoft.Azure.Management.HDInsight.1.3.0\lib\net40\Microsoft.Azure.Management.HDInsight.dll</HintPath>
     </Reference>
     <Reference Include="Microsoft.Azure.Management.HDInsight.Job">
       <SpecificVersion>False</SpecificVersion>
@@ -157,7 +158,7 @@
       <HintPath>..\..\..\packages\Microsoft.Rest.ClientRuntime.2.3.2\lib\net45\Microsoft.Rest.ClientRuntime.dll</HintPath>
       <Private>True</Private>
     </Reference>
-	<Reference Include="Microsoft.Rest.ClientRuntime.Azure, Version=3.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
+    <Reference Include="Microsoft.Rest.ClientRuntime.Azure, Version=3.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
       <HintPath>..\..\..\packages\Microsoft.Rest.ClientRuntime.Azure.3.3.2\lib\net45\Microsoft.Rest.ClientRuntime.Azure.dll</HintPath>
     </Reference>
     <Reference Include="Microsoft.Rest.ClientRuntime.Azure.Authentication, Version=2.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">

src/ResourceManager/HDInsight/Commands.HDInsight/Constants.cs

@@ -36,6 +36,7 @@ public static class CommandNames
             public const string AzureHDInsightDefaultStorage = "AzureRmHDInsightDefaultStorage";
             public const string AzureHDInsightHiveJob = "AzureRmHDInsightHiveJob";
             public const string AzureHDInsightClusterIdentity = "AzureRmHDInsightClusterIdentity";
+            public const string AzureHDInsightComponentVersion = "AzureRmHDInsightComponentVersion";
             public const string Hive = "Hive";
         }
 

src/ResourceManager/HDInsight/Commands.HDInsight/ManagementCommands/AddAzureHDInsightComponentVersionCommand.cs

@@ -0,0 +1,58 @@
+// ----------------------------------------------------------------------------------
+//
+// Copyright Microsoft Corporation
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+// ----------------------------------------------------------------------------------
+
+using Microsoft.Azure.Commands.HDInsight.Commands;
+using Microsoft.Azure.Commands.HDInsight.Models;
+using System.Management.Automation;
+
+namespace Microsoft.Azure.Commands.HDInsight
+{
+    [Cmdlet(
+        VerbsCommon.Add,
+        Constants.CommandNames.AzureHDInsightComponentVersion,
+        SupportsShouldProcess = true),
+    OutputType(typeof(AzureHDInsightConfig))]
+    public class AddAzureHDInsightComponentVersionCommand : HDInsightCmdletBase
+    {
+        #region Input Parameter Definitions
+
+        [Parameter(Position = 0,
+            Mandatory = true,
+            ValueFromPipeline = true,
+            HelpMessage = "The HDInsight cluster configuration to use when creating the new cluster.")]
+        public AzureHDInsightConfig Config { get; set; }
+
+        [Parameter(Position = 1,
+            Mandatory = true,
+            HelpMessage = "The component name whose version should be specified for the new cluster.")]
+        public string ComponentName { get; set; }
+
+        [Parameter(Position = 2,
+            Mandatory = true,
+            HelpMessage = "The version of a specific service in the new cluster.")]
+        public string ComponentVersion { get; set; }
+
+        #endregion
+
+        public override void ExecuteCmdlet()
+        {
+            ConfirmAction("Adding Component Version", "AzureHDInsightConfig",
+                () =>
+                {
+                    Config.ComponentVersion.Add(ComponentName, ComponentVersion);
+                    WriteObject(Config);
+                });
+        }
+    }
+}

src/ResourceManager/HDInsight/Commands.HDInsight/ManagementCommands/NewAzureHDInsightClusterCommand.cs

@@ -140,6 +140,10 @@ var storageAccount in
                 {
                     result.ScriptActions.Add(action.Key, action.Value.Select(a => new AzureHDInsightScriptAction(a)).ToList());
                 }
+                foreach (var component in parameters.ComponentVersion.Where(component => !result.ComponentVersion.ContainsKey(component.Key)))
+                {
+                    result.ComponentVersion.Add(component.Key, component.Value);
+                }
                 return result;
             }
             set
@@ -180,6 +184,10 @@ var storageAccount in
                 {
                     parameters.ScriptActions.Add(action.Key, action.Value.Select(a => a.GetScriptActionFromPSModel()).ToList());
                 }
+                foreach (var component in value.ComponentVersion.Where(component => !parameters.ComponentVersion.ContainsKey(component.Key)))
+                {
+                    parameters.ComponentVersion.Add(component.Key, component.Value);
+                }
             }
         }
 
@@ -240,6 +248,13 @@ public string ClusterType
             set { parameters.ClusterType = value; }
         }
 
+        [Parameter(HelpMessage = "Gets or sets the version for a service in the cluster.")]
+        public Dictionary<string, string> ComponentVersion
+        {
+            get { return parameters.ComponentVersion; }
+            set { parameters.ComponentVersion = value; }
+        }
+
         [Parameter(HelpMessage = "Gets or sets the virtual network guid for this HDInsight cluster.")]
         public string VirtualNetworkId
         {
@@ -310,6 +325,7 @@ public NewAzureHDInsightClusterCommand()
             AdditionalStorageAccounts = new Dictionary<string, string>();
             Configurations = new Dictionary<string, Dictionary<string, string>>();
             ScriptActions = new Dictionary<ClusterNodeType, List<AzureHDInsightScriptAction>>();
+            ComponentVersion = new Dictionary<string, string>();
         }
 
         public override void ExecuteCmdlet()
@@ -352,6 +368,10 @@ var storageAccount in
                 parameters.ScriptActions.Add(action.Key,
                     action.Value.Select(a => a.GetScriptActionFromPSModel()).ToList());
             }
+            foreach (var component in ComponentVersion.Where(component => !parameters.ComponentVersion.ContainsKey(component.Key)))
+            {
+                parameters.ComponentVersion.Add(component.Key, component.Value);
+            }
             if (OozieMetastore != null)
             {
                 var metastore = OozieMetastore;