Add new parameters to set Spark custom configurations

src/ResourceManager/HDInsight/Commands.HDInsight.Test/Commands.HDInsight.Test.csproj

@@ -51,7 +51,7 @@
       <HintPath>..\..\..\packages\Microsoft.Azure.Management.Authorization.0.18.2-preview\lib\net40\Microsoft.Azure.Management.Authorization.dll</HintPath>
     </Reference>
     <Reference Include="Microsoft.Azure.Management.HDInsight, Version=2.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
-      <HintPath>..\..\..\packages\Microsoft.Azure.Management.HDInsight.2.0.3\lib\net40\Microsoft.Azure.Management.HDInsight.dll</HintPath>
+      <HintPath>..\..\..\packages\Microsoft.Azure.Management.HDInsight.2.0.7\lib\net45\Microsoft.Azure.Management.HDInsight.dll</HintPath>
       <Private>True</Private>
     </Reference>
     <Reference Include="Microsoft.Azure.Management.HDInsight.Job">

src/ResourceManager/HDInsight/Commands.HDInsight.Test/UnitTests/ConfigurationTests.cs

@@ -12,7 +12,9 @@
 // limitations under the License.
 // ----------------------------------------------------------------------------------
 
+using System.Collections;
 using Microsoft.Azure.Commands.HDInsight.Models;
+using Microsoft.Azure.Management.HDInsight;
 using Microsoft.WindowsAzure.Commands.ScenarioTest;
 using Moq;
 using Xunit;
@@ -41,6 +43,20 @@ public void CanCreateNewConfigForRServer()
             CreateNewConfig(setEdgeNodeVmSize: true);
         }
 
+        [Fact]
+        [Trait(Category.AcceptanceType, Category.CheckIn)]
+        public void CanAddSparkCustomConfigs()
+        {
+            CustomizeSpark(1);
+        }
+
+        [Fact]
+        [Trait(Category.AcceptanceType, Category.CheckIn)]
+        public void CanAddSpark2CustomConfigs()
+        {
+            CustomizeSpark(2);
+        }
+
         public void CreateNewConfig(bool setEdgeNodeVmSize = false)
         {
             var newconfigcmdlet = new NewAzureHDInsightClusterConfigCommand
@@ -72,5 +88,49 @@ public void CreateNewConfig(bool setEdgeNodeVmSize = false)
                                 c.ScriptActions.Count == 0)),
                 Times.Once);
         }
+
+        private void CustomizeSpark(int sparkVersion)
+        {
+            AzureHDInsightConfig config = new AzureHDInsightConfig();
+
+            Hashtable sparkDefaults = new Hashtable() { { @"spark.executor.instances", "3" } };
+            Hashtable sparkThriftConf = new Hashtable() { { @"spark.executor.cores", "4" } };
+
+            AddAzureHDInsightConfigValuesCommand addConfigValuesCmdlet = new AddAzureHDInsightConfigValuesCommand
+            {
+                CommandRuntime = commandRuntimeMock.Object,
+                HDInsightManagementClient = hdinsightManagementMock.Object,
+                Config = config,
+            };
+
+            if (sparkVersion == 1)
+            {
+                addConfigValuesCmdlet.SparkDefaults = sparkDefaults;
+                addConfigValuesCmdlet.SparkThriftConf = sparkThriftConf;
+            }
+            else
+            {
+                addConfigValuesCmdlet.Spark2Defaults = sparkDefaults;
+                addConfigValuesCmdlet.Spark2ThriftConf = sparkThriftConf;
+            }
+
+            addConfigValuesCmdlet.ExecuteCmdlet();
+
+            commandRuntimeMock.Verify(
+                f =>
+                    f.WriteObject(
+                        It.Is<AzureHDInsightConfig>(
+                            c =>
+                                c.Configurations != null &&
+                                ((sparkVersion == 1 && c.Configurations.ContainsKey(ConfigurationKey.SparkDefaults) &&
+                                c.Configurations[ConfigurationKey.SparkDefaults]["spark.executor.instances"].Equals(sparkDefaults["spark.executor.instances"]) &&
+                                c.Configurations.ContainsKey(ConfigurationKey.SparkThriftConf) &&
+                                c.Configurations[ConfigurationKey.SparkThriftConf]["spark.executor.cores"].Equals(sparkThriftConf["spark.executor.cores"])) ||
+                                (sparkVersion == 2 &&  c.Configurations.ContainsKey(ConfigurationKey.Spark2Defaults) &&
+                                c.Configurations[ConfigurationKey.Spark2Defaults]["spark.executor.instances"].Equals(sparkDefaults["spark.executor.instances"]) &&
+                                c.Configurations.ContainsKey(ConfigurationKey.Spark2ThriftConf) &&
+                                c.Configurations[ConfigurationKey.Spark2ThriftConf]["spark.executor.cores"].Equals(sparkThriftConf["spark.executor.cores"]))))),
+                Times.Once);
+        }
     }
 }

src/ResourceManager/HDInsight/Commands.HDInsight.Test/packages.config

@@ -6,7 +6,7 @@
   <package id="Microsoft.Azure.Gallery" version="2.6.2-preview" targetFramework="net45" />
   <package id="Microsoft.Azure.KeyVault.Core" version="1.0.0" targetFramework="net45" />
   <package id="Microsoft.Azure.Management.Authorization" version="0.18.2-preview" targetFramework="net45" />
-  <package id="Microsoft.Azure.Management.HDInsight" version="2.0.3" targetFramework="net45" />
+  <package id="Microsoft.Azure.Management.HDInsight" version="2.0.7" targetFramework="net45" />
   <package id="Microsoft.Azure.Management.HDInsight.Job" version="2.0.3" targetFramework="net45" />
   <package id="Microsoft.Azure.Management.Resources" version="2.20.0-preview" targetFramework="net45" />
   <package id="Microsoft.Azure.Management.Storage" version="2.4.0-preview" targetFramework="net45" />

src/ResourceManager/HDInsight/Commands.HDInsight/Commands.HDInsight.csproj

@@ -129,7 +129,7 @@
       <HintPath>..\..\..\packages\Microsoft.Azure.Graph.RBAC.3.4.0-preview\lib\net452\Microsoft.Azure.Graph.RBAC.dll</HintPath>
     </Reference>
     <Reference Include="Microsoft.Azure.Management.HDInsight, Version=2.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
-      <HintPath>..\..\..\packages\Microsoft.Azure.Management.HDInsight.2.0.3\lib\net40\Microsoft.Azure.Management.HDInsight.dll</HintPath>
+      <HintPath>..\..\..\packages\Microsoft.Azure.Management.HDInsight.2.0.7\lib\net45\Microsoft.Azure.Management.HDInsight.dll</HintPath>
       <Private>True</Private>
     </Reference>
     <Reference Include="Microsoft.Azure.Management.HDInsight.Job">

src/ResourceManager/HDInsight/Commands.HDInsight/ManagementCommands/AddAzureHDInsightConfigValuesCommand.cs

@@ -82,6 +82,18 @@ public class AddAzureHDInsightConfigValuesCommand : HDInsightCmdletBase
         [Parameter(HelpMessage = "Gets the RServer configurations.")]
         public Hashtable RServer { get; set; }
 
+        [Parameter(HelpMessage = "Gets the Spark Defaults configurations of this HDInsight cluster.", ParameterSetName = "Spark1")]
+        public Hashtable SparkDefaults { get; set; }
+
+        [Parameter(HelpMessage = "Gets the Spark Thrift SparkConf configurations of this HDInsight cluster.", ParameterSetName = "Spark1")]
+        public Hashtable SparkThriftConf { get; set; }
+
+        [Parameter(HelpMessage = "Gets the Spark2 Defaults configurations of this HDInsight cluster.", ParameterSetName = "Spark2")]
+        public Hashtable Spark2Defaults { get; set; }
+
+        [Parameter(HelpMessage = "Gets the Spark2 Thrift SparkConf configurations of this HDInsight cluster.", ParameterSetName = "Spark2")]
+        public Hashtable Spark2ThriftConf { get; set; }
+
         #endregion
 
         public AddAzureHDInsightConfigValuesCommand()
@@ -100,6 +112,10 @@ public AddAzureHDInsightConfigValuesCommand()
             Tez = new Hashtable();
             Hdfs = new Hashtable();
             RServer = new Hashtable();
+            SparkDefaults = new Hashtable();
+            SparkThriftConf = new Hashtable();
+            Spark2Defaults = new Hashtable();
+            Spark2ThriftConf = new Hashtable();
         }
 
         public override void ExecuteCmdlet()
@@ -120,6 +136,10 @@ public override void ExecuteCmdlet()
             AddConfigToConfigurations(Tez, ConfigurationKey.TezSite);
             AddConfigToConfigurations(Hdfs, ConfigurationKey.HdfsSite);
             AddConfigToConfigurations(RServer, RServerConfigurationKey);
+            AddConfigToConfigurations(SparkDefaults, ConfigurationKey.SparkDefaults);
+            AddConfigToConfigurations(SparkThriftConf, ConfigurationKey.SparkThriftConf);
+            AddConfigToConfigurations(Spark2Defaults, ConfigurationKey.Spark2Defaults);
+            AddConfigToConfigurations(Spark2ThriftConf, ConfigurationKey.Spark2ThriftConf);
 
             WriteObject(Config);
         }

src/ResourceManager/HDInsight/Commands.HDInsight/help/Add-AzureRmHDInsightConfigValues.md

@@ -12,11 +12,22 @@ Adds a Hadoop configuration value customization and/or a Hive shared library cus
 
 ## SYNTAX
 
+### Spark1
 ```
 Add-AzureRmHDInsightConfigValues [-Config] <AzureHDInsightConfig> [-Core <Hashtable>] [-HiveSite <Hashtable>]
  [-HiveEnv <Hashtable>] [-OozieSite <Hashtable>] [-OozieEnv <Hashtable>] [-WebHCat <Hashtable>]
  [-HBaseSite <Hashtable>] [-HBaseEnv <Hashtable>] [-Storm <Hashtable>] [-Yarn <Hashtable>]
- [-MapRed <Hashtable>] [-Tez <Hashtable>] [-Hdfs <Hashtable>] [-RServer <Hashtable>] [<CommonParameters>]
+ [-MapRed <Hashtable>] [-Tez <Hashtable>] [-Hdfs <Hashtable>] [-RServer <Hashtable>]
+ [-SparkDefaults <Hashtable>] [-SparkThriftConf <Hashtable>] [<CommonParameters>]
+```
+
+### Spark2
+```
+Add-AzureRmHDInsightConfigValues [-Config] <AzureHDInsightConfig> [-Core <Hashtable>] [-HiveSite <Hashtable>]
+ [-HiveEnv <Hashtable>] [-OozieSite <Hashtable>] [-OozieEnv <Hashtable>] [-WebHCat <Hashtable>]
+ [-HBaseSite <Hashtable>] [-HBaseEnv <Hashtable>] [-Storm <Hashtable>] [-Yarn <Hashtable>]
+ [-MapRed <Hashtable>] [-Tez <Hashtable>] [-Hdfs <Hashtable>] [-RServer <Hashtable>]
+ [-Spark2Defaults <Hashtable>] [-Spark2ThriftConf <Hashtable>] [<CommonParameters>]
 ```
 
 ## DESCRIPTION
@@ -220,7 +231,9 @@ Accept wildcard characters: False
 ```
 
 ### -RServer
-Specifies the RServer configurations. Valid only for RServer clusters.```yaml
+Specifies the RServer configurations. Valid only for RServer clusters.
+
+```yaml
 Type: Hashtable
 Parameter Sets: (All)
 Aliases: 
@@ -232,6 +245,66 @@ Accept pipeline input: False
 Accept wildcard characters: False
 ```
 
+### -Spark2Defaults
+Specifies the Spark2 Defaults configurations of this HDInsight cluster.
+
+```yaml
+Type: Hashtable
+Parameter Sets: Spark2
+Aliases: 
+
+Required: False
+Position: Named
+Default value: None
+Accept pipeline input: False
+Accept wildcard characters: False
+```
+
+### -Spark2ThriftConf
+Specifies the Spark2 Thrift SparkConf configurations of this HDInsight cluster.
+
+```yaml
+Type: Hashtable
+Parameter Sets: Spark2
+Aliases: 
+
+Required: False
+Position: Named
+Default value: None
+Accept pipeline input: False
+Accept wildcard characters: False
+```
+
+### -SparkDefaults
+Specifies the Spark Defaults configurations of this HDInsight cluster.
+
+```yaml
+Type: Hashtable
+Parameter Sets: Spark1
+Aliases: 
+
+Required: False
+Position: Named
+Default value: None
+Accept pipeline input: False
+Accept wildcard characters: False
+```
+
+### -SparkThriftConf
+Specifies the Spark Thrift SparkConf configurations of this HDInsight cluster.
+
+```yaml
+Type: Hashtable
+Parameter Sets: Spark1
+Aliases: 
+
+Required: False
+Position: Named
+Default value: None
+Accept pipeline input: False
+Accept wildcard characters: False
+```
+
 ### -Storm
 Specifies the Storm Site configurations of this HDInsight cluster.
 

src/ResourceManager/HDInsight/Commands.HDInsight/packages.config

@@ -5,7 +5,7 @@
   <package id="Microsoft.Azure.Common.Dependencies" version="1.0.0" targetFramework="net45" />
   <package id="Microsoft.Azure.Graph.RBAC" version="3.4.0-preview" targetFramework="net452" />
   <package id="Microsoft.Azure.KeyVault.Core" version="1.0.0" targetFramework="net45" />
-  <package id="Microsoft.Azure.Management.HDInsight" version="2.0.3" targetFramework="net45" />
+  <package id="Microsoft.Azure.Management.HDInsight" version="2.0.7" targetFramework="net45" />
   <package id="Microsoft.Azure.Management.HDInsight.Job" version="2.0.3" targetFramework="net45" />
   <package id="Microsoft.Azure.Management.Resources" version="2.20.0-preview" targetFramework="net45" />
   <package id="Microsoft.Bcl" version="1.1.9" targetFramework="net45" />