Skip to content

Commit d34efd4

Browse files
cormacpaynecormacpayne
authored
Merge pull request #4172 from jeffersonezra/jeezraNewSparkConfigParameters
Add new parameters to set Spark custom configurations
2 parents 0c00632 + d302769 commit d34efd4

File tree

7 files changed

+159
-6
lines changed

7 files changed

+159
-6
lines changed

‎src/ResourceManager/HDInsight/Commands.HDInsight.Test/Commands.HDInsight.Test.csproj

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,7 @@
5151
<HintPath>..\..\..\packages\Microsoft.Azure.Management.Authorization.0.18.2-preview\lib\net40\Microsoft.Azure.Management.Authorization.dll</HintPath>
5252
</Reference>
5353
<Reference Include="Microsoft.Azure.Management.HDInsight, Version=2.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
54-
<HintPath>..\..\..\packages\Microsoft.Azure.Management.HDInsight.2.0.3\lib\net40\Microsoft.Azure.Management.HDInsight.dll</HintPath>
54+
<HintPath>..\..\..\packages\Microsoft.Azure.Management.HDInsight.2.0.7\lib\net45\Microsoft.Azure.Management.HDInsight.dll</HintPath>
5555
<Private>True</Private>
5656
</Reference>
5757
<Reference Include="Microsoft.Azure.Management.HDInsight.Job">

‎src/ResourceManager/HDInsight/Commands.HDInsight.Test/UnitTests/ConfigurationTests.cs

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,9 @@
1212
// limitations under the License.
1313
// ----------------------------------------------------------------------------------
1414

15+
using System.Collections;
1516
using Microsoft.Azure.Commands.HDInsight.Models;
17+
using Microsoft.Azure.Management.HDInsight;
1618
using Microsoft.WindowsAzure.Commands.ScenarioTest;
1719
using Moq;
1820
using Xunit;
@@ -41,6 +43,20 @@ public void CanCreateNewConfigForRServer()
4143
CreateNewConfig(setEdgeNodeVmSize: true);
4244
}
4345

46+
[Fact]
47+
[Trait(Category.AcceptanceType, Category.CheckIn)]
48+
public void CanAddSparkCustomConfigs()
49+
{
50+
CustomizeSpark(1);
51+
}
52+
53+
[Fact]
54+
[Trait(Category.AcceptanceType, Category.CheckIn)]
55+
public void CanAddSpark2CustomConfigs()
56+
{
57+
CustomizeSpark(2);
58+
}
59+
4460
public void CreateNewConfig(bool setEdgeNodeVmSize = false)
4561
{
4662
var newconfigcmdlet = new NewAzureHDInsightClusterConfigCommand
@@ -72,5 +88,49 @@ public void CreateNewConfig(bool setEdgeNodeVmSize = false)
7288
c.ScriptActions.Count == 0)),
7389
Times.Once);
7490
}
91+
92+
private void CustomizeSpark(int sparkVersion)
93+
{
94+
AzureHDInsightConfig config = new AzureHDInsightConfig();
95+
96+
Hashtable sparkDefaults = new Hashtable() { { @"spark.executor.instances", "3" } };
97+
Hashtable sparkThriftConf = new Hashtable() { { @"spark.executor.cores", "4" } };
98+
99+
AddAzureHDInsightConfigValuesCommand addConfigValuesCmdlet = new AddAzureHDInsightConfigValuesCommand
100+
{
101+
CommandRuntime = commandRuntimeMock.Object,
102+
HDInsightManagementClient = hdinsightManagementMock.Object,
103+
Config = config,
104+
};
105+
106+
if (sparkVersion == 1)
107+
{
108+
addConfigValuesCmdlet.SparkDefaults = sparkDefaults;
109+
addConfigValuesCmdlet.SparkThriftConf = sparkThriftConf;
110+
}
111+
else
112+
{
113+
addConfigValuesCmdlet.Spark2Defaults = sparkDefaults;
114+
addConfigValuesCmdlet.Spark2ThriftConf = sparkThriftConf;
115+
}
116+
117+
addConfigValuesCmdlet.ExecuteCmdlet();
118+
119+
commandRuntimeMock.Verify(
120+
f =>
121+
f.WriteObject(
122+
It.Is<AzureHDInsightConfig>(
123+
c =>
124+
c.Configurations != null &&
125+
((sparkVersion == 1 && c.Configurations.ContainsKey(ConfigurationKey.SparkDefaults) &&
126+
c.Configurations[ConfigurationKey.SparkDefaults]["spark.executor.instances"].Equals(sparkDefaults["spark.executor.instances"]) &&
127+
c.Configurations.ContainsKey(ConfigurationKey.SparkThriftConf) &&
128+
c.Configurations[ConfigurationKey.SparkThriftConf]["spark.executor.cores"].Equals(sparkThriftConf["spark.executor.cores"])) ||
129+
(sparkVersion == 2 && c.Configurations.ContainsKey(ConfigurationKey.Spark2Defaults) &&
130+
c.Configurations[ConfigurationKey.Spark2Defaults]["spark.executor.instances"].Equals(sparkDefaults["spark.executor.instances"]) &&
131+
c.Configurations.ContainsKey(ConfigurationKey.Spark2ThriftConf) &&
132+
c.Configurations[ConfigurationKey.Spark2ThriftConf]["spark.executor.cores"].Equals(sparkThriftConf["spark.executor.cores"]))))),
133+
Times.Once);
134+
}
75135
}
76136
}

‎src/ResourceManager/HDInsight/Commands.HDInsight.Test/packages.config

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66
<package id="Microsoft.Azure.Gallery" version="2.6.2-preview" targetFramework="net45" />
77
<package id="Microsoft.Azure.KeyVault.Core" version="1.0.0" targetFramework="net45" />
88
<package id="Microsoft.Azure.Management.Authorization" version="0.18.2-preview" targetFramework="net45" />
9-
<package id="Microsoft.Azure.Management.HDInsight" version="2.0.3" targetFramework="net45" />
9+
<package id="Microsoft.Azure.Management.HDInsight" version="2.0.7" targetFramework="net45" />
1010
<package id="Microsoft.Azure.Management.HDInsight.Job" version="2.0.3" targetFramework="net45" />
1111
<package id="Microsoft.Azure.Management.Resources" version="2.20.0-preview" targetFramework="net45" />
1212
<package id="Microsoft.Azure.Management.Storage" version="2.4.0-preview" targetFramework="net45" />

‎src/ResourceManager/HDInsight/Commands.HDInsight/Commands.HDInsight.csproj

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -129,7 +129,7 @@
129129
<HintPath>..\..\..\packages\Microsoft.Azure.Graph.RBAC.3.4.0-preview\lib\net452\Microsoft.Azure.Graph.RBAC.dll</HintPath>
130130
</Reference>
131131
<Reference Include="Microsoft.Azure.Management.HDInsight, Version=2.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
132-
<HintPath>..\..\..\packages\Microsoft.Azure.Management.HDInsight.2.0.3\lib\net40\Microsoft.Azure.Management.HDInsight.dll</HintPath>
132+
<HintPath>..\..\..\packages\Microsoft.Azure.Management.HDInsight.2.0.7\lib\net45\Microsoft.Azure.Management.HDInsight.dll</HintPath>
133133
<Private>True</Private>
134134
</Reference>
135135
<Reference Include="Microsoft.Azure.Management.HDInsight.Job">

‎src/ResourceManager/HDInsight/Commands.HDInsight/ManagementCommands/AddAzureHDInsightConfigValuesCommand.cs

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -82,6 +82,18 @@ public class AddAzureHDInsightConfigValuesCommand : HDInsightCmdletBase
8282
[Parameter(HelpMessage = "Gets the RServer configurations.")]
8383
public Hashtable RServer { get; set; }
8484

85+
[Parameter(HelpMessage = "Gets the Spark Defaults configurations of this HDInsight cluster.", ParameterSetName = "Spark1")]
86+
public Hashtable SparkDefaults { get; set; }
87+
88+
[Parameter(HelpMessage = "Gets the Spark Thrift SparkConf configurations of this HDInsight cluster.", ParameterSetName = "Spark1")]
89+
public Hashtable SparkThriftConf { get; set; }
90+
91+
[Parameter(HelpMessage = "Gets the Spark2 Defaults configurations of this HDInsight cluster.", ParameterSetName = "Spark2")]
92+
public Hashtable Spark2Defaults { get; set; }
93+
94+
[Parameter(HelpMessage = "Gets the Spark2 Thrift SparkConf configurations of this HDInsight cluster.", ParameterSetName = "Spark2")]
95+
public Hashtable Spark2ThriftConf { get; set; }
96+
8597
#endregion
8698

8799
public AddAzureHDInsightConfigValuesCommand()
@@ -100,6 +112,10 @@ public AddAzureHDInsightConfigValuesCommand()
100112
Tez = new Hashtable();
101113
Hdfs = new Hashtable();
102114
RServer = new Hashtable();
115+
SparkDefaults = new Hashtable();
116+
SparkThriftConf = new Hashtable();
117+
Spark2Defaults = new Hashtable();
118+
Spark2ThriftConf = new Hashtable();
103119
}
104120

105121
public override void ExecuteCmdlet()
@@ -120,6 +136,10 @@ public override void ExecuteCmdlet()
120136
AddConfigToConfigurations(Tez, ConfigurationKey.TezSite);
121137
AddConfigToConfigurations(Hdfs, ConfigurationKey.HdfsSite);
122138
AddConfigToConfigurations(RServer, RServerConfigurationKey);
139+
AddConfigToConfigurations(SparkDefaults, ConfigurationKey.SparkDefaults);
140+
AddConfigToConfigurations(SparkThriftConf, ConfigurationKey.SparkThriftConf);
141+
AddConfigToConfigurations(Spark2Defaults, ConfigurationKey.Spark2Defaults);
142+
AddConfigToConfigurations(Spark2ThriftConf, ConfigurationKey.Spark2ThriftConf);
123143

124144
WriteObject(Config);
125145
}

‎src/ResourceManager/HDInsight/Commands.HDInsight/help/Add-AzureRmHDInsightConfigValues.md

Lines changed: 75 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -12,11 +12,22 @@ Adds a Hadoop configuration value customization and/or a Hive shared library cus
1212

1313
## SYNTAX
1414

15+
### Spark1
1516
```
1617
Add-AzureRmHDInsightConfigValues [-Config] <AzureHDInsightConfig> [-Core <Hashtable>] [-HiveSite <Hashtable>]
1718
[-HiveEnv <Hashtable>] [-OozieSite <Hashtable>] [-OozieEnv <Hashtable>] [-WebHCat <Hashtable>]
1819
[-HBaseSite <Hashtable>] [-HBaseEnv <Hashtable>] [-Storm <Hashtable>] [-Yarn <Hashtable>]
19-
[-MapRed <Hashtable>] [-Tez <Hashtable>] [-Hdfs <Hashtable>] [-RServer <Hashtable>] [<CommonParameters>]
20+
[-MapRed <Hashtable>] [-Tez <Hashtable>] [-Hdfs <Hashtable>] [-RServer <Hashtable>]
21+
[-SparkDefaults <Hashtable>] [-SparkThriftConf <Hashtable>] [<CommonParameters>]
22+
```
23+
24+
### Spark2
25+
```
26+
Add-AzureRmHDInsightConfigValues [-Config] <AzureHDInsightConfig> [-Core <Hashtable>] [-HiveSite <Hashtable>]
27+
[-HiveEnv <Hashtable>] [-OozieSite <Hashtable>] [-OozieEnv <Hashtable>] [-WebHCat <Hashtable>]
28+
[-HBaseSite <Hashtable>] [-HBaseEnv <Hashtable>] [-Storm <Hashtable>] [-Yarn <Hashtable>]
29+
[-MapRed <Hashtable>] [-Tez <Hashtable>] [-Hdfs <Hashtable>] [-RServer <Hashtable>]
30+
[-Spark2Defaults <Hashtable>] [-Spark2ThriftConf <Hashtable>] [<CommonParameters>]
2031
```
2132

2233
## DESCRIPTION
@@ -220,7 +231,9 @@ Accept wildcard characters: False
220231
```
221232
222233
### -RServer
223-
Specifies the RServer configurations. Valid only for RServer clusters.```yaml
234+
Specifies the RServer configurations. Valid only for RServer clusters.
235+
236+
```yaml
224237
Type: Hashtable
225238
Parameter Sets: (All)
226239
Aliases:
@@ -232,6 +245,66 @@ Accept pipeline input: False
232245
Accept wildcard characters: False
233246
```
234247
248+
### -Spark2Defaults
249+
Specifies the Spark2 Defaults configurations of this HDInsight cluster.
250+
251+
```yaml
252+
Type: Hashtable
253+
Parameter Sets: Spark2
254+
Aliases:
255+
256+
Required: False
257+
Position: Named
258+
Default value: None
259+
Accept pipeline input: False
260+
Accept wildcard characters: False
261+
```
262+
263+
### -Spark2ThriftConf
264+
Specifies the Spark2 Thrift SparkConf configurations of this HDInsight cluster.
265+
266+
```yaml
267+
Type: Hashtable
268+
Parameter Sets: Spark2
269+
Aliases:
270+
271+
Required: False
272+
Position: Named
273+
Default value: None
274+
Accept pipeline input: False
275+
Accept wildcard characters: False
276+
```
277+
278+
### -SparkDefaults
279+
Specifies the Spark Defaults configurations of this HDInsight cluster.
280+
281+
```yaml
282+
Type: Hashtable
283+
Parameter Sets: Spark1
284+
Aliases:
285+
286+
Required: False
287+
Position: Named
288+
Default value: None
289+
Accept pipeline input: False
290+
Accept wildcard characters: False
291+
```
292+
293+
### -SparkThriftConf
294+
Specifies the Spark Thrift SparkConf configurations of this HDInsight cluster.
295+
296+
```yaml
297+
Type: Hashtable
298+
Parameter Sets: Spark1
299+
Aliases:
300+
301+
Required: False
302+
Position: Named
303+
Default value: None
304+
Accept pipeline input: False
305+
Accept wildcard characters: False
306+
```
307+
235308
### -Storm
236309
Specifies the Storm Site configurations of this HDInsight cluster.
237310

‎src/ResourceManager/HDInsight/Commands.HDInsight/packages.config

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
<package id="Microsoft.Azure.Common.Dependencies" version="1.0.0" targetFramework="net45" />
66
<package id="Microsoft.Azure.Graph.RBAC" version="3.4.0-preview" targetFramework="net452" />
77
<package id="Microsoft.Azure.KeyVault.Core" version="1.0.0" targetFramework="net45" />
8-
<package id="Microsoft.Azure.Management.HDInsight" version="2.0.3" targetFramework="net45" />
8+
<package id="Microsoft.Azure.Management.HDInsight" version="2.0.7" targetFramework="net45" />
99
<package id="Microsoft.Azure.Management.HDInsight.Job" version="2.0.3" targetFramework="net45" />
1010
<package id="Microsoft.Azure.Management.Resources" version="2.20.0-preview" targetFramework="net45" />
1111
<package id="Microsoft.Bcl" version="1.1.9" targetFramework="net45" />

0 commit comments

Comments
 (0)