Skip to content

Commit 3c33667

Browse files
markcowlmarkcowl
authored
Merge pull request #3110 from iyerrb/dev
Support creating Spark 2.0 cluster using Azure HDInsight PowerShell
2 parents 02125be + 056de9c commit 3c33667

File tree

10 files changed

+384
-4
lines changed

10 files changed

+384
-4
lines changed

‎src/ResourceManager/HDInsight/Commands.HDInsight.Test/Commands.HDInsight.Test.csproj

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,7 @@
5252
</Reference>
5353
<Reference Include="Microsoft.Azure.Management.HDInsight, Version=1.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
5454
<SpecificVersion>False</SpecificVersion>
55-
<HintPath>..\..\..\packages\Microsoft.Azure.Management.HDInsight.1.0.14-preview\lib\net40\Microsoft.Azure.Management.HDInsight.dll</HintPath>
55+
<HintPath>..\..\..\packages\Microsoft.Azure.Management.HDInsight.1.3.0\lib\net40\Microsoft.Azure.Management.HDInsight.dll</HintPath>
5656
</Reference>
5757
<Reference Include="Microsoft.Azure.Management.HDInsight.Job">
5858
<SpecificVersion>False</SpecificVersion>

‎src/ResourceManager/HDInsight/Commands.HDInsight.Test/UnitTests/NewClusterTests.cs

Lines changed: 104 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -221,5 +221,109 @@ public void CanCreateNewHDInsightCluster_Linux()
221221
clusterout.OperatingSystemType == OSType.Linux)),
222222
Times.Once);
223223
}
224+
225+
[Fact]
226+
[Trait(Category.AcceptanceType, Category.CheckIn)]
227+
public void CanCreateNewHDInsightCluster_LinuxComponentVersion()
228+
{
229+
string sparkClusterType = "Spark";
230+
var componentVersion = new Dictionary<string, string>
231+
{
232+
{"Spark", "2.0"}
233+
};
234+
var componentVersionResponse = "[Spark, 2.0]";
235+
236+
cmdlet.ClusterName = ClusterName;
237+
cmdlet.ResourceGroupName = ResourceGroupName;
238+
cmdlet.ClusterSizeInNodes = ClusterSize;
239+
cmdlet.Location = Location;
240+
cmdlet.HttpCredential = _httpCred;
241+
cmdlet.DefaultStorageAccountName = StorageName;
242+
cmdlet.DefaultStorageAccountKey = StorageKey;
243+
cmdlet.ClusterType = "Spark";
244+
cmdlet.SshCredential = _sshCred;
245+
cmdlet.OSType = OSType.Linux;
246+
cmdlet.ComponentVersion = componentVersion;
247+
248+
var cluster = new Cluster
249+
{
250+
Id = "id",
251+
Name = ClusterName,
252+
Location = Location,
253+
Properties = new ClusterGetProperties
254+
{
255+
ClusterVersion = "3.5",
256+
ClusterState = "Running",
257+
ClusterDefinition = new ClusterDefinition
258+
{
259+
ClusterType = sparkClusterType
260+
},
261+
QuotaInfo = new QuotaInfo
262+
{
263+
CoresUsed = 24
264+
},
265+
OperatingSystemType = OSType.Linux
266+
}
267+
};
268+
269+
270+
cluster.Properties.ClusterDefinition.ComponentVersion = componentVersion;
271+
272+
var coreConfigs = new Dictionary<string, string>
273+
{
274+
{"fs.defaultFS", "wasb://giyertestcsmv2@" + StorageName},
275+
{
276+
"fs.azure.account.key." + StorageName,
277+
StorageKey
278+
}
279+
};
280+
var gatewayConfigs = new Dictionary<string, string>
281+
{
282+
{"restAuthCredential.isEnabled", "true"},
283+
{"restAuthCredential.username", _httpCred.UserName},
284+
{"restAuthCredential.password", _httpCred.Password.ConvertToString()}
285+
};
286+
287+
var configurations = new Dictionary<string, Dictionary<string, string>>
288+
{
289+
{"core-site", coreConfigs},
290+
{"gateway", gatewayConfigs}
291+
};
292+
var serializedConfig = JsonConvert.SerializeObject(configurations);
293+
cluster.Properties.ClusterDefinition.Configurations = serializedConfig;
294+
295+
var getresponse = new ClusterGetResponse { Cluster = cluster };
296+
297+
hdinsightManagementMock.Setup(c => c.CreateNewCluster(ResourceGroupName, ClusterName, It.Is<ClusterCreateParameters>(
298+
parameters =>
299+
parameters.ClusterSizeInNodes == ClusterSize &&
300+
parameters.DefaultStorageAccountName == StorageName &&
301+
parameters.DefaultStorageAccountKey == StorageKey &&
302+
parameters.Location == Location &&
303+
parameters.UserName == _httpCred.UserName &&
304+
parameters.Password == _httpCred.Password.ConvertToString() &&
305+
parameters.ClusterType == sparkClusterType &&
306+
parameters.OSType == OSType.Linux &&
307+
parameters.SshUserName == _sshCred.UserName &&
308+
parameters.SshPassword == _sshCred.Password.ConvertToString() &&
309+
parameters.ComponentVersion["Spark"] == componentVersion["Spark"])))
310+
.Returns(getresponse)
311+
.Verifiable();
312+
313+
cmdlet.ExecuteCmdlet();
314+
315+
commandRuntimeMock.VerifyAll();
316+
commandRuntimeMock.Verify(f => f.WriteObject(It.Is<AzureHDInsightCluster>(
317+
clusterout =>
318+
clusterout.ClusterState == "Running" &&
319+
clusterout.ClusterType == sparkClusterType &&
320+
clusterout.ClusterVersion == "3.5" &&
321+
clusterout.CoresUsed == 24 &&
322+
clusterout.Location == Location &&
323+
clusterout.Name == ClusterName &&
324+
clusterout.OperatingSystemType == OSType.Linux &&
325+
clusterout.ComponentVersion[0] == componentVersionResponse)),
326+
Times.Once);
327+
}
224328
}
225329
}

‎src/ResourceManager/HDInsight/Commands.HDInsight/Commands.HDInsight.csproj

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,7 @@
5050
<Compile Include="JobCommands\UseAzureHDInsightClusterCommand.cs" />
5151
<Compile Include="JobCommands\StopAzureHDInsightJobCommand.cs" />
5252
<Compile Include="JobCommands\StartAzureHDInsightJobCommand.cs" />
53+
<Compile Include="ManagementCommands\AddAzureHDInsightComponentVersionCommand.cs" />
5354
<Compile Include="ManagementCommands\SetAzureHDInsightDefaultStorageCommand.cs" />
5455
<Compile Include="ManagementCommands\AddAzureHDInsightClusterIdentity.cs" />
5556
<Compile Include="ManagementCommands\AddAzureHDInsightStorageCommand.cs" />
@@ -124,7 +125,7 @@
124125
</Reference>
125126
<Reference Include="Microsoft.Azure.Management.HDInsight, Version=1.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
126127
<SpecificVersion>False</SpecificVersion>
127-
<HintPath>..\..\..\packages\Microsoft.Azure.Management.HDInsight.1.0.14-preview\lib\net40\Microsoft.Azure.Management.HDInsight.dll</HintPath>
128+
<HintPath>..\..\..\packages\Microsoft.Azure.Management.HDInsight.1.3.0\lib\net40\Microsoft.Azure.Management.HDInsight.dll</HintPath>
128129
</Reference>
129130
<Reference Include="Microsoft.Azure.Management.HDInsight.Job">
130131
<SpecificVersion>False</SpecificVersion>
@@ -157,7 +158,7 @@
157158
<HintPath>..\..\..\packages\Microsoft.Rest.ClientRuntime.2.3.2\lib\net45\Microsoft.Rest.ClientRuntime.dll</HintPath>
158159
<Private>True</Private>
159160
</Reference>
160-
<Reference Include="Microsoft.Rest.ClientRuntime.Azure, Version=3.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
161+
<Reference Include="Microsoft.Rest.ClientRuntime.Azure, Version=3.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
161162
<HintPath>..\..\..\packages\Microsoft.Rest.ClientRuntime.Azure.3.3.2\lib\net45\Microsoft.Rest.ClientRuntime.Azure.dll</HintPath>
162163
</Reference>
163164
<Reference Include="Microsoft.Rest.ClientRuntime.Azure.Authentication, Version=2.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">

‎src/ResourceManager/HDInsight/Commands.HDInsight/Constants.cs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@ public static class CommandNames
3636
public const string AzureHDInsightDefaultStorage = "AzureRmHDInsightDefaultStorage";
3737
public const string AzureHDInsightHiveJob = "AzureRmHDInsightHiveJob";
3838
public const string AzureHDInsightClusterIdentity = "AzureRmHDInsightClusterIdentity";
39+
public const string AzureHDInsightComponentVersion = "AzureRmHDInsightComponentVersion";
3940
public const string Hive = "Hive";
4041
}
4142

‎src/ResourceManager/HDInsight/Commands.HDInsight/ManagementCommands/AddAzureHDInsightComponentVersionCommand.cs

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
// ----------------------------------------------------------------------------------
2+
//
3+
// Copyright Microsoft Corporation
4+
// Licensed under the Apache License, Version 2.0 (the "License");
5+
// you may not use this file except in compliance with the License.
6+
// You may obtain a copy of the License at
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
// Unless required by applicable law or agreed to in writing, software
9+
// distributed under the License is distributed on an "AS IS" BASIS,
10+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11+
// See the License for the specific language governing permissions and
12+
// limitations under the License.
13+
// ----------------------------------------------------------------------------------
14+
15+
using Microsoft.Azure.Commands.HDInsight.Commands;
16+
using Microsoft.Azure.Commands.HDInsight.Models;
17+
using System.Management.Automation;
18+
19+
namespace Microsoft.Azure.Commands.HDInsight
20+
{
21+
[Cmdlet(
22+
VerbsCommon.Add,
23+
Constants.CommandNames.AzureHDInsightComponentVersion,
24+
SupportsShouldProcess = true),
25+
OutputType(typeof(AzureHDInsightConfig))]
26+
public class AddAzureHDInsightComponentVersionCommand : HDInsightCmdletBase
27+
{
28+
#region Input Parameter Definitions
29+
30+
[Parameter(Position = 0,
31+
Mandatory = true,
32+
ValueFromPipeline = true,
33+
HelpMessage = "The HDInsight cluster configuration to use when creating the new cluster.")]
34+
public AzureHDInsightConfig Config { get; set; }
35+
36+
[Parameter(Position = 1,
37+
Mandatory = true,
38+
HelpMessage = "The component name whose version should be specified for the new cluster.")]
39+
public string ComponentName { get; set; }
40+
41+
[Parameter(Position = 2,
42+
Mandatory = true,
43+
HelpMessage = "The version of a specific service in the new cluster.")]
44+
public string ComponentVersion { get; set; }
45+
46+
#endregion
47+
48+
public override void ExecuteCmdlet()
49+
{
50+
ConfirmAction("Adding Component Version", "AzureHDInsightConfig",
51+
() =>
52+
{
53+
Config.ComponentVersion.Add(ComponentName, ComponentVersion);
54+
WriteObject(Config);
55+
});
56+
}
57+
}
58+
}

‎src/ResourceManager/HDInsight/Commands.HDInsight/ManagementCommands/NewAzureHDInsightClusterCommand.cs

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -140,6 +140,10 @@ var storageAccount in
140140
{
141141
result.ScriptActions.Add(action.Key, action.Value.Select(a => new AzureHDInsightScriptAction(a)).ToList());
142142
}
143+
foreach (var component in parameters.ComponentVersion.Where(component => !result.ComponentVersion.ContainsKey(component.Key)))
144+
{
145+
result.ComponentVersion.Add(component.Key, component.Value);
146+
}
143147
return result;
144148
}
145149
set
@@ -180,6 +184,10 @@ var storageAccount in
180184
{
181185
parameters.ScriptActions.Add(action.Key, action.Value.Select(a => a.GetScriptActionFromPSModel()).ToList());
182186
}
187+
foreach (var component in value.ComponentVersion.Where(component => !parameters.ComponentVersion.ContainsKey(component.Key)))
188+
{
189+
parameters.ComponentVersion.Add(component.Key, component.Value);
190+
}
183191
}
184192
}
185193

@@ -240,6 +248,13 @@ public string ClusterType
240248
set { parameters.ClusterType = value; }
241249
}
242250

251+
[Parameter(HelpMessage = "Gets or sets the version for a service in the cluster.")]
252+
public Dictionary<string, string> ComponentVersion
253+
{
254+
get { return parameters.ComponentVersion; }
255+
set { parameters.ComponentVersion = value; }
256+
}
257+
243258
[Parameter(HelpMessage = "Gets or sets the virtual network guid for this HDInsight cluster.")]
244259
public string VirtualNetworkId
245260
{
@@ -310,6 +325,7 @@ public NewAzureHDInsightClusterCommand()
310325
AdditionalStorageAccounts = new Dictionary<string, string>();
311326
Configurations = new Dictionary<string, Dictionary<string, string>>();
312327
ScriptActions = new Dictionary<ClusterNodeType, List<AzureHDInsightScriptAction>>();
328+
ComponentVersion = new Dictionary<string, string>();
313329
}
314330

315331
public override void ExecuteCmdlet()
@@ -352,6 +368,10 @@ var storageAccount in
352368
parameters.ScriptActions.Add(action.Key,
353369
action.Value.Select(a => a.GetScriptActionFromPSModel()).ToList());
354370
}
371+
foreach (var component in ComponentVersion.Where(component => !parameters.ComponentVersion.ContainsKey(component.Key)))
372+
{
373+
parameters.ComponentVersion.Add(component.Key, component.Value);
374+
}
355375
if (OozieMetastore != null)
356376
{
357377
var metastore = OozieMetastore;

0 commit comments

Comments
 (0)