Skip to content

Commit a6c4f7a

Browse files
author
Abhijith Padmakumar
committed
Updating New-AzureRmHDInsightCluster cmdlet to add new parameter that specifies number of data disks per worker node
1 parent 10354c8 commit a6c4f7a

File tree

5 files changed

+73
-163
lines changed

5 files changed

+73
-163
lines changed

‎src/ResourceManager/HDInsight/Commands.HDInsight.Test/UnitTests/NewClusterTests.cs

Lines changed: 39 additions & 134 deletions
Original file line numberDiff line numberDiff line change
@@ -152,7 +152,6 @@ public void CanCreateNewHDInsightCluster_Linux()
152152
clusterout.Name == ClusterName &&
153153
clusterout.OperatingSystemType == OSType.Linux)),
154154
Times.Once);
155-
156155
}
157156

158157
[Fact]
@@ -206,8 +205,12 @@ public void CanCreateNewHDInsightCluster_Secure_Linux()
206205
Times.Once);
207206
}
208207

209-
private void CreateNewHDInsightCluster(bool addSecurityProfileInresponse = false, bool setEdgeNodeVmSize = false)
208+
private void CreateNewHDInsightCluster(
209+
bool addSecurityProfileInresponse = false,
210+
bool setEdgeNodeVmSize = false,
211+
int workerNodeDataDisks = 0)
210212
{
213+
// Assign cmdlet parameters
211214
cmdlet.ClusterName = ClusterName;
212215
cmdlet.ResourceGroupName = ResourceGroupName;
213216
cmdlet.ClusterSizeInNodes = ClusterSize;
@@ -218,9 +221,13 @@ private void CreateNewHDInsightCluster(bool addSecurityProfileInresponse = false
218221
cmdlet.ClusterType = ClusterType;
219222
cmdlet.SshCredential = _sshCred;
220223
cmdlet.OSType = OSType.Linux;
224+
cmdlet.DisksPerWorkerNode = workerNodeDataDisks;
221225
if (setEdgeNodeVmSize)
226+
{
222227
cmdlet.EdgeNodeSize = "edgeNodeVmSizeSetTest";
228+
}
223229

230+
// Construct cluster Object
224231
var cluster = new Cluster
225232
{
226233
Id = "id",
@@ -238,10 +245,29 @@ private void CreateNewHDInsightCluster(bool addSecurityProfileInresponse = false
238245
{
239246
CoresUsed = 24
240247
},
241-
OperatingSystemType = OSType.Linux
248+
OperatingSystemType = OSType.Linux,
249+
ComputeProfile = new ComputeProfile()
250+
{
251+
Roles = new List<Role>()
252+
}
242253
}
243254
};
244255

256+
if (workerNodeDataDisks > 0)
257+
{
258+
cluster.Properties.ComputeProfile.Roles.Add(new Role()
259+
{
260+
Name = "workernode",
261+
DataDisksGroups = new List<DataDisksGroupProperties>()
262+
{
263+
new DataDisksGroupProperties()
264+
{
265+
DisksPerNode = workerNodeDataDisks
266+
}
267+
}
268+
});
269+
}
270+
245271
if (addSecurityProfileInresponse)
246272
{
247273
cluster.Properties.SecurityProfile = new SecurityProfile()
@@ -286,6 +312,7 @@ private void CreateNewHDInsightCluster(bool addSecurityProfileInresponse = false
286312

287313
var getresponse = new ClusterGetResponse {Cluster = cluster};
288314

315+
// Setup Mocks and verify successfull GET response
289316
hdinsightManagementMock.Setup(
290317
c => c.CreateNewCluster(ResourceGroupName, ClusterName, It.Is<ClusterCreateParameters>(
291318
parameters =>
@@ -300,10 +327,12 @@ private void CreateNewHDInsightCluster(bool addSecurityProfileInresponse = false
300327
parameters.OSType == OSType.Linux &&
301328
parameters.SshUserName == _sshCred.UserName &&
302329
parameters.SshPassword == _sshCred.Password.ConvertToString() &&
303-
((!setEdgeNodeVmSize && parameters.EdgeNodeSize == null) || (setEdgeNodeVmSize && parameters.EdgeNodeSize == "edgeNodeVmSizeSetTest")))))
330+
((!setEdgeNodeVmSize && parameters.EdgeNodeSize == null) || (setEdgeNodeVmSize && parameters.EdgeNodeSize == "edgeNodeVmSizeSetTest")) &&
331+
(workerNodeDataDisks == 0) || (workerNodeDataDisks > 0 && parameters.WorkerNodeDataDisksGroups.First().DisksPerNode == workerNodeDataDisks))))
304332
.Returns(getresponse)
305333
.Verifiable();
306334

335+
// Execute Cmdlet and verify output
307336
cmdlet.ExecuteCmdlet();
308337

309338
commandRuntimeMock.VerifyAll();
@@ -315,7 +344,8 @@ private void CreateNewHDInsightCluster(bool addSecurityProfileInresponse = false
315344
clusterout.CoresUsed == 24 &&
316345
clusterout.Location == Location &&
317346
clusterout.Name == ClusterName &&
318-
clusterout.OperatingSystemType == OSType.Linux)),
347+
clusterout.OperatingSystemType == OSType.Linux &&
348+
(workerNodeDataDisks == 0) || (clusterout.WorkerNodeDataDisksGroups.First().DisksPerNode == workerNodeDataDisks))),
319349
Times.Once);
320350
}
321351

@@ -441,137 +471,12 @@ public void CanCreateNewHDInsightCluster_LinuxComponentVersion()
441471

442472
[Fact]
443473
[Trait(Category.AcceptanceType, Category.CheckIn)]
444-
public void CanCreateNewHDInsightCluster_DataDisksGroups()
474+
public void CanCreateNewHDInsightCluster_Kafka_DataDisks_Linux()
445475
{
446-
var kafkaClusterType = "Kafka";
447-
var hdiVersion = "3.6";
448-
var coresUsed = 24;
449-
var expectedClusterState = "Running";
450-
var dataDisksGroup = new DataDisksGroupProperties()
451-
{
452-
DisksPerNode = 8
453-
};
454-
var workerNodeDataDisksGroups = new List<DataDisksGroupProperties>()
455-
{
456-
dataDisksGroup
457-
};
458-
459-
cmdlet.ClusterName = ClusterName;
460-
cmdlet.ResourceGroupName = ResourceGroupName;
461-
cmdlet.ClusterSizeInNodes = ClusterSize;
462-
cmdlet.Location = Location;
463-
cmdlet.HttpCredential = _httpCred;
464-
cmdlet.DefaultStorageAccountName = StorageName;
465-
cmdlet.DefaultStorageAccountKey = StorageKey;
466-
cmdlet.ClusterType = kafkaClusterType;
467-
cmdlet.SshCredential = _sshCred;
468-
cmdlet.OSType = OSType.Linux;
469-
cmdlet.WorkerNodeDataDisksGroups = workerNodeDataDisksGroups;
476+
ClusterType = "Kafka";
477+
HdiVersion = "3.6";
470478

471-
var cluster = new Cluster
472-
{
473-
Id = "id",
474-
Name = ClusterName,
475-
Location = Location,
476-
Properties = new ClusterGetProperties
477-
{
478-
ClusterVersion = hdiVersion,
479-
ClusterState = expectedClusterState,
480-
ClusterDefinition = new ClusterDefinition
481-
{
482-
ClusterType = kafkaClusterType
483-
},
484-
QuotaInfo = new QuotaInfo
485-
{
486-
CoresUsed = coresUsed
487-
},
488-
OperatingSystemType = OSType.Linux,
489-
ComputeProfile = new ComputeProfile()
490-
{
491-
Roles = new List<Role>()
492-
}
493-
}
494-
};
495-
496-
cluster.Properties.ComputeProfile.Roles.Add(new Role()
497-
{
498-
Name = "workernode",
499-
DataDisksGroups = workerNodeDataDisksGroups
500-
});
501-
502-
var coreConfigs = new Dictionary<string, string>
503-
{
504-
{"fs.defaultFS", "wasb://giyertestcsmv2@" + StorageName},
505-
{
506-
"fs.azure.account.key." + StorageName,
507-
StorageKey
508-
}
509-
};
510-
var gatewayConfigs = new Dictionary<string, string>
511-
{
512-
{"restAuthCredential.isEnabled", "true"},
513-
{"restAuthCredential.username", _httpCred.UserName},
514-
{"restAuthCredential.password", _httpCred.Password.ConvertToString()}
515-
};
516-
517-
var configurations = new Dictionary<string, Dictionary<string, string>>
518-
{
519-
{"core-site", coreConfigs},
520-
{"gateway", gatewayConfigs}
521-
};
522-
var serializedConfig = JsonConvert.SerializeObject(configurations);
523-
cluster.Properties.ClusterDefinition.Configurations = serializedConfig;
524-
525-
var getresponse = new ClusterGetResponse { Cluster = cluster };
526-
527-
hdinsightManagementMock.Setup(c => c.CreateNewCluster(ResourceGroupName, ClusterName, It.Is<ClusterCreateParameters>(
528-
parameters =>
529-
parameters.ClusterSizeInNodes == ClusterSize &&
530-
parameters.DefaultStorageInfo as AzureStorageInfo != null &&
531-
((AzureStorageInfo)parameters.DefaultStorageInfo).StorageAccountName == StorageName &&
532-
((AzureStorageInfo)parameters.DefaultStorageInfo).StorageAccountKey == StorageKey &&
533-
parameters.Location == Location &&
534-
parameters.UserName == _httpCred.UserName &&
535-
parameters.Password == _httpCred.Password.ConvertToString() &&
536-
parameters.ClusterType == kafkaClusterType &&
537-
parameters.OSType == OSType.Linux &&
538-
parameters.SshUserName == _sshCred.UserName &&
539-
parameters.SshPassword == _sshCred.Password.ConvertToString() &&
540-
parameters.WorkerNodeDataDisksGroups.First().DisksPerNode == dataDisksGroup.DisksPerNode)))
541-
.Returns(getresponse)
542-
.Verifiable();
543-
hdinsightManagementMock.Setup(
544-
c => c.CreateNewCluster(ResourceGroupName, ClusterName, It.Is<ClusterCreateParameters>(
545-
parameters =>
546-
parameters.ClusterSizeInNodes == ClusterSize &&
547-
parameters.DefaultStorageInfo as AzureStorageInfo != null &&
548-
((AzureStorageInfo)parameters.DefaultStorageInfo).StorageAccountName == StorageName &&
549-
((AzureStorageInfo)parameters.DefaultStorageInfo).StorageAccountKey == StorageKey &&
550-
parameters.Location == Location &&
551-
parameters.UserName == _httpCred.UserName &&
552-
parameters.Password == _httpCred.Password.ConvertToString() &&
553-
parameters.ClusterType == ClusterType &&
554-
parameters.OSType == OSType.Linux &&
555-
parameters.SshUserName == _sshCred.UserName &&
556-
parameters.SshPassword == _sshCred.Password.ConvertToString())))
557-
.Returns(getresponse)
558-
.Verifiable();
559-
560-
cmdlet.ExecuteCmdlet();
561-
562-
commandRuntimeMock.VerifyAll();
563-
564-
565-
commandRuntimeMock.Verify(f => f.WriteObject(It.Is<AzureHDInsightCluster>(
566-
clusterout =>
567-
clusterout.ClusterState == expectedClusterState &&
568-
clusterout.ClusterType == kafkaClusterType &&
569-
clusterout.ClusterVersion == hdiVersion &&
570-
clusterout.CoresUsed == coresUsed &&
571-
clusterout.Location == Location &&
572-
clusterout.Name == ClusterName &&
573-
clusterout.OperatingSystemType == OSType.Linux)),
574-
Times.Once);
479+
CreateNewHDInsightCluster(workerNodeDataDisks: 8);
575480
}
576481

577482
[Fact]

‎src/ResourceManager/HDInsight/Commands.HDInsight/ManagementCommands/NewAzureHDInsightClusterCommand.cs

Lines changed: 10 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -136,7 +136,8 @@ public AzureHDInsightConfig Config
136136
CertificateFileContents = CertificateFileContents,
137137
CertificateFilePath = CertificateFilePath,
138138
CertificatePassword = CertificatePassword,
139-
SecurityProfile = SecurityProfile
139+
SecurityProfile = SecurityProfile,
140+
DisksPerWorkerNode = DisksPerWorkerNode
140141
};
141142
foreach (
142143
var storageAccount in
@@ -157,10 +158,7 @@ var storageAccount in
157158
{
158159
result.ComponentVersion.Add(component.Key, component.Value);
159160
}
160-
foreach (var dataDisksGroups in parameters.WorkerNodeDataDisksGroups.Where(d => d.DisksPerNode > 0))
161-
{
162-
result.DataDisksGroupProperties.Add(dataDisksGroups);
163-
}
161+
164162
return result;
165163
}
166164
set
@@ -191,6 +189,7 @@ var storageAccount in
191189
ObjectId = value.ObjectId;
192190
CertificatePassword = value.CertificatePassword;
193191
SecurityProfile = value.SecurityProfile;
192+
DisksPerWorkerNode = value.DisksPerWorkerNode;
194193

195194
foreach (
196195
var storageAccount in
@@ -211,10 +210,6 @@ var storageAccount in
211210
{
212211
parameters.ComponentVersion.Add(component.Key, component.Value);
213212
}
214-
foreach (var dataDisksGroups in value.DataDisksGroupProperties.Where(d => d.DisksPerNode > 0))
215-
{
216-
this.parameters.WorkerNodeDataDisksGroups.Add(dataDisksGroups);
217-
}
218213
}
219214
}
220215

@@ -292,13 +287,6 @@ public Dictionary<string, string> ComponentVersion
292287
set { parameters.ComponentVersion = value; }
293288
}
294289

295-
[Parameter(HelpMessage = "Gets or sets the data disks groups for worker node role in the cluster.")]
296-
public List<DataDisksGroupProperties> WorkerNodeDataDisksGroups
297-
{
298-
get { return parameters.WorkerNodeDataDisksGroups; }
299-
set { parameters.WorkerNodeDataDisksGroups = value; }
300-
}
301-
302290
[Parameter(HelpMessage = "Gets or sets the virtual network guid for this HDInsight cluster.")]
303291
public string VirtualNetworkId
304292
{
@@ -363,6 +351,9 @@ public DateTime RdpAccessExpiry
363351
[Parameter(HelpMessage = "Gets or sets Security Profile which is used for creating secure cluster.")]
364352
public AzureHDInsightSecurityProfile SecurityProfile { get; set; }
365353

354+
[Parameter(HelpMessage = "Gets or sets the number of disks for worker node role in the cluster.")]
355+
public int DisksPerWorkerNode { get; set; }
356+
366357
#endregion
367358

368359

@@ -472,14 +463,13 @@ var storageAccount in
472463
};
473464
}
474465

475-
if (WorkerNodeDataDisksGroups != null)
466+
if (DisksPerWorkerNode > 0)
476467
{
477-
var dataDisksGroups = WorkerNodeDataDisksGroups;
478-
this.parameters.WorkerNodeDataDisksGroups = new List<DataDisksGroupProperties>()
468+
parameters.WorkerNodeDataDisksGroups = new List<DataDisksGroupProperties>()
479469
{
480470
new DataDisksGroupProperties()
481471
{
482-
DisksPerNode = dataDisksGroups.First().DisksPerNode
472+
DisksPerNode = DisksPerWorkerNode
483473
}
484474
};
485475
}

‎src/ResourceManager/HDInsight/Commands.HDInsight/Models/Management/AzureHDInsightCluster.cs

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,8 @@ public AzureHDInsightCluster(Cluster cluster)
4848
WorkerNodeDataDisksGroups = new List<DataDisksGroupProperties>();
4949
if (cluster.Properties.ComputeProfile != null && cluster.Properties.ComputeProfile.Roles.Any())
5050
{
51-
foreach (var role in cluster.Properties.ComputeProfile.Roles)
51+
var rolesWithDataDisksGroups = cluster.Properties.ComputeProfile.Roles.Where(x => x.DataDisksGroups != null);
52+
foreach (var role in rolesWithDataDisksGroups)
5253
{
5354
WorkerNodeDataDisksGroups.AddRange(role.DataDisksGroups);
5455
}
@@ -188,7 +189,7 @@ public AzureHDInsightCluster(Cluster cluster, IDictionary<string, string> cluste
188189
public List<string> ComponentVersion { get; set; }
189190

190191
/// <summary>
191-
/// Data Disks Properties for the worker role.
192+
/// Data Disks Group Properties for the Worker Role.
192193
/// </summary>
193194
public List<DataDisksGroupProperties> WorkerNodeDataDisksGroups { get; set; }
194195

‎src/ResourceManager/HDInsight/Commands.HDInsight/Models/Management/AzureHDInsightConfig.cs

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -131,9 +131,9 @@ public class AzureHDInsightConfig
131131
public AzureHDInsightSecurityProfile SecurityProfile { get; set; }
132132

133133
/// <summary>
134-
/// Gets or sets the data disks groups properties for the cluster.
134+
/// Gets or sets the number of disks for worker node role for the cluster.
135135
/// </summary>
136-
public List<DataDisksGroupProperties> DataDisksGroupProperties { get; set; }
136+
public int DisksPerWorkerNode { get; set; }
137137

138138
public AzureHDInsightConfig()
139139
{
@@ -142,7 +142,6 @@ public AzureHDInsightConfig()
142142
Configurations = new Dictionary<string, Hashtable>();
143143
ScriptActions = new Dictionary<ClusterNodeType, List<AzureHDInsightScriptAction>>();
144144
ComponentVersion = new Dictionary<string, string>();
145-
DataDisksGroupProperties = new List<DataDisksGroupProperties>();
146145
}
147146
}
148147
}

0 commit comments

Comments
 (0)