Merge pull request Azure#4678 from metafractal/preview

Update New-AzureRmHDInsightCluster cmdlet to add new parameter that specifies number of data disks per worker node.

Author: cormacpayne

src/ResourceManager/HDInsight/ChangeLog.md

@@ -18,6 +18,8 @@
         - Additional information about change #1
 -->
 ## Current Release
+* Added support for Data Disks property in cluster creation
+    - Added parameter 'WorkerNodeDataDisksGroups' to the New-AzureHDInsightCluster cmdlet
 
 ## Version 3.4.0
 

src/ResourceManager/HDInsight/Commands.HDInsight.Test/UnitTests/NewClusterTests.cs

@@ -20,6 +20,7 @@
 using Moq;
 using Newtonsoft.Json;
 using System.Collections.Generic;
+using System.Linq;
 using System.Management.Automation;
 using Xunit;
 
@@ -151,7 +152,6 @@ public void CanCreateNewHDInsightCluster_Linux()
                     clusterout.Name == ClusterName &&
                     clusterout.OperatingSystemType == OSType.Linux)),
                 Times.Once);
-
         }
 
         [Fact]
@@ -205,8 +205,12 @@ public void CanCreateNewHDInsightCluster_Secure_Linux()
                 Times.Once);
         }
 
-        private void CreateNewHDInsightCluster(bool addSecurityProfileInresponse = false, bool setEdgeNodeVmSize = false)
+        private void CreateNewHDInsightCluster(
+            bool addSecurityProfileInresponse = false, 
+            bool setEdgeNodeVmSize = false,
+            int workerNodeDataDisks = 0)
         {
+            // Assign cmdlet parameters
             cmdlet.ClusterName = ClusterName;
             cmdlet.ResourceGroupName = ResourceGroupName;
             cmdlet.ClusterSizeInNodes = ClusterSize;
@@ -217,9 +221,13 @@ private void CreateNewHDInsightCluster(bool addSecurityProfileInresponse = false
             cmdlet.ClusterType = ClusterType;
             cmdlet.SshCredential = _sshCred;
             cmdlet.OSType = OSType.Linux;
+            cmdlet.DisksPerWorkerNode = workerNodeDataDisks;
             if (setEdgeNodeVmSize)
+            {
                 cmdlet.EdgeNodeSize = "edgeNodeVmSizeSetTest";
+            }
 
+            // Construct cluster Object
             var cluster = new Cluster
             {
                 Id = "id",
@@ -237,10 +245,29 @@ private void CreateNewHDInsightCluster(bool addSecurityProfileInresponse = false
                     {
                         CoresUsed = 24
                     },
-                    OperatingSystemType = OSType.Linux
+                    OperatingSystemType = OSType.Linux,
+                    ComputeProfile = new ComputeProfile()
+                    {
+                        Roles = new List<Role>()
+                    }
                 }
             };
 
+            if (workerNodeDataDisks > 0)
+            {
+                cluster.Properties.ComputeProfile.Roles.Add(new Role()
+                {
+                    Name = "workernode",
+                    DataDisksGroups = new List<DataDisksGroupProperties>()
+                    {
+                        new DataDisksGroupProperties()
+                        {
+                            DisksPerNode = workerNodeDataDisks
+                        }
+                    }
+                });
+            }
+
             if (addSecurityProfileInresponse)
             {
                 cluster.Properties.SecurityProfile = new SecurityProfile()
@@ -285,6 +312,7 @@ private void CreateNewHDInsightCluster(bool addSecurityProfileInresponse = false
 
             var getresponse = new ClusterGetResponse {Cluster = cluster};
 
+            // Setup Mocks and verify successfull GET response
             hdinsightManagementMock.Setup(
                 c => c.CreateNewCluster(ResourceGroupName, ClusterName, It.Is<ClusterCreateParameters>(
                     parameters =>
@@ -299,10 +327,12 @@ private void CreateNewHDInsightCluster(bool addSecurityProfileInresponse = false
                         parameters.OSType == OSType.Linux &&
                         parameters.SshUserName == _sshCred.UserName &&
                         parameters.SshPassword == _sshCred.Password.ConvertToString() &&
-                        ((!setEdgeNodeVmSize && parameters.EdgeNodeSize == null) || (setEdgeNodeVmSize && parameters.EdgeNodeSize == "edgeNodeVmSizeSetTest")))))
+                        ((!setEdgeNodeVmSize && parameters.EdgeNodeSize == null) || (setEdgeNodeVmSize && parameters.EdgeNodeSize == "edgeNodeVmSizeSetTest")) &&
+                        (workerNodeDataDisks == 0) || (workerNodeDataDisks > 0 && parameters.WorkerNodeDataDisksGroups.First().DisksPerNode == workerNodeDataDisks))))
                 .Returns(getresponse)
                 .Verifiable();
 
+            // Execute Cmdlet and verify output
             cmdlet.ExecuteCmdlet();
 
             commandRuntimeMock.VerifyAll();
@@ -314,7 +344,8 @@ private void CreateNewHDInsightCluster(bool addSecurityProfileInresponse = false
                     clusterout.CoresUsed == 24 &&
                     clusterout.Location == Location &&
                     clusterout.Name == ClusterName &&
-                    clusterout.OperatingSystemType == OSType.Linux)),
+                    clusterout.OperatingSystemType == OSType.Linux &&
+                    (workerNodeDataDisks == 0) || (clusterout.WorkerNodeDataDisksGroups.First().DisksPerNode == workerNodeDataDisks))),
                     Times.Once);
         }
 
@@ -438,6 +469,16 @@ public void CanCreateNewHDInsightCluster_LinuxComponentVersion()
                     Times.Once);
         }
 
+        [Fact]
+        [Trait(Category.AcceptanceType, Category.CheckIn)]
+        public void CanCreateNewHDInsightCluster_Kafka_DataDisks_Linux()
+        {
+            ClusterType = "Kafka";
+            HdiVersion = "3.6";
+
+            CreateNewHDInsightCluster(workerNodeDataDisks: 8);
+        }
+
         [Fact]
         [Trait(Category.AcceptanceType, Category.CheckIn)]
         public void TestStorageAccountTypeDefaultsToAzureStorage()

src/ResourceManager/HDInsight/Commands.HDInsight/ManagementCommands/NewAzureHDInsightClusterCommand.cs

@@ -136,7 +136,8 @@ public AzureHDInsightConfig Config
                     CertificateFileContents = CertificateFileContents,
                     CertificateFilePath = CertificateFilePath,
                     CertificatePassword = CertificatePassword,
-                    SecurityProfile = SecurityProfile
+                    SecurityProfile = SecurityProfile,
+                    DisksPerWorkerNode = DisksPerWorkerNode
                 };
                 foreach (
                     var storageAccount in
@@ -157,6 +158,7 @@ var storageAccount in
                 {
                     result.ComponentVersion.Add(component.Key, component.Value);
                 }
+                
                 return result;
             }
             set
@@ -187,6 +189,7 @@ var storageAccount in
                 ObjectId = value.ObjectId;
                 CertificatePassword = value.CertificatePassword;
                 SecurityProfile = value.SecurityProfile;
+                DisksPerWorkerNode = value.DisksPerWorkerNode;
 
                 foreach (
                     var storageAccount in
@@ -348,6 +351,9 @@ public DateTime RdpAccessExpiry
         [Parameter(HelpMessage = "Gets or sets Security Profile which is used for creating secure cluster.")]
         public AzureHDInsightSecurityProfile SecurityProfile { get; set; }
 
+        [Parameter(HelpMessage = "Gets or sets the number of disks for worker node role in the cluster.")]
+        public int DisksPerWorkerNode { get; set; }
+
         #endregion
 
 
@@ -456,7 +462,18 @@ var storageAccount in
                     ClusterUsersGroupDNs = SecurityProfile.ClusterUsersGroupDNs
                 };
             }
-            
+
+            if (DisksPerWorkerNode > 0)
+            {
+                parameters.WorkerNodeDataDisksGroups = new List<DataDisksGroupProperties>()
+                {
+                    new DataDisksGroupProperties()
+                    {
+                        DisksPerNode = DisksPerWorkerNode
+                    }
+                };
+            }
+
             var cluster = HDInsightManagementClient.CreateNewCluster(ResourceGroupName, ClusterName, parameters);
 
             if (cluster != null)

src/ResourceManager/HDInsight/Commands.HDInsight/Models/Management/AzureHDInsightCluster.cs

@@ -45,6 +45,15 @@ public AzureHDInsightCluster(Cluster cluster)
             {
                 ComponentVersion.Add(componentVersion.ToString());
             }
+            WorkerNodeDataDisksGroups = new List<DataDisksGroupProperties>();
+            if (cluster.Properties.ComputeProfile != null && cluster.Properties.ComputeProfile.Roles.Any())
+            {
+                var rolesWithDataDisksGroups = cluster.Properties.ComputeProfile.Roles.Where(x => x.DataDisksGroups != null);
+                foreach (var role in rolesWithDataDisksGroups)
+                {
+                    WorkerNodeDataDisksGroups.AddRange(role.DataDisksGroups);
+                }
+            }
             var clusterSecurityProfile = cluster.Properties.SecurityProfile;
             SecurityProfile = clusterSecurityProfile != null ? new AzureHDInsightSecurityProfile()
             {
@@ -178,6 +187,11 @@ public AzureHDInsightCluster(Cluster cluster, IDictionary<string, string> cluste
         /// Version of a component service in the cluster
         /// </summary>
         public List<string> ComponentVersion { get; set; }
+
+        /// <summary>
+        /// Data Disks Group Properties for the Worker Role.
+        /// </summary>
+        public List<DataDisksGroupProperties> WorkerNodeDataDisksGroups { get; set; }
 
         /// Gets or sets the security profile.
         /// </summary>

src/ResourceManager/HDInsight/Commands.HDInsight/Models/Management/AzureHDInsightConfig.cs

@@ -130,6 +130,11 @@ public class AzureHDInsightConfig
         /// </value>
         public AzureHDInsightSecurityProfile SecurityProfile { get; set; }
 
+        /// <summary>
+        /// Gets or sets the number of disks for worker node role for the cluster.
+        /// </summary>
+        public int DisksPerWorkerNode { get; set; }
+
         public AzureHDInsightConfig()
         {
             ClusterType = Constants.Hadoop;

src/ResourceManager/HDInsight/Commands.HDInsight/help/New-AzureRmHDInsightCluster.md

@@ -1,5 +1,6 @@
 ---
 external help file: Microsoft.Azure.Commands.HDInsight.dll-Help.xml
+Module Name: AzureRM.HDInsight
 ms.assetid: 691AC991-3249-487C-A0DF-C579ED7D00E7
 online version: 
 schema: 2.0.0
@@ -29,7 +30,7 @@ New-AzureRmHDInsightCluster [-Location] <String> [-ResourceGroupName] <String> [
  [-VirtualNetworkId <String>] [-SubnetName <String>] [-OSType <OSType>] [-ClusterTier <Tier>]
  [-SshCredential <PSCredential>] [-SshPublicKey <String>] [-RdpCredential <PSCredential>]
  [-RdpAccessExpiry <DateTime>] [-ObjectId <Guid>] [-CertificatePassword <String>] [-AadTenantId <Guid>]
- [-SecurityProfile <AzureHDInsightSecurityProfile>] [<CommonParameters>]
+ [-SecurityProfile <AzureHDInsightSecurityProfile>] [-DisksPerWorkerNode <Int32>] [<CommonParameters>]
 ```
 
 ### CertificateFilePath
@@ -50,7 +51,7 @@ New-AzureRmHDInsightCluster [-Location] <String> [-ResourceGroupName] <String> [
  [-SshCredential <PSCredential>] [-SshPublicKey <String>] [-RdpCredential <PSCredential>]
  [-RdpAccessExpiry <DateTime>] [-ObjectId <Guid>] [-CertificateFilePath <String>]
  [-CertificatePassword <String>] [-AadTenantId <Guid>] [-SecurityProfile <AzureHDInsightSecurityProfile>]
- [<CommonParameters>]
+ [-DisksPerWorkerNode <Int32>] [<CommonParameters>]
 ```
 
 ### CertificateFileContents
@@ -71,7 +72,7 @@ New-AzureRmHDInsightCluster [-Location] <String> [-ResourceGroupName] <String> [
  [-SshCredential <PSCredential>] [-SshPublicKey <String>] [-RdpCredential <PSCredential>]
  [-RdpAccessExpiry <DateTime>] [-ObjectId <Guid>] [-CertificateFileContents <Byte[]>]
  [-CertificatePassword <String>] [-AadTenantId <Guid>] [-SecurityProfile <AzureHDInsightSecurityProfile>]
- [<CommonParameters>]
+ [-DisksPerWorkerNode <Int32>] [<CommonParameters>]
 ```
 
 ## DESCRIPTION
@@ -86,6 +87,8 @@ The New-AzureHDInsightCluster creates an Azure HDInsight cluster by using the sp
 
 
 
+
+
 ```
 PS C:\&gt; # Primary storage account info
         $storageAccountResourceGroupName = "Group"
@@ -388,6 +391,21 @@ Accept pipeline input: False
 Accept wildcard characters: False
 ```
 
+### -DisksPerWorkerNode
+Specifies the number of disks for worker node role in the cluster.
+
+```yaml
+Type: Int32
+Parameter Sets: (All)
+Aliases: 
+
+Required: False
+Position: Named
+Default value: None
+Accept pipeline input: False
+Accept wildcard characters: False
+```
+
 ### -EdgeNodeSize
 Specifies the size of the virtual machine for the edge node. Use Get-AzureRmVMSize for acceptable VM sizes, and see HDInsight's pricing page. This parameter is valid only for RServer clusters.
 
@@ -709,7 +727,6 @@ This cmdlet supports the common parameters: -Debug, -ErrorAction, -ErrorVariable
 ## INPUTS
 
 ### AzureHDInsightConfig
-
 Parameter 'Config' accepts value of type 'AzureHDInsightConfig' from the pipeline
 
 ## OUTPUTS