Update New-AzureRmHDInsightCluster cmdlet to support data disks groups for workernode during cluster creation.

Author: Abhijith Padmakumar

src/ResourceManager/HDInsight/ChangeLog.md

@@ -18,6 +18,8 @@
         - Additional information about change #1
 -->
 ## Current Release
+* Added support for Data Disks property in cluster creation
+    - Added parameter 'WorkerNodeDataDisksGroups' to the New-AzureHDInsightCluster cmdlet
 
 ## Version 3.3.1
 

src/ResourceManager/HDInsight/Commands.HDInsight.Test/UnitTests/NewClusterTests.cs

@@ -20,6 +20,7 @@
 using Moq;
 using Newtonsoft.Json;
 using System.Collections.Generic;
+using System.Linq;
 using System.Management.Automation;
 using Xunit;
 
@@ -438,6 +439,141 @@ public void CanCreateNewHDInsightCluster_LinuxComponentVersion()
                     Times.Once);
         }
 
+        [Fact]
+        [Trait(Category.AcceptanceType, Category.CheckIn)]
+        public void CanCreateNewHDInsightCluster_DataDisksGroups()
+        {
+            var kafkaClusterType = "Kafka";
+            var hdiVersion = "3.6";
+            var coresUsed = 24;
+            var expectedClusterState = "Running";
+            var dataDisksGroup = new DataDisksGroupProperties()
+            {
+                DisksPerNode = 8
+            };
+            var workerNodeDataDisksGroups = new List<DataDisksGroupProperties>()
+            {
+                dataDisksGroup
+            };
+
+            cmdlet.ClusterName = ClusterName;
+            cmdlet.ResourceGroupName = ResourceGroupName;
+            cmdlet.ClusterSizeInNodes = ClusterSize;
+            cmdlet.Location = Location;
+            cmdlet.HttpCredential = _httpCred;
+            cmdlet.DefaultStorageAccountName = StorageName;
+            cmdlet.DefaultStorageAccountKey = StorageKey;
+            cmdlet.ClusterType = kafkaClusterType;
+            cmdlet.SshCredential = _sshCred;
+            cmdlet.OSType = OSType.Linux;
+            cmdlet.WorkerNodeDataDisksGroups = workerNodeDataDisksGroups;
+
+            var cluster = new Cluster
+            {
+                Id = "id",
+                Name = ClusterName,
+                Location = Location,
+                Properties = new ClusterGetProperties
+                {
+                    ClusterVersion = hdiVersion,
+                    ClusterState = expectedClusterState,
+                    ClusterDefinition = new ClusterDefinition
+                    {
+                        ClusterType = kafkaClusterType
+                    },
+                    QuotaInfo = new QuotaInfo
+                    {
+                        CoresUsed = coresUsed
+                    },
+                    OperatingSystemType = OSType.Linux,
+                    ComputeProfile = new ComputeProfile()
+                    {
+                        Roles = new List<Role>()
+                    }
+                }
+            };
+
+            cluster.Properties.ComputeProfile.Roles.Add(new Role()
+            {
+                Name = "workernode",
+                DataDisksGroups = workerNodeDataDisksGroups
+            });
+
+            var coreConfigs = new Dictionary<string, string>
+            {
+                {"fs.defaultFS", "wasb://giyertestcsmv2@" + StorageName},
+                {
+                    "fs.azure.account.key." + StorageName,
+                    StorageKey
+                }
+            };
+            var gatewayConfigs = new Dictionary<string, string>
+            {
+                {"restAuthCredential.isEnabled", "true"},
+                {"restAuthCredential.username", _httpCred.UserName},
+                {"restAuthCredential.password", _httpCred.Password.ConvertToString()}
+            };
+
+            var configurations = new Dictionary<string, Dictionary<string, string>>
+            {
+                {"core-site", coreConfigs},
+                {"gateway", gatewayConfigs}
+            };
+            var serializedConfig = JsonConvert.SerializeObject(configurations);
+            cluster.Properties.ClusterDefinition.Configurations = serializedConfig;
+
+            var getresponse = new ClusterGetResponse { Cluster = cluster };
+
+            hdinsightManagementMock.Setup(c => c.CreateNewCluster(ResourceGroupName, ClusterName, It.Is<ClusterCreateParameters>(
+                parameters =>
+                    parameters.ClusterSizeInNodes == ClusterSize &&
+                    parameters.DefaultStorageInfo as AzureStorageInfo != null &&
+                    ((AzureStorageInfo)parameters.DefaultStorageInfo).StorageAccountName == StorageName &&
+                    ((AzureStorageInfo)parameters.DefaultStorageInfo).StorageAccountKey == StorageKey &&
+                    parameters.Location == Location &&
+                    parameters.UserName == _httpCred.UserName &&
+                    parameters.Password == _httpCred.Password.ConvertToString() &&
+                    parameters.ClusterType == kafkaClusterType &&
+                    parameters.OSType == OSType.Linux &&
+                    parameters.SshUserName == _sshCred.UserName &&
+                    parameters.SshPassword == _sshCred.Password.ConvertToString() &&
+                    parameters.WorkerNodeDataDisksGroups.First().DisksPerNode == dataDisksGroup.DisksPerNode)))
+            .Returns(getresponse)
+            .Verifiable();
+            hdinsightManagementMock.Setup(
+                c => c.CreateNewCluster(ResourceGroupName, ClusterName, It.Is<ClusterCreateParameters>(
+                    parameters =>
+                        parameters.ClusterSizeInNodes == ClusterSize &&
+                        parameters.DefaultStorageInfo as AzureStorageInfo != null &&
+                        ((AzureStorageInfo)parameters.DefaultStorageInfo).StorageAccountName == StorageName &&
+                        ((AzureStorageInfo)parameters.DefaultStorageInfo).StorageAccountKey == StorageKey &&
+                        parameters.Location == Location &&
+                        parameters.UserName == _httpCred.UserName &&
+                        parameters.Password == _httpCred.Password.ConvertToString() &&
+                        parameters.ClusterType == ClusterType &&
+                        parameters.OSType == OSType.Linux &&
+                        parameters.SshUserName == _sshCred.UserName &&
+                        parameters.SshPassword == _sshCred.Password.ConvertToString())))
+                .Returns(getresponse)
+                .Verifiable();
+
+            cmdlet.ExecuteCmdlet();
+
+            commandRuntimeMock.VerifyAll();
+
+            
+            commandRuntimeMock.Verify(f => f.WriteObject(It.Is<AzureHDInsightCluster>(
+                clusterout =>
+                    clusterout.ClusterState == expectedClusterState &&
+                    clusterout.ClusterType == kafkaClusterType &&
+                    clusterout.ClusterVersion == hdiVersion &&
+                    clusterout.CoresUsed == coresUsed &&
+                    clusterout.Location == Location &&
+                    clusterout.Name == ClusterName &&
+                    clusterout.OperatingSystemType == OSType.Linux)),
+                    Times.Once);
+        }
+
         [Fact]
         [Trait(Category.AcceptanceType, Category.CheckIn)]
         public void TestStorageAccountTypeDefaultsToAzureStorage()

src/ResourceManager/HDInsight/Commands.HDInsight/ManagementCommands/NewAzureHDInsightClusterCommand.cs

@@ -157,6 +157,10 @@ var storageAccount in
                 {
                     result.ComponentVersion.Add(component.Key, component.Value);
                 }
+                foreach (var dataDisksGroups in parameters.WorkerNodeDataDisksGroups.Where(d => d.DisksPerNode > 0))
+                {
+                    result.DataDisksGroupProperties.Add(dataDisksGroups);
+                }
                 return result;
             }
             set
@@ -207,6 +211,10 @@ var storageAccount in
                 {
                     parameters.ComponentVersion.Add(component.Key, component.Value);
                 }
+                foreach (var dataDisksGroups in value.DataDisksGroupProperties.Where(d => d.DisksPerNode > 0))
+                {
+                    this.parameters.WorkerNodeDataDisksGroups.Add(dataDisksGroups);
+                }
             }
         }
 
@@ -284,6 +292,13 @@ public Dictionary<string, string> ComponentVersion
             set { parameters.ComponentVersion = value; }
         }
 
+        [Parameter(HelpMessage = "Gets or sets the data disks groups for worker node role in the cluster.")]
+        public List<DataDisksGroupProperties> WorkerNodeDataDisksGroups
+        {
+            get { return parameters.WorkerNodeDataDisksGroups; }
+            set { parameters.WorkerNodeDataDisksGroups = value; }
+        }
+
         [Parameter(HelpMessage = "Gets or sets the virtual network guid for this HDInsight cluster.")]
         public string VirtualNetworkId
         {
@@ -456,7 +471,19 @@ var storageAccount in
                     ClusterUsersGroupDNs = SecurityProfile.ClusterUsersGroupDNs
                 };
             }
-            
+
+            if (WorkerNodeDataDisksGroups != null)
+            {
+                var dataDisksGroups = WorkerNodeDataDisksGroups;
+                this.parameters.WorkerNodeDataDisksGroups = new List<DataDisksGroupProperties>()
+                {
+                    new DataDisksGroupProperties()
+                    {
+                        DisksPerNode = dataDisksGroups.First().DisksPerNode
+                    }
+                };
+            }
+
             var cluster = HDInsightManagementClient.CreateNewCluster(ResourceGroupName, ClusterName, parameters);
 
             if (cluster != null)

src/ResourceManager/HDInsight/Commands.HDInsight/Models/Management/AzureHDInsightCluster.cs

@@ -45,6 +45,14 @@ public AzureHDInsightCluster(Cluster cluster)
             {
                 ComponentVersion.Add(componentVersion.ToString());
             }
+            WorkerNodeDataDisksGroups = new List<DataDisksGroupProperties>();
+            if (cluster.Properties.ComputeProfile != null && cluster.Properties.ComputeProfile.Roles.Any())
+            {
+                foreach (var role in cluster.Properties.ComputeProfile.Roles)
+                {
+                    WorkerNodeDataDisksGroups.AddRange(role.DataDisksGroups);
+                }
+            }
             var clusterSecurityProfile = cluster.Properties.SecurityProfile;
             SecurityProfile = clusterSecurityProfile != null ? new AzureHDInsightSecurityProfile()
             {
@@ -178,6 +186,11 @@ public AzureHDInsightCluster(Cluster cluster, IDictionary<string, string> cluste
         /// Version of a component service in the cluster
         /// </summary>
         public List<string> ComponentVersion { get; set; }
+
+        /// <summary>
+        /// Data Disks Properties for the worker role.
+        /// </summary>
+        public List<DataDisksGroupProperties> WorkerNodeDataDisksGroups { get; set; }
 
         /// Gets or sets the security profile.
         /// </summary>

src/ResourceManager/HDInsight/Commands.HDInsight/Models/Management/AzureHDInsightConfig.cs

@@ -130,13 +130,19 @@ public class AzureHDInsightConfig
         /// </value>
         public AzureHDInsightSecurityProfile SecurityProfile { get; set; }
 
+        /// <summary>
+        /// Gets or sets the data disks groups properties for the cluster.
+        /// </summary>
+        public List<DataDisksGroupProperties> DataDisksGroupProperties { get; set; }
+
         public AzureHDInsightConfig()
         {
             ClusterType = Constants.Hadoop;
             AdditionalStorageAccounts = new Dictionary<string, string>();
             Configurations = new Dictionary<string, Hashtable>();
             ScriptActions = new Dictionary<ClusterNodeType, List<AzureHDInsightScriptAction>>();
             ComponentVersion = new Dictionary<string, string>();
+            DataDisksGroupProperties = new List<DataDisksGroupProperties>();
         }
     }
 }