Merge pull request #940 from nityasharma/dev

Added storage account information to Get-AzureHDInsightCluster's response

Author: markcowl

src/ResourceManager/HDInsight/Commands.HDInsight/ClusterConfigurationUtils.cs

@@ -0,0 +1,78 @@
+//
+// Copyright Microsoft Corporation
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+// ----------------------------------------------------------------------------------
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+
+namespace Microsoft.Azure.Commands.HDInsight.Models
+{
+    internal class ClusterConfigurationUtils
+    {
+        public static string GetResourceGroupFromClusterId(string clusterId)
+        {
+            // Parse resource group from cluster Id
+            // The code expects Id to be of the format \
+            // /subscriptions/<subscription ID>/resourceGroups/<Resource Group Id>/providers/Microsoft.HDInsight/clusters/<cluster name>
+
+            string resourceGroup = null;
+            int index = clusterId.IndexOf("resourceGroups", StringComparison.OrdinalIgnoreCase);
+
+            if (index >= 0)
+            {
+                index += "resourceGroups".Length;
+                string[] parts = clusterId.Substring(index).Split(new[] { '/' }, StringSplitOptions.RemoveEmptyEntries);
+
+                if (parts.Length > 0)
+                {
+                    resourceGroup = parts[0];
+                }
+            }
+
+            return resourceGroup;
+        }
+
+        public static AzureHDInsightDefaultStorageAccount GetDefaultStorageAccountDetails(
+            IDictionary<string, string> configuration,
+            string version)
+        {
+            string key = version.Equals("2.1") ? Constants.ClusterConfiguration.DefaultStorageAccountNameKeyOld
+                : Constants.ClusterConfiguration.DefaultStorageAccountNameKey;
+
+            string accountAndContainerStr;
+
+            if (configuration.TryGetValue(key, out accountAndContainerStr))
+            {
+                string[] accountAndContainer = accountAndContainerStr.Substring("wasb://".Length).Split('@');
+
+                return new AzureHDInsightDefaultStorageAccount
+                {
+                    StorageContainerName = accountAndContainer[0],
+                    StorageAccountName = accountAndContainer[1],
+                    StorageAccountKey = configuration[Constants.ClusterConfiguration.StorageAccountKeyPrefix + accountAndContainer[1]]
+                };
+            }
+
+            return null;
+        }
+
+        public static List<string> GetAdditionStorageAccounts(IDictionary<string, string> configuration, string defaultAccount)
+        {
+            // Parse the storage account names from the key and exclude the default one
+            return (from key in configuration.Keys
+                    where key.StartsWith(Constants.ClusterConfiguration.StorageAccountKeyPrefix, StringComparison.OrdinalIgnoreCase) &&
+                    !key.EndsWith(defaultAccount, StringComparison.OrdinalIgnoreCase)
+                    select key.Remove(0, Constants.ClusterConfiguration.StorageAccountKeyPrefix.Length)).ToList();
+        }
+    }
+}

src/ResourceManager/HDInsight/Commands.HDInsight/Commands.HDInsight.csproj

@@ -75,10 +75,12 @@
     <Compile Include="Models\Management\AzureHDInsightCluster.cs" />
     <Compile Include="Models\Management\AzureHDInsightConfig.cs" />
     <Compile Include="Models\Job\AzureHdInsightJobManagementClient.cs" />
+    <Compile Include="Models\Management\AzureHDInsightDefaultStorageAccount.cs" />
     <Compile Include="Models\Management\AzureHdInsightManagementClient.cs" />
     <Compile Include="Models\Management\AzureHDInsightMetastoreType.cs" />
     <Compile Include="Models\Management\AzureHDInsightMetastore.cs" />
     <Compile Include="ManagementCommands\NewAzureHDInsightClusterCommand.cs" />
+    <Compile Include="ClusterConfigurationUtils.cs" />
     <Compile Include="Properties\AssemblyInfo.cs" />
   </ItemGroup>
   <ItemGroup>
@@ -90,7 +92,9 @@
       <CopyToOutputDirectory>Always</CopyToOutputDirectory>
     </None>
     <None Include="MSSharedLibKey.snk" />
-    <None Include="packages.config" />
+    <None Include="packages.config">
+      <SubType>Designer</SubType>
+    </None>
   </ItemGroup>
   <ItemGroup>
     <Reference Include="Hyak.Common">

src/ResourceManager/HDInsight/Commands.HDInsight/Constants.cs

@@ -40,5 +40,12 @@ public static class JobDefinitions
             public const string AzureHDInsightMapReduceJobDefinition = "AzureRmHDInsightMapReduceJobDefinition";
             public const string AzureHDInsightStreamingMapReduceJobDefinition = "AzureRmHDInsightStreamingMapReduceJobDefinition";
         }
+
+        public static class ClusterConfiguration
+        {
+            public const string DefaultStorageAccountNameKey = "fs.defaultFS";
+            public const string DefaultStorageAccountNameKeyOld = "fs.default.name";
+            public const string StorageAccountKeyPrefix = "fs.azure.account.key.";
+        }
     }
 }

src/ResourceManager/HDInsight/Commands.HDInsight/ManagementCommands/GetAzureHDInsightClusterCommand.cs

@@ -12,6 +12,7 @@
 // limitations under the License.
 // ----------------------------------------------------------------------------------
 
+using System;
 using System.Collections.Generic;
 using System.Linq;
 using System.Management.Automation;
@@ -44,7 +45,14 @@ public class GetAzureHDInsightCommand : HDInsightCmdletBase
         protected override void ProcessRecord()
         {
             var result = HDInsightManagementClient.GetCluster(ResourceGroupName, ClusterName);
-            var output = result.Select(cluster => new AzureHDInsightCluster(cluster)).ToList();
+
+            var output = result.Select(entry =>
+            {
+                string resourceGroupName = ClusterConfigurationUtils.GetResourceGroupFromClusterId(entry.Id);
+                var configuration = HDInsightManagementClient.GetClusterConfigurations(resourceGroupName, entry.Name, "core-site");
+                return new AzureHDInsightCluster(entry, configuration);
+            }).ToList();
+
             WriteObject(output, true);
         }
     }

src/ResourceManager/HDInsight/Commands.HDInsight/Models/Management/AzureHDInsightCluster.cs

@@ -13,6 +13,7 @@
 // ----------------------------------------------------------------------------------
 
 using System;
+using System.Collections.Generic;
 using System.Linq;
 using Microsoft.Azure.Management.HDInsight.Models;
 
@@ -34,18 +35,35 @@ public AzureHDInsightCluster(Cluster cluster)
                 cluster.Properties.ConnectivityEndpoints.FirstOrDefault(c => c.Name.Equals("HTTPS", StringComparison.OrdinalIgnoreCase));
             HttpEndpoint = httpEndpoint != null ? httpEndpoint.Location : null;
 
+            ResourceGroup = ClusterConfigurationUtils.GetResourceGroupFromClusterId(cluster.Id);
+        }
+
+        public AzureHDInsightCluster(Cluster cluster, IDictionary<string, string> clusterConfiguration)
+            : this(cluster)
+        {
+            if (clusterConfiguration != null)
+            {
+                var defaultAccount = ClusterConfigurationUtils.GetDefaultStorageAccountDetails(
+                clusterConfiguration,
+                cluster.Properties.ClusterVersion);
+
+                DefaultStorageAccount = defaultAccount.StorageAccountName;
+                DefaultStorageContainer = defaultAccount.StorageContainerName;
+
+                AdditionalStorageAccounts = ClusterConfigurationUtils.GetAdditionStorageAccounts(clusterConfiguration, DefaultStorageAccount);
+            }
         }
 
         /// <summary>
         /// The name of the resource.
         /// </summary>
         public string Name { get; set; }
-        
+
         /// <summary>
         /// The ID of the resource.
         /// </summary>
         public string Id { get; set; }
-        
+
         /// <summary>
         /// The location of the resource.
         /// </summary>
@@ -80,5 +98,25 @@ public AzureHDInsightCluster(Cluster cluster)
         /// The endpoint with which to connect to the cluster.
         /// </summary>
         public string HttpEndpoint { get; set; }
+
+        /// <summary>
+        /// Default storage account for this cluster.
+        /// </summary>
+        public string DefaultStorageAccount { get; set; }
+
+        /// <summary>
+        /// Default storage container for this cluster.
+        /// </summary>
+        public string DefaultStorageContainer { get; set; }
+
+        /// <summary>
+        /// Default storage container for this cluster.
+        /// </summary>
+        public string ResourceGroup { get; set; }
+
+        /// <summary>
+        /// Additional storage accounts for this cluster
+        /// </summary>
+        public List<string> AdditionalStorageAccounts { get; set; }
     }
 }

src/ResourceManager/HDInsight/Commands.HDInsight/Models/Management/AzureHDInsightDefaultStorageAccount.cs

@@ -0,0 +1,24 @@
+//
+// Copyright Microsoft Corporation
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+// ----------------------------------------------------------------------------------
+
+namespace Microsoft.Azure.Commands.HDInsight.Models
+{
+    public class AzureHDInsightDefaultStorageAccount
+    {
+        public string StorageAccountName { get; set; }
+
+        public string StorageAccountKey { get; set; }
+
+        public string StorageContainerName { get; set; }
+    }
+}

src/ResourceManager/HDInsight/Commands.HDInsight/Models/Management/AzureHdInsightManagementClient.cs

@@ -59,7 +59,7 @@ public virtual List<Cluster> GetCluster(string resourceGroupName, string cluster
                 var getresponse = Get(resourceGroupName, clusterName);
                 if (getresponse != null)
                 {
-                    result.Add(getresponse.Cluster);   
+                    result.Add(getresponse.Cluster);
                 }
             }
             return result;
@@ -72,7 +72,7 @@ public virtual ClusterListResponse ListClusters()
 
         public virtual ClusterListResponse ListClusters(string resourceGroupName)
         {
-            return HdInsightManagementClient.Clusters.ListByResourceGroup(resourceGroupName);   
+            return HdInsightManagementClient.Clusters.ListByResourceGroup(resourceGroupName);
         }
 
         public virtual ClusterGetResponse Get(string resourceGroupName, string clusterName)
@@ -109,5 +109,22 @@ public virtual CapabilitiesResponse GetCapabilities(string location)
         {
             return HdInsightManagementClient.Clusters.GetCapabilities(location);
         }
+
+        public virtual IDictionary<string, string> GetClusterConfigurations(string resourceGroupName, string clusterName, string configurationName)
+        {
+            Dictionary<string, string> properties = new Dictionary<string, string>();
+
+            if (string.IsNullOrWhiteSpace(resourceGroupName) ||
+                string.IsNullOrWhiteSpace(clusterName) ||
+                string.IsNullOrWhiteSpace(configurationName))
+            {
+                return properties;
+            }
+
+            return HdInsightManagementClient.Clusters.GetClusterConfigurations(
+                resourceGroupName,
+                clusterName,
+                configurationName).Configuration;
+        }
     }
 }