[HDInsight] - Add a cmdlet to update gateway credential

src/HDInsight/HDInsight.Test/HDInsight.Test.csproj

@@ -1,4 +1,4 @@
-<Project Sdk="Microsoft.NET.Sdk">
+<Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
     <PsModuleName>HDInsight</PsModuleName>
@@ -11,7 +11,7 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="Microsoft.Azure.Management.HDInsight" Version="2.0.8" />
+    <PackageReference Include="Microsoft.Azure.Management.HDInsight" Version="2.1.0" />
     <PackageReference Include="Microsoft.Azure.Management.HDInsight.Job" Version="2.0.7" />
   </ItemGroup>
 

src/HDInsight/HDInsight.Test/HDInsightTestBase.cs

@@ -91,6 +91,23 @@ public virtual void SetupManagementClientForJobTests()
             hdinsightManagementMock.Setup(c => c.GetClusterConfigurations(It.IsAny<string>(), It.IsAny<string>(), It.IsAny<string>()))
                 .Returns(configurationResponse)
                 .Verifiable();
+
+            var listConfigurationsResponse = new ClusterListConfigurationsResponse
+            {
+                Configurations = new Dictionary<string, ClusterConfiguration>
+                {
+                    {
+                        "core-site", new ClusterConfiguration
+                        {
+                            Configuration=configurationResponse
+                        }
+                    }
+                }
+            };
+
+            hdinsightManagementMock.Setup(c => c.ListConfigurations(It.IsAny<string>(), It.IsAny<string>()))
+                .Returns(listConfigurationsResponse)
+                .Verifiable();
         }
     }
 }

src/HDInsight/HDInsight.Test/UnitTests/HttpTests.cs

@@ -16,6 +16,7 @@
 using Microsoft.WindowsAzure.Commands.Common;
 using Microsoft.WindowsAzure.Commands.ScenarioTest;
 using Moq;
+using System;
 using System.Management.Automation;
 using System.Net;
 using Xunit;
@@ -24,8 +25,7 @@ namespace Microsoft.Azure.Commands.HDInsight.Test
 {
     public class HttpTests : HDInsightTestBase
     {
-        private GrantAzureHDInsightHttpServicesAccessCommand grantcmdlet;
-        private RevokeAzureHDInsightHttpServicesAccessCommand revokecmdlet;
+        private SetAzureHDInsightGatewayCredentialCommand setcmdlet;
 
         private readonly PSCredential _httpCred;
 
@@ -35,30 +35,25 @@ public HttpTests(Xunit.Abstractions.ITestOutputHelper output)
             base.SetupTestsForManagement();
             _httpCred = new PSCredential("hadoopuser", string.Format("Password1!").ConvertToSecureString());
 
-            grantcmdlet = new GrantAzureHDInsightHttpServicesAccessCommand
+            setcmdlet = new SetAzureHDInsightGatewayCredentialCommand
             {
                 CommandRuntime = commandRuntimeMock.Object,
                 HDInsightManagementClient = hdinsightManagementMock.Object,
-                ClusterName = ClusterName,
+                Name = ClusterName,
                 ResourceGroupName = ResourceGroupName,
                 HttpCredential = _httpCred
             };
-            revokecmdlet = new RevokeAzureHDInsightHttpServicesAccessCommand
-            {
-                CommandRuntime = commandRuntimeMock.Object,
-                HDInsightManagementClient = hdinsightManagementMock.Object,
-                ClusterName = ClusterName,
-                ResourceGroupName = ResourceGroupName
-            };
         }
 
         [Fact]
         [Trait(Category.AcceptanceType, Category.CheckIn)]
-        public void CanGrantHttpAccess()
+        public void CanSetGatewayCredentialSupportsShouldProcess()
         {
+            commandRuntimeMock.Setup(c => c.ShouldProcess(ClusterName, It.IsAny<string>())).Returns(true);
+
             hdinsightManagementMock.Setup(
                 c =>
-                    c.ConfigureHttp(ResourceGroupName, ClusterName,
+                    c.UpdateGatewayCredential(ResourceGroupName, ClusterName,
                         It.Is<HttpSettingsParameters>(
                             param =>
                                 param.HttpUserEnabled && param.HttpUsername == _httpCred.UserName &&
@@ -71,58 +66,58 @@ public void CanGrantHttpAccess()
                 })
                 .Verifiable();
 
-            var connectivitysettings = new HttpConnectivitySettings
+            var gatewayCredential = new HttpConnectivitySettings
             {
                 HttpPassword = _httpCred.Password.ConvertToString(),
                 HttpUserEnabled = true,
                 HttpUsername = _httpCred.UserName,
                 StatusCode = HttpStatusCode.OK
             };
 
-            hdinsightManagementMock.Setup(c => c.GetConnectivitySettings(ResourceGroupName, ClusterName))
-                .Returns(connectivitysettings)
+            hdinsightManagementMock.Setup(c => c.GetGatewaySettings(ResourceGroupName, ClusterName))
+                .Returns(gatewayCredential)
                 .Verifiable();
 
-            grantcmdlet.ExecuteCmdlet();
+            setcmdlet.ExecuteCmdlet();
 
             commandRuntimeMock.VerifyAll();
-            commandRuntimeMock.Verify(f => f.WriteObject(connectivitysettings), Times.Once);
+            commandRuntimeMock.Verify(f => f.WriteObject(gatewayCredential), Times.Once);
         }
 
         [Fact]
         [Trait(Category.AcceptanceType, Category.CheckIn)]
-        public void CanRevokeHttpAccess()
+        public void CanWriteErrorWhenSetGatewayCredentialFailedSupportsProcess()
         {
+            var result = new OperationResource
+            {
+                ErrorInfo = new ErrorInfo { Code = "Ambari Failed Code", Message = "GetAmbariUserFailed" },
+                StatusCode = HttpStatusCode.OK,
+                State = AsyncOperationState.Failed
+            };
+
+            commandRuntimeMock.Setup(c => c.ShouldProcess(ClusterName, It.IsAny<string>())).Returns(true);
+
             hdinsightManagementMock.Setup(
                 c =>
-                    c.ConfigureHttp(ResourceGroupName, ClusterName,
+                    c.UpdateGatewayCredential(ResourceGroupName, ClusterName,
                         It.Is<HttpSettingsParameters>(
                             param =>
-                                !param.HttpUserEnabled &&
-                                string.IsNullOrEmpty(param.HttpPassword) &&
-                                string.IsNullOrEmpty(param.HttpUsername))))
-                .Returns(new OperationResource
-                {
-                    ErrorInfo = null,
-                    StatusCode = HttpStatusCode.OK,
-                    State = AsyncOperationState.Succeeded
-                })
-                .Verifiable();
-
-            var connectivitysettings = new HttpConnectivitySettings
-            {
-                HttpUserEnabled = false,
-                StatusCode = HttpStatusCode.OK
-            };
-
-            hdinsightManagementMock.Setup(c => c.GetConnectivitySettings(ResourceGroupName, ClusterName))
-                .Returns(connectivitysettings)
+                                param.HttpUserEnabled && param.HttpUsername == _httpCred.UserName &&
+                                param.HttpPassword == _httpCred.Password.ConvertToString())))
+                .Returns(result)
                 .Verifiable();
 
-            revokecmdlet.ExecuteCmdlet();
+            setcmdlet.ExecuteCmdlet();
 
             commandRuntimeMock.VerifyAll();
-            commandRuntimeMock.Verify(f => f.WriteObject(connectivitysettings), Times.Once);
+            commandRuntimeMock.Verify(
+                f =>
+                    f.WriteError(It.Is<ErrorRecord>(
+                        record =>
+                            record.Exception.Message == $"{result.ErrorInfo.Code}: {result.ErrorInfo.Message}" &&
+                            string.IsNullOrEmpty(record.FullyQualifiedErrorId) &&
+                            record.CategoryInfo.Category == ErrorCategory.InvalidArgument)),
+                Times.Once);
         }
     }
 }

src/HDInsight/HDInsight/Az.HDInsight.psd1

@@ -89,9 +89,8 @@ CmdletsToExport = 'Get-AzHDInsightJob', 'New-AzHDInsightSqoopJobDefinition',
                'Add-AzHDInsightScriptAction', 'Add-AzHDInsightMetastore', 
                'Add-AzHDInsightConfigValue', 'Get-AzHDInsightProperty', 
                'Revoke-AzHDInsightRdpServicesAccess', 
-               'Revoke-AzHDInsightHttpServicesAccess', 
                'Grant-AzHDInsightRdpServicesAccess', 
-               'Grant-AzHDInsightHttpServicesAccess', 
+               'Set-AzHDInsightGatewayCredential', 
                'New-AzHDInsightClusterConfig', 'Remove-AzHDInsightCluster', 
                'Set-AzHDInsightClusterSize', 
                'Get-AzHDInsightPersistedScriptAction', 

src/HDInsight/HDInsight/ChangeLog.md

@@ -18,6 +18,14 @@
         - Additional information about change #1
 -->
 ## Upcoming Release
+* Removed two cmdlets:
+    - Grant-AzHDInsightHttpServicesAccess
+    - Revoke-AzHDInsightHttpServicesAccess
+* Added a new cmdlet Set-AzHDInsightGatewayCredential to replace Grant-AzHDInsightHttpServicesAccess
+* Update cmdlet Get-AzHDInsightJobOutput to distinguish reader role and hdinsight operator role:
+    - Users with reader role need to specify `DefaultStorageAccountKey` parameter explicitly, otherwise error occurs.
+	- Users with hdinsight operator role will not be affected.
+
 
 ## Version 1.1.0
 * Updated cmdlets with plural nouns to singular, and deprecated plural names.

src/HDInsight/HDInsight/ClusterConfigurationUtils.cs

@@ -53,6 +53,7 @@ public static AzureHDInsightDefaultStorageAccount GetDefaultStorageAccountDetail
             string defaultFSUrl;
             const string AdlPrefix = "adl://";
             const string WasbPrefix = "wasb://";
+            const string SecureWasbPrefix = "wasbs://";
 
             if (coreSiteConfiguration.TryGetValue(key, out defaultFSUrl))
             {
@@ -78,15 +79,26 @@ public static AzureHDInsightDefaultStorageAccount GetDefaultStorageAccountDetail
                         resourceUri: resourceUri
                     );
                 }
-                else if (defaultFSUrl.StartsWith(WasbPrefix))
+                else if (defaultFSUrl.StartsWith(WasbPrefix) || defaultFSUrl.StartsWith(SecureWasbPrefix))
                 {
-                    string[] accountAndContainer = defaultFSUrl.Substring(WasbPrefix.Length).Split('@');
+                    string[] accountAndContainer;
+                    if (defaultFSUrl.StartsWith(WasbPrefix))
+                    {
+                        accountAndContainer = defaultFSUrl.Substring(WasbPrefix.Length).Split('@');
+                    }
+                    else
+                    {
+                        accountAndContainer = defaultFSUrl.Substring(SecureWasbPrefix.Length).Split('@');
+                    }
+
+                    string storageAccountKey;
+                    coreSiteConfiguration.TryGetValue(Constants.ClusterConfiguration.StorageAccountKeyPrefix + accountAndContainer[1], out storageAccountKey);
 
                     return new AzureHDInsightWASBDefaultStorageAccount
                     (
                         storageContainerName: accountAndContainer[0],
                         storageAccountName: accountAndContainer[1],
-                        storageAccountKey: coreSiteConfiguration[Constants.ClusterConfiguration.StorageAccountKeyPrefix + accountAndContainer[1]]
+                        storageAccountKey: storageAccountKey
                     );
                 }
                 else

src/HDInsight/HDInsight/HDInsight.csproj

@@ -1,4 +1,4 @@
-<Project Sdk="Microsoft.NET.Sdk">
+<Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
     <PsModuleName>HDInsight</PsModuleName>
@@ -11,7 +11,7 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="Microsoft.Azure.Management.HDInsight" Version="2.0.8" />
+    <PackageReference Include="Microsoft.Azure.Management.HDInsight" Version="2.1.0" />
     <PackageReference Include="Microsoft.Azure.Management.HDInsight.Job" Version="2.0.7" />
     <PackageReference Include="WindowsAzure.Storage" Version="9.3.0" />
   </ItemGroup>

src/HDInsight/HDInsight/JobCommands/GetAzureHDInsightJobOutputCommand.cs

@@ -17,14 +17,20 @@
 using Microsoft.Azure.Commands.HDInsight.Models;
 using Microsoft.Azure.Commands.HDInsight.Models.Job;
 using Microsoft.Azure.Commands.ResourceManager.Common.ArgumentCompleters;
+using Microsoft.Azure.Management.HDInsight;
 using Microsoft.Azure.Management.HDInsight.Job.Models;
+using Microsoft.Azure.Management.HDInsight.Models;
 using Microsoft.WindowsAzure.Commands.Common;
+using Microsoft.WindowsAzure.Commands.Common.CustomAttributes;
+using System;
 using System.IO;
+using System.Linq;
 using System.Management.Automation;
 
 namespace Microsoft.Azure.Commands.HDInsight
 {
-    [Cmdlet("Get", ResourceManager.Common.AzureRMConstants.AzureRMPrefix + "HDInsightJobOutput"),OutputType(typeof(string))]
+    [GenericBreakingChange("Users with reader role need to specify `DefaultStorageAccountKey` parameter explicitly, otherwise error occurs.", "2.0.0", "05/06/2019")]
+    [Cmdlet("Get", ResourceManager.Common.AzureRMConstants.AzureRMPrefix + "HDInsightJobOutput"), OutputType(typeof(string))]
     public class GetAzureHDInsightJobOutputCommand : HDInsightCmdletBase
     {
         #region Input Parameter Definitions
@@ -131,20 +137,53 @@ private static string Convert(Stream stream)
             return text;
         }
 
+        private string GetStorageAccountKey(string resourceGroupName, string clusterName)
+        {
+            string storageAccountKey = null;
+            string errorMessage = "Fails to retrieve storage account key. Please specify DefaultStorageAccountKey explicitly.";
+            const string AuthorizationFailedCode = "AuthorizationFailed";
+
+            try
+            {
+                ClusterConfiguration coreSiteClusterConfiguration;
+                HDInsightManagementClient.ListConfigurations(resourceGroupName, clusterName).Configurations.TryGetValue(ConfigurationKey.CoreSite, out coreSiteClusterConfiguration);
+                coreSiteClusterConfiguration?.Configuration.TryGetValue(Constants.ClusterConfiguration.StorageAccountKeyPrefix + DefaultStorageAccountName, out storageAccountKey);
+            }
+            catch (CloudException cloudEx)
+            {
+                if (cloudEx.Error.Code == AuthorizationFailedCode)
+                {
+                    errorMessage = "Insufficient permissions to retrieve storage account key. Please specify DefaultStorageAccountKey explicitly.";
+                }
+            }
+            catch (Exception ex)
+            {
+                errorMessage = errorMessage + " Reason: " + ex.Message;
+            }
+
+            if (storageAccountKey == null)
+            {
+                throw new CloudException(errorMessage);
+            }
+
+            return storageAccountKey;
+        }
+
         internal IStorageAccess GetDefaultStorageAccess(string resourceGroupName, string clusterName)
         {
             var StorageAccountSuffix = "";
-    
+
             if (DefaultContainer == null && DefaultStorageAccountName == null && DefaultStorageAccountKey == null)
             {
                 var DefaultStorageAccount = GetDefaultStorageAccount(resourceGroupName, clusterName);
 
                 var wasbAccount = DefaultStorageAccount as AzureHDInsightWASBDefaultStorageAccount;
+
                 if (wasbAccount != null)
                 {
                     DefaultContainer = wasbAccount.StorageContainerName;
                     DefaultStorageAccountName = wasbAccount.StorageAccountName;
-                    DefaultStorageAccountKey = wasbAccount.StorageAccountKey;
+                    DefaultStorageAccountKey = GetStorageAccountKey(resourceGroupName, clusterName);
                     StorageAccountSuffix = DefaultContext.Environment.StorageEndpointSuffix;
                 }
                 else
@@ -154,7 +193,7 @@ internal IStorageAccess GetDefaultStorageAccess(string resourceGroupName, string
 
             }
 
-            return new AzureStorageAccess(DefaultStorageAccountName, DefaultStorageAccountKey, DefaultContainer,StorageAccountSuffix);
+            return new AzureStorageAccess(DefaultStorageAccountName, DefaultStorageAccountKey, DefaultContainer, StorageAccountSuffix);
         }
     }
 }