1. Merge az.hdinsight-preview and master branch for ignite release

2. Remove some breaking change notice and related alias
3. modify the release notes in changelog and psd1 file

Author: Zhenyu Zhou

src/HDInsight/HDInsight.Test/ScenarioTests/Common.ps1

@@ -35,7 +35,7 @@ Create cluster
 function Create-Cluster{
     param(
       [string] $clusterName="hdi-ps-test",
-      [string] $location="North Central US",
+      [string] $location="West US",
       [string] $resourceGroupName="group-ps-test",
       [string] $clusterType="Spark",
       [string] $storageAccountName="storagepstest"

src/HDInsight/HDInsight.Test/ScenarioTests/HDInsightClusterTests.cs

@@ -14,9 +14,6 @@
 
 using Microsoft.Azure.ServiceManagement.Common.Models;
 using Microsoft.WindowsAzure.Commands.ScenarioTest;
-using System;
-using System.Collections.Generic;
-using System.Text;
 using Xunit;
 using Xunit.Abstractions;
 

src/HDInsight/HDInsight.Test/SessionRecords/Commands.HDInsight.Test.ScenarioTests.HDInsightClusterTests/TestClusterRelatedCommands.json

@@ -12,6 +12,7 @@
 // limitations under the License.
 // ----------------------------------------------------------------------------------
 
+using Microsoft.Azure.Commands.HDInsight.Models.Management;
 using Microsoft.Azure.Management.HDInsight.Models;
 using Microsoft.WindowsAzure.Commands.ScenarioTest;
 using Moq;
@@ -42,18 +43,21 @@ public GetPropertiesTests(Xunit.Abstractions.ITestOutputHelper output)
         public void CanGetProperties()
         {
             var features = new string[] { "feature1", "feature2" };
-            var versions = new Dictionary<string, AzHDInsightVersionsCapability> { { "key", new VersionsCapability() } };
+            var versions = new Dictionary<string, VersionsCapability> { { "key", new VersionsCapability() } };
             var vm = new Dictionary<string, VmSizesCapability> { { "key1", new VmSizesCapability() } };
             var regions = new Dictionary<string, RegionsCapability> { { "eastus", new RegionsCapability() } };
-            var propertiesResponse = new CapabilitiesResponse
+            var capabilitiesResult = new CapabilitiesResult
             {
                 Features = features,
                 Versions = versions,
                 VmSizes = vm,
                 Regions = regions
             };
-            hdinsightManagementMock.Setup(c => c.GetCapabilities(Location))
-                .Returns(propertiesResponse)
+
+            var propertiesResponse = new AzureHDInsightCapabilities(capabilitiesResult);
+
+            hdinsightManagementMock.Setup(c => c.GetProperties(Location))
+                .Returns(capabilitiesResult)
                 .Verifiable();
 
             cmdlet.ExecuteCmdlet();
@@ -62,10 +66,12 @@ public void CanGetProperties()
             commandRuntimeMock.Verify(
                 f =>
                     f.WriteObject(
-                        It.Is<CapabilitiesResponse>(
+                        It.Is<AzureHDInsightCapabilities>(
                             resp =>
-                                resp.Features == features && resp.Regions == regions &&
-                                resp.Versions == versions && resp.VmSizes == vm)),
+                                resp.Features == propertiesResponse.Features
+                                && resp.Regions["eastus"].Available == propertiesResponse.Regions["eastus"].Available
+                                && resp.Versions.Count == propertiesResponse.Versions.Count 
+                                && resp.VmSizes["key1"].Available == propertiesResponse.VmSizes["key1"].Available), true),
                 Times.Once);
         }
     }

src/HDInsight/HDInsight.Test/SessionRecords/Commands.HDInsight.Test.ScenarioTests.HDInsightMonitoringTests/TestMonitoringRelatedCommands.json

@@ -12,6 +12,7 @@
 // limitations under the License.
 // ----------------------------------------------------------------------------------
 
+using Microsoft.Azure.Commands.HDInsight.Models.Management;
 using Microsoft.Azure.Management.HDInsight.Models;
 using Microsoft.WindowsAzure.Commands.Common;
 using Microsoft.WindowsAzure.Commands.ScenarioTest;
@@ -69,7 +70,14 @@ public void CanSetGatewayCredentialSupportsShouldProcess()
             setcmdlet.ExecuteCmdlet();
 
             commandRuntimeMock.VerifyAll();
-            commandRuntimeMock.Verify(f => f.WriteObject(gatewaySettings), Times.Once);
+            commandRuntimeMock.Verify(
+                f => f.WriteObject(
+                    It.Is<AzureHDInsightGatewaySettings>(
+                        rsp =>
+                            rsp.IsCredentialEnabled == gatewaySettings.IsCredentialEnabled
+                            && rsp.UserName == gatewaySettings.UserName
+                            && rsp.Password == gatewaySettings.Password)),
+                Times.Once);
         }
     }
 }

src/HDInsight/HDInsight.Test/SessionRecords/Commands.HDInsight.Test.ScenarioTests.HDInsightScriptActionTests/TestScriptActionRelatedCommands.json

@@ -12,7 +12,7 @@
 # RootModule = ''
 
 # Version number of this module.
-ModuleVersion = '3.0.0'
+ModuleVersion = '2.0.2'
 
 # Supported PSEditions
 CompatiblePSEditions = 'Core', 'Desktop'
@@ -133,7 +133,9 @@ PrivateData = @{
         # IconUri = ''
 
         # ReleaseNotes of this module
-        ReleaseNotes = '* Changed Microsoft.Azure.Management.HDInsight from 2.1.0 to 5.1.0
+        ReleaseNotes = '* Fixed the bug that customer will get "Not a valid Base-64 string" error when using Get-AzHDInsightCluster to get the cluster with ADLSGen1 storage.
+* Add a parameter named "ApplicationId" to three cmdlets Add-AzHDInsightClusterIdentity, New-AzHDInsightClusterConfig and New-AzHDInsightCluster so that customer can provide the service principal application id for accessing Azure Data Lake.
+* Changed Microsoft.Azure.Management.HDInsight from 2.1.0 to 5.1.0
 * Removed five cmdlets:
     - Get-AzHDInsightOMS
     - Enable-AzHDInsightOMS
@@ -151,8 +153,8 @@ PrivateData = @{
 *  - Changed the output type of Get-AzHDInsightProperties from  CapabilitiesResponse to AzureHDInsightCapabilities.
 *  - Changed the output type of Remove-AzHDInsightCluster from ClusterGetResponse to bool.
 *  - Changed the output type of Set-AzHDInsightGatewaySettings HttpConnectivitySettings to GatewaySettings.
-* Added Scenario Test Framework and added some scenario test cases.'
-* Removed alias 'Add-AzHDInsightConfigValues' and 'Get-AzHDInsightProperties'.
+* Added Scenario Test Framework and added some scenario test cases.
+* Removed alias "Add-AzHDInsightConfigValues" and "Get-AzHDInsightProperties".'
 
         # Prerelease string of this module
         # Prerelease = ''

src/HDInsight/HDInsight.Test/SessionRecords/Commands.HDInsight.Test.ScenarioTests.HDInsightSetGatewayCredentialTest/TestSetGatewayCredential.json

@@ -18,6 +18,8 @@
         - Additional information about change #1
 -->
 ## Upcoming Release
+* Fixed the bug that customer will get "Not a valid Base-64 string" error when using Get-AzHDInsightCluster to get the cluster with ADLSGen1 storage.
+* Add a parameter named "ApplicationId" to three cmdlets Add-AzHDInsightClusterIdentity, New-AzHDInsightClusterConfig and New-AzHDInsightCluster so that customer can provide the service principal application id for accessing Azure Data Lake.
 * Changed Microsoft.Azure.Management.HDInsight from 2.1.0 to 5.1.0
 * Removed five cmdlets:
     - Get-AzHDInsightOMS

src/HDInsight/HDInsight.Test/UnitTests/GetPropertiesTests.cs

@@ -17,20 +17,16 @@
 using Microsoft.Azure.Commands.HDInsight.Models;
 using Microsoft.Azure.Commands.HDInsight.Models.Job;
 using Microsoft.Azure.Commands.ResourceManager.Common.ArgumentCompleters;
-using Microsoft.Azure.Management.HDInsight;
 using Microsoft.Azure.Management.HDInsight.Job.Models;
 using Microsoft.Azure.Management.HDInsight.Models;
 using Microsoft.WindowsAzure.Commands.Common;
-using Microsoft.WindowsAzure.Commands.Common.CustomAttributes;
 using System;
 using System.Collections.Generic;
 using System.IO;
-using System.Linq;
 using System.Management.Automation;
 
 namespace Microsoft.Azure.Commands.HDInsight
 {
-    [GenericBreakingChange("Users with reader role need to specify `DefaultStorageAccountKey` parameter explicitly, otherwise error occurs.", "2.0.0", "05/06/2019")]
     [Cmdlet("Get", ResourceManager.Common.AzureRMConstants.AzureRMPrefix + "HDInsightJobOutput"), OutputType(typeof(string))]
     public class GetAzureHDInsightJobOutputCommand : HDInsightCmdletBase
     {

src/HDInsight/HDInsight.Test/UnitTests/HttpTests.cs

@@ -14,8 +14,6 @@
 
 using Microsoft.Azure.Commands.HDInsight.Commands;
 using Microsoft.Azure.Commands.ResourceManager.Common.ArgumentCompleters;
-using Microsoft.Azure.Management.HDInsight.Models;
-using Microsoft.WindowsAzure.Commands.Common.CustomAttributes;
 using System.Management.Automation;
 
 namespace Microsoft.Azure.Commands.HDInsight

src/HDInsight/HDInsight/Az.HDInsight.psd1

@@ -16,11 +16,7 @@
 using Microsoft.Azure.Commands.HDInsight.Commands;
 using Microsoft.Azure.Management.HDInsight.Models;
 using System.Management.Automation;
-using System.IO;
-using System.Reflection;
-using Microsoft.Azure.Commands.Common.Authentication.Abstractions;
 using Microsoft.Azure.Commands.ResourceManager.Common.ArgumentCompleters;
-using Microsoft.WindowsAzure.Commands.Common.CustomAttributes;
 
 namespace Microsoft.Azure.Commands.HDInsight
 {

src/HDInsight/HDInsight/ChangeLog.md

@@ -15,7 +15,6 @@
 using Microsoft.Azure.Commands.HDInsight.Commands;
 using Microsoft.Azure.Commands.HDInsight.Models;
 using Microsoft.Azure.Commands.ResourceManager.Common.ArgumentCompleters;
-using System.Collections.Generic;
 using System.Linq;
 using System.Management.Automation;
 

src/HDInsight/HDInsight/JobCommands/GetAzureHDInsightJobOutputCommand.cs

@@ -15,7 +15,6 @@
 using Microsoft.Azure.Commands.HDInsight.Commands;
 using Microsoft.Azure.Commands.HDInsight.Models.Management;
 using Microsoft.Azure.Commands.ResourceManager.Common.ArgumentCompleters;
-using Microsoft.WindowsAzure.Commands.Common.CustomAttributes;
 using System.Management.Automation;
 
 namespace Microsoft.Azure.Commands.HDInsight

src/HDInsight/HDInsight/ManagementCommands/DisableAzureHDInsightMonitoringCommand.cs

@@ -15,8 +15,6 @@
 using Microsoft.Azure.Commands.HDInsight.Commands;
 using Microsoft.Azure.Commands.HDInsight.Models.Management;
 using Microsoft.Azure.Commands.ResourceManager.Common.ArgumentCompleters;
-using Microsoft.Azure.Management.HDInsight.Models;
-using Microsoft.WindowsAzure.Commands.Common.CustomAttributes;
 using System.Management.Automation;
 
 namespace Microsoft.Azure.Commands.HDInsight

src/HDInsight/HDInsight/ManagementCommands/EnableAzureHDInsightMonitoringCommand.cs

@@ -14,8 +14,6 @@
 
 using Microsoft.Azure.Commands.HDInsight.Commands;
 using Microsoft.Azure.Commands.ResourceManager.Common.ArgumentCompleters;
-using Microsoft.Azure.Management.HDInsight.Models;
-using Microsoft.WindowsAzure.Commands.Common.CustomAttributes;
 using System.Management.Automation;
 
 namespace Microsoft.Azure.Commands.HDInsight
@@ -35,6 +33,9 @@ public class RemoveAzureHDInsightCommand : HDInsightCmdletBase
         [ResourceGroupCompleter]
         public string ResourceGroupName { get; set; }
 
+        [Parameter(Mandatory = false)]
+        public SwitchParameter PassThru { get; set; }
+
         #endregion
 
         public override void ExecuteCmdlet()
@@ -46,7 +47,10 @@ public override void ExecuteCmdlet()
 
             HDInsightManagementClient.DeleteCluster(ResourceGroupName, ClusterName);
 
-            WriteObject(true);
+            if (this.PassThru.IsPresent)
+            {
+                WriteObject(true);
+            }
         }
     }
 }

src/HDInsight/HDInsight/ManagementCommands/GetAzureHDInsightClusterCommand.cs

@@ -15,6 +15,7 @@
 using Hyak.Common;
 using Microsoft.Azure.Commands.HDInsight.Commands;
 using Microsoft.Azure.Commands.HDInsight.Models;
+using Microsoft.Azure.Commands.HDInsight.Models.Management;
 using Microsoft.Azure.Commands.ResourceManager.Common.ArgumentCompleters;
 using Microsoft.Azure.Management.HDInsight.Models;
 using Microsoft.Azure.Management.Internal.Resources.Utilities.Models;
@@ -24,7 +25,7 @@
 
 namespace Microsoft.Azure.Commands.HDInsight
 {
-    [Cmdlet(VerbsCommon.Set, ResourceManager.Common.AzureRMConstants.AzureRMPrefix + "HDInsightGatewayCredential", DefaultParameterSetName = SetByNameParameterSet, SupportsShouldProcess = true), OutputType(typeof(GatewaySettings))]
+    [Cmdlet(VerbsCommon.Set, ResourceManager.Common.AzureRMConstants.AzureRMPrefix + "HDInsightGatewayCredential", DefaultParameterSetName = SetByNameParameterSet, SupportsShouldProcess = true), OutputType(typeof(AzureHDInsightGatewaySettings))]
     public class SetAzureHDInsightGatewayCredentialCommand : HDInsightCmdletBase
     {
         private const string SetByNameParameterSet = "SetByNameParameterSet";
@@ -126,7 +127,7 @@ public override void ExecuteCmdlet()
             if (ShouldProcess(Name, "set gateway http credential"))
             {
                 HDInsightManagementClient.UpdateGatewayCredential(ResourceGroupName, Name, updateGatewaySettingsParameters);
-                WriteObject(HDInsightManagementClient.GetGatewaySettings(ResourceGroupName, Name));
+                WriteObject(new AzureHDInsightGatewaySettings(HDInsightManagementClient.GetGatewaySettings(ResourceGroupName, Name)));
             }
         }
     }

src/HDInsight/HDInsight/ManagementCommands/GetAzureHDInsightMonitoringCommand.cs

@@ -13,10 +13,8 @@
 // ----------------------------------------------------------------------------------
 
 using Microsoft.Azure.Management.HDInsight.Models;
-using System;
 using System.Collections.Generic;
 using System.Linq;
-using System.Text;
 
 namespace Microsoft.Azure.Commands.HDInsight.Models.Management
 {

src/HDInsight/HDInsight/ManagementCommands/GetAzureHDInsightPropertiesCommand.cs

@@ -0,0 +1,43 @@
+// ----------------------------------------------------------------------------------
+//
+// Copyright Microsoft Corporation
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+// ----------------------------------------------------------------------------------
+
+using Microsoft.Azure.Management.HDInsight.Models;
+
+namespace Microsoft.Azure.Commands.HDInsight.Models.Management
+{
+    public class AzureHDInsightGatewaySettings
+    {
+        public AzureHDInsightGatewaySettings(GatewaySettings gatewaySettings)
+        {
+            this.IsCredentialEnabled = gatewaySettings.IsCredentialEnabled;
+            this.UserName = gatewaySettings.UserName;
+            this.Password = gatewaySettings.Password;
+        }
+
+        /// <summary>
+        /// indicates whether or not the gateway settings based authorization is enabled.
+        /// </summary>
+        public string IsCredentialEnabled { get; set; }
+
+        /// <summary>
+        /// the gateway settings user name.
+        /// </summary>
+        public string UserName { get; set; }
+
+        /// <summary>
+        /// the gateway settings user password.
+        /// </summary>
+        public string Password { get; set; }
+    }
+}

src/HDInsight/HDInsight/ManagementCommands/RemoveAzureHDInsightClusterCommand.cs

@@ -1,6 +1,4 @@
-using Microsoft.Azure.Management.HDInsight.Models;
-using System;
-// ----------------------------------------------------------------------------------
+// ----------------------------------------------------------------------------------
 //
 // Copyright Microsoft Corporation
 // Licensed under the Apache License, Version 2.0 (the "License");
@@ -14,8 +12,8 @@
 // limitations under the License.
 // ----------------------------------------------------------------------------------
 
+using Microsoft.Azure.Management.HDInsight.Models;
 using System.Collections.Generic;
-using System.Text;
 
 namespace Microsoft.Azure.Commands.HDInsight.Models.Management
 {
@@ -26,6 +24,9 @@ public AzureHDInsightRegionsCapability(RegionsCapability regionsCapability)
             this.Available = regionsCapability?.Available;
         }
 
+        /// <summary>
+        /// Available region capability.
+        /// </summary>
         public IList<string> Available { get; set; }
     }
 }

src/HDInsight/HDInsight/ManagementCommands/SetAzureHDInsightGatewayCredentialCommand.cs

@@ -26,13 +26,7 @@ The **Disable-AzHDInsightMonitoring** cmdlet disables monitoring in a Azure HDIn
 ```
 PS C:\> Disable-AzHDInsightMonitoring -Name testcluster
 
-ErrorInfo  :
-
-State      : Succeeded
-
-RequestId  : 1417ad86-d055-48cd-9d68-a5c19a212a3a
-
-StatusCode : OK
+True
 ```
 
 Monitoring will be disabled on the HDInsight cluster and relevant logs will stop flowing to the monitoring workspace.
@@ -41,13 +35,7 @@ Monitoring will be disabled on the HDInsight cluster and relevant logs will stop
 ```
 PS C:\> Disable-AzHDInsightMonitoring -Name testcluster -ResourceGroupName testrg
 
-ErrorInfo  :
-
-State      : Succeeded
-
-RequestId  : 1417ad86-d055-48cd-9d68-a5c19a212a3a
-
-StatusCode : OK
+True
 ```
 
 Monitoring will be disabled on the HDInsight cluster and relevant logs will stop flowing to the monitoring workspace.
@@ -138,7 +126,7 @@ This cmdlet supports the common parameters: -Debug, -ErrorAction, -ErrorVariable
 
 ## OUTPUTS
 
-### Microsoft.Azure.Management.HDInsight.Models.OperationResource
+### System.Boolean
 
 ## NOTES