Merge branch 'master' of https://github.com/Azure/azure-powershell into remove-nonCanonical-cmdlet-in-AzureRM.Example.psm1#9737

Author: VeryEarly

src/Advisor/Advisor/ChangeLog.md

@@ -19,6 +19,7 @@
         - Additional information about change #1
 -->
 ## Upcoming Release
+* Added new category "Operational Excellence" to Get-AzAdvisorRecommendation cmdlet.
 
 ## Version 1.0.1
 * Fixed miscellaneous typos across module

src/Advisor/Advisor/Cmdlets/GetAzAdvisorRecommendation.cs

@@ -52,7 +52,7 @@ public class GetAzAdvisorRecommendation : ResourceAdvisorBaseCmdlet
         /// </summary>
         [Parameter(ParameterSetName = "IdParameterSet", Mandatory = false, HelpMessage = "Category of the recommendation")]
         [Parameter(ParameterSetName = "NameParameterSet", Mandatory = false, HelpMessage = "Category of the recommendation")]
-        [ValidateSet("Cost", "HighAvailability", "Performance", "Security")]
+        [ValidateSet("Cost", "HighAvailability", "OperationalExcellence", "Performance", "Security")]
         public string Category { get; set; }
 
         /// <summary>
@@ -96,4 +96,4 @@ public override void ExecuteCmdlet()
             this.WriteObject(results, true);
         }
     }
-}
+}

src/Advisor/Advisor/Cmdlets/Utilities/RecommendationHelper.cs

@@ -51,8 +51,8 @@ public static List<PsAzureAdvisorResourceRecommendationBase> RecommendationFilte
 
             List<PsAzureAdvisorResourceRecommendationBase> filteredList = new List<PsAzureAdvisorResourceRecommendationBase>();
 
-            // Filter by category only if its a valid input
-            if (Category.Cost.Equals(category) || Category.HighAvailability.Equals(category) || Category.Performance.Equals(category) || Category.Security.Equals(category))
+            // Filter by category
+            if (!string.IsNullOrEmpty(category))
             {
                 // If resourceGroup filtering is as well specified 
                 if (!string.IsNullOrEmpty(resourceGroup))

src/Advisor/Advisor/help/Get-AzAdvisorRecommendation.md

@@ -80,7 +80,7 @@ Category of the recommendation
 Type: String
 Parameter Sets: (All)
 Aliases:
-Accepted values: Cost, HighAvailability, Performance, Security
+Accepted values: Cost, HighAvailability, Performance, Security, OperationalExcellence
 
 Required: False
 Position: Named

src/DataFactory/DataFactoryV2/help/Az.DataFactory.md

@@ -11,6 +11,9 @@ Locale: en-US
 Azure Data Factory V2 is the data integration platform that goes beyond Azure Data Factory V1's orchestration and batch-processing of time-series log data, with a general purpose app model supporting modern data warehousing patterns and scenarios, lift-and-shift SSIS, and data-driven SaaS applications. Compose and manage reliable and secure data integration workflows at scale. Use native ADF data connectors and Integration Runtimes to move and transform cloud and on-premises data that can be unstructured, semi-structured, and structured with Hadoop, Azure Data Lake, Spark, SQL Server, Cosmos DB and many other data platforms.
 
 ## Az.DataFactory Cmdlets
+### [Add-AzDataFactoryV2DataFlowDebugSessionPackage](Add-AzDataFactoryV2DataFlowDebugSessionPackage.md)
+Add data flow resource and its dependencies into specific data flow debug session.
+
 ### [Get-AzDataFactory](Get-AzDataFactory.md)
 Gets information about Data Factories.
 
@@ -50,6 +53,12 @@ Gets information about activity runs for a pipeline run.
 ### [Get-AzDataFactoryV2Dataset](Get-AzDataFactoryV2Dataset.md)
 Gets information about datasets in Data Factory.
 
+### [Get-AzDataFactoryV2DataFlow](Get-AzDataFactoryV2DataFlow.md)
+Gets information about data flows in Data Factory.
+
+### [Get-AzDataFactoryV2DataFlowDebugSession](Get-AzDataFactoryV2DataFlowDebugSession.md)
+Get all active data flow debug sessions by Azure Data Factory.
+
 ### [Get-AzDataFactoryV2IntegrationRuntime](Get-AzDataFactoryV2IntegrationRuntime.md)
 Gets information about integration runtime resources.
 
@@ -80,8 +89,11 @@ Returns information about trigger runs.
 ### [Invoke-AzDataFactoryV2IntegrationRuntimeUpgrade](Invoke-AzDataFactoryV2IntegrationRuntimeUpgrade.md)
 Upgrades self-hosted integration runtime.
 
+### [Invoke-AzDataFactoryV2DataFlowDebugSessionCommand](Invoke-AzDataFactoryV2DataFlowDebugSessionCommand.md)
+Invoke debug action in data flow debug session.
+
 ### [Invoke-AzDataFactoryV2Pipeline](Invoke-AzDataFactoryV2Pipeline.md)
-  Invokes a pipeline to start a run for it.
+Invokes a pipeline to start a run for it.
 
 ### [New-AzDataFactory](New-AzDataFactory.md)
 Creates a data factory.
@@ -137,6 +149,9 @@ Removes a data factory.
 ### [Remove-AzDataFactoryV2Dataset](Remove-AzDataFactoryV2Dataset.md)
 Removes a dataset from Data Factory.
 
+### [Remove-AzDataFactoryV2DataFlow](Remove-AzDataFactoryV2DataFlow.md)
+Removes a data flow from Data Factory.
+
 ### [Remove-AzDataFactoryV2IntegrationRuntime](Remove-AzDataFactoryV2IntegrationRuntime.md)
 Removes an integration runtime.
 
@@ -173,6 +188,9 @@ Creates a data factory.
 ### [Set-AzDataFactoryV2Dataset](Set-AzDataFactoryV2Dataset.md)
 Creates a dataset in Data Factory.
 
+### [Set-AzDataFactoryV2DataFlow](Set-AzDataFactoryV2DataFlow.md)
+Creates a data flow in Data Factory.
+
 ### [Set-AzDataFactoryV2IntegrationRuntime](Set-AzDataFactoryV2IntegrationRuntime.md)
 Updates an integration runtime.
 
@@ -191,6 +209,9 @@ Starts a managed dedicated integration runtime.
 ### [Start-AzDataFactoryV2Trigger](Start-AzDataFactoryV2Trigger.md)
 Starts a trigger in a data factory.
 
+### [Start-AzDataFactoryV2DataFlowDebugSession](Start-AzDataFactoryV2DataFlowDebugSession.md)
+Starts a data flow debug session in Azure Data Factory.
+
 ### [Stop-AzDataFactoryV2IntegrationRuntime](Stop-AzDataFactoryV2IntegrationRuntime.md)
 Stops a managed dedicated integration runtime.
 
@@ -200,6 +221,9 @@ Stops a pipeline run in a data factory.
 ### [Stop-AzDataFactoryV2Trigger](Stop-AzDataFactoryV2Trigger.md)
 Stops a trigger in a data factory.
 
+### [Stop-AzDataFactoryV2DataFlowDebugSession](Stop-AzDataFactoryV2DataFlowDebugSession.md)
+Stops a data flow debug session in Azure Data Factory.
+
 ### [Suspend-AzDataFactoryPipeline](Suspend-AzDataFactoryPipeline.md)
 Suspends a pipeline in Azure Data Factory.
 

src/HDInsight/HDInsight.Test/DataLakeStoreDefaultFilesystemTests.cs

@@ -32,6 +32,7 @@ public class DataLakeStoreDefaultFilesystemTests : HDInsightTestBase
         private const string StorageName = "dummystorage.azuredatalakestore.net";
         private const int ClusterSize = 4;
         private Guid ObjectId = new Guid("11111111-1111-1111-1111-111111111111");
+        private Guid ApplicationId = new Guid("11111111-1111-1111-1111-111111111111");
         private Guid AadTenantId = new Guid("11111111-1111-1111-1111-111111111111");
         private string Certificate = "";
         private string CertificatePassword = "";
@@ -61,6 +62,7 @@ public void CanCreateClusterConfigWithDataLakeStoreParameters()
                 CommandRuntime = commandRuntimeMock.Object,
                 HDInsightManagementClient = hdinsightManagementMock.Object,
                 ObjectId = ObjectId,
+                ApplicationId = ApplicationId,
                 CertificateFilePath = Certificate,
                 AadTenantId = AadTenantId,
                 CertificatePassword = CertificatePassword,
@@ -76,6 +78,7 @@ public void CanCreateClusterConfigWithDataLakeStoreParameters()
                                 c.AADTenantId == AadTenantId &&
                                 c.CertificatePassword == CertificatePassword &&
                                 c.ObjectId == ObjectId &&
+                                c.ApplicationId == ApplicationId &&
                                 c.CertificateFilePath == Certificate &&
                                 c.DefaultStorageAccountName == StorageName
                                 )),
@@ -91,6 +94,7 @@ public void CanCreateDataLakeClusterWithCertificateFileContents()
                 CommandRuntime = commandRuntimeMock.Object,
                 HDInsightManagementClient = hdinsightManagementMock.Object,
                 ObjectId = ObjectId,
+                ApplicationId = ApplicationId,
                 CertificateFileContents = CertificateFileContents,
                 AadTenantId = AadTenantId,
                 CertificatePassword = CertificatePassword,
@@ -106,6 +110,7 @@ public void CanCreateDataLakeClusterWithCertificateFileContents()
                                 c.AADTenantId == AadTenantId &&
                                 c.CertificatePassword == CertificatePassword &&
                                 c.ObjectId == ObjectId &&
+                                c.ApplicationId == ApplicationId &&
                                 c.CertificateFileContents == CertificateFileContents &&
                                 c.DefaultStorageAccountName == StorageName
                                 )),
@@ -121,6 +126,7 @@ public void ShouldThrowIfCertificateOptionsAreNotPassed()
                 CommandRuntime = commandRuntimeMock.Object,
                 HDInsightManagementClient = hdinsightManagementMock.Object,
                 ObjectId = ObjectId,
+                ApplicationId = ApplicationId,
                 AadTenantId = AadTenantId,
                 CertificatePassword = CertificatePassword
             };
@@ -137,6 +143,7 @@ public void ShouldThrowIfBothCertificateOptionsArePassed()
                 CommandRuntime = commandRuntimeMock.Object,
                 HDInsightManagementClient = hdinsightManagementMock.Object,
                 ObjectId = ObjectId,
+                ApplicationId = ApplicationId,
                 AadTenantId = AadTenantId,
                 CertificatePassword = CertificatePassword,
                 CertificateFileContents = CertificateFileContents,

src/HDInsight/HDInsight.Test/UnitTests/DataLakeStoreTests.cs

@@ -30,6 +30,7 @@ public class DataLakeStoreTests : HDInsightTestBase
     {
         private NewAzureHDInsightClusterCommand cmdlet;
         private Guid ObjectId = new Guid("11111111-1111-1111-1111-111111111111");
+        private Guid ApplicationId = new Guid("11111111-1111-1111-1111-111111111111");
         private Guid AadTenantId = new Guid("11111111-1111-1111-1111-111111111111");
         private string Certificate = "";
         private string CertificatePassword = "";
@@ -59,6 +60,7 @@ public void CanCreateClusterConfigWithDataLakeStoreParameters()
                 CommandRuntime = commandRuntimeMock.Object,
                 HDInsightManagementClient = hdinsightManagementMock.Object,
                 ObjectId = ObjectId,
+                ApplicationId = ApplicationId,
                 CertificateFilePath = Certificate,
                 AadTenantId = AadTenantId,
                 CertificatePassword = CertificatePassword
@@ -73,6 +75,7 @@ public void CanCreateClusterConfigWithDataLakeStoreParameters()
                                 c.AADTenantId == AadTenantId &&
                                 c.CertificatePassword == CertificatePassword &&
                                 c.ObjectId == ObjectId &&
+                                c.ApplicationId == ApplicationId &&
                                 c.CertificateFilePath == Certificate
                                 )),
                 Times.Once);
@@ -87,6 +90,7 @@ public void CanCreateDataLakeClusterWithCertificationFileContents()
                 CommandRuntime = commandRuntimeMock.Object,
                 HDInsightManagementClient = hdinsightManagementMock.Object,
                 ObjectId = ObjectId,
+                ApplicationId = ApplicationId,
                 CertificateFileContents = CertificateFileContents,
                 AadTenantId = AadTenantId,
                 CertificatePassword = CertificatePassword
@@ -101,6 +105,7 @@ public void CanCreateDataLakeClusterWithCertificationFileContents()
                                 c.AADTenantId == AadTenantId &&
                                 c.CertificatePassword == CertificatePassword &&
                                 c.ObjectId == ObjectId &&
+                                c.ApplicationId == ApplicationId &&
                                 c.CertificateFileContents == CertificateFileContents
                                 )),
                 Times.Once);
@@ -115,6 +120,7 @@ public void ShouldThrowIfCertificateOptionsAreNotPassed()
                 CommandRuntime = commandRuntimeMock.Object,
                 HDInsightManagementClient = hdinsightManagementMock.Object,
                 ObjectId = ObjectId,
+                ApplicationId = ApplicationId,
                 AadTenantId = AadTenantId,
                 CertificatePassword = CertificatePassword
             };
@@ -131,6 +137,7 @@ public void ShouldThrowIfBothCertificateOptionsArePassed()
                 CommandRuntime = commandRuntimeMock.Object,
                 HDInsightManagementClient = hdinsightManagementMock.Object,
                 ObjectId = ObjectId,
+                ApplicationId = ApplicationId,
                 AadTenantId = AadTenantId,
                 CertificatePassword = CertificatePassword,
                 CertificateFileContents = CertificateFileContents,

src/HDInsight/HDInsight/ChangeLog.md

@@ -18,6 +18,8 @@
         - Additional information about change #1
 -->
 ## Upcoming Release
+* Fixed the bug that customer will get "Not a valid Base-64 string" error when using Get-AzHDInsightCluster to get the cluster with ADLSGen1 storage.
+* Add a parameter named "ApplicationId" to three cmdlets Add-AzHDInsightClusterIdentity, New-AzHDInsightClusterConfig and New-AzHDInsightCluster so that customer can provide the service principal application id for accessing Azure Data Lake.
 
 ## Version 2.0.2
 * Call out breaking changes

src/HDInsight/HDInsight/ClusterConfigurationUtils.cs

@@ -74,7 +74,7 @@ public static AzureHDInsightDefaultStorageAccount GetDefaultStorageAccountDetail
                         storageRootPath: storageRootPath,
                         applicationId: appId,
                         tenantId: tenantId,
-                        certificateContents: Convert.FromBase64String(certContents ?? ""),
+                        certificateContents: null,
                         certificatePassword: certPassword,
                         resourceUri: resourceUri
                     );

src/HDInsight/HDInsight/ManagementCommands/AddAzureHDInsightClusterIdentity.cs

@@ -61,6 +61,11 @@ public class AddAzureHDInsightClusterIdentity : HDInsightCmdletBase
             HelpMessage = "The Service Principal AAD Tenant Id for accessing Azure Data Lake.")]
         public Guid AadTenantId { get; set; }
 
+        [Parameter(Position = 5,
+            Mandatory = false,
+            HelpMessage = "The Service Principal Application Id for accessing Azure Data Lake.")]
+        public Guid ApplicationId { get; set; }
+
         #endregion
 
         public override void ExecuteCmdlet()
@@ -82,6 +87,7 @@ public override void ExecuteCmdlet()
             }
 
             Config.ObjectId = ObjectId;
+            Config.ApplicationId = ApplicationId;
             Config.AADTenantId = AadTenantId;
             Config.CertificatePassword = CertificatePassword;
 

src/HDInsight/HDInsight/ManagementCommands/NewAzureHDInsightClusterCommand.cs

@@ -130,6 +130,7 @@ public AzureHDInsightConfig Config
                     HiveMetastore = HiveMetastore,
                     OozieMetastore = OozieMetastore,
                     ObjectId = ObjectId,
+                    ApplicationId = ApplicationId,
                     AADTenantId = AadTenantId,
                     CertificateFileContents = CertificateFileContents,
                     CertificateFilePath = CertificateFilePath,
@@ -185,6 +186,7 @@ var storageAccount in
                 CertificateFilePath = value.CertificateFilePath;
                 AadTenantId = value.AADTenantId;
                 ObjectId = value.ObjectId;
+                ApplicationId = value.ApplicationId;
                 CertificatePassword = value.CertificatePassword;
                 SecurityProfile = value.SecurityProfile;
                 DisksPerWorkerNode = value.DisksPerWorkerNode;
@@ -332,6 +334,9 @@ public DateTime RdpAccessExpiry
         [Parameter(HelpMessage = "Gets or sets the Service Principal Object Id for accessing Azure Data Lake.")]
         public Guid ObjectId { get; set; }
 
+        [Parameter(HelpMessage = "Gets or sets the Service Principal Application Id for accessing Azure Data Lake.")]
+        public Guid ApplicationId { get; set; }
+
         [Parameter(HelpMessage = "Gets or sets the Service Principal Certificate file path for accessing Azure Data Lake.",
             ParameterSetName = CertificateFilePathSet)]
         public string CertificateFilePath { get; set; }
@@ -434,7 +439,7 @@ var storageAccount in
                     CertificateFileContents = File.ReadAllBytes(CertificateFilePath);
                 }
                 var servicePrincipal = new Management.HDInsight.Models.ServicePrincipal(
-                    GetApplicationId(), GetTenantId(AadTenantId), CertificateFileContents,
+                    GetApplicationId(ApplicationId), GetTenantId(AadTenantId), CertificateFileContents,
                     CertificatePassword);
 
                 parameters.Principal = servicePrincipal;
@@ -504,20 +509,34 @@ private Guid GetTenantId(Guid tenantId)
             return new Guid(tenantIdStr);
         }
 
-        //Get ApplicationId for the given ObjectId.
-        private Guid GetApplicationId()
+        //Get ApplicationId of Service Principal if user doesn't provide this parameter
+        private Guid GetApplicationId(Guid applicationId)
         {
+            if (applicationId != Guid.Empty)
+            {
+                return applicationId;
+            }
+
             GraphRbacManagementClient graphClient = AzureSession.Instance.ClientFactory.CreateArmClient<GraphRbacManagementClient>(
                 DefaultProfile.DefaultContext, AzureEnvironment.Endpoint.Graph);
 
             graphClient.TenantID = DefaultProfile.DefaultContext.Tenant.Id.ToString();
 
-            Microsoft.Azure.Graph.RBAC.Version1_6.Models.ServicePrincipal sp = graphClient.ServicePrincipals.Get(ObjectId.ToString());
+            Microsoft.Azure.Graph.RBAC.Version1_6.Models.ServicePrincipal sp=null;
+            try
+            {
+                sp = graphClient.ServicePrincipals.Get(ObjectId.ToString());
+            }
+            catch(Microsoft.Azure.Graph.RBAC.Version1_6.Models.GraphErrorException e)
+            {
+                string errorMessage = e.Message + ". Please specify Application Id explicitly by providing ApplicationId parameter and retry.";
+                throw new Microsoft.Azure.Graph.RBAC.Version1_6.Models.GraphErrorException(errorMessage);
+            }
 
-            var applicationId = Guid.Empty;
-            Guid.TryParse(sp.AppId, out applicationId);
-            Debug.Assert(applicationId != Guid.Empty);
-            return applicationId;
+            var spApplicationId = Guid.Empty;
+            Guid.TryParse(sp.AppId, out spApplicationId);
+            Debug.Assert(spApplicationId != Guid.Empty);
+            return spApplicationId;
         }
     }
 }

src/HDInsight/HDInsight/ManagementCommands/NewAzureHDInsightClusterConfigCommand.cs

@@ -113,6 +113,13 @@ public Guid ObjectId
             set { _config.ObjectId = value; }
         }
 
+        [Parameter(HelpMessage = "Gets or sets the Service Principal Application Id for accessing Azure Data Lake.")]
+        public Guid ApplicationId
+        {
+            get { return _config.ApplicationId; }
+            set { _config.ApplicationId = value; }
+        }
+
         [Parameter(HelpMessage = "Gets or sets the Service Principal Certificate file contents for accessing Azure Data Lake.")]
         public byte[] CertificateFileContents
         {

src/HDInsight/HDInsight/Models/Management/AzureHDInsightConfig.cs

@@ -92,6 +92,11 @@ public class AzureHDInsightConfig
         /// </summary>
         public Guid ObjectId { get; set; }
 
+        /// <summary>
+        /// Gets Application id of the service principal. 
+        /// </summary>
+        public Guid ApplicationId { get; set; }
+
         /// <summary>
         /// Gets the file path of the client certificate file contents associated with the service principal.
         /// </summary>