Merge pull request #10306 from aim-for-better/FixResource404WhenCreateCluster

[HDInsight] Fix Error Operation returned an invalid status code 'NotFound' when using New-AzHDInsightCluster with ADLS

Author: wyunchi-ms

src/HDInsight/HDInsight.Test/DataLakeStoreDefaultFilesystemTests.cs

@@ -32,6 +32,7 @@ public class DataLakeStoreDefaultFilesystemTests : HDInsightTestBase
         private const string StorageName = "dummystorage.azuredatalakestore.net";
         private const int ClusterSize = 4;
         private Guid ObjectId = new Guid("11111111-1111-1111-1111-111111111111");
+        private Guid ApplicationId = new Guid("11111111-1111-1111-1111-111111111111");
         private Guid AadTenantId = new Guid("11111111-1111-1111-1111-111111111111");
         private string Certificate = "";
         private string CertificatePassword = "";
@@ -61,6 +62,7 @@ public void CanCreateClusterConfigWithDataLakeStoreParameters()
                 CommandRuntime = commandRuntimeMock.Object,
                 HDInsightManagementClient = hdinsightManagementMock.Object,
                 ObjectId = ObjectId,
+                ApplicationId = ApplicationId,
                 CertificateFilePath = Certificate,
                 AadTenantId = AadTenantId,
                 CertificatePassword = CertificatePassword,
@@ -76,6 +78,7 @@ public void CanCreateClusterConfigWithDataLakeStoreParameters()
                                 c.AADTenantId == AadTenantId &&
                                 c.CertificatePassword == CertificatePassword &&
                                 c.ObjectId == ObjectId &&
+                                c.ApplicationId == ApplicationId &&
                                 c.CertificateFilePath == Certificate &&
                                 c.DefaultStorageAccountName == StorageName
                                 )),
@@ -91,6 +94,7 @@ public void CanCreateDataLakeClusterWithCertificateFileContents()
                 CommandRuntime = commandRuntimeMock.Object,
                 HDInsightManagementClient = hdinsightManagementMock.Object,
                 ObjectId = ObjectId,
+                ApplicationId = ApplicationId,
                 CertificateFileContents = CertificateFileContents,
                 AadTenantId = AadTenantId,
                 CertificatePassword = CertificatePassword,
@@ -106,6 +110,7 @@ public void CanCreateDataLakeClusterWithCertificateFileContents()
                                 c.AADTenantId == AadTenantId &&
                                 c.CertificatePassword == CertificatePassword &&
                                 c.ObjectId == ObjectId &&
+                                c.ApplicationId == ApplicationId &&
                                 c.CertificateFileContents == CertificateFileContents &&
                                 c.DefaultStorageAccountName == StorageName
                                 )),
@@ -121,6 +126,7 @@ public void ShouldThrowIfCertificateOptionsAreNotPassed()
                 CommandRuntime = commandRuntimeMock.Object,
                 HDInsightManagementClient = hdinsightManagementMock.Object,
                 ObjectId = ObjectId,
+                ApplicationId = ApplicationId,
                 AadTenantId = AadTenantId,
                 CertificatePassword = CertificatePassword
             };
@@ -137,6 +143,7 @@ public void ShouldThrowIfBothCertificateOptionsArePassed()
                 CommandRuntime = commandRuntimeMock.Object,
                 HDInsightManagementClient = hdinsightManagementMock.Object,
                 ObjectId = ObjectId,
+                ApplicationId = ApplicationId,
                 AadTenantId = AadTenantId,
                 CertificatePassword = CertificatePassword,
                 CertificateFileContents = CertificateFileContents,

src/HDInsight/HDInsight.Test/UnitTests/DataLakeStoreTests.cs

@@ -30,6 +30,7 @@ public class DataLakeStoreTests : HDInsightTestBase
     {
         private NewAzureHDInsightClusterCommand cmdlet;
         private Guid ObjectId = new Guid("11111111-1111-1111-1111-111111111111");
+        private Guid ApplicationId = new Guid("11111111-1111-1111-1111-111111111111");
         private Guid AadTenantId = new Guid("11111111-1111-1111-1111-111111111111");
         private string Certificate = "";
         private string CertificatePassword = "";
@@ -59,6 +60,7 @@ public void CanCreateClusterConfigWithDataLakeStoreParameters()
                 CommandRuntime = commandRuntimeMock.Object,
                 HDInsightManagementClient = hdinsightManagementMock.Object,
                 ObjectId = ObjectId,
+                ApplicationId = ApplicationId,
                 CertificateFilePath = Certificate,
                 AadTenantId = AadTenantId,
                 CertificatePassword = CertificatePassword
@@ -73,6 +75,7 @@ public void CanCreateClusterConfigWithDataLakeStoreParameters()
                                 c.AADTenantId == AadTenantId &&
                                 c.CertificatePassword == CertificatePassword &&
                                 c.ObjectId == ObjectId &&
+                                c.ApplicationId == ApplicationId &&
                                 c.CertificateFilePath == Certificate
                                 )),
                 Times.Once);
@@ -87,6 +90,7 @@ public void CanCreateDataLakeClusterWithCertificationFileContents()
                 CommandRuntime = commandRuntimeMock.Object,
                 HDInsightManagementClient = hdinsightManagementMock.Object,
                 ObjectId = ObjectId,
+                ApplicationId = ApplicationId,
                 CertificateFileContents = CertificateFileContents,
                 AadTenantId = AadTenantId,
                 CertificatePassword = CertificatePassword
@@ -101,6 +105,7 @@ public void CanCreateDataLakeClusterWithCertificationFileContents()
                                 c.AADTenantId == AadTenantId &&
                                 c.CertificatePassword == CertificatePassword &&
                                 c.ObjectId == ObjectId &&
+                                c.ApplicationId == ApplicationId &&
                                 c.CertificateFileContents == CertificateFileContents
                                 )),
                 Times.Once);
@@ -115,6 +120,7 @@ public void ShouldThrowIfCertificateOptionsAreNotPassed()
                 CommandRuntime = commandRuntimeMock.Object,
                 HDInsightManagementClient = hdinsightManagementMock.Object,
                 ObjectId = ObjectId,
+                ApplicationId = ApplicationId,
                 AadTenantId = AadTenantId,
                 CertificatePassword = CertificatePassword
             };
@@ -131,6 +137,7 @@ public void ShouldThrowIfBothCertificateOptionsArePassed()
                 CommandRuntime = commandRuntimeMock.Object,
                 HDInsightManagementClient = hdinsightManagementMock.Object,
                 ObjectId = ObjectId,
+                ApplicationId = ApplicationId,
                 AadTenantId = AadTenantId,
                 CertificatePassword = CertificatePassword,
                 CertificateFileContents = CertificateFileContents,

src/HDInsight/HDInsight/ChangeLog.md

@@ -19,6 +19,7 @@
 -->
 ## Upcoming Release
 * Fixed the bug that customer will get "Not a valid Base-64 string" error when using Get-AzHDInsightCluster to get the cluster with ADLSGen1 storage.
+* Add a parameter named "ApplicationId" to three cmdlets Add-AzHDInsightClusterIdentity, New-AzHDInsightClusterConfig and New-AzHDInsightCluster so that customer can provide the service principal application id for accessing Azure Data Lake.
 
 ## Version 2.0.2
 * Call out breaking changes

src/HDInsight/HDInsight/ManagementCommands/AddAzureHDInsightClusterIdentity.cs

@@ -61,6 +61,11 @@ public class AddAzureHDInsightClusterIdentity : HDInsightCmdletBase
             HelpMessage = "The Service Principal AAD Tenant Id for accessing Azure Data Lake.")]
         public Guid AadTenantId { get; set; }
 
+        [Parameter(Position = 5,
+            Mandatory = false,
+            HelpMessage = "The Service Principal Application Id for accessing Azure Data Lake.")]
+        public Guid ApplicationId { get; set; }
+
         #endregion
 
         public override void ExecuteCmdlet()
@@ -82,6 +87,7 @@ public override void ExecuteCmdlet()
             }
 
             Config.ObjectId = ObjectId;
+            Config.ApplicationId = ApplicationId;
             Config.AADTenantId = AadTenantId;
             Config.CertificatePassword = CertificatePassword;
 

src/HDInsight/HDInsight/ManagementCommands/NewAzureHDInsightClusterCommand.cs

@@ -130,6 +130,7 @@ public AzureHDInsightConfig Config
                     HiveMetastore = HiveMetastore,
                     OozieMetastore = OozieMetastore,
                     ObjectId = ObjectId,
+                    ApplicationId = ApplicationId,
                     AADTenantId = AadTenantId,
                     CertificateFileContents = CertificateFileContents,
                     CertificateFilePath = CertificateFilePath,
@@ -185,6 +186,7 @@ var storageAccount in
                 CertificateFilePath = value.CertificateFilePath;
                 AadTenantId = value.AADTenantId;
                 ObjectId = value.ObjectId;
+                ApplicationId = value.ApplicationId;
                 CertificatePassword = value.CertificatePassword;
                 SecurityProfile = value.SecurityProfile;
                 DisksPerWorkerNode = value.DisksPerWorkerNode;
@@ -332,6 +334,9 @@ public DateTime RdpAccessExpiry
         [Parameter(HelpMessage = "Gets or sets the Service Principal Object Id for accessing Azure Data Lake.")]
         public Guid ObjectId { get; set; }
 
+        [Parameter(HelpMessage = "Gets or sets the Service Principal Application Id for accessing Azure Data Lake.")]
+        public Guid ApplicationId { get; set; }
+
         [Parameter(HelpMessage = "Gets or sets the Service Principal Certificate file path for accessing Azure Data Lake.",
             ParameterSetName = CertificateFilePathSet)]
         public string CertificateFilePath { get; set; }
@@ -434,7 +439,7 @@ var storageAccount in
                     CertificateFileContents = File.ReadAllBytes(CertificateFilePath);
                 }
                 var servicePrincipal = new Management.HDInsight.Models.ServicePrincipal(
-                    GetApplicationId(), GetTenantId(AadTenantId), CertificateFileContents,
+                    GetApplicationId(ApplicationId), GetTenantId(AadTenantId), CertificateFileContents,
                     CertificatePassword);
 
                 parameters.Principal = servicePrincipal;
@@ -504,20 +509,34 @@ private Guid GetTenantId(Guid tenantId)
             return new Guid(tenantIdStr);
         }
 
-        //Get ApplicationId for the given ObjectId.
-        private Guid GetApplicationId()
+        //Get ApplicationId of Service Principal if user doesn't provide this parameter
+        private Guid GetApplicationId(Guid applicationId)
         {
+            if (applicationId != Guid.Empty)
+            {
+                return applicationId;
+            }
+
             GraphRbacManagementClient graphClient = AzureSession.Instance.ClientFactory.CreateArmClient<GraphRbacManagementClient>(
                 DefaultProfile.DefaultContext, AzureEnvironment.Endpoint.Graph);
 
             graphClient.TenantID = DefaultProfile.DefaultContext.Tenant.Id.ToString();
 
-            Microsoft.Azure.Graph.RBAC.Version1_6.Models.ServicePrincipal sp = graphClient.ServicePrincipals.Get(ObjectId.ToString());
+            Microsoft.Azure.Graph.RBAC.Version1_6.Models.ServicePrincipal sp=null;
+            try
+            {
+                sp = graphClient.ServicePrincipals.Get(ObjectId.ToString());
+            }
+            catch(Microsoft.Azure.Graph.RBAC.Version1_6.Models.GraphErrorException e)
+            {
+                string errorMessage = e.Message + ". Please specify Application Id explicitly by providing ApplicationId parameter and retry.";
+                throw new Microsoft.Azure.Graph.RBAC.Version1_6.Models.GraphErrorException(errorMessage);
+            }
 
-            var applicationId = Guid.Empty;
-            Guid.TryParse(sp.AppId, out applicationId);
-            Debug.Assert(applicationId != Guid.Empty);
-            return applicationId;
+            var spApplicationId = Guid.Empty;
+            Guid.TryParse(sp.AppId, out spApplicationId);
+            Debug.Assert(spApplicationId != Guid.Empty);
+            return spApplicationId;
         }
     }
 }

src/HDInsight/HDInsight/ManagementCommands/NewAzureHDInsightClusterConfigCommand.cs

@@ -113,6 +113,13 @@ public Guid ObjectId
             set { _config.ObjectId = value; }
         }
 
+        [Parameter(HelpMessage = "Gets or sets the Service Principal Application Id for accessing Azure Data Lake.")]
+        public Guid ApplicationId
+        {
+            get { return _config.ApplicationId; }
+            set { _config.ApplicationId = value; }
+        }
+
         [Parameter(HelpMessage = "Gets or sets the Service Principal Certificate file contents for accessing Azure Data Lake.")]
         public byte[] CertificateFileContents
         {

src/HDInsight/HDInsight/Models/Management/AzureHDInsightConfig.cs

@@ -92,6 +92,11 @@ public class AzureHDInsightConfig
         /// </summary>
         public Guid ObjectId { get; set; }
 
+        /// <summary>
+        /// Gets Application id of the service principal. 
+        /// </summary>
+        public Guid ApplicationId { get; set; }
+
         /// <summary>
         /// Gets the file path of the client certificate file contents associated with the service principal.
         /// </summary>

src/HDInsight/HDInsight/help/Add-AzHDInsightClusterIdentity.md

@@ -1,4 +1,4 @@
----
+---
 external help file: Microsoft.Azure.PowerShell.Cmdlets.HDInsight.dll-Help.xml
 Module Name: Az.HDInsight
 ms.assetid: A40AB6AB-D3CB-4A6C-B614-0B22085759DA
@@ -17,14 +17,14 @@ Adds a cluster identity to a cluster configuration object.
 ```
 Add-AzHDInsightClusterIdentity [-Config] <AzureHDInsightConfig> [-ObjectId] <Guid>
  [-CertificateFilePath] <String> [-CertificatePassword] <String> [[-AadTenantId] <Guid>]
- [-DefaultProfile <IAzureContextContainer>] [<CommonParameters>]
+ [-ApplicationId <Guid>] [-DefaultProfile <IAzureContextContainer>] [<CommonParameters>]
 ```
 
 ### CertificateFileContents
 ```
 Add-AzHDInsightClusterIdentity [-Config] <AzureHDInsightConfig> [-ObjectId] <Guid>
  [-CertificateFileContents] <Byte[]> [-CertificatePassword] <String> [[-AadTenantId] <Guid>]
- [-DefaultProfile <IAzureContextContainer>] [<CommonParameters>]
+ [-ApplicationId <Guid>] [-DefaultProfile <IAzureContextContainer>] [<CommonParameters>]
 ```
 
 ## DESCRIPTION
@@ -52,6 +52,7 @@ PS C:\> $clusterCreds = Get-Credential
 # Cluster Identity values
 PS C:\> $tenantId = (Get-AzContext).Tenant.TenantId
 PS C:\> $objectId = "<Azure AD Service Principal Object ID>"
+PS C:\> $applicationId = "<Azure AD Service Principal Application ID>"
 PS C:\> $certificateFilePath = "<Path to Azure AD Service Principal Certificate>"
 PS C:\> $certificatePassword = "<Password for Azure AD Service Principal Certificate>"
 
@@ -60,11 +61,12 @@ PS C:\> New-AzHDInsightClusterConfig `
             | Add-AzHDInsightClusterIdentity `
                 -AadTenantId $tenantId `
                 -ObjectId $objectId `
+                -Application $applicationId
                 -CertificateFilePath $certificateFilePath `
                 -CertificatePassword $certificatePassword `
             | New-AzHDInsightCluster `
                 -ClusterType Hadoop `
-                -OSType Windows `
+                -OSType Linux `
                 -ClusterSizeInNodes 4 `
                 -ResourceGroupName $clusterResourceGroupName `
                 -ClusterName $clusterName `
@@ -94,6 +96,21 @@ Accept pipeline input: False
 Accept wildcard characters: False
 ```
 
+### -ApplicationId
+The Service Principal Application Id for accessing Azure Data Lake.
+
+```yaml
+Type: System.Guid
+Parameter Sets: (All)
+Aliases:
+
+Required: False
+Position: Named
+Default value: None
+Accept pipeline input: False
+Accept wildcard characters: False
+```
+
 ### -CertificateFileContents
 Specifies file contents of the certificate that will be used when accessing Azure Data Lake Store.
 
@@ -189,7 +206,7 @@ Accept wildcard characters: False
 ```
 
 ### CommonParameters
-This cmdlet supports the common parameters: -Debug, -ErrorAction, -ErrorVariable, -InformationAction, -InformationVariable, -OutVariable, -OutBuffer, -PipelineVariable, -Verbose, -WarningAction, and -WarningVariable. For more information, see about_CommonParameters (http://go.microsoft.com/fwlink/?LinkID=113216).
+This cmdlet supports the common parameters: -Debug, -ErrorAction, -ErrorVariable, -InformationAction, -InformationVariable, -OutVariable, -OutBuffer, -PipelineVariable, -Verbose, -WarningAction, and -WarningVariable. For more information, see [about_CommonParameters](http://go.microsoft.com/fwlink/?LinkID=113216).
 
 ## INPUTS
 

src/HDInsight/HDInsight/help/New-AzHDInsightCluster.md

@@ -1,4 +1,4 @@
----
+---
 external help file: Microsoft.Azure.PowerShell.Cmdlets.HDInsight.dll-Help.xml
 Module Name: Az.HDInsight
 ms.assetid: 691AC991-3249-487C-A0DF-C579ED7D00E7
@@ -29,8 +29,8 @@ New-AzHDInsightCluster [-Location] <String> [-ResourceGroupName] <String> [-Clus
  [-ComponentVersion <System.Collections.Generic.Dictionary`2[System.String,System.String]>]
  [-VirtualNetworkId <String>] [-SubnetName <String>] [-OSType <OSType>] [-ClusterTier <Tier>]
  [-SshCredential <PSCredential>] [-SshPublicKey <String>] [-RdpCredential <PSCredential>]
- [-RdpAccessExpiry <DateTime>] [-ObjectId <Guid>] [-CertificatePassword <String>] [-AadTenantId <Guid>]
- [-SecurityProfile <AzureHDInsightSecurityProfile>] [-DisksPerWorkerNode <Int32>]
+ [-RdpAccessExpiry <DateTime>] [-ObjectId <Guid>] [-ApplicationId <Guid>] [-CertificatePassword <String>]
+ [-AadTenantId <Guid>] [-SecurityProfile <AzureHDInsightSecurityProfile>] [-DisksPerWorkerNode <Int32>]
  [-DefaultProfile <IAzureContextContainer>] [<CommonParameters>]
 ```
 
@@ -50,7 +50,7 @@ New-AzHDInsightCluster [-Location] <String> [-ResourceGroupName] <String> [-Clus
  [-ComponentVersion <System.Collections.Generic.Dictionary`2[System.String,System.String]>]
  [-VirtualNetworkId <String>] [-SubnetName <String>] [-OSType <OSType>] [-ClusterTier <Tier>]
  [-SshCredential <PSCredential>] [-SshPublicKey <String>] [-RdpCredential <PSCredential>]
- [-RdpAccessExpiry <DateTime>] [-ObjectId <Guid>] [-CertificateFilePath <String>]
+ [-RdpAccessExpiry <DateTime>] [-ObjectId <Guid>] [-ApplicationId <Guid>] [-CertificateFilePath <String>]
  [-CertificatePassword <String>] [-AadTenantId <Guid>] [-SecurityProfile <AzureHDInsightSecurityProfile>]
  [-DisksPerWorkerNode <Int32>] [-DefaultProfile <IAzureContextContainer>] [<CommonParameters>]
 ```
@@ -71,7 +71,7 @@ New-AzHDInsightCluster [-Location] <String> [-ResourceGroupName] <String> [-Clus
  [-ComponentVersion <System.Collections.Generic.Dictionary`2[System.String,System.String]>]
  [-VirtualNetworkId <String>] [-SubnetName <String>] [-OSType <OSType>] [-ClusterTier <Tier>]
  [-SshCredential <PSCredential>] [-SshPublicKey <String>] [-RdpCredential <PSCredential>]
- [-RdpAccessExpiry <DateTime>] [-ObjectId <Guid>] [-CertificateFileContents <Byte[]>]
+ [-RdpAccessExpiry <DateTime>] [-ObjectId <Guid>] [-ApplicationId <Guid>] [-CertificateFileContents <Byte[]>]
  [-CertificatePassword <String>] [-AadTenantId <Guid>] [-SecurityProfile <AzureHDInsightSecurityProfile>]
  [-DisksPerWorkerNode <Int32>] [-DefaultProfile <IAzureContextContainer>] [<CommonParameters>]
 ```
@@ -149,6 +149,21 @@ Accept pipeline input: False
 Accept wildcard characters: False
 ```
 
+### -ApplicationId
+Gets or sets the Service Principal Application Id for accessing Azure Data Lake.
+
+```yaml
+Type: System.Guid
+Parameter Sets: (All)
+Aliases:
+
+Required: False
+Position: Named
+Default value: None
+Accept pipeline input: False
+Accept wildcard characters: False
+```
+
 ### -CertificateFileContents
 Specifies file contents of the certificate that will be used when accessing Azure Data Lake Store.
 
@@ -730,7 +745,7 @@ Accept wildcard characters: False
 ```
 
 ### CommonParameters
-This cmdlet supports the common parameters: -Debug, -ErrorAction, -ErrorVariable, -InformationAction, -InformationVariable, -OutVariable, -OutBuffer, -PipelineVariable, -Verbose, -WarningAction, and -WarningVariable. For more information, see about_CommonParameters (http://go.microsoft.com/fwlink/?LinkID=113216).
+This cmdlet supports the common parameters: -Debug, -ErrorAction, -ErrorVariable, -InformationAction, -InformationVariable, -OutVariable, -OutBuffer, -PipelineVariable, -Verbose, -WarningAction, and -WarningVariable. For more information, see [about_CommonParameters](http://go.microsoft.com/fwlink/?LinkID=113216).
 
 ## INPUTS