ADLA and ADLS Update: Compute Policy, Job Recurrence, App ID and Key Rotation

src/Common/Commands.Common.Authentication.Abstractions/AzureEnvironment.cs

@@ -52,6 +52,7 @@ public class AzureEnvironment : IAzureEnvironment
                     AzureDataLakeAnalyticsCatalogAndJobEndpointSuffix = AzureEnvironmentConstants.AzureDataLakeAnalyticsCatalogAndJobEndpointSuffix,
                     AzureDataLakeStoreFileSystemEndpointSuffix = AzureEnvironmentConstants.AzureDataLakeStoreFileSystemEndpointSuffix,
                     GraphEndpointResourceId = AzureEnvironmentConstants.AzureGraphEndpoint,
+                    DataLakeEndpointResourceId = AzureEnvironmentConstants.AzureDataLakeServiceEndpointResourceId,
                     AdTenant = "Common"
                 }
             },
@@ -222,6 +223,11 @@ public AzureEnvironment(IAzureEnvironment other)
         /// </summary>
         public string GraphEndpointResourceId { get; set; }
 
+        /// <summary>
+        /// The token audience required for communicating with the Azure Active Directory Data Lake service in this environment
+        /// </summary>
+        public string DataLakeEndpointResourceId { get; set; }
+
         /// <summary>
         /// The domain name suffix for Azure DataLake Catalog and Job services created in this environment
         /// </summary>
@@ -268,7 +274,8 @@ public static class Endpoint
                 PublishSettingsFileUrl = "PublishSettingsFileUrl",
                 ManagementPortalUrl = "ManagementPortalUrl",
                 AzureDataLakeAnalyticsCatalogAndJobEndpointSuffix = "AzureDataLakeAnalyticsCatalogAndJobEndpointSuffix",
-                AzureDataLakeStoreFileSystemEndpointSuffix = "AzureDataLakeStoreFileSystemEndpointSuffix";
+                AzureDataLakeStoreFileSystemEndpointSuffix = "AzureDataLakeStoreFileSystemEndpointSuffix",
+                DataLakeEndpointResourceId = "DataLakeEndpointResourceId";
 
         }
     }

src/Common/Commands.Common.Authentication.Abstractions/AzureEnvironmentConstants.cs

@@ -120,5 +120,10 @@ public static class AzureEnvironmentConstants
         /// </summary>
         public const string AzureDataLakeAnalyticsCatalogAndJobEndpointSuffix = "azuredatalakeanalytics.net";
         public const string AzureDataLakeStoreFileSystemEndpointSuffix = "azuredatalakestore.net";
+
+        /// <summary>
+        /// The token audience for authorizing DataLake requests
+        /// </summary>
+        public const string AzureDataLakeServiceEndpointResourceId = "https://datalake.azure.net";
     }
 }

src/Common/Commands.Common.Authentication.Abstractions/Extensions/AzureEnvironmentExtensions.cs

@@ -62,6 +62,9 @@ public static bool TryGetEndpointUrl(this IAzureEnvironment environment, string
                 case AzureEnvironment.Endpoint.GraphEndpointResourceId:
                     endpoint = new Uri(environment.GraphEndpointResourceId);
                     break;
+                case AzureEnvironment.Endpoint.DataLakeEndpointResourceId:
+                    endpoint = new Uri(environment.DataLakeEndpointResourceId);
+                    break;
                 default:
                     result = false;
                     break;
@@ -220,6 +223,9 @@ public static void SetEndpoint(this IAzureEnvironment environment, string endpoi
                     case AzureEnvironment.Endpoint.AzureDataLakeStoreFileSystemEndpointSuffix:
                         environment.AzureDataLakeStoreFileSystemEndpointSuffix = propertyValue;
                         break;
+                    case AzureEnvironment.Endpoint.DataLakeEndpointResourceId:
+                        environment.DataLakeEndpointResourceId = propertyValue;
+                        break;
                     case AzureEnvironment.Endpoint.ActiveDirectory:
                         environment.ActiveDirectoryAuthority = propertyValue;
                         break;
@@ -258,6 +264,12 @@ public static string GetTokenAudience(this IAzureEnvironment environment, string
             {
                 resource = AzureEnvironment.Endpoint.GraphEndpointResourceId;
             }
+            else if (targetEndpoint == AzureEnvironment.Endpoint.AzureDataLakeAnalyticsCatalogAndJobEndpointSuffix ||
+                targetEndpoint == AzureEnvironment.Endpoint.AzureDataLakeStoreFileSystemEndpointSuffix ||
+                targetEndpoint == AzureEnvironment.Endpoint.DataLakeEndpointResourceId)
+            {
+                resource = AzureEnvironment.Endpoint.DataLakeEndpointResourceId;
+            }
 
             return resource;
         }

src/Common/Commands.Common.Authentication.Abstractions/Interfaces/IAzureEnvironment.cs

@@ -102,6 +102,11 @@ public interface IAzureEnvironment : IExtensibleModel
         /// </summary>
         string GraphEndpointResourceId { get; set; }
 
+        /// <summary>
+        /// The token audience required to authenticate with the Azure Active Directory Data Lake services
+        /// </summary>
+        string DataLakeEndpointResourceId { get; set; }
+
         /// <summary>
         ///  The domain name suffix for Azure DataLake Catalog and Job services
         /// </summary>

src/ResourceManager/Common/Commands.ScenarioTests.ResourceManager.Common/ProfileClient.cs

@@ -847,6 +847,7 @@ private IAzureEnvironment MergeEnvironmentProperties(IAzureEnvironment environme
                 GraphEndpointResourceId = environment1.GraphEndpointResourceId ?? environment2.GraphEndpointResourceId,
                 AdTenant = environment1.AdTenant ?? environment2.AdTenant,
                 AzureDataLakeStoreFileSystemEndpointSuffix = environment1.AzureDataLakeStoreFileSystemEndpointSuffix ?? environment2.AzureDataLakeStoreFileSystemEndpointSuffix,
+                DataLakeEndpointResourceId = environment1.DataLakeEndpointResourceId ?? environment2.DataLakeEndpointResourceId,
                 AzureKeyVaultServiceEndpointResourceId = environment1.AzureKeyVaultServiceEndpointResourceId ?? environment2.AzureKeyVaultServiceEndpointResourceId,
                 GraphUrl = environment1.GraphUrl ?? environment2.GraphUrl,
                 ManagementPortalUrl = environment1.ManagementPortalUrl ?? environment2.ManagementPortalUrl,

src/ResourceManager/DataLakeAnalytics/AzureRM.DataLakeAnalytics.psd1

@@ -96,7 +96,13 @@ CmdletsToExport = 'Get-AzureRmDataLakeAnalyticsDataSource',
                'Add-AzureRmDataLakeAnalyticsFirewallRule', 
                'Get-AzureRmDataLakeAnalyticsFirewallRule', 
                'Set-AzureRmDataLakeAnalyticsFirewallRule', 
-               'Remove-AzureRmDataLakeAnalyticsFirewallRule'
+               'Remove-AzureRmDataLakeAnalyticsFirewallRule',
+               'New-AzureRmDataLakeAnalyticsComputePolicy',
+               'Get-AzureRmDataLakeAnalyticsComputePolicy',
+               'Update-AzureRmDataLakeAnalyticsComputePolicy',
+               'Remove-AzureRmDataLakeAnalyticsComputePolicy',
+               'Get-AzureRmDataLakeAnalyticsJobPipeline',
+               'Get-AzureRmDataLakeAnalyticsJobRecurrence'
 
 # Variables to export from this module
 # VariablesToExport = @()
@@ -112,7 +118,10 @@ AliasesToExport = 'Get-AdlAnalyticsDataSource', 'New-AdlCatalogCredential',
                'Get-AdlAnalyticsAccount', 'Submit-AdlJob', 'New-AdlAnalyticsAccount', 
                'Remove-AdlAnalyticsAccount', 'Set-AdlAnalyticsAccount', 
                'Add-AdlAnalyticsFirewallRule', 'Get-AdlAnalyticsFirewallRule', 
-               'Set-AdlAnalyticsFirewallRule', 'Remove-AdlAnalyticsFirewallRule'
+               'Set-AdlAnalyticsFirewallRule', 'Remove-AdlAnalyticsFirewallRule',
+               'New-AdlAnalyticsComputePolicy', 'Get-AdlAnalyticsComputePolicy',
+               'Update-AdlAnalyticsComputePolicy', 'Remove-AdlAnalyticsComputePolicy',
+               'Get-AdlJobPipeline', 'Get-AdlJobRecurrence'
 
 # DSC resources to export from this module
 # DscResourcesToExport = @()

src/ResourceManager/DataLakeAnalytics/ChangeLog.md

@@ -18,7 +18,18 @@
         - Additional information about change #1
 -->
 ## Current Release
-
+* Add support for Compute Policy CRUD through the following commands:
+    - New-AzureRMDataLakeAnalyticsComputePolicy
+    - Get-AzureRMDataLakeAnalyticsComputePolicy
+    - Remove-AzureRMDataLakeAnalyticsComputePolicy
+    - Update-AzureRMDataLakeAnalyticsComputePolicy
+* Add support for job relationship metadata for help with recurring jobs and job pipelines. The following commands were updated or added:
+    - Submit-AzureRMDataLakeAnalyticsJob
+    - Get-AzureRMDataLakeAnalyticsJob
+    - Get-AzureRMDataLakeAnalyticsJobRecurrence
+    - Get-AzureRMDataLakeAnalyticsJobPipeline
+* Updated the token audience for job and catalog APIs to use the correct Data Lake specific audience instead of the Azure Resource audience.
+
 ## Version 3.1.0
 
 ## Version 3.0.1

src/ResourceManager/DataLakeAnalytics/Commands.DataLakeAnalytics.Test/Commands.DataLakeAnalytics.Test.csproj

@@ -62,13 +62,14 @@
       <HintPath>..\..\..\packages\Microsoft.Azure.Management.Authorization.2.0.0\lib\net40\Microsoft.Azure.Management.Authorization.dll</HintPath>
       <Private>True</Private>
     </Reference>
-    <Reference Include="Microsoft.Azure.Management.DataLake.Analytics, Version=2.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
+    <Reference Include="Microsoft.Azure.Management.DataLake.Analytics, Version=3.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
       <SpecificVersion>False</SpecificVersion>
-      <HintPath>..\..\..\packages\Microsoft.Azure.Management.DataLake.Analytics.2.2.0-preview\lib\net452\Microsoft.Azure.Management.DataLake.Analytics.dll</HintPath>
+      <HintPath>..\..\..\packages\Microsoft.Azure.Management.DataLake.Analytics.3.0.0\lib\net452\Microsoft.Azure.Management.DataLake.Analytics.dll</HintPath>
       <Private>True</Private>
     </Reference>
-    <Reference Include="Microsoft.Azure.Management.DataLake.Store">
-      <HintPath>..\..\..\packages\Microsoft.Azure.Management.DataLake.Store.2.1.2-preview\lib\net452\Microsoft.Azure.Management.DataLake.Store.dll</HintPath>
+    <Reference Include="Microsoft.Azure.Management.DataLake.Store, Version=2.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
+      <SpecificVersion>False</SpecificVersion>
+      <HintPath>..\..\..\packages\Microsoft.Azure.Management.DataLake.Store.2.2.0\lib\net452\Microsoft.Azure.Management.DataLake.Store.dll</HintPath>
       <Private>True</Private>
     </Reference>
     <Reference Include="Microsoft.Azure.Management.ResourceManager, Version=1.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
@@ -103,12 +104,12 @@
     </Reference>
     <Reference Include="Microsoft.Rest.ClientRuntime, Version=2.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
       <SpecificVersion>False</SpecificVersion>
-      <HintPath>..\..\..\packages\Microsoft.Rest.ClientRuntime.2.3.6\lib\net45\Microsoft.Rest.ClientRuntime.dll</HintPath>
+      <HintPath>..\..\..\packages\Microsoft.Rest.ClientRuntime.2.3.8\lib\net452\Microsoft.Rest.ClientRuntime.dll</HintPath>
       <Private>True</Private>
     </Reference>
     <Reference Include="Microsoft.Rest.ClientRuntime.Azure, Version=3.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
       <SpecificVersion>False</SpecificVersion>
-      <HintPath>..\..\..\packages\Microsoft.Rest.ClientRuntime.Azure.3.3.5\lib\net45\Microsoft.Rest.ClientRuntime.Azure.dll</HintPath>
+      <HintPath>..\..\..\packages\Microsoft.Rest.ClientRuntime.Azure.3.3.7\lib\net452\Microsoft.Rest.ClientRuntime.Azure.dll</HintPath>
       <Private>True</Private>
     </Reference>
     <Reference Include="Microsoft.Rest.ClientRuntime.Azure.Authentication, Version=2.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
@@ -233,12 +234,18 @@
     <None Include="SessionRecords\Microsoft.Azure.Commands.DataLakeAnalytics.Test.ScenarioTests.AdlaAliasTests\TestAdlaCatalog.json">
       <CopyToOutputDirectory>Always</CopyToOutputDirectory>
     </None>
+    <None Include="SessionRecords\Microsoft.Azure.Commands.DataLakeAnalytics.Test.ScenarioTests.AdlaAliasTests\TestAdlaComputePolicy.json">
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+    </None>
     <None Include="SessionRecords\Microsoft.Azure.Commands.DataLakeAnalytics.Test.ScenarioTests.AdlaAliasTests\TestAdlaFirewallRules.json">
-      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
     </None>
     <None Include="SessionRecords\Microsoft.Azure.Commands.DataLakeAnalytics.Test.ScenarioTests.AdlaAliasTests\TestAdlaJob.json">
       <CopyToOutputDirectory>Always</CopyToOutputDirectory>
     </None>
+    <None Include="SessionRecords\Microsoft.Azure.Commands.DataLakeAnalytics.Test.ScenarioTests.AdlaAliasTests\TestAdlaJobRelationships.json">
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+    </None>
     <None Include="SessionRecords\Microsoft.Azure.Commands.DataLakeAnalytics.Test.ScenarioTests.AdlaAliasTests\TestNegativeAdlaAccount.json">
       <CopyToOutputDirectory>Always</CopyToOutputDirectory>
     </None>
@@ -254,12 +261,18 @@
     <None Include="SessionRecords\Microsoft.Azure.Commands.DataLakeAnalytics.Test.ScenarioTests.AdlaTests\TestAdlaCatalog.json">
       <CopyToOutputDirectory>Always</CopyToOutputDirectory>
     </None>
+    <None Include="SessionRecords\Microsoft.Azure.Commands.DataLakeAnalytics.Test.ScenarioTests.AdlaTests\TestAdlaComputePolicy.json">
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+    </None>
     <None Include="SessionRecords\Microsoft.Azure.Commands.DataLakeAnalytics.Test.ScenarioTests.AdlaTests\TestAdlaFirewallRules.json">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </None>
     <None Include="SessionRecords\Microsoft.Azure.Commands.DataLakeAnalytics.Test.ScenarioTests.AdlaTests\TestAdlaJob.json">
       <CopyToOutputDirectory>Always</CopyToOutputDirectory>
     </None>
+    <None Include="SessionRecords\Microsoft.Azure.Commands.DataLakeAnalytics.Test.ScenarioTests.AdlaTests\TestAdlaJobRelationships.json">
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+    </None>
     <None Include="SessionRecords\Microsoft.Azure.Commands.DataLakeAnalytics.Test.ScenarioTests.AdlaTests\TestNegativeAdlaAccount.json">
       <CopyToOutputDirectory>Always</CopyToOutputDirectory>
     </None>

src/ResourceManager/DataLakeAnalytics/Commands.DataLakeAnalytics.Test/ScenarioTests/AdlaAliasTests.cs

@@ -54,6 +54,16 @@ public void TestAdlaFirewallRules()
                     AdlaTestsBase.resourceGroupLocation));
         }
 
+        [Fact]
+        [Trait(Category.AcceptanceType, Category.CheckIn)]
+        public void TestAdlaComputePolicy()
+        {
+            AdlaTestsBase.NewInstance.RunPsTest(false,
+                string.Format(
+                    "Test-DataLakeAnalyticsComputePolicy -location '{0}'",
+                    AdlaTestsBase.resourceGroupLocation));
+        }
+
         [Fact]
         [Trait(Category.AcceptanceType, Category.CheckIn)]
         public void TestAdlaCatalog()
@@ -74,6 +84,16 @@ public void TestAdlaJob()
                     AdlaTestsBase.resourceGroupLocation));
         }
 
+        [Fact]
+        [Trait(Category.AcceptanceType, Category.CheckIn)]
+        public void TestAdlaJobRelationships()
+        {
+            AdlaTestsBase.NewInstance.RunPsTest(false,
+                string.Format(
+                    "Test-DataLakeAnalyticsJobRelationships -location '{0}'",
+                    AdlaTestsBase.resourceGroupLocation));
+        }
+
         [Fact]
         [Trait(Category.AcceptanceType, Category.CheckIn)]
         public void TestNegativeAdlaAccount()