Skip to content

Commit a714fae

Browse files
cormacpaynecormacpayne
authored
Merge pull request #4136 from begoldsm/job_recurrence
ADLA and ADLS Update: Compute Policy, Job Recurrence, App ID and Key Rotation
2 parents 485cd61 + de3e782 commit a714fae

File tree

73 files changed

+87946
-11916
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

73 files changed

+87946
-11916
lines changed

‎src/Common/Commands.Common.Authentication.Abstractions/AzureEnvironment.cs

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,7 @@ public class AzureEnvironment : IAzureEnvironment
5252
AzureDataLakeAnalyticsCatalogAndJobEndpointSuffix = AzureEnvironmentConstants.AzureDataLakeAnalyticsCatalogAndJobEndpointSuffix,
5353
AzureDataLakeStoreFileSystemEndpointSuffix = AzureEnvironmentConstants.AzureDataLakeStoreFileSystemEndpointSuffix,
5454
GraphEndpointResourceId = AzureEnvironmentConstants.AzureGraphEndpoint,
55+
DataLakeEndpointResourceId = AzureEnvironmentConstants.AzureDataLakeServiceEndpointResourceId,
5556
AdTenant = "Common"
5657
}
5758
},
@@ -222,6 +223,11 @@ public AzureEnvironment(IAzureEnvironment other)
222223
/// </summary>
223224
public string GraphEndpointResourceId { get; set; }
224225

226+
/// <summary>
227+
/// The token audience required for communicating with the Azure Active Directory Data Lake service in this environment
228+
/// </summary>
229+
public string DataLakeEndpointResourceId { get; set; }
230+
225231
/// <summary>
226232
/// The domain name suffix for Azure DataLake Catalog and Job services created in this environment
227233
/// </summary>
@@ -268,7 +274,8 @@ public static class Endpoint
268274
PublishSettingsFileUrl = "PublishSettingsFileUrl",
269275
ManagementPortalUrl = "ManagementPortalUrl",
270276
AzureDataLakeAnalyticsCatalogAndJobEndpointSuffix = "AzureDataLakeAnalyticsCatalogAndJobEndpointSuffix",
271-
AzureDataLakeStoreFileSystemEndpointSuffix = "AzureDataLakeStoreFileSystemEndpointSuffix";
277+
AzureDataLakeStoreFileSystemEndpointSuffix = "AzureDataLakeStoreFileSystemEndpointSuffix",
278+
DataLakeEndpointResourceId = "DataLakeEndpointResourceId";
272279

273280
}
274281
}

‎src/Common/Commands.Common.Authentication.Abstractions/AzureEnvironmentConstants.cs

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -120,5 +120,10 @@ public static class AzureEnvironmentConstants
120120
/// </summary>
121121
public const string AzureDataLakeAnalyticsCatalogAndJobEndpointSuffix = "azuredatalakeanalytics.net";
122122
public const string AzureDataLakeStoreFileSystemEndpointSuffix = "azuredatalakestore.net";
123+
124+
/// <summary>
125+
/// The token audience for authorizing DataLake requests
126+
/// </summary>
127+
public const string AzureDataLakeServiceEndpointResourceId = "https://datalake.azure.net";
123128
}
124129
}

‎src/Common/Commands.Common.Authentication.Abstractions/Extensions/AzureEnvironmentExtensions.cs

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,9 @@ public static bool TryGetEndpointUrl(this IAzureEnvironment environment, string
6262
case AzureEnvironment.Endpoint.GraphEndpointResourceId:
6363
endpoint = new Uri(environment.GraphEndpointResourceId);
6464
break;
65+
case AzureEnvironment.Endpoint.DataLakeEndpointResourceId:
66+
endpoint = new Uri(environment.DataLakeEndpointResourceId);
67+
break;
6568
default:
6669
result = false;
6770
break;
@@ -220,6 +223,9 @@ public static void SetEndpoint(this IAzureEnvironment environment, string endpoi
220223
case AzureEnvironment.Endpoint.AzureDataLakeStoreFileSystemEndpointSuffix:
221224
environment.AzureDataLakeStoreFileSystemEndpointSuffix = propertyValue;
222225
break;
226+
case AzureEnvironment.Endpoint.DataLakeEndpointResourceId:
227+
environment.DataLakeEndpointResourceId = propertyValue;
228+
break;
223229
case AzureEnvironment.Endpoint.ActiveDirectory:
224230
environment.ActiveDirectoryAuthority = propertyValue;
225231
break;
@@ -258,6 +264,12 @@ public static string GetTokenAudience(this IAzureEnvironment environment, string
258264
{
259265
resource = AzureEnvironment.Endpoint.GraphEndpointResourceId;
260266
}
267+
else if (targetEndpoint == AzureEnvironment.Endpoint.AzureDataLakeAnalyticsCatalogAndJobEndpointSuffix ||
268+
targetEndpoint == AzureEnvironment.Endpoint.AzureDataLakeStoreFileSystemEndpointSuffix ||
269+
targetEndpoint == AzureEnvironment.Endpoint.DataLakeEndpointResourceId)
270+
{
271+
resource = AzureEnvironment.Endpoint.DataLakeEndpointResourceId;
272+
}
261273

262274
return resource;
263275
}

‎src/Common/Commands.Common.Authentication.Abstractions/Interfaces/IAzureEnvironment.cs

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -102,6 +102,11 @@ public interface IAzureEnvironment : IExtensibleModel
102102
/// </summary>
103103
string GraphEndpointResourceId { get; set; }
104104

105+
/// <summary>
106+
/// The token audience required to authenticate with the Azure Active Directory Data Lake services
107+
/// </summary>
108+
string DataLakeEndpointResourceId { get; set; }
109+
105110
/// <summary>
106111
/// The domain name suffix for Azure DataLake Catalog and Job services
107112
/// </summary>

‎src/ResourceManager/Common/Commands.ScenarioTests.ResourceManager.Common/ProfileClient.cs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -847,6 +847,7 @@ private IAzureEnvironment MergeEnvironmentProperties(IAzureEnvironment environme
847847
GraphEndpointResourceId = environment1.GraphEndpointResourceId ?? environment2.GraphEndpointResourceId,
848848
AdTenant = environment1.AdTenant ?? environment2.AdTenant,
849849
AzureDataLakeStoreFileSystemEndpointSuffix = environment1.AzureDataLakeStoreFileSystemEndpointSuffix ?? environment2.AzureDataLakeStoreFileSystemEndpointSuffix,
850+
DataLakeEndpointResourceId = environment1.DataLakeEndpointResourceId ?? environment2.DataLakeEndpointResourceId,
850851
AzureKeyVaultServiceEndpointResourceId = environment1.AzureKeyVaultServiceEndpointResourceId ?? environment2.AzureKeyVaultServiceEndpointResourceId,
851852
GraphUrl = environment1.GraphUrl ?? environment2.GraphUrl,
852853
ManagementPortalUrl = environment1.ManagementPortalUrl ?? environment2.ManagementPortalUrl,

‎src/ResourceManager/DataLakeAnalytics/AzureRM.DataLakeAnalytics.psd1

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -96,7 +96,13 @@ CmdletsToExport = 'Get-AzureRmDataLakeAnalyticsDataSource',
9696
'Add-AzureRmDataLakeAnalyticsFirewallRule',
9797
'Get-AzureRmDataLakeAnalyticsFirewallRule',
9898
'Set-AzureRmDataLakeAnalyticsFirewallRule',
99-
'Remove-AzureRmDataLakeAnalyticsFirewallRule'
99+
'Remove-AzureRmDataLakeAnalyticsFirewallRule',
100+
'New-AzureRmDataLakeAnalyticsComputePolicy',
101+
'Get-AzureRmDataLakeAnalyticsComputePolicy',
102+
'Update-AzureRmDataLakeAnalyticsComputePolicy',
103+
'Remove-AzureRmDataLakeAnalyticsComputePolicy',
104+
'Get-AzureRmDataLakeAnalyticsJobPipeline',
105+
'Get-AzureRmDataLakeAnalyticsJobRecurrence'
100106

101107
# Variables to export from this module
102108
# VariablesToExport = @()
@@ -112,7 +118,10 @@ AliasesToExport = 'Get-AdlAnalyticsDataSource', 'New-AdlCatalogCredential',
112118
'Get-AdlAnalyticsAccount', 'Submit-AdlJob', 'New-AdlAnalyticsAccount',
113119
'Remove-AdlAnalyticsAccount', 'Set-AdlAnalyticsAccount',
114120
'Add-AdlAnalyticsFirewallRule', 'Get-AdlAnalyticsFirewallRule',
115-
'Set-AdlAnalyticsFirewallRule', 'Remove-AdlAnalyticsFirewallRule'
121+
'Set-AdlAnalyticsFirewallRule', 'Remove-AdlAnalyticsFirewallRule',
122+
'New-AdlAnalyticsComputePolicy', 'Get-AdlAnalyticsComputePolicy',
123+
'Update-AdlAnalyticsComputePolicy', 'Remove-AdlAnalyticsComputePolicy',
124+
'Get-AdlJobPipeline', 'Get-AdlJobRecurrence'
116125

117126
# DSC resources to export from this module
118127
# DscResourcesToExport = @()

‎src/ResourceManager/DataLakeAnalytics/ChangeLog.md

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,18 @@
1818
- Additional information about change #1
1919
-->
2020
## Current Release
21-
21+
* Add support for Compute Policy CRUD through the following commands:
22+
- New-AzureRMDataLakeAnalyticsComputePolicy
23+
- Get-AzureRMDataLakeAnalyticsComputePolicy
24+
- Remove-AzureRMDataLakeAnalyticsComputePolicy
25+
- Update-AzureRMDataLakeAnalyticsComputePolicy
26+
* Add support for job relationship metadata for help with recurring jobs and job pipelines. The following commands were updated or added:
27+
- Submit-AzureRMDataLakeAnalyticsJob
28+
- Get-AzureRMDataLakeAnalyticsJob
29+
- Get-AzureRMDataLakeAnalyticsJobRecurrence
30+
- Get-AzureRMDataLakeAnalyticsJobPipeline
31+
* Updated the token audience for job and catalog APIs to use the correct Data Lake specific audience instead of the Azure Resource audience.
32+
2233
## Version 3.1.0
2334

2435
## Version 3.0.1

‎src/ResourceManager/DataLakeAnalytics/Commands.DataLakeAnalytics.Test/Commands.DataLakeAnalytics.Test.csproj

Lines changed: 20 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -62,13 +62,14 @@
6262
<HintPath>..\..\..\packages\Microsoft.Azure.Management.Authorization.2.0.0\lib\net40\Microsoft.Azure.Management.Authorization.dll</HintPath>
6363
<Private>True</Private>
6464
</Reference>
65-
<Reference Include="Microsoft.Azure.Management.DataLake.Analytics, Version=2.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
65+
<Reference Include="Microsoft.Azure.Management.DataLake.Analytics, Version=3.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
6666
<SpecificVersion>False</SpecificVersion>
67-
<HintPath>..\..\..\packages\Microsoft.Azure.Management.DataLake.Analytics.2.2.0-preview\lib\net452\Microsoft.Azure.Management.DataLake.Analytics.dll</HintPath>
67+
<HintPath>..\..\..\packages\Microsoft.Azure.Management.DataLake.Analytics.3.0.0\lib\net452\Microsoft.Azure.Management.DataLake.Analytics.dll</HintPath>
6868
<Private>True</Private>
6969
</Reference>
70-
<Reference Include="Microsoft.Azure.Management.DataLake.Store">
71-
<HintPath>..\..\..\packages\Microsoft.Azure.Management.DataLake.Store.2.1.2-preview\lib\net452\Microsoft.Azure.Management.DataLake.Store.dll</HintPath>
70+
<Reference Include="Microsoft.Azure.Management.DataLake.Store, Version=2.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
71+
<SpecificVersion>False</SpecificVersion>
72+
<HintPath>..\..\..\packages\Microsoft.Azure.Management.DataLake.Store.2.2.0\lib\net452\Microsoft.Azure.Management.DataLake.Store.dll</HintPath>
7273
<Private>True</Private>
7374
</Reference>
7475
<Reference Include="Microsoft.Azure.Management.ResourceManager, Version=1.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
@@ -103,12 +104,12 @@
103104
</Reference>
104105
<Reference Include="Microsoft.Rest.ClientRuntime, Version=2.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
105106
<SpecificVersion>False</SpecificVersion>
106-
<HintPath>..\..\..\packages\Microsoft.Rest.ClientRuntime.2.3.6\lib\net45\Microsoft.Rest.ClientRuntime.dll</HintPath>
107+
<HintPath>..\..\..\packages\Microsoft.Rest.ClientRuntime.2.3.8\lib\net452\Microsoft.Rest.ClientRuntime.dll</HintPath>
107108
<Private>True</Private>
108109
</Reference>
109110
<Reference Include="Microsoft.Rest.ClientRuntime.Azure, Version=3.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
110111
<SpecificVersion>False</SpecificVersion>
111-
<HintPath>..\..\..\packages\Microsoft.Rest.ClientRuntime.Azure.3.3.5\lib\net45\Microsoft.Rest.ClientRuntime.Azure.dll</HintPath>
112+
<HintPath>..\..\..\packages\Microsoft.Rest.ClientRuntime.Azure.3.3.7\lib\net452\Microsoft.Rest.ClientRuntime.Azure.dll</HintPath>
112113
<Private>True</Private>
113114
</Reference>
114115
<Reference Include="Microsoft.Rest.ClientRuntime.Azure.Authentication, Version=2.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
@@ -233,12 +234,18 @@
233234
<None Include="SessionRecords\Microsoft.Azure.Commands.DataLakeAnalytics.Test.ScenarioTests.AdlaAliasTests\TestAdlaCatalog.json">
234235
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
235236
</None>
237+
<None Include="SessionRecords\Microsoft.Azure.Commands.DataLakeAnalytics.Test.ScenarioTests.AdlaAliasTests\TestAdlaComputePolicy.json">
238+
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
239+
</None>
236240
<None Include="SessionRecords\Microsoft.Azure.Commands.DataLakeAnalytics.Test.ScenarioTests.AdlaAliasTests\TestAdlaFirewallRules.json">
237-
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
241+
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
238242
</None>
239243
<None Include="SessionRecords\Microsoft.Azure.Commands.DataLakeAnalytics.Test.ScenarioTests.AdlaAliasTests\TestAdlaJob.json">
240244
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
241245
</None>
246+
<None Include="SessionRecords\Microsoft.Azure.Commands.DataLakeAnalytics.Test.ScenarioTests.AdlaAliasTests\TestAdlaJobRelationships.json">
247+
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
248+
</None>
242249
<None Include="SessionRecords\Microsoft.Azure.Commands.DataLakeAnalytics.Test.ScenarioTests.AdlaAliasTests\TestNegativeAdlaAccount.json">
243250
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
244251
</None>
@@ -254,12 +261,18 @@
254261
<None Include="SessionRecords\Microsoft.Azure.Commands.DataLakeAnalytics.Test.ScenarioTests.AdlaTests\TestAdlaCatalog.json">
255262
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
256263
</None>
264+
<None Include="SessionRecords\Microsoft.Azure.Commands.DataLakeAnalytics.Test.ScenarioTests.AdlaTests\TestAdlaComputePolicy.json">
265+
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
266+
</None>
257267
<None Include="SessionRecords\Microsoft.Azure.Commands.DataLakeAnalytics.Test.ScenarioTests.AdlaTests\TestAdlaFirewallRules.json">
258268
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
259269
</None>
260270
<None Include="SessionRecords\Microsoft.Azure.Commands.DataLakeAnalytics.Test.ScenarioTests.AdlaTests\TestAdlaJob.json">
261271
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
262272
</None>
273+
<None Include="SessionRecords\Microsoft.Azure.Commands.DataLakeAnalytics.Test.ScenarioTests.AdlaTests\TestAdlaJobRelationships.json">
274+
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
275+
</None>
263276
<None Include="SessionRecords\Microsoft.Azure.Commands.DataLakeAnalytics.Test.ScenarioTests.AdlaTests\TestNegativeAdlaAccount.json">
264277
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
265278
</None>

‎src/ResourceManager/DataLakeAnalytics/Commands.DataLakeAnalytics.Test/ScenarioTests/AdlaAliasTests.cs

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,16 @@ public void TestAdlaFirewallRules()
5454
AdlaTestsBase.resourceGroupLocation));
5555
}
5656

57+
[Fact]
58+
[Trait(Category.AcceptanceType, Category.CheckIn)]
59+
public void TestAdlaComputePolicy()
60+
{
61+
AdlaTestsBase.NewInstance.RunPsTest(false,
62+
string.Format(
63+
"Test-DataLakeAnalyticsComputePolicy -location '{0}'",
64+
AdlaTestsBase.resourceGroupLocation));
65+
}
66+
5767
[Fact]
5868
[Trait(Category.AcceptanceType, Category.CheckIn)]
5969
public void TestAdlaCatalog()
@@ -74,6 +84,16 @@ public void TestAdlaJob()
7484
AdlaTestsBase.resourceGroupLocation));
7585
}
7686

87+
[Fact]
88+
[Trait(Category.AcceptanceType, Category.CheckIn)]
89+
public void TestAdlaJobRelationships()
90+
{
91+
AdlaTestsBase.NewInstance.RunPsTest(false,
92+
string.Format(
93+
"Test-DataLakeAnalyticsJobRelationships -location '{0}'",
94+
AdlaTestsBase.resourceGroupLocation));
95+
}
96+
7797
[Fact]
7898
[Trait(Category.AcceptanceType, Category.CheckIn)]
7999
public void TestNegativeAdlaAccount()

0 commit comments

Comments
 (0)