Skip to content

Commit 04ae7cd

Browse files
markcowlmarkcowl
authored
Merge pull request #8945 from Azure/task-add-httpclientsdk-to-az
Update the ADLS sdk to use httpclient, integrate with azure framework
2 parents 563486e + e343297 commit 04ae7cd

29 files changed

+33305
-14226
lines changed

‎src/DataLakeStore/DataLakeStore.Test/ScenarioTests/AdlMockContext.cs

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
using Microsoft.Azure.Test.HttpRecorder;
2+
using Microsoft.Rest.ClientRuntime.Azure.TestFramework;
3+
using System.Net.Http;
4+
5+
namespace Microsoft.Azure.Commands.DataLake.Test.ScenarioTests
6+
{
7+
/// <summary>
8+
/// This class is needed because MockContext does not expose AddHandlers api. And we need to call
9+
/// that api to get the mock handlers and pass in to our ADLS client
10+
/// </summary>
11+
public class AdlMockContext : MockContext
12+
{
13+
public new static MockContext Start(
14+
string className,
15+
[System.Runtime.CompilerServices.CallerMemberName]
16+
string methodName= "testframework_failed")
17+
{
18+
var context = new AdlMockContext();
19+
if (HttpMockServer.FileSystemUtilsObject == null)
20+
{
21+
HttpMockServer.FileSystemUtilsObject = new Microsoft.Azure.Test.HttpRecorder.FileSystemUtils();
22+
}
23+
HttpMockServer.Initialize(className, methodName);
24+
if (HttpMockServer.Mode != HttpRecorderMode.Playback)
25+
{
26+
context.disposed = false;
27+
}
28+
29+
return context;
30+
}
31+
32+
public DelegatingHandler[] GetDelegatingHAndlersForDataPlane(TestEnvironment currentEnvironment,
33+
params DelegatingHandler[] existingHandlers)
34+
{
35+
return base.AddHandlers(currentEnvironment, existingHandlers);
36+
}
37+
}
38+
}

‎src/DataLakeStore/DataLakeStore.Test/ScenarioTests/AdlMockDelegatingHandler.cs

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
using Newtonsoft.Json;
2+
using System.Net.Http;
3+
using System.Threading;
4+
using System.Threading.Tasks;
5+
6+
namespace Microsoft.Azure.Commands.DataLake.Test.ScenarioTests
7+
{
8+
/// <summary>
9+
/// Mock handler for azure test framework will read the output from httpclienthandler
10+
/// and indent them while writing them to json files. This is needed to non-indent the output of
11+
/// httpresponsemessage that we get from their mock handler
12+
/// </summary>
13+
public class AdlMockDelegatingHandler : DelegatingHandler
14+
{
15+
public static bool IsJson(string content)
16+
{
17+
content = content.Trim();
18+
return content.StartsWith("{") && content.EndsWith("}")
19+
|| content.StartsWith("[") && content.EndsWith("]");
20+
}
21+
protected override async Task<HttpResponseMessage> SendAsync(HttpRequestMessage request,
22+
CancellationToken cancellationToken)
23+
{
24+
var response = await base.SendAsync(request, cancellationToken).ConfigureAwait(false);
25+
26+
var str = await response.Content.ReadAsStringAsync();
27+
if (IsJson(str))
28+
{
29+
try
30+
{
31+
object parsedJson = JsonConvert.DeserializeObject(str);
32+
response.Content = new StringContent(JsonConvert.SerializeObject(parsedJson));
33+
return response;
34+
}
35+
catch
36+
{
37+
// can't parse JSON, return the original string
38+
return response;
39+
}
40+
}
41+
42+
return response;
43+
}
44+
}
45+
}

‎src/DataLakeStore/DataLakeStore.Test/ScenarioTests/AdlsAliasTests.cs

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,8 @@ namespace Microsoft.Azure.Commands.DataLakeStore.Test.ScenarioTests
1818
{
1919
using Microsoft.WindowsAzure.Commands.ScenarioTest;
2020
using ServiceManagement.Common.Models;
21+
using System.IO;
22+
using System.Reflection;
2123
using Xunit;
2224

2325
public class AdlsAliasTests : AdlsTestsBase
@@ -69,7 +71,9 @@ public void TestAdlsTrustedIdProvider()
6971
[Trait(Category.AcceptanceType, Category.CheckIn)]
7072
public void TestAdlsFileSystem()
7173
{
72-
NewInstance.RunPsTest(_logger, string.Format("Test-DataLakeStoreFileSystem -fileToCopy '{0}' -location '{1}'", ".\\ScenarioTests\\" + this.GetType().Name + ".ps1", AdlsTestsBase.ResourceGroupLocation));
74+
var workingPath = Path.GetDirectoryName(new Uri(Assembly.GetExecutingAssembly().CodeBase).AbsolutePath);
75+
var testLocation = Path.Combine(workingPath, "ScenarioTests", (this.GetType().Name + ".ps1"));
76+
NewInstance.RunPsTest(_logger, string.Format("Test-DataLakeStoreFileSystem -fileToCopy '{0}' -location '{1}'", testLocation, AdlsTestsBase.ResourceGroupLocation));
7377
}
7478

7579
[Fact]

‎src/DataLakeStore/DataLakeStore.Test/ScenarioTests/AdlsAliasTests.ps1

Lines changed: 28 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -502,7 +502,7 @@ function Test-DataLakeStoreFileSystem
502502
$resourceGroupName = Get-ResourceGroupName
503503
$accountName = Get-DataLakeStoreAccountName
504504
New-AzResourceGroup -Name $resourceGroupName -Location $location
505-
$accountCreated = New-AdlStore -ResourceGroupName $resourceGroupName -Name $accountName -Location $location
505+
$accountCreated = New-AdlStore -ResourceGroupName $resourceGroupName -Name $accountName -Location $location -DisableEncryption
506506

507507
Assert-AreEqual $accountName $accountCreated.Name
508508
Assert-AreEqual $location $accountCreated.Location
@@ -701,17 +701,16 @@ function Test-DataLakeStoreFileSystem
701701
Assert-AreEqual $result.FileCount 1
702702

703703
# Export DiskUsage
704-
$targetFile = "/DuOutputFile"
705-
Export-AdlStoreChildItemProperties -Account $accountName -Path $summaryFolder -OutputPath $targetFile -GetDiskUsage -IncludeFile -SaveToAdl
706-
$result = Get-AzDataLakeStoreItem -Account $accountName -path $targetFile
704+
$targetFile = Join-Path $currentDir "DuOutput"
705+
Export-AdlStoreChildItemProperties -Account $accountName -Path $summaryFolder -OutputPath $targetFile -GetDiskUsage -IncludeFile
706+
$result = Get-Item -Path $targetFile
707707
Assert-NotNull $result "No file was created on export properties"
708+
Remove-Item -Path $targetFile
708709

709710
# delete a file
710711
Assert-True {Remove-AdlStoreItem -Account $accountName -paths "$moveFolder/movefile.txt" -force -passthru } "Remove File Failed"
711712
Assert-Throws {Get-AdlStoreItem -Account $accountName -path $moveFile}
712-
Assert-True {Remove-AdlStoreItem -Account $accountName -paths $targetFile -force -passthru } "Remove File Failed"
713-
Assert-Throws {Get-AdlStoreItem -Account $accountName -path $targetFile}
714-
713+
715714
# delete a folder
716715
Assert-True {Remove-AdlStoreItem -Account $accountName -paths $moveFolder -force -recurse -passthru} "Remove folder failed"
717716
Assert-Throws {Get-AdlStoreItem -Account $accountName -path $moveFolder}
@@ -755,7 +754,7 @@ function Test-DataLakeStoreFileSystemPermissions
755754
$resourceGroupName = Get-ResourceGroupName
756755
$accountName = Get-DataLakeStoreAccountName
757756
New-AzResourceGroup -Name $resourceGroupName -Location $location
758-
$accountCreated = New-AdlStore -ResourceGroupName $resourceGroupName -Name $accountName -Location $location
757+
$accountCreated = New-AdlStore -ResourceGroupName $resourceGroupName -Name $accountName -Location $location -DisableEncryption
759758

760759
Assert-AreEqual $accountName $accountCreated.Name
761760
Assert-AreEqual $location $accountCreated.Location
@@ -857,10 +856,11 @@ function Test-DataLakeStoreFileSystemPermissions
857856
Assert-AreEqual $($currentCount+1) $result.Count
858857

859858
# Export Acl
860-
$targetFile = "/aclOutputFile"
861-
Export-AdlStoreChildItemProperties -Account $accountName -Path "/" -OutputPath $targetFile -GetAcl -IncludeFile -SaveToAdl
862-
$result = Get-AzDataLakeStoreItem -Account $accountName -path $targetFile
859+
$targetFile = "./ScenarioTests/acloutput"
860+
Export-AdlStoreChildItemProperties -Account $accountName -Path "/" -OutputPath $targetFile -GetAcl -IncludeFile
861+
$result = Get-Item -Path $targetFile
863862
Assert-NotNull $result "No file was created on export properties"
863+
Remove-Item -Path $targetFile
864864

865865
#Recursive Acl remove
866866
Remove-AdlStoreItemAclEntry -Account $accountName -path "/" -AceType User -Id $aceUserId -Recurse
@@ -1014,8 +1014,7 @@ function Test-AdlsEnumerateAndRestoreDeletedItem
10141014
$resourceGroupName = Get-ResourceGroupName
10151015
$accountName = Get-DataLakeStoreAccountName
10161016
New-AzureRmResourceGroup -Name $resourceGroupName -Location $location
1017-
$accountCreated = New-AzureRMDataLakeStoreAccount -ResourceGroupName $resourceGroupName -Name $accountName -Location $location
1018-
1017+
$accountCreated = New-AdlStore -ResourceGroupName $resourceGroupName -Name $accountName -Location $location
10191018
Assert-AreEqual $accountName $accountCreated.Name
10201019
Assert-AreEqual $location $accountCreated.Location
10211020
Assert-AreEqual "Microsoft.DataLakeStore/accounts" $accountCreated.Type
@@ -1024,7 +1023,7 @@ function Test-AdlsEnumerateAndRestoreDeletedItem
10241023
# In loop to check if account exists
10251024
for ($i = 0; $i -le 60; $i++)
10261025
{
1027-
[array]$accountGet = Get-AzureRMDataLakeStoreAccount -ResourceGroupName $resourceGroupName -Name $accountName
1026+
[array]$accountGet = Get-AdlStore -ResourceGroupName $resourceGroupName -Name $accountName
10281027
if ($accountGet[0].ProvisioningState -like "Succeeded")
10291028
{
10301029
Assert-AreEqual $accountName $accountGet[0].Name
@@ -1040,22 +1039,23 @@ function Test-AdlsEnumerateAndRestoreDeletedItem
10401039
}
10411040

10421041
# define all the files and folders
1043-
$folderToCreate1 = "/adlfolder1"
1044-
$folderToCreate2 = "/adlfolder2"
1045-
$fileToCreate1 = "/adlfolder1/adlfile1"
1046-
$fileToCreate2 = "/adlfolder2/adlfile2"
1042+
# define all the files and folders
1043+
$folderToCreate1 = "/adlfolderTest1"
1044+
$folderToCreate2 = "/adlfolderTest2"
1045+
$fileToCreate1 = "/adlfolderTest1/adlfile1"
1046+
$fileToCreate2 = "/adlfolderTest2/adlfile2"
10471047

10481048
# Create and get Empty folder
1049-
$result = New-AzureRMDataLakeStoreItem -Account $accountName -path $folderToCreate1 -Folder
1049+
$result = New-AdlStoreItem -Account $accountName -path $folderToCreate1 -Folder
10501050
Assert-NotNull $result "No value was returned on folder creation"
10511051

1052-
$result = New-AzureRMDataLakeStoreItem -Account $accountName -path $folderToCreate2 -Folder
1052+
$result = New-AdlStoreItem -Account $accountName -path $folderToCreate2 -Folder
10531053
Assert-NotNull $result "No value was returned on folder creation"
10541054

10551055
# Create and get Empty File
1056-
$result = New-AzureRMDataLakeStoreItem -Account $accountName -path $fileToCreate1
1056+
$result = New-AdlStoreItem -Account $accountName -path $fileToCreate1
10571057
Assert-NotNull $result "No value was returned on empty file creation"
1058-
$result = New-AzureRMDataLakeStoreItem -Account $accountName -path $fileToCreate2
1058+
$result = New-AdlStoreItem -Account $accountName -path $fileToCreate2
10591059
Assert-NotNull $result "No value was returned on empty file creation"
10601060

10611061
# delete a file
@@ -1065,28 +1065,28 @@ function Test-AdlsEnumerateAndRestoreDeletedItem
10651065
Assert-Throws {Get-AdlStoreItem -Account $accountName -path $fileToCreate2}
10661066

10671067
# search delete folder
1068-
$out = Get-AdlStoreDeletedItem -Account $accountName -filter "adlfolder1" -Count 1000
1068+
$out = Get-AdlStoreDeletedItem -Account $accountName -filter "adlfolderTest1" -Count 1000
10691069
foreach($item in $out)
10701070
{
10711071
Assert-True { Restore-AdlStoreDeletedItem -Account $accountName -Path $item.TrashDirPath -Destination $item.OriginalPath -Type "file" -Force -Passthru}
10721072
}
10731073

1074-
$out = Get-AdlStoreDeletedItem -Account $accountName -filter "adlfolder2" -Count 1000
1074+
$out = Get-AdlStoreDeletedItem -Account $accountName -filter "adlfolderTest2" -Count 1000
10751075
foreach($item in $out)
10761076
{
10771077
Assert-True { Restore-AdlStoreDeletedItem -Account $accountName $item -Force -Passthru}
10781078
}
10791079

10801080
# Delete Data Lake account
1081-
Assert-True {Remove-AzureRMDataLakeStoreAccount -ResourceGroupName $resourceGroupName -Name $accountName -Force -PassThru} "Remove Account failed."
1081+
Assert-True {Remove-AdlStore -ResourceGroupName $resourceGroupName -Name $accountName -Force -PassThru} "Remove Account failed."
10821082

10831083
# Verify that it is gone by trying to get it again
1084-
Assert-Throws {Get-AzureRMDataLakeStoreAccount -ResourceGroupName $resourceGroupName -Name $accountName}
1084+
Assert-Throws {Get-AdlStore -ResourceGroupName $resourceGroupName -Name $accountName}
10851085
}
10861086
finally
10871087
{
10881088
# cleanup the resource group that was used in case it still exists. This is a best effort task, we ignore failures here.
1089-
Invoke-HandledCmdlet -Command {Remove-AzureRMDataLakeStoreAccount -ResourceGroupName $resourceGroupName -Name $accountName -Force -ErrorAction SilentlyContinue} -IgnoreFailures
1090-
Invoke-HandledCmdlet -Command {Remove-AzureRmResourceGroup -Name $resourceGroupName -Force -ErrorAction SilentlyContinue} -IgnoreFailures
1089+
Invoke-HandledCmdlet -Command {Remove-AdlStore -ResourceGroupName $resourceGroupName -Name $accountName -Force -ErrorAction SilentlyContinue} -IgnoreFailures
1090+
Invoke-HandledCmdlet -Command {Remove-AzResourceGroup -Name $resourceGroupName -Force -ErrorAction SilentlyContinue} -IgnoreFailures
10911091
}
10921092
}

‎src/DataLakeStore/DataLakeStore.Test/ScenarioTests/AdlsTests.cs

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,10 @@ namespace Microsoft.Azure.Commands.DataLakeStore.Test.ScenarioTests
1616
{
1717
using Microsoft.WindowsAzure.Commands.ScenarioTest;
1818
using ServiceManagement.Common.Models;
19+
using System;
20+
using System.Reflection;
1921
using Xunit;
22+
using System.IO;
2023

2124
public class AdlsTests : AdlsTestsBase
2225
{
@@ -67,7 +70,9 @@ public void TestAdlsAccountTiers()
6770
[Trait(Category.AcceptanceType, Category.CheckIn)]
6871
public void TestAdlsFileSystem()
6972
{
70-
NewInstance.RunPsTest(_logger, string.Format("Test-DataLakeStoreFileSystem -fileToCopy '{0}' -location '{1}'", ".\\ScenarioTests\\" + this.GetType().Name + ".ps1", AdlsTestsBase.ResourceGroupLocation));
73+
var workingPath = Path.GetDirectoryName(new Uri(Assembly.GetExecutingAssembly().CodeBase).AbsolutePath);
74+
var testLocation = Path.Combine(workingPath, "ScenarioTests", (this.GetType().Name + ".ps1"));
75+
NewInstance.RunPsTest(_logger, string.Format("Test-DataLakeStoreFileSystem -fileToCopy '{0}' -location '{1}'", testLocation, AdlsTestsBase.ResourceGroupLocation));
7176
}
7277

7378
[Fact]

0 commit comments

Comments
 (0)