Skip to content

Commit 221534c

Browse files
cormacpaynecormacpayne
authored
Merge pull request #5860 from cormacpayne/adls-data-plane
Merge adls-data-plane into preview
2 parents d1e9b73 + edb8e6b commit 221534c

File tree

73 files changed

+1717
-1499
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

73 files changed

+1717
-1499
lines changed

‎src/ResourceManager/Common/Commands.ScenarioTests.ResourceManager.Common/Assert.ps1

Lines changed: 22 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -277,7 +277,28 @@ function Assert-AreEqual
277277

278278
return $true
279279
}
280-
280+
###################
281+
#
282+
# Verify that if actual falls in an acceptable range of expected
283+
#
284+
# param [long] $expected : The expected number
285+
# param [long] $actual : The actual number
286+
# param [long] $interval : The acceptable offset either side of the expected
287+
# param [string] $message : The message to return if the given objects are not equal
288+
####################
289+
function Assert-NumAreInRange
290+
{
291+
param([long] $expected, [long] $actual, [long] $interval, [string] $message)
292+
if (!$message)
293+
{
294+
$message = "Assertion failed because expected '$expected' does not fall in accepted range of 'interval' of actual '$actual'"
295+
}
296+
if(!($actual -ge ($expected-$interval) -and $actual -le ($expected+$interval)))
297+
{
298+
throw $message
299+
}
300+
return $true
301+
}
281302
###################
282303
#
283304
# Verify that two given arrays are equal

‎src/ResourceManager/DataLakeStore/AzureRM.DataLakeStore.Netcore.psd1

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -54,7 +54,7 @@ PowerShellVersion = '5.1'
5454
RequiredModules = @(@{ModuleName = 'AzureRM.Profile.Netcore'; ModuleVersion = '0.10.0'; })
5555

5656
# Assemblies that must be loaded prior to importing this module
57-
RequiredAssemblies = '.\Microsoft.Azure.Management.DataLake.Store.dll'
57+
RequiredAssemblies = '.\Microsoft.Azure.Management.DataLake.Store.dll','.\Microsoft.Azure.DataLake.Store.dll','.\NLog.dll'
5858

5959
# Script files (.ps1) that are run in the caller's environment prior to importing this module.
6060
# ScriptsToProcess = @()

‎src/ResourceManager/DataLakeStore/AzureRM.DataLakeStore.psd1

Lines changed: 5 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -54,7 +54,7 @@ CLRVersion = '4.0'
5454
RequiredModules = @(@{ModuleName = 'AzureRM.Profile'; ModuleVersion = '4.5.0'; })
5555

5656
# Assemblies that must be loaded prior to importing this module
57-
RequiredAssemblies = '.\Microsoft.Azure.Management.DataLake.Store.dll'
57+
RequiredAssemblies = '.\Microsoft.Azure.Management.DataLake.Store.dll','.\Microsoft.Azure.DataLake.Store.dll','.\NLog.dll'
5858

5959
# Script files (.ps1) that are run in the caller's environment prior to importing this module.
6060
# ScriptsToProcess = @()
@@ -63,8 +63,7 @@ RequiredAssemblies = '.\Microsoft.Azure.Management.DataLake.Store.dll'
6363
# TypesToProcess = @()
6464

6565
# Format files (.ps1xml) to be loaded when importing this module
66-
FormatsToProcess =
67-
'.\Microsoft.Azure.Commands.DataLakeStoreFileSystem.format.ps1xml'
66+
FormatsToProcess = '.\Microsoft.Azure.Commands.DataLakeStoreFileSystem.format.ps1xml'
6867

6968
# Modules to import as nested modules of the module specified in RootModule/ModuleToProcess
7069
NestedModules = @('.\Microsoft.Azure.Commands.DataLakeStore.dll')
@@ -154,10 +153,10 @@ PrivateData = @{
154153
# ReleaseNotes of this module
155154
ReleaseNotes = '* Corrected usage of ''Login-AzureRmAccount'' to use ''Connect-AzureRmAccount''
156155
* Corrected the error message of ''Test-AzureRmDataLakeStoreAccount'' when running this cmdlet without having logged in with ''Login-AzureRmAccount'''
157-
158-
# Prerelease string of this module
156+
157+
# Prerelease string of this module
159158
# Prerelease = ''
160-
159+
161160
# Flag to indicate whether the module requires explicit user acceptance for install/update
162161
# RequireLicenseAcceptance = $false
163162

‎src/ResourceManager/DataLakeStore/ChangeLog.md

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,13 @@
1919
-->
2020
## Current Release
2121
* Updated to the latest version of the Azure ClientRuntime
22+
* Add debug functionality
23+
* Update the version of the ADLS dataplane SDK to 1.1.2
24+
* Export-AzureRmDataLakeStoreItem (https://github.com/Azure/azure-powershell/blob/adls-data-plane/src/ResourceManager/DataLakeStore/documentation/upcoming-breaking-changes.md) - Deprecated parameters PerFileThreadCount, ConcurrentFileCount and introduced parameter Concurrency
25+
* Import-AzureRMDataLakeStoreItem (https://github.com/Azure/azure-powershell/blob/adls-data-plane/src/ResourceManager/DataLakeStore/documentation/upcoming-breaking-changes.md) -Deprecated parametersPerFileThreadCount, ConcurrentFileCount and introduced parameter Concurrency
26+
* Get-AzureRMDataLakeStoreItemContent - Fixed the tail behavior for contents greater than 4MB
27+
* Set-AzureRMDataLakeStoreItemExpiry - Introduced new parameter set SetRelativeExpiry for setting relative expiration time
28+
* Remove-AzureRmDataLakeStoreItem (https://github.com/Azure/azure-powershell/blob/adls-data-plane/src/ResourceManager/DataLakeStore/documentation/upcoming-breaking-changes.md) - Deprecated parameter Clean.
2229

2330
## Version 5.1.1
2431
* Corrected usage of 'Login-AzureRmAccount' to use 'Connect-AzureRmAccount'

‎src/ResourceManager/DataLakeStore/Commands.DataLakeStore.Test/Commands.DataLakeStore.Test.csproj

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -188,6 +188,10 @@
188188
<Project>{5ee72c53-1720-4309-b54b-5fb79703195f}</Project>
189189
<Name>Commands.Common</Name>
190190
</ProjectReference>
191+
<ProjectReference Include="..\Commands.DataLakeStore\Commands.DataLakeStore.csproj">
192+
<Project>{8aab43e6-e8f6-4f91-af8a-6a63cd78ef1e}</Project>
193+
<Name>Commands.DataLakeStore</Name>
194+
</ProjectReference>
191195
</ItemGroup>
192196
<ItemGroup>
193197
<None Include="MSSharedLibKey.snk" />

‎src/ResourceManager/DataLakeStore/Commands.DataLakeStore.Test/ScenarioTests/AdlsAliasTests.cs

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,8 @@
1212
// limitations under the License.
1313
// ----------------------------------------------------------------------------------
1414

15+
using System;
16+
1517
namespace Microsoft.Azure.Commands.DataLakeStore.Test.ScenarioTests
1618
{
1719
using Microsoft.WindowsAzure.Commands.ScenarioTest;

‎src/ResourceManager/DataLakeStore/Commands.DataLakeStore.Test/ScenarioTests/AdlsAliasTests.ps1

Lines changed: 28 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -424,38 +424,46 @@ function Test-DataLakeStoreFileSystem
424424
$result = Get-AdlStoreItem -Account $accountName -path $folderToCreate
425425
Assert-NotNull $result "No value was returned on folder get"
426426
Assert-AreEqual "Directory" $result.Type
427+
427428
# Create and get Empty File
428429
$result = New-AdlStoreItem -Account $accountName -path $emptyFilePath
429430
Assert-NotNull $result "No value was returned on empty file creation"
430431
$result = Get-AdlStoreItem -Account $accountName -path $emptyFilePath
432+
$emptyFileCreationDate=$result.LastWriteTime # To be used later
431433
Assert-NotNull $result "No value was returned on empty file get"
432434
Assert-AreEqual "File" $result.Type
433435
Assert-AreEqual 0 $result.Length
436+
434437
# Create and get file with content
435438
$result = New-AdlStoreItem -Account $accountName -path $contentFilePath -Value $content
436439
Assert-NotNull $result "No value was returned on content file creation"
437440
$result = Get-AdlStoreItem -Account $accountName -path $contentFilePath
438441
Assert-NotNull $result "No value was returned on content file get"
439442
Assert-AreEqual "File" $result.Type
440443
Assert-AreEqual $content.length $result.Length
441-
# set and validate expiration for a file
444+
445+
# set absolute expiration for content file
442446
Assert-True {253402300800000 -ge $result.ExpirationTime -or 0 -le $result.ExpirationTime} # validate that expiration is currently max value
443447
[DateTimeOffset]$timeToUse = [Microsoft.Azure.Test.HttpRecorder.HttpMockServer]::GetVariable("absoluteTime", [DateTimeOffset]::UtcNow.AddSeconds(120))
444448
$result = Set-AdlStoreItemExpiry -Account $accountName -path $contentFilePath -Expiration $timeToUse
445-
Assert-AreEqual $timeToUse.UtcTicks $result.Expiration.UtcTicks
449+
Assert-NumAreInRange $timeToUse.UtcTicks $result.Expiration.UtcTicks 500000 # range of 50 milliseconds
450+
446451
# set it back to "never expire"
447452
$result = Set-AdlStoreItemExpiry -Account $accountName -path $contentFilePath
448453
Assert-True {253402300800000 -ge $result.ExpirationTime -or 0 -le $result.ExpirationTime} # validate that expiration is currently max value
454+
449455
# list files
450456
$result = Get-AdlStoreChildItem -Account $accountName -path $folderToCreate
451457
Assert-NotNull $result "No value was returned on folder list"
452458
Assert-AreEqual 2 $result.length
459+
453460
# add content to empty file
454461
Add-AdlStoreItemContent -Account $accountName -Path $emptyFilePath -Value $content
455462
$result = Get-AdlStoreItem -Account $accountName -path $emptyFilePath
456463
Assert-NotNull $result "No value was returned on empty file get with content added"
457464
Assert-AreEqual "File" $result.Type
458465
Assert-AreEqual $content.length $result.Length
466+
459467
# concat files
460468
$result = Join-AdlStoreItem -Account $accountName -Paths $emptyFilePath,$contentFilePath -Destination $concatFile
461469
Assert-NotNull $result "No value was returned on concat file"
@@ -506,6 +514,7 @@ function Test-DataLakeStoreFileSystem
506514
Assert-NotNull $result "No value was returned on import file get"
507515
Assert-AreEqual "File" $result.Type
508516
Assert-AreEqual $localFileInfo.length $result.Length
517+
509518
# download file
510519
$currentDir = Split-Path $fileToCopy
511520
$targetFile = Join-Path $currentDir "adlspstestdownload.txt"
@@ -527,6 +536,7 @@ function Test-DataLakeStoreFileSystem
527536
Assert-AreEqual "File" $result.Type
528537
Assert-AreEqual $($content.length*2) $result.Length
529538
Assert-Throws {Get-AdlStoreItem -Account $accountName -path $concatFile}
539+
530540
# move a folder
531541
$result = Move-AdlStoreItem -Account $accountName -Path $folderToCreate -Destination $moveFolder
532542
Assert-NotNull $result "No value was returned on move folder"
@@ -535,9 +545,11 @@ function Test-DataLakeStoreFileSystem
535545
Assert-AreEqual "Directory" $result.Type
536546
Assert-AreEqual 0 $result.Length
537547
Assert-Throws {Get-AdlStoreItem -Account $accountName -path $folderToCreate}
548+
538549
# delete a file
539550
Assert-True {Remove-AdlStoreItem -Account $accountName -paths "$moveFolder/movefile.txt" -force -passthru } "Remove File Failed"
540551
Assert-Throws {Get-AdlStoreItem -Account $accountName -path $moveFile}
552+
541553
# delete a folder
542554
Assert-True {Remove-AdlStoreItem -Account $accountName -paths $moveFolder -force -recurse -passthru} "Remove folder failed"
543555
Assert-Throws {Get-AdlStoreItem -Account $accountName -path $moveFolder}
@@ -601,6 +613,17 @@ function Test-DataLakeStoreFileSystemPermissions
601613
# define the permissions to add/remove
602614
$aceUserId = "027c28d5-c91d-49f0-98c5-d10134b169b3"
603615

616+
#set owner
617+
New-AdlStoreItem -Account $accountName -Path "/temp"
618+
$prevOwner=Get-AdlStoreItemOwner -Account $accountName -Path "/temp" -Type User
619+
$prevGroup=Get-AdlStoreItemOwner -Account $accountName -Path "/temp" -Type Group
620+
$currentOwner=Set-AdlStoreItemOwner -Account $accountName -Path "/temp" -Type User -Id $aceUserId -PassThru
621+
$currentGroup=Get-AdlStoreItemOwner -Account $accountName -Path "/temp" -Type Group
622+
Assert-AreEqual $aceUserId $currentOwner
623+
Assert-AreNotEqual $prevOwner $currentOwner
624+
Assert-AreEqual $prevGroup $currentGroup
625+
Remove-AdlStoreItem -Account $accountName -paths "/temp" -force
626+
604627
# Set and get all the permissions
605628
$result = Get-AdlStoreItemAclEntry -Account $accountName -path "/"
606629
Assert-NotNull $result "Did not get any result from ACL get"
@@ -635,14 +658,17 @@ function Test-DataLakeStoreFileSystemPermissions
635658
Set-AdlStoreItemAclEntry -Account $accountName -path "/" -AceType User -Id $aceUserId -Permissions All
636659
$result = Get-AdlStoreItemAclEntry -Account $accountName -path "/"
637660
Assert-AreEqual $($currentCount+1) $result.Count
661+
638662
# remove a specific permission with friendly remove
639663
Remove-AdlStoreItemAclEntry -Account $accountName -path "/" -AceType User -Id $aceUserId
640664
$result = Get-AdlStoreItemAclEntry -Account $accountName -path "/"
641665
Assert-AreEqual $($currentCount) $result.Count
666+
642667
# set and get a specific permission with the ACE string
643668
Set-AdlStoreItemAclEntry -Account $accountName -path "/" -Acl $([string]::Format("user:{0}:rwx", $aceUserId))
644669
$result = Get-AdlStoreItemAclEntry -Account $accountName -path "/"
645670
Assert-AreEqual $($currentCount+1) $result.Count
671+
646672
# remove a specific permission with the ACE string
647673
Remove-AdlStoreItemAclEntry -Account $accountName -path "/" -Acl $([string]::Format("user:{0}:---", $aceUserId))
648674
$result = Get-AdlStoreItemAclEntry -Account $accountName -path "/"

‎src/ResourceManager/DataLakeStore/Commands.DataLakeStore.Test/ScenarioTests/AdlsTests.ps1

Lines changed: 31 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -423,38 +423,46 @@ function Test-DataLakeStoreFileSystem
423423
$result = Get-AzureRMDataLakeStoreItem -Account $accountName -path $folderToCreate
424424
Assert-NotNull $result "No value was returned on folder get"
425425
Assert-AreEqual "Directory" $result.Type
426+
426427
# Create and get Empty File
427428
$result = New-AzureRMDataLakeStoreItem -Account $accountName -path $emptyFilePath
428429
Assert-NotNull $result "No value was returned on empty file creation"
429430
$result = Get-AzureRMDataLakeStoreItem -Account $accountName -path $emptyFilePath
431+
$emptyFileCreationDate=$result.LastWriteTime # To be used later
430432
Assert-NotNull $result "No value was returned on empty file get"
431433
Assert-AreEqual "File" $result.Type
432434
Assert-AreEqual 0 $result.Length
435+
433436
# Create and get file with content
434437
$result = New-AzureRMDataLakeStoreItem -Account $accountName -path $contentFilePath -Value $content
435438
Assert-NotNull $result "No value was returned on content file creation"
436439
$result = Get-AzureRMDataLakeStoreItem -Account $accountName -path $contentFilePath
437440
Assert-NotNull $result "No value was returned on content file get"
438441
Assert-AreEqual "File" $result.Type
439442
Assert-AreEqual $content.length $result.Length
440-
# set and validate expiration for a file
443+
444+
# set absolute expiration for content file
441445
Assert-True {253402300800000 -ge $result.ExpirationTime -or 0 -le $result.ExpirationTime} # validate that expiration is currently max value
442446
[DateTimeOffset]$timeToUse = [Microsoft.Azure.Test.HttpRecorder.HttpMockServer]::GetVariable("absoluteTime", [DateTimeOffset]::UtcNow.AddSeconds(120))
443-
$result = Set-AzureRmDataLakeStoreItemExpiry -Account $accountName -path $contentFilePath -Expiration $timeToUse
444-
Assert-AreEqual $timeToUse.UtcTicks $result.Expiration.UtcTicks
447+
$result = Set-AdlStoreItemExpiry -Account $accountName -path $contentFilePath -Expiration $timeToUse
448+
Assert-NumAreInRange $timeToUse.UtcTicks $result.Expiration.UtcTicks 500000 # range of 50 milliseconds
449+
445450
# set it back to "never expire"
446-
$result = Set-AzureRmDataLakeStoreItemExpiry -Account $accountName -path $contentFilePath
451+
$result = Set-AdlStoreItemExpiry -Account $accountName -path $contentFilePath
447452
Assert-True {253402300800000 -ge $result.ExpirationTime -or 0 -le $result.ExpirationTime} # validate that expiration is currently max value
453+
448454
# list files
449455
$result = Get-AzureRMDataLakeStoreChildItem -Account $accountName -path $folderToCreate
450456
Assert-NotNull $result "No value was returned on folder list"
451457
Assert-AreEqual 2 $result.length
458+
452459
# add content to empty file
453460
Add-AzureRMDataLakeStoreItemContent -Account $accountName -Path $emptyFilePath -Value $content
454461
$result = Get-AzureRMDataLakeStoreItem -Account $accountName -path $emptyFilePath
455462
Assert-NotNull $result "No value was returned on empty file get with content added"
456463
Assert-AreEqual "File" $result.Type
457464
Assert-AreEqual $content.length $result.Length
465+
458466
# concat files
459467
$result = Join-AzureRMDataLakeStoreItem -Account $accountName -Paths $emptyFilePath,$contentFilePath -Destination $concatFile
460468
Assert-NotNull $result "No value was returned on concat file"
@@ -505,6 +513,7 @@ function Test-DataLakeStoreFileSystem
505513
Assert-NotNull $result "No value was returned on import file get"
506514
Assert-AreEqual "File" $result.Type
507515
Assert-AreEqual $localFileInfo.length $result.Length
516+
508517
# download file
509518
$currentDir = Split-Path $fileToCopy
510519
$targetFile = Join-Path $currentDir "adlspstestdownload.txt"
@@ -517,7 +526,7 @@ function Test-DataLakeStoreFileSystem
517526
$downloadedFileInfo = Get-ChildItem $targetFile
518527
Assert-AreEqual $($content.length*2) $downloadedFileInfo.length
519528
Remove-Item -path $targetFile -force -confirm:$false
520-
529+
521530
# move a file
522531
$result = Move-AzureRMDataLakeStoreItem -Account $accountName -Path $concatFile -Destination $moveFile
523532
Assert-NotNull $result "No value was returned on move file"
@@ -526,6 +535,7 @@ function Test-DataLakeStoreFileSystem
526535
Assert-AreEqual "File" $result.Type
527536
Assert-AreEqual $($content.length*2) $result.Length
528537
Assert-Throws {Get-AzureRMDataLakeStoreItem -Account $accountName -path $concatFile}
538+
529539
# move a folder
530540
$result = Move-AzureRMDataLakeStoreItem -Account $accountName -Path $folderToCreate -Destination $moveFolder
531541
Assert-NotNull $result "No value was returned on move folder"
@@ -534,9 +544,11 @@ function Test-DataLakeStoreFileSystem
534544
Assert-AreEqual "Directory" $result.Type
535545
Assert-AreEqual 0 $result.Length
536546
Assert-Throws {Get-AzureRMDataLakeStoreItem -Account $accountName -path $folderToCreate}
547+
537548
# delete a file
538549
Assert-True {Remove-AzureRMDataLakeStoreItem -Account $accountName -paths "$moveFolder/movefile.txt" -force -passthru } "Remove File Failed"
539550
Assert-Throws {Get-AzureRMDataLakeStoreItem -Account $accountName -path $moveFile}
551+
540552
# delete a folder
541553
Assert-True {Remove-AzureRMDataLakeStoreItem -Account $accountName -paths $moveFolder -force -recurse -passthru} "Remove folder failed"
542554
Assert-Throws {Get-AzureRMDataLakeStoreItem -Account $accountName -path $moveFolder}
@@ -600,6 +612,17 @@ function Test-DataLakeStoreFileSystemPermissions
600612
# define the permissions to add/remove
601613
$aceUserId = "027c28d5-c91d-49f0-98c5-d10134b169b3"
602614

615+
#set owner
616+
New-AdlStoreItem -Account $accountName -Path "/temp"
617+
$prevOwner=Get-AdlStoreItemOwner -Account $accountName -Path "/temp" -Type User
618+
$prevGroup=Get-AdlStoreItemOwner -Account $accountName -Path "/temp" -Type Group
619+
$currentOwner=Set-AdlStoreItemOwner -Account $accountName -Path "/temp" -Type User -Id $aceUserId -PassThru
620+
$currentGroup=Get-AdlStoreItemOwner -Account $accountName -Path "/temp" -Type Group
621+
Assert-AreEqual $aceUserId $currentOwner
622+
Assert-AreNotEqual $prevOwner $currentOwner
623+
Assert-AreEqual $prevGroup $currentGroup
624+
Remove-AdlStoreItem -Account $accountName -paths "/temp" -force
625+
603626
# Set and get all the permissions
604627
$result = Get-AzureRMDataLakeStoreItemAclEntry -Account $accountName -path "/"
605628
Assert-NotNull $result "Did not get any result from ACL get"
@@ -636,14 +659,17 @@ function Test-DataLakeStoreFileSystemPermissions
636659
Set-AzureRMDataLakeStoreItemAclEntry -Account $accountName -path "/" -AceType User -Id $aceUserId -Permissions All
637660
$result = Get-AzureRMDataLakeStoreItemAclEntry -Account $accountName -path "/"
638661
Assert-AreEqual $($currentCount+1) $result.Count
662+
639663
# remove a specific permission with friendly remove
640664
Remove-AzureRMDataLakeStoreItemAclEntry -Account $accountName -path "/" -AceType User -Id $aceUserId
641665
$result = Get-AzureRMDataLakeStoreItemAclEntry -Account $accountName -path "/"
642666
Assert-AreEqual $($currentCount) $result.Count
667+
643668
# set and get a specific permission with the ACE string
644669
Set-AzureRMDataLakeStoreItemAclEntry -Account $accountName -path "/" -Acl $([string]::Format("user:{0}:rwx", $aceUserId))
645670
$result = Get-AzureRMDataLakeStoreItemAclEntry -Account $accountName -path "/"
646671
Assert-AreEqual $($currentCount+1) $result.Count
672+
647673
# remove a specific permission with the ACE string
648674
Remove-AzureRMDataLakeStoreItemAclEntry -Account $accountName -path "/" -Acl $([string]::Format("user:{0}:---", $aceUserId))
649675
$result = Get-AzureRMDataLakeStoreItemAclEntry -Account $accountName -path "/"

0 commit comments

Comments
 (0)