Skip to content

Commit d003249

Browse files
markcowlmarkcowl
authored
Merge pull request #4986 from rahuldutta90/preview
Integrating the new custom dataplane sdk for ADLS to the powershell cmdlets
2 parents e1dbc15 + ad92197 commit d003249

File tree

68 files changed

+1690
-1495
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

68 files changed

+1690
-1495
lines changed

‎src/ResourceManager/Common/Commands.ScenarioTests.ResourceManager.Common/Assert.ps1

Lines changed: 22 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -277,7 +277,28 @@ function Assert-AreEqual
277277

278278
return $true
279279
}
280-
280+
###################
281+
#
282+
# Verify that if actual falls in an acceptable range of expected
283+
#
284+
# param [long] $expected : The expected number
285+
# param [long] $actual : The actual number
286+
# param [long] $interval : The acceptable offset either side of the expected
287+
# param [string] $message : The message to return if the given objects are not equal
288+
####################
289+
function Assert-NumAreInRange
290+
{
291+
param([long] $expected, [long] $actual, [long] $interval, [string] $message)
292+
if (!$message)
293+
{
294+
$message = "Assertion failed because expected '$expected' does not fall in accepted range of 'interval' of actual '$actual'"
295+
}
296+
if(!($actual -ge ($expected-$interval) -and $actual -le ($expected+$interval)))
297+
{
298+
throw $message
299+
}
300+
return $true
301+
}
281302
###################
282303
#
283304
# Verify that two given arrays are equal

‎src/ResourceManager/DataLakeStore/ChangeLog.md

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,42 @@
1919
-->
2020
## Current Release
2121

22+
* Moved from AutoRest sdk to our version of Dataplane sdk for Azure data lake.
23+
* Refactored following cmdlets to use the new version of SDK:
24+
* Add-AzureRmDataLakeStoreItemContent
25+
* Export-AzureRmDataLakeStoreItem
26+
* Get-AzureRmDataLakeStoreChildItem
27+
* Get-AzureRmDataLakeStoreItem
28+
* Get-AzureRmDataLakeStoreItemAclEntry
29+
* Get-AzureRmDataLakeStoreItemContent
30+
* Get-AzureRmDataLakeStoreItemOwner
31+
* Get-AzureRmDataLakeStoreItemPermission
32+
* Import-AzureRmDataLakeStoreItem
33+
* Join-AzureRmDataLakeStoreItem
34+
* Move-AzureRmDataLakeStoreItem
35+
* New-AzureRmDataLakeStoreItem
36+
* Remove-AzureRmDataLakeStoreItem
37+
* Remove-AzureRmDataLakeStoreItemAcl
38+
* Remove-AzureRmDataLakeStoreItemAclEntry
39+
* Set-AzureRmDataLakeStoreItemAcl
40+
* Set-AzureRmDataLakeStoreItemAclEntry
41+
* Set-AzureRmDataLakeStoreItemExpiry
42+
* Set-AzureRmDataLakeStoreItemOwner
43+
* Set-AzureRmDataLakeStoreItemPermission
44+
* Test-AzureRmDataLakeStoreItem
45+
* Refactored following models to use models from new SDK:
46+
* DataLakeStoreEnums
47+
* DataLakeStoreFileSystemClient
48+
* DataLakeStoreItem
49+
* DataLakeStoreItemAce
50+
* DataLakeStorePathInstance
51+
* Moved the commands and models related to the dataplane into separate folders
52+
* Export-AzureRmDataLakeStoreItem (documented in the breaking changes) - Deprecated parameters PerFileThreadCount, ConcurrentFileCount and introduced parameter Concurrency
53+
* Import-AzureRMDataLakeStoreItem (documented in the breaking changes) -Deprecated parametersPerFileThreadCount, ConcurrentFileCount and introduced parameter Concurrency
54+
* Get-AzureRMDataLakeStoreItemContent - Fixed the tail behavior for contents greater than 4MB
55+
* Set-AzureRMDataLakeStoreItemExpiry - Introduced new parameter set for setting relative expiration time
56+
* Remove-AzureRmDataLakeStoreItem (documented in the breaking changes) - Deprecated parameter Clean.
57+
2258
## Version 5.0.0
2359
* NOTE: This is a breaking change release. Please see the migration guide (https://aka.ms/azps-migration-guide) for a full list of breaking changes introduced.
2460
* Removed the Obsolete Properties fields in PSDataLakeStoreAccount.cs and its assoicated files

‎src/ResourceManager/DataLakeStore/Commands.DataLakeStore.Test/Commands.DataLakeStore.Test.csproj

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -196,6 +196,10 @@
196196
<Project>{5ee72c53-1720-4309-b54b-5fb79703195f}</Project>
197197
<Name>Commands.Common</Name>
198198
</ProjectReference>
199+
<ProjectReference Include="..\Commands.DataLakeStore\Commands.DataLakeStore.csproj">
200+
<Project>{8aab43e6-e8f6-4f91-af8a-6a63cd78ef1e}</Project>
201+
<Name>Commands.DataLakeStore</Name>
202+
</ProjectReference>
199203
</ItemGroup>
200204
<ItemGroup>
201205
<None Include="MSSharedLibKey.snk" />

‎src/ResourceManager/DataLakeStore/Commands.DataLakeStore.Test/ScenarioTests/AdlsAliasTests.cs

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,8 @@
1212
// limitations under the License.
1313
// ----------------------------------------------------------------------------------
1414

15+
using System;
16+
1517
namespace Microsoft.Azure.Commands.DataLakeStore.Test.ScenarioTests
1618
{
1719
using Microsoft.WindowsAzure.Commands.ScenarioTest;

‎src/ResourceManager/DataLakeStore/Commands.DataLakeStore.Test/ScenarioTests/AdlsAliasTests.ps1

Lines changed: 28 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -424,38 +424,46 @@ function Test-DataLakeStoreFileSystem
424424
$result = Get-AdlStoreItem -Account $accountName -path $folderToCreate
425425
Assert-NotNull $result "No value was returned on folder get"
426426
Assert-AreEqual "Directory" $result.Type
427+
427428
# Create and get Empty File
428429
$result = New-AdlStoreItem -Account $accountName -path $emptyFilePath
429430
Assert-NotNull $result "No value was returned on empty file creation"
430431
$result = Get-AdlStoreItem -Account $accountName -path $emptyFilePath
432+
$emptyFileCreationDate=$result.LastWriteTime # To be used later
431433
Assert-NotNull $result "No value was returned on empty file get"
432434
Assert-AreEqual "File" $result.Type
433435
Assert-AreEqual 0 $result.Length
436+
434437
# Create and get file with content
435438
$result = New-AdlStoreItem -Account $accountName -path $contentFilePath -Value $content
436439
Assert-NotNull $result "No value was returned on content file creation"
437440
$result = Get-AdlStoreItem -Account $accountName -path $contentFilePath
438441
Assert-NotNull $result "No value was returned on content file get"
439442
Assert-AreEqual "File" $result.Type
440443
Assert-AreEqual $content.length $result.Length
441-
# set and validate expiration for a file
444+
445+
# set absolute expiration for content file
442446
Assert-True {253402300800000 -ge $result.ExpirationTime -or 0 -le $result.ExpirationTime} # validate that expiration is currently max value
443447
[DateTimeOffset]$timeToUse = [Microsoft.Azure.Test.HttpRecorder.HttpMockServer]::GetVariable("absoluteTime", [DateTimeOffset]::UtcNow.AddSeconds(120))
444448
$result = Set-AdlStoreItemExpiry -Account $accountName -path $contentFilePath -Expiration $timeToUse
445-
Assert-AreEqual $timeToUse.UtcTicks $result.Expiration.UtcTicks
449+
Assert-NumAreInRange $timeToUse.UtcTicks $result.Expiration.UtcTicks 500000 # range of 50 milliseconds
450+
446451
# set it back to "never expire"
447452
$result = Set-AdlStoreItemExpiry -Account $accountName -path $contentFilePath
448453
Assert-True {253402300800000 -ge $result.ExpirationTime -or 0 -le $result.ExpirationTime} # validate that expiration is currently max value
454+
449455
# list files
450456
$result = Get-AdlStoreChildItem -Account $accountName -path $folderToCreate
451457
Assert-NotNull $result "No value was returned on folder list"
452458
Assert-AreEqual 2 $result.length
459+
453460
# add content to empty file
454461
Add-AdlStoreItemContent -Account $accountName -Path $emptyFilePath -Value $content
455462
$result = Get-AdlStoreItem -Account $accountName -path $emptyFilePath
456463
Assert-NotNull $result "No value was returned on empty file get with content added"
457464
Assert-AreEqual "File" $result.Type
458465
Assert-AreEqual $content.length $result.Length
466+
459467
# concat files
460468
$result = Join-AdlStoreItem -Account $accountName -Paths $emptyFilePath,$contentFilePath -Destination $concatFile
461469
Assert-NotNull $result "No value was returned on concat file"
@@ -506,6 +514,7 @@ function Test-DataLakeStoreFileSystem
506514
Assert-NotNull $result "No value was returned on import file get"
507515
Assert-AreEqual "File" $result.Type
508516
Assert-AreEqual $localFileInfo.length $result.Length
517+
509518
# download file
510519
$currentDir = Split-Path $fileToCopy
511520
$targetFile = Join-Path $currentDir "adlspstestdownload.txt"
@@ -527,6 +536,7 @@ function Test-DataLakeStoreFileSystem
527536
Assert-AreEqual "File" $result.Type
528537
Assert-AreEqual $($content.length*2) $result.Length
529538
Assert-Throws {Get-AdlStoreItem -Account $accountName -path $concatFile}
539+
530540
# move a folder
531541
$result = Move-AdlStoreItem -Account $accountName -Path $folderToCreate -Destination $moveFolder
532542
Assert-NotNull $result "No value was returned on move folder"
@@ -535,9 +545,11 @@ function Test-DataLakeStoreFileSystem
535545
Assert-AreEqual "Directory" $result.Type
536546
Assert-AreEqual 0 $result.Length
537547
Assert-Throws {Get-AdlStoreItem -Account $accountName -path $folderToCreate}
548+
538549
# delete a file
539550
Assert-True {Remove-AdlStoreItem -Account $accountName -paths "$moveFolder/movefile.txt" -force -passthru } "Remove File Failed"
540551
Assert-Throws {Get-AdlStoreItem -Account $accountName -path $moveFile}
552+
541553
# delete a folder
542554
Assert-True {Remove-AdlStoreItem -Account $accountName -paths $moveFolder -force -recurse -passthru} "Remove folder failed"
543555
Assert-Throws {Get-AdlStoreItem -Account $accountName -path $moveFolder}
@@ -601,6 +613,17 @@ function Test-DataLakeStoreFileSystemPermissions
601613
# define the permissions to add/remove
602614
$aceUserId = "027c28d5-c91d-49f0-98c5-d10134b169b3"
603615

616+
#set owner
617+
New-AdlStoreItem -Account $accountName -Path "/temp"
618+
$prevOwner=Get-AdlStoreItemOwner -Account $accountName -Path "/temp" -Type User
619+
$prevGroup=Get-AdlStoreItemOwner -Account $accountName -Path "/temp" -Type Group
620+
$currentOwner=Set-AdlStoreItemOwner -Account $accountName -Path "/temp" -Type User -Id $aceUserId -PassThru
621+
$currentGroup=Get-AdlStoreItemOwner -Account $accountName -Path "/temp" -Type Group
622+
Assert-AreEqual $aceUserId $currentOwner
623+
Assert-AreNotEqual $prevOwner $currentOwner
624+
Assert-AreEqual $prevGroup $currentGroup
625+
Remove-AdlStoreItem -Account $accountName -paths "/temp" -force
626+
604627
# Set and get all the permissions
605628
$result = Get-AdlStoreItemAclEntry -Account $accountName -path "/"
606629
Assert-NotNull $result "Did not get any result from ACL get"
@@ -635,14 +658,17 @@ function Test-DataLakeStoreFileSystemPermissions
635658
Set-AdlStoreItemAclEntry -Account $accountName -path "/" -AceType User -Id $aceUserId -Permissions All
636659
$result = Get-AdlStoreItemAclEntry -Account $accountName -path "/"
637660
Assert-AreEqual $($currentCount+1) $result.Count
661+
638662
# remove a specific permission with friendly remove
639663
Remove-AdlStoreItemAclEntry -Account $accountName -path "/" -AceType User -Id $aceUserId
640664
$result = Get-AdlStoreItemAclEntry -Account $accountName -path "/"
641665
Assert-AreEqual $($currentCount) $result.Count
666+
642667
# set and get a specific permission with the ACE string
643668
Set-AdlStoreItemAclEntry -Account $accountName -path "/" -Acl $([string]::Format("user:{0}:rwx", $aceUserId))
644669
$result = Get-AdlStoreItemAclEntry -Account $accountName -path "/"
645670
Assert-AreEqual $($currentCount+1) $result.Count
671+
646672
# remove a specific permission with the ACE string
647673
Remove-AdlStoreItemAclEntry -Account $accountName -path "/" -Acl $([string]::Format("user:{0}:---", $aceUserId))
648674
$result = Get-AdlStoreItemAclEntry -Account $accountName -path "/"

‎src/ResourceManager/DataLakeStore/Commands.DataLakeStore.Test/ScenarioTests/AdlsTests.ps1

Lines changed: 31 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -423,38 +423,46 @@ function Test-DataLakeStoreFileSystem
423423
$result = Get-AzureRMDataLakeStoreItem -Account $accountName -path $folderToCreate
424424
Assert-NotNull $result "No value was returned on folder get"
425425
Assert-AreEqual "Directory" $result.Type
426+
426427
# Create and get Empty File
427428
$result = New-AzureRMDataLakeStoreItem -Account $accountName -path $emptyFilePath
428429
Assert-NotNull $result "No value was returned on empty file creation"
429430
$result = Get-AzureRMDataLakeStoreItem -Account $accountName -path $emptyFilePath
431+
$emptyFileCreationDate=$result.LastWriteTime # To be used later
430432
Assert-NotNull $result "No value was returned on empty file get"
431433
Assert-AreEqual "File" $result.Type
432434
Assert-AreEqual 0 $result.Length
435+
433436
# Create and get file with content
434437
$result = New-AzureRMDataLakeStoreItem -Account $accountName -path $contentFilePath -Value $content
435438
Assert-NotNull $result "No value was returned on content file creation"
436439
$result = Get-AzureRMDataLakeStoreItem -Account $accountName -path $contentFilePath
437440
Assert-NotNull $result "No value was returned on content file get"
438441
Assert-AreEqual "File" $result.Type
439442
Assert-AreEqual $content.length $result.Length
440-
# set and validate expiration for a file
443+
444+
# set absolute expiration for content file
441445
Assert-True {253402300800000 -ge $result.ExpirationTime -or 0 -le $result.ExpirationTime} # validate that expiration is currently max value
442446
[DateTimeOffset]$timeToUse = [Microsoft.Azure.Test.HttpRecorder.HttpMockServer]::GetVariable("absoluteTime", [DateTimeOffset]::UtcNow.AddSeconds(120))
443-
$result = Set-AzureRmDataLakeStoreItemExpiry -Account $accountName -path $contentFilePath -Expiration $timeToUse
444-
Assert-AreEqual $timeToUse.UtcTicks $result.Expiration.UtcTicks
447+
$result = Set-AdlStoreItemExpiry -Account $accountName -path $contentFilePath -Expiration $timeToUse
448+
Assert-NumAreInRange $timeToUse.UtcTicks $result.Expiration.UtcTicks 500000 # range of 50 milliseconds
449+
445450
# set it back to "never expire"
446-
$result = Set-AzureRmDataLakeStoreItemExpiry -Account $accountName -path $contentFilePath
451+
$result = Set-AdlStoreItemExpiry -Account $accountName -path $contentFilePath
447452
Assert-True {253402300800000 -ge $result.ExpirationTime -or 0 -le $result.ExpirationTime} # validate that expiration is currently max value
453+
448454
# list files
449455
$result = Get-AzureRMDataLakeStoreChildItem -Account $accountName -path $folderToCreate
450456
Assert-NotNull $result "No value was returned on folder list"
451457
Assert-AreEqual 2 $result.length
458+
452459
# add content to empty file
453460
Add-AzureRMDataLakeStoreItemContent -Account $accountName -Path $emptyFilePath -Value $content
454461
$result = Get-AzureRMDataLakeStoreItem -Account $accountName -path $emptyFilePath
455462
Assert-NotNull $result "No value was returned on empty file get with content added"
456463
Assert-AreEqual "File" $result.Type
457464
Assert-AreEqual $content.length $result.Length
465+
458466
# concat files
459467
$result = Join-AzureRMDataLakeStoreItem -Account $accountName -Paths $emptyFilePath,$contentFilePath -Destination $concatFile
460468
Assert-NotNull $result "No value was returned on concat file"
@@ -505,6 +513,7 @@ function Test-DataLakeStoreFileSystem
505513
Assert-NotNull $result "No value was returned on import file get"
506514
Assert-AreEqual "File" $result.Type
507515
Assert-AreEqual $localFileInfo.length $result.Length
516+
508517
# download file
509518
$currentDir = Split-Path $fileToCopy
510519
$targetFile = Join-Path $currentDir "adlspstestdownload.txt"
@@ -517,7 +526,7 @@ function Test-DataLakeStoreFileSystem
517526
$downloadedFileInfo = Get-ChildItem $targetFile
518527
Assert-AreEqual $($content.length*2) $downloadedFileInfo.length
519528
Remove-Item -path $targetFile -force -confirm:$false
520-
529+
521530
# move a file
522531
$result = Move-AzureRMDataLakeStoreItem -Account $accountName -Path $concatFile -Destination $moveFile
523532
Assert-NotNull $result "No value was returned on move file"
@@ -526,6 +535,7 @@ function Test-DataLakeStoreFileSystem
526535
Assert-AreEqual "File" $result.Type
527536
Assert-AreEqual $($content.length*2) $result.Length
528537
Assert-Throws {Get-AzureRMDataLakeStoreItem -Account $accountName -path $concatFile}
538+
529539
# move a folder
530540
$result = Move-AzureRMDataLakeStoreItem -Account $accountName -Path $folderToCreate -Destination $moveFolder
531541
Assert-NotNull $result "No value was returned on move folder"
@@ -534,9 +544,11 @@ function Test-DataLakeStoreFileSystem
534544
Assert-AreEqual "Directory" $result.Type
535545
Assert-AreEqual 0 $result.Length
536546
Assert-Throws {Get-AzureRMDataLakeStoreItem -Account $accountName -path $folderToCreate}
547+
537548
# delete a file
538549
Assert-True {Remove-AzureRMDataLakeStoreItem -Account $accountName -paths "$moveFolder/movefile.txt" -force -passthru } "Remove File Failed"
539550
Assert-Throws {Get-AzureRMDataLakeStoreItem -Account $accountName -path $moveFile}
551+
540552
# delete a folder
541553
Assert-True {Remove-AzureRMDataLakeStoreItem -Account $accountName -paths $moveFolder -force -recurse -passthru} "Remove folder failed"
542554
Assert-Throws {Get-AzureRMDataLakeStoreItem -Account $accountName -path $moveFolder}
@@ -600,6 +612,17 @@ function Test-DataLakeStoreFileSystemPermissions
600612
# define the permissions to add/remove
601613
$aceUserId = "027c28d5-c91d-49f0-98c5-d10134b169b3"
602614

615+
#set owner
616+
New-AdlStoreItem -Account $accountName -Path "/temp"
617+
$prevOwner=Get-AdlStoreItemOwner -Account $accountName -Path "/temp" -Type User
618+
$prevGroup=Get-AdlStoreItemOwner -Account $accountName -Path "/temp" -Type Group
619+
$currentOwner=Set-AdlStoreItemOwner -Account $accountName -Path "/temp" -Type User -Id $aceUserId -PassThru
620+
$currentGroup=Get-AdlStoreItemOwner -Account $accountName -Path "/temp" -Type Group
621+
Assert-AreEqual $aceUserId $currentOwner
622+
Assert-AreNotEqual $prevOwner $currentOwner
623+
Assert-AreEqual $prevGroup $currentGroup
624+
Remove-AdlStoreItem -Account $accountName -paths "/temp" -force
625+
603626
# Set and get all the permissions
604627
$result = Get-AzureRMDataLakeStoreItemAclEntry -Account $accountName -path "/"
605628
Assert-NotNull $result "Did not get any result from ACL get"
@@ -636,14 +659,17 @@ function Test-DataLakeStoreFileSystemPermissions
636659
Set-AzureRMDataLakeStoreItemAclEntry -Account $accountName -path "/" -AceType User -Id $aceUserId -Permissions All
637660
$result = Get-AzureRMDataLakeStoreItemAclEntry -Account $accountName -path "/"
638661
Assert-AreEqual $($currentCount+1) $result.Count
662+
639663
# remove a specific permission with friendly remove
640664
Remove-AzureRMDataLakeStoreItemAclEntry -Account $accountName -path "/" -AceType User -Id $aceUserId
641665
$result = Get-AzureRMDataLakeStoreItemAclEntry -Account $accountName -path "/"
642666
Assert-AreEqual $($currentCount) $result.Count
667+
643668
# set and get a specific permission with the ACE string
644669
Set-AzureRMDataLakeStoreItemAclEntry -Account $accountName -path "/" -Acl $([string]::Format("user:{0}:rwx", $aceUserId))
645670
$result = Get-AzureRMDataLakeStoreItemAclEntry -Account $accountName -path "/"
646671
Assert-AreEqual $($currentCount+1) $result.Count
672+
647673
# remove a specific permission with the ACE string
648674
Remove-AzureRMDataLakeStoreItemAclEntry -Account $accountName -path "/" -Acl $([string]::Format("user:{0}:---", $aceUserId))
649675
$result = Get-AzureRMDataLakeStoreItemAclEntry -Account $accountName -path "/"

0 commit comments

Comments
 (0)