Skip to content

Commit 9fc95b9

Browse files
bluewwblueww
committed
[Storage] Support DatalakeGen2 , and upgrade Blob to Track2 SDK
1 parent 29bfe01 commit 9fc95b9

File tree

62 files changed

+5034
-139
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

62 files changed

+5034
-139
lines changed

‎src/Storage/Storage.Management.Test/ScenarioTests/StorageDataPlaneTests.cs

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -83,5 +83,12 @@ public void TestCommon()
8383
{
8484
TestController.NewInstance.RunPsTest(_logger, $"Test-Common -ResourceGroupName \"{resourceGroupName}\" -StorageAccountName \"{storageAccountName}\"");
8585
}
86+
87+
[Fact]
88+
[Trait(Category.RunType, Category.LiveOnly)]
89+
public void TestDatalakeGen2()
90+
{
91+
TestController.NewInstance.RunPsTest(_logger, $"Test-DatalakeGen2 -ResourceGroupName \"{resourceGroupName}\" -StorageAccountName \"{storageAccountName}\"");
92+
}
8693
}
8794
}

‎src/Storage/Storage.Management.Test/ScenarioTests/StorageDataPlaneTests.ps1

Lines changed: 167 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -207,6 +207,26 @@ function Test-Blob
207207
Assert-AreEqual $blob.Count 2
208208
Get-AzStorageBlob -Container $containerName -Blob $objectName2 -Context $storageContext | Remove-AzStorageBlob -Force
209209

210+
#check XSCL Track2 Items works for container
211+
$container = Get-AzStorageContainer $containerName -Context $storageContext
212+
$containerProperties = $container.BlobContainerClient.GetProperties().Value
213+
Assert-AreEqual $container.BlobContainerProperties.ETag $containerProperties.ETag
214+
Set-AzStorageContainerAcl $containerName -Context $storageContext -Permission Blob
215+
$containerProperties = $container.BlobContainerClient.GetProperties().Value
216+
Assert-AreNotEqual $container.BlobContainerProperties.ETag $containerProperties.ETag
217+
$container.FetchAttributes()
218+
Assert-AreEqual $container.BlobContainerProperties.ETag $containerProperties.ETag
219+
220+
#check XSCL Track2 Items works for Blob
221+
$blob = Get-AzStorageBlob -Container $containerName -Blob $objectName1 -Context $storageContext
222+
$blobProperties = $blob.BlobClient.GetProperties().Value
223+
Assert-AreEqual $blob.BlobProperties.ETag $blobProperties.ETag
224+
Set-AzStorageBlobContent -File $localSrcFile -Container $containerName -Blob $objectName1 -Force -Context $storageContext
225+
$blobProperties = $blob.BlobClient.GetProperties().Value
226+
Assert-AreNotEqual $blob.BlobProperties.ETag $blobProperties.ETag
227+
$blob.FetchAttributes()
228+
Assert-AreEqual $blob.BlobProperties.ETag $blobProperties.ETag
229+
210230
# Copy blob to the same container, but with a different name.
211231
Start-AzStorageBlobCopy -srcContainer $containerName -SrcBlob $objectName1 -DestContainer $containerName -DestBlob $objectName2 -StandardBlobTier $StandardBlobTier -RehydratePriority High -Context $storageContext -DestContext $storageContext
212232
Get-AzStorageBlobCopyState -Container $containerName -Blob $objectName2 -Context $storageContext
@@ -684,6 +704,153 @@ function Test-Common
684704
}
685705
}
686706

707+
<#
708+
.SYNOPSIS
709+
Tests DatalakeGen-only related commands.
710+
#>
711+
function Test-DatalakeGen2
712+
{
713+
Param(
714+
[Parameter(Mandatory = $True)]
715+
[string]
716+
$StorageAccountName,
717+
[Parameter(Mandatory = $True)]
718+
[string]
719+
$ResourceGroupName
720+
)
721+
722+
New-TestResourceGroupAndStorageAccount -ResourceGroupName $ResourceGroupName -StorageAccountName $StorageAccountName -EnableHNFS $true
723+
724+
try{
725+
726+
$storageAccountKeyValue = $(Get-AzStorageAccountKey -ResourceGroupName $ResourceGroupName -Name $StorageAccountName)[0].Value
727+
$storageContext = New-AzStorageContext -StorageAccountName $StorageAccountName -StorageAccountKey $storageAccountKeyValue
728+
729+
$localSrcFile = "localsrcDatalakeGen2testfile.psd1" #The file need exist before test, and should be 512 bytes aligned
730+
New-Item $localSrcFile -ItemType File -Force
731+
$localDestFile = "localdestDatalakeGen2testfile.txt"
732+
733+
$filesystemName = "adlsgen2testfilesystem"
734+
$directoryPath1 = "dir1"
735+
$directoryPath2 = "dir2"
736+
$directoryPath3 = "dir3"
737+
$filePath1 = "dir1/Item1.txt"
738+
$filePath2 = "dir2/Item2.txt"
739+
$filePath3 = "dir2/Item3.txt"
740+
$ContentType = "image/jpeg"
741+
$ContentMD5 = "i727sP7HigloQDsqadNLHw=="
742+
743+
# Create FileSystem (actually a container)
744+
New-AzDatalakeGen2FileSystem $filesystemName -Context $storageContext
745+
746+
# Create folders
747+
$dir1 = New-AzDataLakeGen2Item -Context $storageContext -FileSystem $filesystemName -Path $directoryPath1 -Directory -Permission rwxrwxrwx -Umask ---rwx--- -Property @{"ContentEncoding" = "UDF8"; "CacheControl" = "READ"} -Metadata @{"tag1" = "value1"; "tag2" = "value2" }
748+
Assert-AreEqual $dir1.Path $directoryPath1
749+
Assert-AreEqual $dir1.Permissions.ToSymbolicPermissions() "rwx---rwx"
750+
$dir2 = New-AzDataLakeGen2Item -Context $storageContext -FileSystem $filesystemName -Path $directoryPath2 -Directory
751+
752+
# Create (upload) File
753+
$t = New-AzDataLakeGen2Item -Context $storageContext -FileSystem $filesystemName -Path $filePath1 -Source $localSrcFile -Force -AsJob
754+
$t | wait-job
755+
Assert-AreEqual $t.State "Completed"
756+
Assert-AreEqual $t.Error $null
757+
$file2 = New-AzDataLakeGen2Item -Context $storageContext -FileSystem $filesystemName -Path $filePath2 -Source $localSrcFile -Permission rwxrwxrwx -Umask ---rwx--- -Property @{"ContentType" = $ContentType; "ContentMD5" = $ContentMD5} -Metadata @{"tag1" = "value1"; "tag2" = "value2" }
758+
Assert-AreEqual $file2.Path $filePath2
759+
Assert-AreEqual $file2.Properties.ContentType $ContentType
760+
Assert-AreEqual $file2.Properties.Metadata.Count 2
761+
Assert-AreEqual $file2.Permissions.ToSymbolicPermissions() "rwx---rwx"
762+
763+
# update Blob and Directory
764+
$ContentType = "application/octet-stream"
765+
$ContentMD5 = "NW/H9Zxr2md6L1/yhNKdew=="
766+
$ContentEncoding = "UDF8"
767+
## create ACL with 3 ACEs
768+
$acl = New-AzDataLakeGen2ItemAclObject -AccessControlType user -Permission rw-
769+
$acl = New-AzDataLakeGen2ItemAclObject -AccessControlType group -Permission rw- -InputObject $acl
770+
$acl = New-AzDataLakeGen2ItemAclObject -AccessControlType other -Permission "-wx" -InputObject $acl
771+
##Update File with pipeline
772+
$file1 = Get-AzDataLakeGen2Item -Context $storageContext -FileSystem $filesystemName -Path $filePath1 | Update-AzDataLakeGen2Item `
773+
-Acl $acl `
774+
-Property @{"ContentType" = $ContentType; "ContentMD5" = $ContentMD5} `
775+
-Metadata @{"tag1" = "value1"; "tag2" = "value2" } `
776+
-Permission rw-rw--wx `
777+
-Owner '$superuser' `
778+
-Group '$superuser'
779+
$file1 = Get-AzDataLakeGen2Item -Context $storageContext -FileSystem $filesystemName -Path $filePath1
780+
Assert-AreEqual $file1.Path $filePath1
781+
Assert-AreEqual $file1.Permissions.ToSymbolicPermissions() "rw-rw--wx"
782+
Assert-AreEqual $file1.Properties.ContentType $ContentType
783+
Assert-AreEqual $file1.Properties.Metadata.Count 2
784+
Assert-AreEqual $file1.Owner '$superuser'
785+
Assert-AreEqual $file1.Group '$superuser'
786+
## Update Directory
787+
$dir1 = Update-AzDataLakeGen2Item -Context $storageContext -FileSystem $filesystemName -Path $directoryPath1 `
788+
-Acl $acl `
789+
-Property @{"ContentEncoding" = $ContentEncoding} `
790+
-Metadata @{"tag1" = "value1"; "tag2" = "value2" } `
791+
-Permission rw-rw--wx `
792+
-Owner '$superuser' `
793+
-Group '$superuser'
794+
$dir1 = Get-AzDataLakeGen2Item -Context $storageContext -FileSystem $filesystemName -Path $directoryPath1
795+
Assert-AreEqual $dir1.Path $directoryPath1
796+
Assert-AreEqual $dir1.Permissions.ToSymbolicPermissions() "rw-rw--wx"
797+
Assert-AreEqual $dir1.Properties.ContentEncoding $ContentEncoding
798+
Assert-AreEqual $dir1.Properties.Metadata.Count 3 # inlucde "hdi_isfolder" which is handle by server
799+
Assert-AreEqual $dir1.Owner '$superuser'
800+
Assert-AreEqual $dir1.Group '$superuser'
801+
802+
#list Items
803+
## List direct Items from FileSystem
804+
$items = Get-AzDataLakeGen2ChildItem -Context $storageContext -FileSystem $filesystemName -FetchPermission
805+
Assert-AreEqual $items.Count 2
806+
Assert-NotNull $items[0].Permissions
807+
$items = Get-AzDataLakeGen2ChildItem -Context $storageContext -FileSystem $filesystemName -Recurse
808+
Assert-AreEqual $items.Count 4
809+
Assert-Null $items[0].Permissions
810+
811+
#download File
812+
$t = Get-AzDataLakeGen2ItemContent -Context $storageContext -FileSystem $filesystemName -Path $filePath1 -Destination $localDestFile -AsJob -Force
813+
$t | Wait-Job
814+
Assert-AreEqual $t.State "Completed"
815+
Assert-AreEqual $t.Error $null
816+
Assert-AreEqual (Get-FileHash -Path $localDestFile -Algorithm MD5).Hash (Get-FileHash -Path $localSrcFile -Algorithm MD5).Hash
817+
818+
# Move Items
819+
## Move File
820+
$file3 = Move-AzDataLakeGen2Item -Context $storageContext -FileSystem $filesystemName -Path $filePath2 -DestFileSystem $filesystemName -DestPath $filePath3 -Force
821+
$file3 = Get-AzDataLakeGen2Item -Context $storageContext -FileSystem $filesystemName -Path $filePath3
822+
Assert-AreEqual $file3.Path $filePath3
823+
Assert-AreEqual $file3.Permissions $file2.Permissions
824+
$file2 = $file3 | Move-AzDataLakeGen2Item -DestFileSystem $filesystemName -DestPath $filePath2
825+
$file2 = Get-AzDataLakeGen2Item -Context $storageContext -FileSystem $filesystemName -Path $filePath2
826+
Assert-AreEqual $file2.Path $filePath2
827+
Assert-AreEqual $file2.Permissions $file3.Permissions
828+
## Move Folder
829+
$dir3 = Move-AzDataLakeGen2Item -Context $storageContext -FileSystem $filesystemName -Path $directoryPath1 -DestFileSystem $filesystemName -DestPath $directoryPath3
830+
$dir3 = Get-AzDataLakeGen2Item -Context $storageContext -FileSystem $filesystemName -Path $directoryPath3
831+
Assert-AreEqual $dir3.Path $directoryPath3
832+
Assert-AreEqual $dir3.Permissions $dir1.Permissions
833+
$dir1 = $dir3 | Move-AzDataLakeGen2Item -DestFileSystem $filesystemName -DestPath $directoryPath1
834+
$dir1 = Get-AzDataLakeGen2Item -Context $storageContext -FileSystem $filesystemName -Path $directoryPath1
835+
Assert-AreEqual $dir1.Path $directoryPath1
836+
837+
# Remove Items
838+
Remove-AzDataLakeGen2Item -Context $storageContext -FileSystem $filesystemName -Path $filePath1 -Force
839+
Remove-AzDataLakeGen2Item -Context $storageContext -FileSystem $filesystemName -Path $directoryPath1 -Force
840+
841+
# Clean Storage Account
842+
Get-AzDataLakeGen2ChildItem -Context $storageContext -FileSystem $filesystemName | Remove-AzDataLakeGen2Item -Force
843+
844+
# remove File system (actually a container)
845+
Remove-AzDatalakeGen2FileSystem $filesystemName -Context $storageContext
846+
847+
}
848+
finally
849+
{
850+
Clean-ResourceGroup $ResourceGroupName
851+
}
852+
}
853+
687854
function New-TestResourceGroupAndStorageAccount
688855
{
689856
Param(

‎src/Storage/Storage.Management.Test/Storage.Management.Test.csproj

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,8 @@
1111
</PropertyGroup>
1212

1313
<ItemGroup>
14+
<PackageReference Include="Azure.Storage.Blobs" Version="12.4.0" />
15+
<PackageReference Include="Azure.Storage.Files.DataLake" Version="12.0.0" />
1416
<PackageReference Include="Microsoft.Azure.Management.Storage" Version="14.5.0" />
1517
</ItemGroup>
1618

‎src/Storage/Storage.Management/Az.Storage.psd1

Lines changed: 13 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -66,7 +66,10 @@ RequiredAssemblies = 'Microsoft.Azure.Management.Storage.dll',
6666
'Microsoft.Azure.Storage.DataMovement.dll',
6767
'Microsoft.Azure.DocumentDB.Core.dll', 'Microsoft.OData.Core.dll',
6868
'Microsoft.OData.Edm.dll', 'Microsoft.Spatial.dll',
69-
'Microsoft.Azure.KeyVault.Core.dll'
69+
'Microsoft.Azure.KeyVault.Core.dll','Azure.Storage.Blobs.dll',
70+
'Azure.Storage.Common.dll', 'Azure.Storage.Files.DataLake.dll',
71+
'Azure.Core.dll', 'Microsoft.Bcl.AsyncInterfaces.dll',
72+
'System.Text.Json.dll', 'System.Threading.Tasks.Extensions.dll'
7073

7174
# Script files (.ps1) that are run in the caller's environment prior to importing this module.
7275
# ScriptsToProcess = @()
@@ -158,15 +161,22 @@ CmdletsToExport = 'Get-AzStorageAccount', 'Get-AzStorageAccountKey',
158161
'Revoke-AzStorageAccountUserDelegationKeys',
159162
'Get-AzStorageFileHandle', 'Close-AzStorageFileHandle',
160163
'New-AzRmStorageShare', 'Remove-AzRmStorageShare',
161-
'Get-AzRmStorageShare', 'Update-AzRmStorageShare'
164+
'Get-AzRmStorageShare', 'Update-AzRmStorageShare',
165+
'Get-AzDataLakeGen2ChildItem', 'Get-AzDataLakeGen2Item',
166+
'New-AzDataLakeGen2Item', 'Move-AzDataLakeGen2Item',
167+
'Remove-AzDataLakeGen2Item', 'Update-AzDataLakeGen2Item',
168+
'Set-AzDataLakeGen2ItemAclObject', 'Get-AzDataLakeGen2ItemContent'
169+
162170

163171
# Variables to export from this module
164172
# VariablesToExport = @()
165173

166174
# Aliases to export from this module, for best performance, do not use wildcards and do not delete the entry, use an empty array if there are no aliases to export.
167175
AliasesToExport = 'Get-AzStorageContainerAcl', 'Start-CopyAzureStorageBlob',
168176
'Stop-CopyAzureStorageBlob', 'Enable-AzStorageSoftDelete',
169-
'Disable-AzStorageSoftDelete'
177+
'Disable-AzStorageSoftDelete',
178+
'New-AzDatalakeGen2FileSystem', 'Remove-AzDatalakeGen2FileSystem',
179+
'Get-AzDatalakeGen2FileSystem', 'New-AzDataLakeGen2ItemAclObject'
170180

171181
# DSC resources to export from this module
172182
# DscResourcesToExport = @()

‎src/Storage/Storage.Management/ChangeLog.md

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,15 @@
2222
* Support new SkuName StandardGZRS, StandardRAGZRS when create/update Storage account
2323
- New-AzStorageAccount
2424
- Set-AzStorageAccount
25+
* Support DataLake Gen2
26+
- New-AzDataLakeGen2Item
27+
- Get-AzDataLakeGen2Item
28+
- Get-AzDataLakeGen2ChildItem
29+
- Move-AzDataLakeGen2Item
30+
- Set-AzDataLakeGen2ItemAclObject
31+
- Update-AzDataLakeGen2Item
32+
- Get-AzDataLakeGen2ItemContent
33+
- Remove-AzDataLakeGen2Item
2534

2635
## Version 1.13.0
2736
* Supported AllowProtectedAppendWrite in ImmutabilityPolicy

‎src/Storage/Storage.Management/Storage.Management.csproj

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -15,11 +15,12 @@
1515
</PropertyGroup>
1616
<ItemGroup>
1717
<PackageReference Include="Microsoft.Azure.Management.Storage" Version="14.5.0" />
18-
<PackageReference Include="Microsoft.Azure.Storage.Blob" Version="11.1.0" />
19-
<PackageReference Include="Microsoft.Azure.Storage.File" Version="11.1.0" />
20-
<PackageReference Include="Microsoft.Azure.Storage.Queue" Version="11.1.0" />
18+
<PackageReference Include="Microsoft.Azure.Storage.Blob" Version="11.1.1" />
19+
<PackageReference Include="Microsoft.Azure.Storage.File" Version="11.1.1" />
20+
<PackageReference Include="Microsoft.Azure.Storage.Queue" Version="11.1.1" />
2121
</ItemGroup>
22-
<ItemGroup>
22+
23+
<ItemGroup>
2324
<Compile Include="$(StorageToolsPath)Adapters\ARM.Storage.6\ARMStorageProvider.cs" Link="Common\ARMStorageProvider.cs" />
2425
<Compile Include="$(StorageToolsPath)Adapters\ARM.Storage.6\ARMStorageService.cs" Link="Common\ARMStorageService.cs" />
2526
</ItemGroup>

‎src/Storage/Storage.Management/help/Az.Storage.md

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,15 @@ Enable delete retention policy for the Azure Storage Blob service.
4141
### [Enable-AzStorageStaticWebsite](Enable-AzStorageStaticWebsite.md)
4242
Enable static website for the Azure Storage account.
4343

44+
### [Get-AzDataLakeGen2ChildItem](Get-AzDataLakeGen2ChildItem.md)
45+
Lists sub directorys and files from a directory or filesystem root.
46+
47+
### [Get-AzDataLakeGen2Item](Get-AzDataLakeGen2Item.md)
48+
Gets the details of a file or directory in a filesystem.
49+
50+
### [Get-AzDataLakeGen2ItemContent](Get-AzDataLakeGen2ItemContent.md)
51+
Download a file.
52+
4453
### [Get-AzRmStorageContainer](Get-AzRmStorageContainer.md)
4554
Gets or lists Storage blob containers
4655

@@ -131,6 +140,12 @@ Gets the Storage resource usage of the current subscription.
131140
### [Lock-AzRmStorageContainerImmutabilityPolicy](Lock-AzRmStorageContainerImmutabilityPolicy.md)
132141
Locks ImmutabilityPolicy of a Storage blob containers
133142

143+
### [Move-AzDataLakeGen2Item](Move-AzDataLakeGen2Item.md)
144+
Move a file or directory to another a file or directory in same Storage account.
145+
146+
### [New-AzDataLakeGen2Item](New-AzDataLakeGen2Item.md)
147+
Create a file or directory in a filesystem.
148+
134149
### [New-AzRmStorageContainer](New-AzRmStorageContainer.md)
135150
Creates a Storage blob container
136151

@@ -200,6 +215,9 @@ Generates an SAS token for an Azure Storage table.
200215
### [New-AzStorageTableStoredAccessPolicy](New-AzStorageTableStoredAccessPolicy.md)
201216
Creates a stored access policy for an Azure storage table.
202217

218+
### [Remove-AzDataLakeGen2Item](Remove-AzDataLakeGen2Item.md)
219+
Remove a file or directory.
220+
203221
### [Remove-AzRmStorageContainer](Remove-AzRmStorageContainer.md)
204222
Removes a Storage blob container
205223

@@ -263,6 +281,9 @@ Revoke all User Delegation keys of a Storage account.
263281
### [Set-AzCurrentStorageAccount](Set-AzCurrentStorageAccount.md)
264282
Modifies the current Storage account of the specified subscription.
265283

284+
### [Set-AzDataLakeGen2ItemAclObject](Set-AzDataLakeGen2ItemAclObject.md)
285+
Creates/Updates a DataLake gen2 item ACL object, which can be used in Update-AzDataLakeGen2Item cmdlet.
286+
266287
### [Set-AzRmStorageContainerImmutabilityPolicy](Set-AzRmStorageContainerImmutabilityPolicy.md)
267288
Creates or updates ImmutabilityPolicy of a Storage blob containers
268289

@@ -320,6 +341,9 @@ Stops a copy operation.
320341
### [Stop-AzStorageFileCopy](Stop-AzStorageFileCopy.md)
321342
Stops a copy operation to the specified destination file.
322343

344+
### [Update-AzDataLakeGen2Item](Update-AzDataLakeGen2Item.md)
345+
Update a file or directory on properties, metadata, permission, ACL, and owner.
346+
323347
### [Update-AzRmStorageContainer](Update-AzRmStorageContainer.md)
324348
Modifies a Storage blob container
325349

0 commit comments

Comments
 (0)