Skip to content

Commit 41132b7

Browse files
markcowlmarkcowl
authored
Merge pull request #5911 from Azure/adls-data-plane-newfeatures
Adls data plane newfeatures
2 parents 518f424 + 70c8eda commit 41132b7

20 files changed

+862
-39
lines changed

‎src/ResourceManager/DataLakeStore/AzureRM.DataLakeStore.psd1

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -102,7 +102,9 @@ CmdletsToExport = 'Get-AzureRmDataLakeStoreTrustedIdProvider',
102102
'Set-AzureRmDataLakeStoreItemExpiry',
103103
'Set-AzureRmDataLakeStoreItemOwner',
104104
'Set-AzureRmDataLakeStoreItemPermission',
105-
'Test-AzureRmDataLakeStoreAccount', 'Test-AzureRmDataLakeStoreItem'
105+
'Test-AzureRmDataLakeStoreAccount', 'Test-AzureRmDataLakeStoreItem',
106+
'Export-AzureRmDataLakeStoreChildItemProperties',
107+
'Get-AzureRmDataLakeStoreChildItemSummary'
106108

107109
# Variables to export from this module
108110
# VariablesToExport = @()
@@ -122,7 +124,7 @@ AliasesToExport = 'Get-AdlStoreTrustedIdProvider', 'Remove-AdlStoreTrustedIdProv
122124
'Remove-AdlStoreItemAcl', 'Remove-AdlStoreItemAclEntry',
123125
'Set-AdlStoreItemAclEntry', 'Set-AdlStore', 'Set-AdlStoreItemAcl',
124126
'Set-AdlStoreItemExpiry', 'Set-AdlStoreItemOwner',
125-
'Set-AdlStoreItemPermission', 'Test-AdlStore', 'Test-AdlStoreItem'
127+
'Set-AdlStoreItemPermission', 'Test-AdlStore', 'Test-AdlStoreItem', 'Get-AdlStoreChildItemSummary','Export-AdlStoreChildItemProperties'
126128

127129
# DSC resources to export from this module
128130
# DscResourcesToExport = @()

‎src/ResourceManager/DataLakeStore/ChangeLog.md

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,12 @@
1818
- Additional information about change #1
1919
-->
2020
## Current Release
21+
* Add new feature of recursive Acl Change to Remove-AzureRmDataLakeStoreItemAclEntry, Set-AzureRmDataLakeStoreItemAclEntry, Set-AzureRmDataLakeStoreItemAcl
22+
* Add new cmdlet for retrieving the content summary under a directory
23+
* Add new cmdlet for retrieving the disk usage and Acl dump
2124
* Set minimum dependency of module to PowerShell 5.0
25+
* Correct return type of Set-AzureRmDataLakeStoreItemAcl bool to IEnumerable<DataLakeStoreItemAce>
26+
* Correct return type of Set-AzureRmDataLakeStoreItemAclEntry bool to IEnumerable<DataLakeStoreItemAce>
2227
* Breaking changes in Export-AzureRmDataLakeStoreItem, Import-AzureRmDataLakeStoreItem, Remove-AzureRmDataLakeStoreItem
2328

2429
## Version 5.2.0

‎src/ResourceManager/DataLakeStore/Commands.DataLakeStore/Commands.DataLakeStore.csproj

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,8 @@
5656
<Compile Include="Commands\GetAzureRmDataLakeStoreTrustedIdProvider.cs" />
5757
<Compile Include="Commands\RemoveAzureRmDataLakeStoreTrustedIdProvider.cs" />
5858
<Compile Include="Commands\RemoveAzureRmDataLakeStoreFirewallRule.cs" />
59+
<Compile Include="DataPlaneCommands\ExportAzureRmDataLakeStoreChildItemProperties.cs" />
60+
<Compile Include="DataPlaneCommands\GetAzureRmDataLakeStoreChildItemSummary.cs" />
5961
<Compile Include="DataPlaneCommands\SetAzureRmDataLakeStoreItemExpiry.cs" />
6062
<Compile Include="Commands\SetAzureRmDataLakeStoreTrustedIdProvider.cs" />
6163
<Compile Include="Commands\AddAzureRmDataLakeStoreTrustedIdProvider.cs" />
@@ -89,6 +91,7 @@
8991
<Compile Include="DataPlaneCommands\TestAzureRmDataLakeStoreItem.cs" />
9092
<Compile Include="DataPlaneModels\AdlsClientFactory.cs" />
9193
<Compile Include="DataPlaneModels\AdlsLoggerTarget.cs" />
94+
<Compile Include="DataPlaneModels\DataLakeStoreChildItemSummary.cs" />
9295
<Compile Include="Models\DataLakeStoreClient.cs" />
9396
<Compile Include="Models\DataLakeStoreCmdletBase.cs" />
9497
<Compile Include="DataPlaneModels\DataLakeStoreEnums.cs" />

‎src/ResourceManager/DataLakeStore/Commands.DataLakeStore/DataPlaneCommands/ExportAzureRmDataLakeStoreChildItemProperties.cs

Lines changed: 97 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,97 @@
1+
using System.Management.Automation;
2+
using Microsoft.Azure.Commands.DataLakeStore.Models;
3+
4+
namespace Microsoft.Azure.Commands.DataLakeStore
5+
{
6+
[Cmdlet(VerbsData.Export, "AzureRmDataLakeStoreChildItemProperties", SupportsShouldProcess = true), OutputType(typeof(bool))]
7+
[Alias("Export-AdlStoreChildItemProperties")]
8+
public class ExportAzureRmDataLakeStoreChildItemProperties : DataLakeStoreFileSystemCmdletBase
9+
{
10+
internal const string BaseParameterSetName = "GetDiskUsage";
11+
internal const string GetAclParameterSetName = "GetAclDump";
12+
internal const string GetBothParameterSetName = "GetAllProperties";
13+
14+
[Parameter(ValueFromPipelineByPropertyName = true, Position = 0, Mandatory = true,
15+
HelpMessage = "The Data Lake Store account to execute the filesystem operation in")]
16+
[ValidateNotNullOrEmpty]
17+
[Alias("AccountName")]
18+
public string Account { get; set; }
19+
20+
[Parameter(ValueFromPipelineByPropertyName = true, Position = 1, Mandatory = true,
21+
HelpMessage =
22+
"The path in the specified Data Lake account whose properties need to be retrieved. Can be a file or folder " +
23+
"In the format '/folder/file.txt', " +
24+
"where the first '/' after the DNS indicates the root of the file system.")]
25+
[ValidateNotNull]
26+
public DataLakeStorePathInstance Path { get; set; }
27+
28+
[Parameter(ValueFromPipelineByPropertyName = true, Position = 2, Mandatory = true,
29+
HelpMessage = "Path to output file. Can be a Local path or Adl Path. By default"+
30+
" it is local. If SaveToAdl is pecified then it is an ADL path in the same account")]
31+
[ValidateNotNullOrEmpty]
32+
public string OutputPath { get; set; }
33+
34+
[Parameter(ValueFromPipelineByPropertyName = true, Mandatory = false,
35+
HelpMessage =
36+
"Save output to ADL Store, in the same account. The OutputPath parameter should be the full ADL path to output to." +
37+
"Default is to save to a local file. In that case, OutputPath specifies the pathto local file.")]
38+
public SwitchParameter SaveToAdl { get; set; }
39+
40+
[Parameter(ValueFromPipelineByPropertyName = true, Mandatory = false,
41+
HelpMessage =
42+
"Show stats at file level (default is to show directory-level info only)")]
43+
public SwitchParameter IncludeFile { get; set; }
44+
45+
[Parameter(ValueFromPipelineByPropertyName = true, Mandatory = false,
46+
HelpMessage = "Maximum depth from the Path specified, to which disk usage or acl is displayed")]
47+
public int MaximumDepth { get; set; } = int.MaxValue;
48+
49+
[Parameter(ValueFromPipelineByPropertyName = true, Mandatory = false,
50+
HelpMessage =
51+
"Number of files/directories processed in parallel. Optional: a reasonable default will be selected")]
52+
public int Concurrency { get; set; } = -1;
53+
54+
[Parameter(ValueFromPipelineByPropertyName = true, ParameterSetName = BaseParameterSetName, Mandatory = true,
55+
HelpMessage = "Retrieves the disk usage starting from the Path specified")]
56+
[Parameter(ValueFromPipelineByPropertyName = true, ParameterSetName = GetBothParameterSetName, Mandatory = true,
57+
HelpMessage = "Retrieves the disk usage starting from the Path specified")]
58+
public SwitchParameter GetDiskUsage { get; set; }
59+
60+
[Parameter(ValueFromPipelineByPropertyName = true, ParameterSetName = GetAclParameterSetName, Mandatory = true,
61+
HelpMessage = "Retrieves the acl starting from the Path specified")]
62+
[Parameter(ValueFromPipelineByPropertyName = true, ParameterSetName = GetBothParameterSetName, Mandatory = true,
63+
HelpMessage = "Retrieves the acl starting from the Path specified")]
64+
public SwitchParameter GetAcl { get; set; }
65+
66+
[Parameter(ValueFromPipelineByPropertyName = true, ParameterSetName = GetAclParameterSetName, Mandatory = false,
67+
HelpMessage = "Do not show directory subtree if the ACLs are the same throughout the entire subtree. This " +
68+
"makes it easier to see only the paths up to which the ACLs differ." +
69+
"For example if all files and folders under /a/b are the same, do not show the subtree" +
70+
"under /a/b, and just output /a/b with 'True' in the Consistent ACL column" +
71+
"Cannot be set if IncludeFiles is not set, because consistent Acl cannot be determined without retrieving acls for the files.")]
72+
[Parameter(ValueFromPipelineByPropertyName = true, ParameterSetName = GetBothParameterSetName, Mandatory = false,
73+
HelpMessage = "Do not show directory subtree if the ACLs are the same throughout the entire subtree. This " +
74+
"makes it easier to see only the paths up to which the ACLs differ." +
75+
"For example if all files and folders under /a/b are the same, do not show the subtree" +
76+
"under /a/b, and just output /a/b with 'True' in the Consistent ACL column" +
77+
"Cannot be set if IncludeFiles is not set, because consistent Acl cannot be determined without retrieving acls for the files.")]
78+
public SwitchParameter HideConsistentAcl { get; set; }
79+
80+
[Parameter(ValueFromPipelineByPropertyName = true, Mandatory = false,
81+
HelpMessage =
82+
"Indicates a boolean response should be returned indicating the result of the delete operation."
83+
)]
84+
public SwitchParameter PassThru { get; set; }
85+
86+
public override void ExecuteCmdlet()
87+
{
88+
DataLakeStoreFileSystemClient.GetFileProperties(Account, Path.TransformedPath, GetAcl, OutputPath,
89+
GetDiskUsage, !SaveToAdl, Concurrency, IncludeFile, HideConsistentAcl, MaximumDepth);
90+
if (PassThru)
91+
{
92+
WriteObject(true);
93+
}
94+
}
95+
96+
}
97+
}

‎src/ResourceManager/DataLakeStore/Commands.DataLakeStore/DataPlaneCommands/GetAzureRmDataLakeStoreChildItem.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ public class GetAzureDataLakeStoreChildItem : DataLakeStoreFileSystemCmdletBase
2929
[Alias("AccountName")]
3030
public string Account { get; set; }
3131

32-
[Parameter(ValueFromPipelineByPropertyName = true, Position = 0, Mandatory = true,
32+
[Parameter(ValueFromPipelineByPropertyName = true, Position = 1, Mandatory = true,
3333
HelpMessage =
3434
"The path in the specified Data Lake account that should be retrieve. Can be a file or folder " +
3535
"In the format '/folder/file.txt', " +

‎src/ResourceManager/DataLakeStore/Commands.DataLakeStore/DataPlaneCommands/GetAzureRmDataLakeStoreChildItemSummary.cs

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
using System.Management.Automation;
2+
using Microsoft.Azure.Commands.DataLakeStore.Models;
3+
4+
namespace Microsoft.Azure.Commands.DataLakeStore
5+
{
6+
[Cmdlet(VerbsCommon.Get, "AzureRmDataLakeStoreChildItemSummary", SupportsShouldProcess = true), OutputType(typeof(DataLakeStoreChildItemSummary))]
7+
[Alias("Get-AdlStoreChildItemSummary")]
8+
public class GetAzureRmDataLakeStoreChildItemSummary : DataLakeStoreFileSystemCmdletBase
9+
{
10+
[Parameter(ValueFromPipelineByPropertyName = true, Position = 0, Mandatory = true,
11+
HelpMessage = "The Data Lake Store account to execute the filesystem operation in")]
12+
[ValidateNotNullOrEmpty]
13+
[Alias("AccountName")]
14+
public string Account { get; set; }
15+
16+
[Parameter(ValueFromPipelineByPropertyName = true, Position = 0, Mandatory = true,
17+
HelpMessage =
18+
"The path in the specified Data Lake account that should be retrieve. Can be a file or folder " +
19+
"In the format '/folder/file.txt', " +
20+
"where the first '/' after the DNS indicates the root of the file system.")]
21+
[ValidateNotNull]
22+
public DataLakeStorePathInstance Path { get; set; }
23+
24+
[Parameter(ValueFromPipelineByPropertyName = true, Mandatory = false,
25+
HelpMessage =
26+
"Number of files/directories processed in parallel. Optional: a reasonable default will be selected"
27+
)]
28+
public int Concurrency { get; set; } = -1;
29+
30+
public override void ExecuteCmdlet()
31+
{
32+
var toReturn = DataLakeStoreFileSystemClient.GetContentSummary(Path.TransformedPath, Account, Concurrency, CmdletCancellationToken);
33+
WriteObject(new DataLakeStoreChildItemSummary(toReturn));
34+
}
35+
}
36+
}

‎src/ResourceManager/DataLakeStore/Commands.DataLakeStore/DataPlaneCommands/RemoveAzureRmDataLakeStoreItemAclEntry.cs

Lines changed: 20 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
using System.Linq;
2020
using System.Management.Automation;
2121
using Microsoft.Azure.DataLake.Store.Acl;
22+
using Microsoft.Azure.DataLake.Store.AclTools;
2223

2324
namespace Microsoft.Azure.Commands.DataLakeStore
2425
{
@@ -84,6 +85,15 @@ public class RemoveAzureDataLakeStoreItemAclEntry : DataLakeStoreFileSystemCmdle
8485
)]
8586
public SwitchParameter PassThru { get; set; }
8687

88+
[Parameter(ValueFromPipelineByPropertyName = true, Mandatory = false, HelpMessage = "Indicates the ACL to be removed recursively to the child subdirectories and files")]
89+
public SwitchParameter Recurse { get; set; }
90+
91+
[Parameter(ValueFromPipelineByPropertyName = true, Mandatory = false,
92+
HelpMessage =
93+
"Number of files/directories processed in parallel. Optional: a reasonable default will be selected"
94+
)]
95+
public int Concurrency { get; set; } = -1;
96+
8797
public override void ExecuteCmdlet()
8898
{
8999
var aclSpec = ParameterSetName.Equals(BaseParameterSetName)
@@ -95,7 +105,16 @@ public override void ExecuteCmdlet()
95105
Path.OriginalPath,
96106
() =>
97107
{
98-
DataLakeStoreFileSystemClient.RemoveAclEntries(Path.TransformedPath, Account, aclSpec);
108+
if (Recurse)
109+
{
110+
DataLakeStoreFileSystemClient.ChangeAclRecursively(Path.TransformedPath,
111+
Account, aclSpec, RequestedAclType.RemoveAcl, Concurrency);
112+
}
113+
else
114+
{
115+
DataLakeStoreFileSystemClient.RemoveAclEntries(Path.TransformedPath, Account, aclSpec);
116+
}
117+
99118
if (PassThru)
100119
{
101120
WriteObject(true);

‎src/ResourceManager/DataLakeStore/Commands.DataLakeStore/DataPlaneCommands/SetAzureRmDataLakeStoreItemAcl.cs

Lines changed: 31 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -16,14 +16,16 @@
1616
using Microsoft.Azure.Commands.DataLakeStore.Properties;
1717
using System.Management.Automation;
1818
using System.Linq;
19+
using Microsoft.Azure.DataLake.Store.AclTools;
1920

2021
namespace Microsoft.Azure.Commands.DataLakeStore
2122
{
2223
[Cmdlet(VerbsCommon.Set, "AzureRmDataLakeStoreItemAcl", SupportsShouldProcess = true),
23-
OutputType(typeof(bool))]
24+
OutputType(typeof(DataLakeStoreItemAce))]
2425
[Alias("Set-AdlStoreItemAcl")]
2526
public class SetAzureDataLakeStoreItemAcl : DataLakeStoreFileSystemCmdletBase
2627
{
28+
2729
[Parameter(ValueFromPipelineByPropertyName = true, Position = 0, Mandatory = true,
2830
HelpMessage = "The DataLakeStore account to execute the filesystem operation in")]
2931
[ValidateNotNullOrEmpty]
@@ -51,25 +53,44 @@ public class SetAzureDataLakeStoreItemAcl : DataLakeStoreFileSystemCmdletBase
5153
)]
5254
public SwitchParameter PassThru { get; set; }
5355

56+
[Parameter(ValueFromPipelineByPropertyName = true, Mandatory = false,
57+
HelpMessage =
58+
"Indicates the ACL to be set recursively to the child subdirectories and files"
59+
)]
60+
public SwitchParameter Recurse { get; set; }
61+
62+
[Parameter(ValueFromPipelineByPropertyName = true, Mandatory = false,
63+
HelpMessage =
64+
"Indicates the number of files/directories processed in parallel for recursive processing. Default will be computed as a best effort based on system specification."
65+
)]
66+
public int Concurrency { get; set; } = -1;
67+
5468
public override void ExecuteCmdlet()
5569
{
5670
WriteWarning(Resources.IncorrectOutputTypeWarning);
5771
ConfirmAction(
5872
string.Format(Resources.SetDataLakeStoreItemAcl, Path.OriginalPath),
5973
Path.OriginalPath,
6074
() =>
61-
{
62-
DataLakeStoreFileSystemClient.SetAcl(
63-
Path.TransformedPath,
64-
Account,
65-
Acl.Select(entry=>entry.ParseDataLakeStoreItemAce()).ToList());
75+
{
76+
if (Recurse)
77+
{
78+
DataLakeStoreFileSystemClient.ChangeAclRecursively(Path.TransformedPath,
79+
Account,
80+
Acl.Select(entry => entry.ParseDataLakeStoreItemAce()).ToList(), RequestedAclType.SetAcl, Concurrency);
81+
}
82+
else
83+
{
84+
DataLakeStoreFileSystemClient.SetAcl(
85+
Path.TransformedPath,
86+
Account,
87+
Acl.Select(entry => entry.ParseDataLakeStoreItemAce()).ToList());
88+
}
6689

6790
if (PassThru)
6891
{
69-
var toReturn = DataLakeStoreFileSystemClient.GetAclStatus(Path.TransformedPath,
70-
Account).Entries.Select(entry=>new DataLakeStoreItemAce(entry));
71-
72-
WriteObject(toReturn);
92+
WriteObject(DataLakeStoreFileSystemClient.GetAclStatus(Path.TransformedPath,
93+
Account).Entries.Select(entry => new DataLakeStoreItemAce(entry)));
7394
}
7495
});
7596
}

0 commit comments

Comments
 (0)