Skip to content

Commit 65e7e6c

Browse files
cormacpaynecormacpayne
authored
Merge pull request #5989 from Azure/adls-data-plane-breakingchanges
Breaking Change: Remove PErFileThreadCount and ConcurrentFileCount fr…
2 parents 2470034 + 3f14bc2 commit 65e7e6c

File tree

45 files changed

+217
-438
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

45 files changed

+217
-438
lines changed

‎src/ResourceManager/DataLakeStore/ChangeLog.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
-->
2020
## Current Release
2121
* Set minimum dependency of module to PowerShell 5.0
22+
* Breaking changes in Export-AzureRmDataLakeStoreItem, Import-AzureRmDataLakeStoreItem, Remove-AzureRmDataLakeStoreItem
2223

2324
## Version 5.2.0
2425
* Updated to the latest version of the Azure ClientRuntime

‎src/ResourceManager/DataLakeStore/Commands.DataLakeStore/DataPlaneCommands/ExportAzureRmDataLakeStoreItem.cs

Lines changed: 1 addition & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -76,26 +76,6 @@ public class ExportAzureDataLakeStoreItem : DataLakeStoreFileSystemCmdletBase
7676
ParameterSetName = DiagnosticParameterSetName)]
7777
public SwitchParameter Resume { get; set; }
7878

79-
[Obsolete("Parameter PerFileThreadCount of ExportAzureRmDataLakeStoreItem is deprecated. This parameter will be removed in future releases. Please use Concurrency parameter instead.")]
80-
[Parameter(ValueFromPipelineByPropertyName = true, Position = 5, Mandatory = false,
81-
HelpMessage = "Indicates the maximum number of threads to use per file. Default will be computed as a best effort based on folder and file size",
82-
ParameterSetName = BaseParameterSetName)]
83-
[Parameter(ValueFromPipelineByPropertyName = true, Position = 5, Mandatory = false,
84-
HelpMessage = "Indicates the maximum number of threads to use per file. Default will be computed as a best effort based on folder and file size",
85-
ParameterSetName = DiagnosticParameterSetName)]
86-
public int PerFileThreadCount { get; set; } = -1;
87-
88-
[Obsolete("Parameter ConcurrentFileCount of ExportAzureRmDataLakeStoreItem is deprecated. This parameter will be removed in future releases. Please use Concurrency parameter instead.")]
89-
[Parameter(ValueFromPipelineByPropertyName = true, Position = 6, Mandatory = false,
90-
HelpMessage =
91-
"Indicates the maximum number of files to download in parallel for a folder download. Default will be computed as a best effort based on folder and file size",
92-
ParameterSetName = BaseParameterSetName)]
93-
[Parameter(ValueFromPipelineByPropertyName = true, Position = 6, Mandatory = false,
94-
HelpMessage =
95-
"Indicates the maximum number of files to download in parallel for a folder download. Default will be computed as a best effort based on folder and file size",
96-
ParameterSetName = DiagnosticParameterSetName)]
97-
public int ConcurrentFileCount { get; set; } = -1;
98-
9979
[Parameter(ValueFromPipelineByPropertyName = true, Position = 7, Mandatory = false,
10080
HelpMessage = "Indicates that, if the file or folder exists, it should be overwritten",
10181
ParameterSetName = BaseParameterSetName)]
@@ -128,14 +108,6 @@ public class ExportAzureDataLakeStoreItem : DataLakeStoreFileSystemCmdletBase
128108

129109
public override void ExecuteCmdlet()
130110
{
131-
if (ConcurrentFileCount != -1)
132-
{
133-
WriteWarning(Resources.IncorrectConcurrentFileCountWarning);
134-
}
135-
if (PerFileThreadCount != -1)
136-
{
137-
WriteWarning(Resources.IncorrectPerFileThreadCountWarning);
138-
}
139111
// We will let this throw itself if the path they give us is invalid
140112
var powerShellReadyPath = SessionState.Path.GetUnresolvedProviderPathFromPSPath(Destination);
141113
ConfirmAction(
@@ -152,23 +124,7 @@ public override void ExecuteCmdlet()
152124
DataLakeStoreFileSystemClient.SetupFileLogging(DiagnosticLogLevel, diagnosticPath);
153125
}
154126

155-
int threadCount;
156-
if (Concurrency > 0)
157-
{
158-
threadCount = Concurrency;
159-
}
160-
else if (ConcurrentFileCount > 0 && PerFileThreadCount <= 0)
161-
{
162-
threadCount = ConcurrentFileCount;
163-
}
164-
else if (ConcurrentFileCount <= 0 && PerFileThreadCount > 0)
165-
{
166-
threadCount = PerFileThreadCount;
167-
}
168-
else
169-
{
170-
threadCount = Math.Min(PerFileThreadCount * ConcurrentFileCount, DataLakeStoreFileSystemClient.ImportExportMaxThreads);
171-
}
127+
int threadCount = Concurrency;
172128
DataLakeStoreFileSystemClient.BulkCopy(powerShellReadyPath, Account,
173129
Path.TransformedPath, CmdletCancellationToken, threadCount, Recurse, Force, Resume, true, this);
174130
WriteObject(powerShellReadyPath);

‎src/ResourceManager/DataLakeStore/Commands.DataLakeStore/DataPlaneCommands/ImportAzureRmDataLakeStoreItem.cs

Lines changed: 1 addition & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -87,25 +87,6 @@ public class ImportAzureDataLakeStoreItem : DataLakeStoreFileSystemCmdletBase
8787
ParameterSetName = DiagnosticParameterSetName)]
8888
public SwitchParameter ForceBinary { get; set; }
8989

90-
[Obsolete("Parameter PerFileThreadCount of ImportAzureRmDataLakeStoreItem is deprecated. This parameter will be removed in future releases. Please use Concurrency parameter instead.")]
91-
[Parameter(ValueFromPipelineByPropertyName = true, Position = 6, Mandatory = false,
92-
HelpMessage =
93-
"Indicates the maximum number of threads to use per file. Default will be computed as a best effort based on folder and file size",
94-
ParameterSetName = BaseParameterSetName)]
95-
[Parameter(ValueFromPipelineByPropertyName = true, Position = 6, Mandatory = false,
96-
HelpMessage = "Indicates the maximum number of threads to use per file. Default will be computed as a best effort based on folder and file size",
97-
ParameterSetName = DiagnosticParameterSetName)]
98-
public int PerFileThreadCount { get; set; } = -1;
99-
100-
[Obsolete("Parameter ConcurrentFileCount of ImportAzureRmDataLakeStoreItem is deprecated. This parameter will be removed in future releases. Please use Concurrency parameter instead.")]
101-
[Parameter(ValueFromPipelineByPropertyName = true, Position = 7, Mandatory = false,
102-
HelpMessage = "Indicates the maximum number of files to upload in parallel for a folder upload. Default will be computed as a best effort based on folder and file size",
103-
ParameterSetName = BaseParameterSetName)]
104-
[Parameter(ValueFromPipelineByPropertyName = true, Position = 7, Mandatory = false,
105-
HelpMessage = "Indicates the maximum number of files to upload in parallel for a folder upload. Default will be computed as a best effort based on folder and file size",
106-
ParameterSetName = DiagnosticParameterSetName)]
107-
public int ConcurrentFileCount { get; set; } = -1;
108-
10990
[Parameter(ValueFromPipelineByPropertyName = true, Position = 8, Mandatory = false,
11091
HelpMessage = "Indicates that, if the file or folder exists, it should be overwritten",
11192
ParameterSetName = BaseParameterSetName)]
@@ -137,14 +118,6 @@ public class ImportAzureDataLakeStoreItem : DataLakeStoreFileSystemCmdletBase
137118

138119
public override void ExecuteCmdlet()
139120
{
140-
if (ConcurrentFileCount != -1)
141-
{
142-
WriteWarning(Resources.IncorrectConcurrentFileCountWarning);
143-
}
144-
if (PerFileThreadCount != -1)
145-
{
146-
WriteWarning(Resources.IncorrectPerFileThreadCountWarning);
147-
}
148121
var powerShellSourcePath = SessionState.Path.GetUnresolvedProviderPathFromPSPath(Path);
149122
ConfirmAction(
150123
Resources.UploadFileMessage,
@@ -160,24 +133,7 @@ public override void ExecuteCmdlet()
160133
DataLakeStoreFileSystemClient.SetupFileLogging(DiagnosticLogLevel, diagnosticPath);
161134
}
162135

163-
int threadCount;
164-
// If concurrency is specified then accept that
165-
if (Concurrency > 0)
166-
{
167-
threadCount = Concurrency;
168-
}
169-
else if (ConcurrentFileCount > 0 && PerFileThreadCount <= 0)
170-
{
171-
threadCount = ConcurrentFileCount;
172-
}
173-
else if (ConcurrentFileCount <= 0 && PerFileThreadCount > 0)
174-
{
175-
threadCount = PerFileThreadCount;
176-
}
177-
else
178-
{
179-
threadCount = Math.Min(PerFileThreadCount * ConcurrentFileCount, DataLakeStoreFileSystemClient.ImportExportMaxThreads);
180-
}
136+
int threadCount = Concurrency;
181137
DataLakeStoreFileSystemClient.BulkCopy(Destination.TransformedPath, Account,
182138
powerShellSourcePath, CmdletCancellationToken, threadCount, Recurse, Force, Resume, false, this, ForceBinary);
183139
// only attempt to write output if this cmdlet hasn't been cancelled.

‎src/ResourceManager/DataLakeStore/Commands.DataLakeStore/DataPlaneCommands/RemoveAzureRmDataLakeStoreItem.cs

Lines changed: 0 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -12,11 +12,9 @@
1212
// limitations under the License.
1313
// ----------------------------------------------------------------------------------
1414

15-
using System;
1615
using Microsoft.Azure.Commands.DataLakeStore.Models;
1716
using Microsoft.Azure.Commands.DataLakeStore.Properties;
1817
using System.Management.Automation;
19-
using Microsoft.Azure.DataLake.Store;
2018

2119
namespace Microsoft.Azure.Commands.DataLakeStore
2220
{
@@ -41,12 +39,6 @@ public class RemoveAzureDataLakeStoreItem : DataLakeStoreFileSystemCmdletBase
4139
HelpMessage = "Indicates the user wants a recursive delete of the folder.")]
4240
public SwitchParameter Recurse { get; set; }
4341

44-
[Obsolete("Parameter Clean of RemoveAzureDataLakeStoreItem is deprecated. This parameter will be removed in future releases.")]
45-
[Parameter(ValueFromPipelineByPropertyName = true, Position = 3, Mandatory = false,
46-
HelpMessage =
47-
"Indicates the user wants to remove all of the contents of the folder, but not the folder itself")]
48-
public SwitchParameter Clean { get; set; }
49-
5042
[Parameter(ValueFromPipelineByPropertyName = true, Position = 4, Mandatory = false,
5143
HelpMessage =
5244
"Indicates delete should be performed without prompting."
@@ -61,17 +53,9 @@ public class RemoveAzureDataLakeStoreItem : DataLakeStoreFileSystemCmdletBase
6153

6254
public override void ExecuteCmdlet()
6355
{
64-
if (Clean)
65-
{
66-
WriteWarning(Resources.IncorrectCleanWarning);
67-
}
6856
bool success = true;
6957
foreach (var path in Paths)
7058
{
71-
DirectoryEntryType testClean;
72-
var pathExists = DataLakeStoreFileSystemClient.TestFileOrFolderExistence(path.TransformedPath,
73-
Account, out testClean);
74-
7559
ConfirmAction(
7660
Force.IsPresent,
7761
string.Format(Resources.RemovingDataLakeStoreItem, path.OriginalPath),
@@ -83,11 +67,6 @@ public override void ExecuteCmdlet()
8367
success &&
8468
DataLakeStoreFileSystemClient.DeleteFileOrFolder(path.TransformedPath, Account,
8569
Recurse);
86-
if (pathExists && testClean == DirectoryEntryType.DIRECTORY && Clean)
87-
{
88-
// recreate the directory as an empty directory if clean was specified.
89-
DataLakeStoreFileSystemClient.CreateDirectory(path.TransformedPath, Account);
90-
}
9170
if (PassThru)
9271
{
9372
WriteObject(success);

‎src/ResourceManager/DataLakeStore/Commands.DataLakeStore/Properties/Resources.Designer.cs

Lines changed: 1 addition & 28 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎src/ResourceManager/DataLakeStore/Commands.DataLakeStore/Properties/Resources.resx

Lines changed: 0 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -168,15 +168,6 @@
168168
<data name="IncorrectOutputTypeWarning" xml:space="preserve">
169169
<value>The output type defined for this cmdlet is incorrect and will be updated to reflect what is actually returned (and defined in the help) in a future release.</value>
170170
</data>
171-
<data name="IncorrectPerFileThreadCountWarning" xml:space="preserve">
172-
<value>Parameter PerFileThreadCount is deprecated. This parameter will be removed in future releases. Please use Concurrency parameter instead.</value>
173-
</data>
174-
<data name="IncorrectConcurrentFileCountWarning" xml:space="preserve">
175-
<value>Parameter ConcurrentFileCount is deprecated. This parameter will be removed in future releases. Please use Concurrency parameter instead.</value>
176-
</data>
177-
<data name="IncorrectCleanWarning" xml:space="preserve">
178-
<value>Parameter Clean is deprecated. This parameter will be removed in future releases.</value>
179-
</data>
180171
<data name="InvalidAce" xml:space="preserve">
181172
<value>Invalid ACE returned. Expected ACE format: '[default]:&lt;scope&gt;:&lt;identity&gt;:&lt;permissions&gt;'. Actual ACE: {0}</value>
182173
</data>

‎src/ResourceManager/DataLakeStore/Commands.DataLakeStore/help/Add-AzureRmDataLakeStoreFirewallRule.md

Lines changed: 5 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
---
1+
---
22
external help file: Microsoft.Azure.Commands.DataLakeStore.dll-Help.xml
33
Module Name: AzureRM.DataLakeStore
44
ms.assetid: C6FD4734-720C-4C8C-9B58-EDB331DD6415
@@ -69,7 +69,7 @@ The end of the valid ip range for the firewall rule
6969
```yaml
7070
Type: String
7171
Parameter Sets: (All)
72-
Aliases:
72+
Aliases:
7373

7474
Required: True
7575
Position: 3
@@ -84,7 +84,7 @@ The name of the firewall rule to add.
8484
```yaml
8585
Type: String
8686
Parameter Sets: (All)
87-
Aliases:
87+
Aliases:
8888

8989
Required: True
9090
Position: 1
@@ -99,7 +99,7 @@ Name of resource group under which the account to add the firewall rule is.
9999
```yaml
100100
Type: String
101101
Parameter Sets: (All)
102-
Aliases:
102+
Aliases:
103103

104104
Required: False
105105
Position: 4
@@ -114,7 +114,7 @@ The start of the valid ip range for the firewall rule
114114
```yaml
115115
Type: String
116116
Parameter Sets: (All)
117-
Aliases:
117+
Aliases:
118118

119119
Required: True
120120
Position: 2
@@ -167,4 +167,3 @@ The firewall rule that was added.
167167
## NOTES
168168
169169
## RELATED LINKS
170-

‎src/ResourceManager/DataLakeStore/Commands.DataLakeStore/help/Add-AzureRmDataLakeStoreItemContent.md

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
---
1+
---
22
external help file: Microsoft.Azure.Commands.DataLakeStore.dll-Help.xml
33
Module Name: AzureRM.DataLakeStore
44
ms.assetid: B008028D-27FC-4469-BE71-54F7218C068B
@@ -82,7 +82,7 @@ The acceptable values for this parameter are:
8282
```yaml
8383
Type: FileSystemCmdletProviderEncoding
8484
Parameter Sets: (All)
85-
Aliases:
85+
Aliases:
8686
Accepted values: Unknown, String, Unicode, Byte, BigEndianUnicode, UTF8, UTF7, UTF32, Ascii, Default, Oem, BigEndianUTF32
8787

8888
Required: False
@@ -98,7 +98,7 @@ Specifies the Data Lake Store path of the item to modify, starting with the root
9898
```yaml
9999
Type: DataLakeStorePathInstance
100100
Parameter Sets: (All)
101-
Aliases:
101+
Aliases:
102102

103103
Required: True
104104
Position: 1
@@ -113,7 +113,7 @@ Specifies the content to add to the item.
113113
```yaml
114114
Type: Object
115115
Parameter Sets: (All)
116-
Aliases:
116+
Aliases:
117117

118118
Required: True
119119
Position: 2

‎src/ResourceManager/DataLakeStore/Commands.DataLakeStore/help/Add-AzureRmDataLakeStoreTrustedIdProvider.md

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
---
1+
---
22
external help file: Microsoft.Azure.Commands.DataLakeStore.dll-Help.xml
33
Module Name: AzureRM.DataLakeStore
44
ms.assetid: 5C788778-58A4-4798-AB66-1D3562BB9338
@@ -69,7 +69,7 @@ The name of the trusted identity provider to add
6969
```yaml
7070
Type: String
7171
Parameter Sets: (All)
72-
Aliases:
72+
Aliases:
7373

7474
Required: True
7575
Position: 1
@@ -84,7 +84,7 @@ The valid trusted provider endpoint in the format: https://sts.windows.net/\<pro
8484
```yaml
8585
Type: String
8686
Parameter Sets: (All)
87-
Aliases:
87+
Aliases:
8888

8989
Required: True
9090
Position: 2
@@ -99,7 +99,7 @@ Name of resource group under which the account to add the trusted identity provi
9999
```yaml
100100
Type: String
101101
Parameter Sets: (All)
102-
Aliases:
102+
Aliases:
103103

104104
Required: False
105105
Position: 3
@@ -152,4 +152,3 @@ The added Trusted Identity Provider.
152152
## NOTES
153153
154154
## RELATED LINKS
155-

‎src/ResourceManager/DataLakeStore/Commands.DataLakeStore/help/AzureRM.DataLakeStore.md

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
---
1+
---
22
Module Name: AzureRM.DataLakeStore
33
Module Guid: 90dfd814-abce-4e1f-99b6-fe16760c079a
44
Download Help Link: None

0 commit comments

Comments
 (0)