Merge pull request #7718 from markcowl/dlsnewt

markcowl · web-flow · commit dec8784e3181 · 2018-10-31T14:26:35.000-07:00
Update DataLakeStore data plane library
diff --git a/src/ResourceManager/DataLakeStore/Commands.DataLakeStore.Test/packages.config b/src/ResourceManager/DataLakeStore/Commands.DataLakeStore.Test/packages.config
@@ -19,7 +19,6 @@
   <package id="Microsoft.Rest.ClientRuntime.Azure.TestFramework" version="1.5.1-preview" targetFramework="net45" />
   <package id="Microsoft.WindowsAzure.Management" version="4.1.1" targetFramework="net45" />
   <package id="Moq" version="4.2.1510.2205" targetFramework="net45" />
-  <package id="Newtonsoft.Json" version="6.0.8" targetFramework="net452" />
   <package id="xunit" version="2.1.0" targetFramework="net45" />
   <package id="xunit.abstractions" version="2.0.0" targetFramework="net45" />
   <package id="xunit.assert" version="2.1.0" targetFramework="net45" />
diff --git a/src/ResourceManager/DataLakeStore/Commands.DataLakeStore/ChangeLog.md b/src/ResourceManager/DataLakeStore/Commands.DataLakeStore/ChangeLog.md
@@ -18,6 +18,8 @@
         - Additional information about change #1
 -->
 ## Current Release
+* Update the DataLake package to 1.1.10.
+* Add default Concurrency to multithreaded operations.
 
 ## Version 6.2.0
 * Adding support for Virtual Network Rules
diff --git a/src/ResourceManager/DataLakeStore/Commands.DataLakeStore/Commands.DataLakeStore.Netcore.csproj b/src/ResourceManager/DataLakeStore/Commands.DataLakeStore/Commands.DataLakeStore.Netcore.csproj
@@ -32,7 +32,7 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="Microsoft.Azure.DataLake.Store" Version="1.1.9" />
+    <PackageReference Include="Microsoft.Azure.DataLake.Store" Version="1.1.10" />
     <PackageReference Include="Microsoft.Azure.Management.DataLake.Store" Version="2.4.2-preview" />
     <PackageReference Include="NLog" Version="4.5.0" />
     <PackageReference Include="System.Net.Requests" Version="4.3.0" />
diff --git a/src/ResourceManager/DataLakeStore/Commands.DataLakeStore/Commands.DataLakeStore.csproj b/src/ResourceManager/DataLakeStore/Commands.DataLakeStore/Commands.DataLakeStore.csproj
@@ -39,7 +39,7 @@
   <ItemGroup>
     <Reference Include="Microsoft.Azure.DataLake.Store, Version=1.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
       <SpecificVersion>False</SpecificVersion>
-      <HintPath>..\..\..\packages\Microsoft.Azure.DataLake.Store.1.1.9\lib\net452\Microsoft.Azure.DataLake.Store.dll</HintPath>
+      <HintPath>..\..\..\packages\Microsoft.Azure.DataLake.Store.1.1.10\lib\net452\Microsoft.Azure.DataLake.Store.dll</HintPath>
       <Private>True</Private>
     </Reference>
     <Reference Include="Microsoft.Azure.Management.DataLake.Store, Version=2.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
diff --git a/src/ResourceManager/DataLakeStore/Commands.DataLakeStore/DataPlaneCommands/ExportAzureRmDataLakeStoreChildItemProperties.cs b/src/ResourceManager/DataLakeStore/Commands.DataLakeStore/DataPlaneCommands/ExportAzureRmDataLakeStoreChildItemProperties.cs
@@ -85,8 +85,11 @@ public class ExportAzureRmDataLakeStoreChildItemProperties : DataLakeStoreFileSy
 
         public override void ExecuteCmdlet()
         {
+            // Currently SDK default thread calculation is not correct, so pass a default thread count
+            int threadCount = Concurrency == -1 ? DataLakeStoreFileSystemClient.ImportExportMaxThreads : Concurrency;
+
             DataLakeStoreFileSystemClient.GetFileProperties(Account, Path.TransformedPath, GetAcl, OutputPath,
-                GetDiskUsage, !SaveToAdl, Concurrency, IncludeFile, HideConsistentAcl, MaximumDepth, this, CmdletCancellationToken);
+                GetDiskUsage, !SaveToAdl, threadCount, IncludeFile, HideConsistentAcl, MaximumDepth, this, CmdletCancellationToken);
             if (PassThru)
             {
                 WriteObject(true);
diff --git a/src/ResourceManager/DataLakeStore/Commands.DataLakeStore/DataPlaneCommands/ExportAzureRmDataLakeStoreItem.cs b/src/ResourceManager/DataLakeStore/Commands.DataLakeStore/DataPlaneCommands/ExportAzureRmDataLakeStoreItem.cs
@@ -123,7 +123,8 @@ public override void ExecuteCmdlet()
                             DataLakeStoreFileSystemClient.SetupFileLogging(DiagnosticLogLevel, diagnosticPath);
                         }
 
-                        int threadCount = Concurrency;
+                        // Currently SDK default thread calculation is not correct, so pass a default thread count
+                        int threadCount = Concurrency == -1 ? DataLakeStoreFileSystemClient.ImportExportMaxThreads : Concurrency;
                         DataLakeStoreFileSystemClient.BulkCopy(powerShellReadyPath, Account,
                             Path.TransformedPath, CmdletCancellationToken, threadCount, Recurse, Force, Resume, true, this);
                         WriteObject(powerShellReadyPath);
diff --git a/src/ResourceManager/DataLakeStore/Commands.DataLakeStore/DataPlaneCommands/ImportAzureRmDataLakeStoreItem.cs b/src/ResourceManager/DataLakeStore/Commands.DataLakeStore/DataPlaneCommands/ImportAzureRmDataLakeStoreItem.cs
@@ -132,7 +132,9 @@ public override void ExecuteCmdlet()
                             DataLakeStoreFileSystemClient.SetupFileLogging(DiagnosticLogLevel, diagnosticPath);
                         }
 
-                        int threadCount = Concurrency;
+                        // Currently SDK default thread calculation is not correct, so pass a default thread count
+                        int threadCount = Concurrency == -1 ? DataLakeStoreFileSystemClient.ImportExportMaxThreads : Concurrency;
+
                         DataLakeStoreFileSystemClient.BulkCopy(Destination.TransformedPath, Account,
                             powerShellSourcePath, CmdletCancellationToken, threadCount, Recurse, Force, Resume, false, this, ForceBinary);
                         // only attempt to write output if this cmdlet hasn't been cancelled.
diff --git a/src/ResourceManager/DataLakeStore/Commands.DataLakeStore/DataPlaneCommands/RemoveAzureRmDataLakeStoreItemAclEntry.cs b/src/ResourceManager/DataLakeStore/Commands.DataLakeStore/DataPlaneCommands/RemoveAzureRmDataLakeStoreItemAclEntry.cs
@@ -110,8 +110,11 @@ public override void ExecuteCmdlet()
                 {
                     if (Recurse)
                     {
+                        // Currently SDK default thread calculation is not correct, so pass a default thread count
+                        int threadCount = Concurrency == -1 ? DataLakeStoreFileSystemClient.ImportExportMaxThreads : Concurrency;
+
                         DataLakeStoreFileSystemClient.ChangeAclRecursively(Path.TransformedPath,
-                            Account, aclSpec, RequestedAclType.RemoveAcl, Concurrency, this, ShowProgress, CmdletCancellationToken);
+                            Account, aclSpec, RequestedAclType.RemoveAcl, threadCount, this, ShowProgress, CmdletCancellationToken);
                     }
                     else
                     {
diff --git a/src/ResourceManager/DataLakeStore/Commands.DataLakeStore/DataPlaneCommands/SetAzureRmDataLakeStoreItemAcl.cs b/src/ResourceManager/DataLakeStore/Commands.DataLakeStore/DataPlaneCommands/SetAzureRmDataLakeStoreItemAcl.cs
@@ -80,9 +80,12 @@ public override void ExecuteCmdlet()
                 {
                         if (Recurse)
                         {
-                            DataLakeStoreFileSystemClient.ChangeAclRecursively(Path.TransformedPath,
+                        // Currently SDK default thread calculation is not correct, so pass a default thread count
+                        int threadCount = Concurrency == -1 ? DataLakeStoreFileSystemClient.ImportExportMaxThreads : Concurrency;
+
+                        DataLakeStoreFileSystemClient.ChangeAclRecursively(Path.TransformedPath,
                                 Account,
-                                Acl.Select(entry => entry.ParseDataLakeStoreItemAce()).ToList(), RequestedAclType.SetAcl, Concurrency, this, ShowProgress, CmdletCancellationToken);
+                                Acl.Select(entry => entry.ParseDataLakeStoreItemAce()).ToList(), RequestedAclType.SetAcl, threadCount, this, ShowProgress, CmdletCancellationToken);
                         }
                         else
                         {
diff --git a/src/ResourceManager/DataLakeStore/Commands.DataLakeStore/DataPlaneCommands/SetAzureRmDataLakeStoreItemAclEntry.cs b/src/ResourceManager/DataLakeStore/Commands.DataLakeStore/DataPlaneCommands/SetAzureRmDataLakeStoreItemAclEntry.cs
@@ -122,10 +122,12 @@ public override void ExecuteCmdlet()
                 {
                     if (Recurse)
                     {
+                        // Currently SDK default thread calculation is not correct, so pass a default thread count
+                        int threadCount = Concurrency == -1 ? DataLakeStoreFileSystemClient.ImportExportMaxThreads : Concurrency;
 
                         DataLakeStoreFileSystemClient.ChangeAclRecursively(Path.TransformedPath,
                             Account,
-                            aclSpec, RequestedAclType.ModifyAcl, Concurrency, this, ShowProgress, CmdletCancellationToken);
+                            aclSpec, RequestedAclType.ModifyAcl, threadCount, this, ShowProgress, CmdletCancellationToken);
                     }
                     else
                     {
diff --git a/src/ResourceManager/DataLakeStore/Commands.DataLakeStore/DataPlaneModels/DataLakeStoreFileSystemClient.cs b/src/ResourceManager/DataLakeStore/Commands.DataLakeStore/DataPlaneModels/DataLakeStoreFileSystemClient.cs
@@ -46,7 +46,7 @@ public class DataLakeStoreFileSystemClient
         private readonly Random _uniqueActivityIdGenerator;
         private const int MaxConnectionLimit = 1000;
         private const long NeverExpireValue = 253402300800000;
-        internal const int ImportExportMaxThreads = 256;
+        internal const int ImportExportMaxThreads = 128;
         private readonly LoggingConfiguration _adlsLoggerConfig;
         private readonly bool _isDebugEnabled;
         private const int DebugMessageFlushThreshold = 500;
diff --git a/src/ResourceManager/DataLakeStore/Commands.DataLakeStore/packages.config b/src/ResourceManager/DataLakeStore/Commands.DataLakeStore/packages.config
@@ -1,6 +1,6 @@
 ﻿<?xml version="1.0" encoding="utf-8"?>
 <packages>
-  <package id="Microsoft.Azure.DataLake.Store" version="1.1.9" targetFramework="net452" />
+  <package id="Microsoft.Azure.DataLake.Store" version="1.1.10" targetFramework="net452" />
   <package id="Microsoft.Azure.Management.DataLake.Store" version="2.4.2-preview" targetFramework="net452" />
   <package id="NLog" version="4.4.12" targetFramework="net452" />
 </packages>

Original file line number	Diff line number	Diff line change
`@@ -85,8 +85,11 @@ public class ExportAzureRmDataLakeStoreChildItemProperties : DataLakeStoreFileSy`
`85`	`85`
`86`	`86`	`public override void ExecuteCmdlet()`
`87`	`87`	`{`
	`88`	`+ // Currently SDK default thread calculation is not correct, so pass a default thread count`
	`89`	`+ int threadCount = Concurrency == -1 ? DataLakeStoreFileSystemClient.ImportExportMaxThreads : Concurrency;`
	`90`	`+`
`88`	`91`	`DataLakeStoreFileSystemClient.GetFileProperties(Account, Path.TransformedPath, GetAcl, OutputPath,`
`89`		`- GetDiskUsage, !SaveToAdl, Concurrency, IncludeFile, HideConsistentAcl, MaximumDepth, this, CmdletCancellationToken);`
	`92`	`+ GetDiskUsage, !SaveToAdl, threadCount, IncludeFile, HideConsistentAcl, MaximumDepth, this, CmdletCancellationToken);`
`90`	`93`	`if (PassThru)`
`91`	`94`	`{`
`92`	`95`	`WriteObject(true);`
Original file line number	Diff line number	Diff line change
`@@ -123,7 +123,8 @@ public override void ExecuteCmdlet()`
`123`	`123`	`DataLakeStoreFileSystemClient.SetupFileLogging(DiagnosticLogLevel, diagnosticPath);`
`124`	`124`	`}`
`125`	`125`
`126`		`- int threadCount = Concurrency;`
	`126`	`+ // Currently SDK default thread calculation is not correct, so pass a default thread count`
	`127`	`+ int threadCount = Concurrency == -1 ? DataLakeStoreFileSystemClient.ImportExportMaxThreads : Concurrency;`
`127`	`128`	`DataLakeStoreFileSystemClient.BulkCopy(powerShellReadyPath, Account,`
`128`	`129`	`Path.TransformedPath, CmdletCancellationToken, threadCount, Recurse, Force, Resume, true, this);`
`129`	`130`	`WriteObject(powerShellReadyPath);`
Original file line number	Diff line number	Diff line change
`@@ -110,8 +110,11 @@ public override void ExecuteCmdlet()`
`110`	`110`	`{`
`111`	`111`	`if (Recurse)`
`112`	`112`	`{`
	`113`	`+ // Currently SDK default thread calculation is not correct, so pass a default thread count`
	`114`	`+ int threadCount = Concurrency == -1 ? DataLakeStoreFileSystemClient.ImportExportMaxThreads : Concurrency;`
	`115`	`+`
`113`	`116`	`DataLakeStoreFileSystemClient.ChangeAclRecursively(Path.TransformedPath,`
`114`		`- Account, aclSpec, RequestedAclType.RemoveAcl, Concurrency, this, ShowProgress, CmdletCancellationToken);`
	`117`	`+ Account, aclSpec, RequestedAclType.RemoveAcl, threadCount, this, ShowProgress, CmdletCancellationToken);`
`115`	`118`	`}`
`116`	`119`	`else`
`117`	`120`	`{`
Original file line number	Diff line number	Diff line change
`@@ -80,9 +80,12 @@ public override void ExecuteCmdlet()`
`80`	`80`	`{`
`81`	`81`	`if (Recurse)`
`82`	`82`	`{`
`83`		`- DataLakeStoreFileSystemClient.ChangeAclRecursively(Path.TransformedPath,`
	`83`	`+ // Currently SDK default thread calculation is not correct, so pass a default thread count`
	`84`	`+ int threadCount = Concurrency == -1 ? DataLakeStoreFileSystemClient.ImportExportMaxThreads : Concurrency;`
	`85`	`+`
	`86`	`+ DataLakeStoreFileSystemClient.ChangeAclRecursively(Path.TransformedPath,`
`84`	`87`	`Account,`
`85`		`- Acl.Select(entry => entry.ParseDataLakeStoreItemAce()).ToList(), RequestedAclType.SetAcl, Concurrency, this, ShowProgress, CmdletCancellationToken);`
	`88`	`+ Acl.Select(entry => entry.ParseDataLakeStoreItemAce()).ToList(), RequestedAclType.SetAcl, threadCount, this, ShowProgress, CmdletCancellationToken);`
`86`	`89`	`}`
`87`	`90`	`else`
`88`	`91`	`{`
Original file line number	Diff line number	Diff line change
`@@ -122,10 +122,12 @@ public override void ExecuteCmdlet()`
`122`	`122`	`{`
`123`	`123`	`if (Recurse)`
`124`	`124`	`{`
	`125`	`+ // Currently SDK default thread calculation is not correct, so pass a default thread count`
	`126`	`+ int threadCount = Concurrency == -1 ? DataLakeStoreFileSystemClient.ImportExportMaxThreads : Concurrency;`
`125`	`127`
`126`	`128`	`DataLakeStoreFileSystemClient.ChangeAclRecursively(Path.TransformedPath,`
`127`	`129`	`Account,`
`128`		`- aclSpec, RequestedAclType.ModifyAcl, Concurrency, this, ShowProgress, CmdletCancellationToken);`
	`130`	`+ aclSpec, RequestedAclType.ModifyAcl, threadCount, this, ShowProgress, CmdletCancellationToken);`
`129`	`131`	`}`
`130`	`132`	`else`
`131`	`133`	`{`