Merge pull request #5573 from Azure/adls-data-plane-task-addDebugFunctionality

Adls data plane task add debug functionality

Author: cormacpayne

src/ResourceManager/DataLakeStore/ChangeLog.md

@@ -18,15 +18,10 @@
         - Additional information about change #1
 -->
 ## Current Release
-
-* Update the version of the ADLS dataplane SDK to 1.1.2 
-
-## Version 5.1.1
-* Corrected usage of 'Login-AzureRmAccount' to use 'Connect-AzureRmAccount'
-* Corrected the error message of 'Test-AzureRmDataLakeStoreAccount' when running this cmdlet without having logged in with 'Login-AzureRmAccount'
+* Add debug functionality
+* Update the version of the ADLS dataplane SDK to 1.1.2
 
 ## Version 5.2.0-preview
-
 * Export-AzureRmDataLakeStoreItem (https://github.com/Azure/azure-powershell/blob/adls-data-plane/src/ResourceManager/DataLakeStore/documentation/upcoming-breaking-changes.md) - Deprecated parameters PerFileThreadCount, ConcurrentFileCount and introduced parameter Concurrency
 * Import-AzureRMDataLakeStoreItem (https://github.com/Azure/azure-powershell/blob/adls-data-plane/src/ResourceManager/DataLakeStore/documentation/upcoming-breaking-changes.md) -Deprecated parametersPerFileThreadCount, ConcurrentFileCount and introduced parameter Concurrency
 * Get-AzureRMDataLakeStoreItemContent - Fixed the tail behavior for contents greater than 4MB

src/ResourceManager/DataLakeStore/Commands.DataLakeStore/Commands.DataLakeStore.csproj

@@ -38,7 +38,7 @@
   <ItemGroup>
     <Reference Include="Microsoft.Azure.DataLake.Store, Version=1.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
       <SpecificVersion>False</SpecificVersion>
-      <HintPath>..\..\..\packages\Microsoft.Azure.DataLake.Store.1.1.2\lib\net452\Microsoft.Azure.DataLake.Store.dll</HintPath>
+      <HintPath>..\..\..\packages\Microsoft.Azure.DataLake.Store.1.1.4\lib\net452\Microsoft.Azure.DataLake.Store.dll</HintPath>
       <Private>True</Private>
     </Reference>
     <Reference Include="Microsoft.Azure.Management.DataLake.Store, Version=2.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
@@ -89,6 +89,7 @@
     <Compile Include="Commands\TestAzureRmDataLakeStoreAccount.cs" />
     <Compile Include="DataPlaneCommands\TestAzureRmDataLakeStoreItem.cs" />
     <Compile Include="DataPlaneModels\AdlsClientFactory.cs" />
+    <Compile Include="DataPlaneModels\AdlsLoggerTarget.cs" />
     <Compile Include="Models\DataLakeStoreClient.cs" />
     <Compile Include="Models\DataLakeStoreCmdletBase.cs" />
     <Compile Include="DataPlaneModels\DataLakeStoreEnums.cs" />

src/ResourceManager/DataLakeStore/Commands.DataLakeStore/DataPlaneCommands/ExportAzureRmDataLakeStoreItem.cs

@@ -149,7 +149,7 @@ public override void ExecuteCmdlet()
                         {
                             var diagnosticPath =
                                 SessionState.Path.GetUnresolvedProviderPathFromPSPath(DiagnosticLogPath);
-                            DataLakeStoreFileSystemClient.SetupLogging(DiagnosticLogLevel, diagnosticPath);
+                            DataLakeStoreFileSystemClient.SetupFileLogging(DiagnosticLogLevel, diagnosticPath);
                         }
 
                         int threadCount;

src/ResourceManager/DataLakeStore/Commands.DataLakeStore/DataPlaneCommands/ImportAzureRmDataLakeStoreItem.cs

@@ -157,7 +157,7 @@ public override void ExecuteCmdlet()
                         {
                             var diagnosticPath =
                                 SessionState.Path.GetUnresolvedProviderPathFromPSPath(DiagnosticLogPath);
-                            DataLakeStoreFileSystemClient.SetupLogging(DiagnosticLogLevel, diagnosticPath);
+                            DataLakeStoreFileSystemClient.SetupFileLogging(DiagnosticLogLevel, diagnosticPath);
                         }
 
                         int threadCount;

src/ResourceManager/DataLakeStore/Commands.DataLakeStore/DataPlaneModels/AdlsLoggerTarget.cs

@@ -0,0 +1,22 @@
+using System.Collections.Concurrent;
+using NLog;
+using NLog.Targets;
+
+namespace Microsoft.Azure.Commands.DataLakeStore.Models
+{
+    /// <summary>
+    /// NLog is used by the ADLS dataplane sdk to log debug messages. We can create a custom target
+    /// which basically queues the debug data to the ConcurrentQueue for debug messages.
+    /// https://github.com/NLog/NLog/wiki/How-to-write-a-custom-target
+    /// </summary>
+    [Target("AdlsLogger")]
+    internal sealed class AdlsLoggerTarget : TargetWithLayout
+    {
+        internal ConcurrentQueue<string> DebugMessageQueue;
+        protected override void Write(LogEventInfo logEvent)
+        {
+            string logMessage = Layout.Render(logEvent);
+            DebugMessageQueue?.Enqueue(logMessage);
+        }
+    }
+}

src/ResourceManager/DataLakeStore/Commands.DataLakeStore/DataPlaneModels/DataLakeStoreFileSystemClient.cs

@@ -45,8 +45,17 @@ public class DataLakeStoreFileSystemClient
         private const int MaxConnectionLimit = 1000;
         private const long NeverExpireValue = 253402300800000;
         internal const int ImportExportMaxThreads = 256;
+        private readonly LoggingConfiguration _adlsLoggerConfig;
+
         #region Constructors
 
+        static DataLakeStoreFileSystemClient()
+        {
+            // Registering the custom target class
+            Target.Register<AdlsLoggerTarget>("AdlsLogger"); //generic
+            LogManager.ReconfigExistingLoggers();
+        }
+
         public DataLakeStoreFileSystemClient(IAzureContext context)
         {
             if (context != null)
@@ -68,6 +77,29 @@ public DataLakeStoreFileSystemClient(IAzureContext context)
             ServicePointManager.UseNagleAlgorithm = false;
         }
 
+        public DataLakeStoreFileSystemClient(IAzureContext context, DataLakeStoreFileSystemCmdletBase cmdlet) : this(context)
+        {
+            _adlsLoggerConfig = new LoggingConfiguration();
+            
+            // Custom target that logs the debug messages from the SDK to the powershell framework's debug message queue
+            var adlsTarget = new AdlsLoggerTarget{ 
+                DebugMessageQueue = cmdlet.DebugMessages
+            };
+
+            // Add the target to the configuration
+            _adlsLoggerConfig.AddTarget("logger", adlsTarget);
+
+            //Logs all patterns of debug messages
+            var rule = new LoggingRule("adls.dotnet.*", NLog.LogLevel.Debug, adlsTarget);
+            _adlsLoggerConfig.LoggingRules.Add(rule);
+            
+            var powershellLoggingRule= new LoggingRule("adls.powershell.WebTransport", NLog.LogLevel.Debug, adlsTarget);
+            _adlsLoggerConfig.LoggingRules.Add(powershellLoggingRule);
+
+            // Enable the NLog configuration to use this
+            LogManager.Configuration = _adlsLoggerConfig;
+        }
+
         #endregion
 
         #region File and Folder Permissions Operations
@@ -495,28 +527,22 @@ public void AppendToFile(string filePath, string accountName, byte[] contents)
             }
         }
         /// <summary>
-        /// Setsup Nlog logging
+        /// Setsup Nlog logging to a file
         /// </summary>
         /// <param name="level">Logging level- debug or error</param>
         /// <param name="fileName">Path of file where logging will be done</param>
-        public void SetupLogging(LogLevel level, string fileName)
+        public void SetupFileLogging(LogLevel level, string fileName)
         {
-            // Step 1. Create configuration object 
-            var config = new LoggingConfiguration();
-
-            // Step 2. Create targets and add them to the configuration 
             var fileTarget = new FileTarget();
-            config.AddTarget("file", fileTarget);
+            _adlsLoggerConfig.AddTarget("file", fileTarget);
 
-            // Step 3. Set target properties 
             fileTarget.FileName = fileName;
-            fileTarget.Layout = "${message}";
+            
+            var rule = new LoggingRule("adls.dotnet.*", NLog.LogLevel.Debug, fileTarget);
+            _adlsLoggerConfig.LoggingRules.Add(rule);
 
-            // Step 4. Define rules
-            var rule2 = new LoggingRule("asdl.dotnet.*", NLog.LogLevel.Debug, fileTarget);
-            config.LoggingRules.Add(rule2);
-            // Step 5. Activate the configuration
-            LogManager.Configuration = config;
+            //Re-enable the configuration
+            LogManager.Configuration = _adlsLoggerConfig;
         }
         /// <summary>
         /// Performs the bulk copy and tracks the progress

src/ResourceManager/DataLakeStore/Commands.DataLakeStore/DataPlaneModels/DataLakeStoreFileSystemCmdletBase.cs

@@ -34,7 +34,7 @@ public DataLakeStoreFileSystemClient DataLakeStoreFileSystemClient
         {
             get {
             return _dataLakeFileSystemClient ??
-                   (_dataLakeFileSystemClient = new DataLakeStoreFileSystemClient(DefaultProfile.DefaultContext));
+                   (_dataLakeFileSystemClient = new DataLakeStoreFileSystemClient(DefaultProfile.DefaultContext, this));
         }
 
 

src/ResourceManager/DataLakeStore/Commands.DataLakeStore/packages.config

@@ -1,6 +1,6 @@
 <?xml version="1.0" encoding="utf-8"?>
 <packages>
-  <package id="Microsoft.Azure.DataLake.Store" version="1.1.2" targetFramework="net452" />
+  <package id="Microsoft.Azure.DataLake.Store" version="1.1.4" targetFramework="net452" />
   <package id="Microsoft.Azure.Management.DataLake.Store" version="2.3.0-preview" targetFramework="net452" />
   <package id="NLog" version="4.4.12" targetFramework="net452" />
 </packages>