Added option of Byte encoding for New-AzDataLakeStoreItem, Add-AzDAtaLakeStoreItemContent, Get-AzDAtaLakeStoreItemContent

src/DataLakeStore/DataLakeStore.Test/ScenarioTests/AdlsAliasTests.cs

@@ -73,7 +73,7 @@ public void TestAdlsFileSystem()
         {
             var workingPath = Path.GetDirectoryName(new Uri(Assembly.GetExecutingAssembly().CodeBase).AbsolutePath);
             var testLocation = Path.Combine(workingPath, "ScenarioTests", (this.GetType().Name + ".ps1"));
-            NewInstance.RunPsTest(_logger, string.Format("Test-DataLakeStoreFileSystem -fileToCopy '{0}' -location '{1}'", testLocation, AdlsTestsBase.ResourceGroupLocation));
+            NewInstance.RunPsTest(_logger, string.Format("Test-DataLakeStoreFileSystem -fileToCopy '{0}' -location '{1}'", testLocation, AdlsTestsBase.TestFileSystemResourceGroupLocation));
         }
 
         [Fact]

src/DataLakeStore/DataLakeStore.Test/ScenarioTests/AdlsAliasTests.ps1

@@ -647,6 +647,13 @@ function Test-DataLakeStoreFileSystem
 		Assert-AreEqual 3 $headTailResult[0]
 		Assert-AreEqual 4 $headTailResult[1]
 
+        #Create a file with byte and read it
+        $byteDataFile="/byteData/filetest.txt"
+        [byte[]] $byteData = 1,2,3,4,5
+        New-AdlStoreItem -Account $accountName -Path $byteDataFile -Force -Value $byteData -Encoding Byte
+        $result = Get-AdlStoreItemContent -Account $accountName -path $byteDataFile -Encoding Byte
+        Assert-True {@(Compare-Object $byteData $result -SyncWindow 0).Length -eq 0}
+
 		# Import and get file
 		$localFileInfo = Get-ChildItem $fileToCopy
 		$result = Import-AdlStoreItem -Account $accountName -Path $fileToCopy -Destination $importFile

src/DataLakeStore/DataLakeStore.Test/ScenarioTests/AdlsTests.cs

@@ -66,13 +66,13 @@ public void TestAdlsAccountTiers()
             NewInstance.RunPsTest(_logger, string.Format("Test-DataLakeStoreAccountTiers -location '{0}'", AdlsTestsBase.ResourceGroupLocation));
         }
 
-        [Fact(Skip="This case has been unstable on Linux for approximately 20% of failure. Needs investigation.")]
+        [Fact(Skip = "This case has been unstable on Linux for approximately 20% of failure. Needs investigation.")]
         [Trait(Category.AcceptanceType, Category.CheckIn)]
         public void TestAdlsFileSystem()
         {
             var workingPath = Path.GetDirectoryName(new Uri(Assembly.GetExecutingAssembly().CodeBase).AbsolutePath);
             var testLocation = Path.Combine(workingPath, "ScenarioTests", (this.GetType().Name + ".ps1"));
-            NewInstance.RunPsTest(_logger, string.Format("Test-DataLakeStoreFileSystem -fileToCopy '{0}' -location '{1}'", testLocation, AdlsTestsBase.ResourceGroupLocation));
+            NewInstance.RunPsTest(_logger, string.Format("Test-DataLakeStoreFileSystem -fileToCopy '{0}' -location '{1}'", testLocation, AdlsTestsBase.TestFileSystemResourceGroupLocation));
         }
 
         [Fact]

src/DataLakeStore/DataLakeStore.Test/ScenarioTests/AdlsTests.ps1

@@ -646,6 +646,13 @@ function Test-DataLakeStoreFileSystem
 		Assert-AreEqual 3 $headTailResult[0]
 		Assert-AreEqual 4 $headTailResult[1]
 
+        #Create a file with byte and read it
+        $byteDataFile="/byteData/filetest.txt"
+        [byte[]] $byteData = 1,2,3,4,5
+        New-AzDataLakeStoreItem -Account $accountName -Path $byteDataFile -Force -Value $byteData -Encoding Byte
+        $result = Get-AzDataLakeStoreItemContent -Account $accountName -path $byteDataFile -Encoding Byte
+        Assert-True {@(Compare-Object $byteData $result -SyncWindow 0).Length -eq 0}
+
 		# Import and get file
 		$localFileInfo = Get-ChildItem $fileToCopy
 		$result = Import-AzDataLakeStoreItem -Account $accountName -Path $fileToCopy -Destination $importFile

src/DataLakeStore/DataLakeStore.Test/ScenarioTests/AdlsTestsBase.cs

@@ -38,6 +38,7 @@ public class AdlsTestsBase : RMTestBase
         private readonly EnvironmentSetupHelper _helper;
 
         internal const string ResourceGroupLocation = "westus";
+        internal const string TestFileSystemResourceGroupLocation = "eastus2";
 
         public NewResourceManagementClient NewResourceManagementClient { get; private set; }
 

src/DataLakeStore/DataLakeStore/ChangeLog.md

@@ -18,6 +18,7 @@
         - Additional information about change #1
 -->
 ## Upcoming Release
+Added option of Byte encoding for New-AzDataLakeStoreItem, Add-AzDAtaLakeStoreItemContent, Get-AzDAtaLakeStoreItemContent
 
 ## Version 1.2.7
 * Added reference to System.Buffers explicitly in csproj and psd1.

src/DataLakeStore/DataLakeStore/DataPlaneCommands/AddAzureRmDataLakeStoreItemContent.cs

@@ -47,7 +47,7 @@ public class AddAzureDataLakeStoreItemContent : DataLakeStoreFileSystemCmdletBas
             HelpMessage =
                 "Optionally indicates the encoding for the content being uploaded as part of 'Value'. Default is UTF8")]
         [ArgumentToEncodingTransformation]
-        [PSArgumentCompleter(EncodingUtils.Unknown, EncodingUtils.String, EncodingUtils.Unicode, EncodingUtils.BigEndianUnicode, EncodingUtils.Ascii, EncodingUtils.Utf8, EncodingUtils.Utf7, EncodingUtils.Utf32, EncodingUtils.Default, EncodingUtils.Oem, EncodingUtils.BigEndianUtf32)]
+        [PSArgumentCompleter(EncodingUtils.Unknown, EncodingUtils.String, EncodingUtils.Unicode, EncodingUtils.BigEndianUnicode, EncodingUtils.Ascii, EncodingUtils.Utf8, EncodingUtils.Utf7, EncodingUtils.Utf32, EncodingUtils.Default, EncodingUtils.Oem, EncodingUtils.BigEndianUtf32, EncodingUtils.Byte)]
         public Encoding Encoding { get; set; } = Encoding.UTF8;
 
         public override void ExecuteCmdlet()

src/DataLakeStore/DataLakeStore/DataPlaneCommands/GetAzureRmDataLakeStoreItemContent.cs

@@ -87,7 +87,7 @@ public class GetAzureDataLakeStoreContent : DataLakeStoreFileSystemCmdletBase
             Mandatory = false,
             HelpMessage = "Optionally indicates the encoding for the content being downloaded. Default is UTF8")]
         [ArgumentToEncodingTransformation]
-        [PSArgumentCompleter(EncodingUtils.Unknown, EncodingUtils.String, EncodingUtils.Unicode, EncodingUtils.BigEndianUnicode, EncodingUtils.Ascii, EncodingUtils.Utf8, EncodingUtils.Utf7, EncodingUtils.Utf32, EncodingUtils.Default, EncodingUtils.Oem, EncodingUtils.BigEndianUtf32)]
+        [PSArgumentCompleter(EncodingUtils.Unknown, EncodingUtils.String, EncodingUtils.Unicode, EncodingUtils.BigEndianUnicode, EncodingUtils.Ascii, EncodingUtils.Utf8, EncodingUtils.Utf7, EncodingUtils.Utf32, EncodingUtils.Default, EncodingUtils.Oem, EncodingUtils.BigEndianUtf32, EncodingUtils.Byte)]
 
         public Encoding Encoding { get; set; } = Encoding.UTF8;
 
@@ -97,6 +97,10 @@ public class GetAzureDataLakeStoreContent : DataLakeStoreFileSystemCmdletBase
 
         public override void ExecuteCmdlet()
         {
+            var useByteEncoding = UsingByteEncoding(Encoding);
+            // Byte encoding is not possible to use for reading rows from the stream. It has to be a defined encoding.
+            // Previously also we used to fall back to UTF8 in case of reading head rows and tail rows. 
+            var fallBackEncoding = useByteEncoding ? Encoding.UTF8 : Encoding;
             if (ParameterSetName.Equals(BaseParameterSetName, StringComparison.OrdinalIgnoreCase))
             {
                 ConfirmAction(
@@ -137,19 +141,25 @@ public override void ExecuteCmdlet()
                             {
                                 Array.Resize(ref byteArray, (int)totalLengthRead);
                             }
-
-                            WriteObject(BytesToString(byteArray, Encoding));
+                            if (useByteEncoding)
+                            {
+                                WriteObject(byteArray);
+                            }
+                            else
+                            {
+                                WriteObject(BytesToString(byteArray, Encoding));
+                            }
 
                         }
                     });
             }
             else if (ParameterSetName.Equals(HeadRowParameterSetName, StringComparison.OrdinalIgnoreCase))
             {
-                WriteObject(DataLakeStoreFileSystemClient.GetStreamRows(Path.TransformedPath, Account, Head, Encoding), true);
+                WriteObject(DataLakeStoreFileSystemClient.GetStreamRows(Path.TransformedPath, Account, Head, fallBackEncoding), true);
             }
             else
             {
-                WriteObject(DataLakeStoreFileSystemClient.GetStreamRows(Path.TransformedPath, Account, Tail, Encoding, true), true);
+                WriteObject(DataLakeStoreFileSystemClient.GetStreamRows(Path.TransformedPath, Account, Tail, fallBackEncoding, true), true);
             }
         }
     }

src/DataLakeStore/DataLakeStore/DataPlaneCommands/NewAzureRmDataLakeStoreItem.cs

@@ -48,7 +48,7 @@ public class NewAzureDataLakeStoreItem : DataLakeStoreFileSystemCmdletBase
             HelpMessage =
                 "Optionally indicates the encoding for the content being uploaded as part of 'Value'. Default is UTF8")]
         [ArgumentToEncodingTransformation]
-        [PSArgumentCompleter(EncodingUtils.Unknown, EncodingUtils.String, EncodingUtils.Unicode, EncodingUtils.BigEndianUnicode, EncodingUtils.Ascii, EncodingUtils.Utf8, EncodingUtils.Utf7, EncodingUtils.Utf32, EncodingUtils.Default, EncodingUtils.Oem, EncodingUtils.BigEndianUtf32)]
+        [PSArgumentCompleter(EncodingUtils.Unknown, EncodingUtils.String, EncodingUtils.Unicode, EncodingUtils.BigEndianUnicode, EncodingUtils.Ascii, EncodingUtils.Utf8, EncodingUtils.Utf7, EncodingUtils.Utf32, EncodingUtils.Default, EncodingUtils.Oem, EncodingUtils.BigEndianUtf32, EncodingUtils.Byte)]
         public Encoding Encoding { get; set; } = Encoding.UTF8;
 
         [Parameter(ValueFromPipelineByPropertyName = true, Position = 3, Mandatory = false,

src/DataLakeStore/DataLakeStore/DataPlaneModels/ByteEncoding.cs

@@ -0,0 +1,38 @@
+using System;
+using System.Text;
+
+namespace Microsoft.Azure.Commands.DataLakeStore.Models
+{
+    internal class ByteEncoding : Encoding
+    {
+        public override int GetByteCount(char[] chars, int index, int count)
+        {
+            throw new NotImplementedException();
+        }
+
+        public override int GetBytes(char[] chars, int charIndex, int charCount, byte[] bytes, int byteIndex)
+        {
+            throw new NotImplementedException();
+        }
+
+        public override int GetCharCount(byte[] bytes, int index, int count)
+        {
+            throw new NotImplementedException();
+        }
+
+        public override int GetChars(byte[] bytes, int byteIndex, int byteCount, char[] chars, int charIndex)
+        {
+            throw new NotImplementedException();
+        }
+
+        public override int GetMaxByteCount(int charCount)
+        {
+            throw new NotImplementedException();
+        }
+
+        public override int GetMaxCharCount(int byteCount)
+        {
+            throw new NotImplementedException();
+        }
+    }
+}

src/DataLakeStore/DataLakeStore/DataPlaneModels/DataLakeStoreFileSystemCmdletBase.cs

@@ -61,7 +61,15 @@ protected override void StopProcessing()
         }
 
         #region cmdlet helpers from the FilesystemProvider
-
+        /// <summary>
+        /// Checks whether type is ByteEncoding
+        /// </summary>
+        /// <param name="encoding"></param>
+        /// <returns></returns>
+        internal static bool UsingByteEncoding(Encoding encoding)
+        {
+            return encoding.GetType() == typeof(ByteEncoding);
+        }
         /// <summary>
         /// Converts the stream type string into an Encoding
         /// </summary>
@@ -85,13 +93,52 @@ private static byte[] GetBytes(string content, Encoding encoding)
 
         internal static byte[] GetBytes(object content, Encoding encoding)
         {
-            var contentString = content as string;
-            if (contentString == null)
+            if (UsingByteEncoding(encoding))
             {
-                throw new CloudException(Resources.InvalidContent);
+                // first attempt to convert it directly into a byte array	
+                var byteArray = content as byte[];
+                if (byteArray != null)
+                {
+                    return byteArray;
+                }
+                /*
+                 *  [byte[]] $byteData = 1,2,3,4,5
+                 * $MyList = [System.Collections.Generic.List[object]]::new()
+                 * $MyList.Add($byteData[0])
+                 * $MyList.Add($byteData[1])
+                 * And then pass $MyList.ToArray() this will pass object[] containing bytes
+                 */
+                // attempt to convert the object into an object array	
+                var contentArray = content as object[];
+                if (contentArray == null)
+                {
+                    throw new CloudException(Resources.InvalidEncoding);
+                }
+
+                // now, for each element in the content array, ensure it is of type byte	
+                var byteList = new List<byte>();
+                foreach (var entry in contentArray)
+                {
+                    if (!(entry is byte))
+                    {
+                        throw new CloudException(Resources.InvalidEncoding);
+                    }
+
+                    byteList.Add((byte)entry);
+                }
+
+                return byteList.ToArray();
             }
+            else
+            {
+                var contentString = content as string;
+                if (contentString == null)
+                {
+                    throw new CloudException(Resources.InvalidContent);
+                }
 
-            return GetBytes(contentString, encoding);
+                return GetBytes(contentString, encoding);
+            }
         }
 
         internal static string BytesToString(byte[] content, Encoding encoding)

src/DataLakeStore/DataLakeStore/DataPlaneModels/EncodingUtils.cs

@@ -19,8 +19,7 @@ internal static class EncodingUtils
         internal const string Default = "default";
         internal const string Oem = "oem";
         internal const string BigEndianUtf32 = "bigendianutf32";
-
-
+        internal const string Byte = "byte";
     }
 
     internal sealed class ArgumentToEncodingTransformationAttribute : ArgumentTransformationAttribute
@@ -54,6 +53,8 @@ public override object Transform(EngineIntrinsics engineIntrinsics, object input
                             var oemCP = NativeMethods.GetOEMCP();
                             return Encoding.GetEncoding((int)oemCP);
                         }
+                    case EncodingUtils.Byte:
+                        return new ByteEncoding();
                     default:
                         // Default to unicode encoding
                         throw new ArgumentException($"{encodingName} is not a supported Encoding type");