Skip to content

Commit 97ad57e

Browse files
MiYanniMiYanni
authored
Merge pull request #6598 from Azure/task-adl-fixencodingparameter
Task adl fixencodingparameter
2 parents 24e9cb7 + 863cc3b commit 97ad57e

File tree

14 files changed

+157
-172
lines changed

14 files changed

+157
-172
lines changed

‎src/ResourceManager/DataLakeStore/Commands.DataLakeStore.Test/ScenarioTests/AdlsAliasTests.ps1

Lines changed: 20 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -528,9 +528,14 @@ function Test-DataLakeStoreFileSystem
528528
}
529529

530530
# define all the files and folders to create
531+
$encodingFolder="/encodingFolder"
531532
$folderToCreate = "/adlspstestfolder"
532533
$emptyFilePath = "$folderToCreate\emptyfile.txt" # have one where the slash is in the wrong direction to make sure they get fixed.
533534
$contentFilePath = "$folderToCreate/contentfile.txt"
535+
$unicodeContentFilePath="$encodingFolder/unicodecontentfile.txt"
536+
$unicodetext="I am unicode text"
537+
$utf32ContentFilePath="$encodingFolder/utf32contentfile.txt"
538+
$utf32text="I am utf32 text"
534539
$concatFile = "$folderToCreate/concatfile.txt"
535540
$moveFile = "$folderToCreate/movefile.txt"
536541
$movefolder = "/adlspstestmovefolder"
@@ -565,6 +570,19 @@ function Test-DataLakeStoreFileSystem
565570
Assert-AreEqual "File" $result.Type
566571
Assert-AreEqual $content.length $result.Length
567572

573+
#Create empty file and add unicode content
574+
$result = New-AdlStoreItem -Account $accountName -path $unicodeContentFilePath
575+
Assert-NotNull $result "No value was returned on content file creation"
576+
Add-AdlStoreItemContent -Account $accountName -Path $unicodeContentFilePath -Value $unicodetext -Encoding Unicode
577+
$retrievedContent = Get-AdlStoreItemContent -Account $accountName -Path $unicodeContentFilePath -Encoding Unicode
578+
Assert-AreEqual $unicodetext $retrievedContent
579+
580+
#Create utf32 file with content
581+
$result = New-AdlStoreItem -Account $accountName -path $utf32ContentFilePath -Value $utf32text -Encoding UTF32
582+
Assert-NotNull $result "No value was returned on content file creation"
583+
$retrievedContent = Get-AdlStoreItemContent -Account $accountName -Path $utf32ContentFilePath -Encoding UTF32
584+
Assert-AreEqual $utf32text $retrievedContent
585+
568586
# set absolute expiration for content file
569587
Assert-True {253402300800000 -ge $result.ExpirationTime -or 0 -le $result.ExpirationTime} # validate that expiration is currently max value
570588
[DateTimeOffset]$timeToUse = [Microsoft.Azure.Test.HttpRecorder.HttpMockServer]::GetVariable("absoluteTime", [DateTimeOffset]::UtcNow.AddSeconds(120))
@@ -699,7 +717,8 @@ function Test-DataLakeStoreFileSystem
699717
Assert-Throws {Get-AdlStoreItem -Account $accountName -path $moveFolder}
700718
Assert-True {Remove-AdlStoreItem -Account $accountName -paths $summaryFolder -force -recurse -passthru} "Remove folder failed"
701719
Assert-Throws {Get-AdlStoreItem -Account $accountName -path $summaryFolder}
702-
720+
Assert-True {Remove-AdlStoreItem -Account $accountName -paths $encodingFolder -force -recurse -passthru} "Remove folder failed"
721+
703722
# Delete Data Lake account
704723
Assert-True {Remove-AdlStore -ResourceGroupName $resourceGroupName -Name $accountName -Force -PassThru} "Remove Account failed."
705724

‎src/ResourceManager/DataLakeStore/Commands.DataLakeStore.Test/ScenarioTests/AdlsTests.ps1

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -527,9 +527,14 @@ function Test-DataLakeStoreFileSystem
527527
}
528528

529529
# define all the files and folders to create
530+
$encodingFolder="/encodingFolder"
530531
$folderToCreate = "/adlspstestfolder"
531532
$emptyFilePath = "$folderToCreate\emptyfile.txt" # have one where the slash is in the wrong direction to make sure they get fixed.
532533
$contentFilePath = "$folderToCreate/contentfile.txt"
534+
$unicodeContentFilePath="$encodingFolder/unicodecontentfile.txt"
535+
$unicodetext="I am unicode text"
536+
$utf32ContentFilePath="$encodingFolder/utf32contentfile.txt"
537+
$utf32text="I am utf32 text"
533538
$concatFile = "$folderToCreate/concatfile.txt"
534539
$moveFile = "$folderToCreate/movefile.txt"
535540
$movefolder = "/adlspstestmovefolder"
@@ -564,6 +569,19 @@ function Test-DataLakeStoreFileSystem
564569
Assert-AreEqual "File" $result.Type
565570
Assert-AreEqual $content.length $result.Length
566571

572+
#Create empty file and add unicode content
573+
$result = New-AzureRMDataLakeStoreItem -Account $accountName -path $unicodeContentFilePath
574+
Assert-NotNull $result "No value was returned on content file creation"
575+
Add-AzureRmDataLakeStoreItemContent -Account $accountName -Path $unicodeContentFilePath -Value $unicodetext -Encoding Unicode
576+
$retrievedContent = Get-AzureRMDataLakeStoreItemContent -Account $accountName -Path $unicodeContentFilePath -Encoding Unicode
577+
Assert-AreEqual $unicodetext $retrievedContent
578+
579+
#Create utf32 file with content
580+
$result = New-AzureRMDataLakeStoreItem -Account $accountName -path $utf32ContentFilePath -Value $utf32text -Encoding UTF32
581+
Assert-NotNull $result "No value was returned on content file creation"
582+
$retrievedContent = Get-AzureRMDataLakeStoreItemContent -Account $accountName -Path $utf32ContentFilePath -Encoding UTF32
583+
Assert-AreEqual $utf32text $retrievedContent
584+
567585
# set absolute expiration for content file
568586
Assert-True {253402300800000 -ge $result.ExpirationTime -or 0 -le $result.ExpirationTime} # validate that expiration is currently max value
569587
[DateTimeOffset]$timeToUse = [Microsoft.Azure.Test.HttpRecorder.HttpMockServer]::GetVariable("absoluteTime", [DateTimeOffset]::UtcNow.AddSeconds(120))
@@ -697,6 +715,7 @@ function Test-DataLakeStoreFileSystem
697715
Assert-Throws {Get-AzureRMDataLakeStoreItem -Account $accountName -path $moveFolder}
698716
Assert-True {Remove-AzureRMDataLakeStoreItem -Account $accountName -paths $summaryFolder -force -recurse -passthru} "Remove folder failed"
699717
Assert-Throws {Get-AzureRMDataLakeStoreItem -Account $accountName -path $summaryFolder}
718+
Assert-True {Remove-AzureRMDataLakeStoreItem -Account $accountName -paths $encodingFolder -force -recurse -passthru} "Remove folder failed"
700719

701720
# Delete Data Lake account
702721
Assert-True {Remove-AzureRMDataLakeStoreAccount -ResourceGroupName $resourceGroupName -Name $accountName -Force -PassThru} "Remove Account failed."

‎src/ResourceManager/DataLakeStore/Commands.DataLakeStore/ChangeLog.md

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,4 +18,5 @@
1818
- Additional information about change #1
1919
-->
2020
## Current Release
21-
* Update the sdk version of dataplane to 1.1.13
21+
* Update the sdk version of dataplane to 1.1.13
22+
* Change the type of Encoding parameter to system.Encoding for commandlets: New-AzureRmDataLakeStoreItem, Add-AzureRmDataLakeStoreItemContent, Get-AzureRmDataLakeStoreItemContent to make it compatible to .netcore

‎src/ResourceManager/DataLakeStore/Commands.DataLakeStore/Commands.DataLakeStore.csproj

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -95,6 +95,7 @@
9595
<Compile Include="DataPlaneModels\AdlsClientFactory.cs" />
9696
<Compile Include="DataPlaneModels\AdlsLoggerTarget.cs" />
9797
<Compile Include="DataPlaneModels\DataLakeStoreChildItemSummary.cs" />
98+
<Compile Include="DataPlaneModels\EncodingUtils.cs" />
9899
<Compile Include="Models\DataLakeStoreClient.cs" />
99100
<Compile Include="Models\DataLakeStoreCmdletBase.cs" />
100101
<Compile Include="DataPlaneModels\DataLakeStoreEnums.cs" />

‎src/ResourceManager/DataLakeStore/Commands.DataLakeStore/DataPlaneCommands/AddAzureRmDataLakeStoreItemContent.cs

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,8 @@
1414

1515
using Microsoft.Azure.Commands.DataLakeStore.Models;
1616
using System.Management.Automation;
17+
using System.Text;
18+
using Microsoft.Azure.Commands.ResourceManager.Common.ArgumentCompleters;
1719

1820
namespace Microsoft.Azure.Commands.DataLakeStore
1921
{
@@ -44,7 +46,9 @@ public class AddAzureDataLakeStoreItemContent : DataLakeStoreFileSystemCmdletBas
4446
[Parameter(ValueFromPipelineByPropertyName = true, Position = 3, Mandatory = false,
4547
HelpMessage =
4648
"Optionally indicates the encoding for the content being uploaded as part of 'Value'. Default is UTF8")]
47-
public FileSystemCmdletProviderEncoding Encoding { get; set; } = FileSystemCmdletProviderEncoding.UTF8;
49+
[ArgumentToEncodingTransformation]
50+
[PSArgumentCompleter(EncodingUtils.Unknown, EncodingUtils.String, EncodingUtils.Unicode, EncodingUtils.BigEndianUnicode, EncodingUtils.Ascii, EncodingUtils.Utf8, EncodingUtils.Utf7, EncodingUtils.Utf32, EncodingUtils.Default, EncodingUtils.Oem, EncodingUtils.BigEndianUtf32)]
51+
public Encoding Encoding { get; set; } = Encoding.UTF8;
4852

4953
public override void ExecuteCmdlet()
5054
{

‎src/ResourceManager/DataLakeStore/Commands.DataLakeStore/DataPlaneCommands/GetAzureRmDataLakeStoreItemContent.cs

Lines changed: 11 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,8 @@
1717
using System;
1818
using System.IO;
1919
using System.Management.Automation;
20+
using System.Text;
21+
using Microsoft.Azure.Commands.ResourceManager.Common.ArgumentCompleters;
2022

2123
namespace Microsoft.Azure.Commands.DataLakeStore
2224
{
@@ -84,7 +86,10 @@ public class GetAzureDataLakeStoreContent : DataLakeStoreFileSystemCmdletBase
8486
[Parameter(ValueFromPipelineByPropertyName = true, Position = 3, ParameterSetName = TailRowParameterSetName,
8587
Mandatory = false,
8688
HelpMessage = "Optionally indicates the encoding for the content being downloaded. Default is UTF8")]
87-
public FileSystemCmdletProviderEncoding Encoding { get; set; } = FileSystemCmdletProviderEncoding.UTF8;
89+
[ArgumentToEncodingTransformation]
90+
[PSArgumentCompleter(EncodingUtils.Unknown, EncodingUtils.String, EncodingUtils.Unicode, EncodingUtils.BigEndianUnicode, EncodingUtils.Ascii, EncodingUtils.Utf8, EncodingUtils.Utf7, EncodingUtils.Utf32, EncodingUtils.Default, EncodingUtils.Oem, EncodingUtils.BigEndianUtf32)]
91+
92+
public Encoding Encoding { get; set; } = Encoding.UTF8;
8893

8994
[Parameter(ValueFromPipelineByPropertyName = true, Position = 5, ParameterSetName = BaseParameterSetName, Mandatory = false,
9095
HelpMessage = "If the length parameter is not specified or is less than or equal to zero, force returns all content of the file, otherwise it does nothing.")]
@@ -132,26 +137,19 @@ public override void ExecuteCmdlet()
132137
{
133138
Array.Resize(ref byteArray, (int)totalLengthRead);
134139
}
135-
if (UsingByteEncoding(Encoding))
136-
{
137-
WriteObject(byteArray);
138-
}
139-
else
140-
{
141-
WriteObject(BytesToString(byteArray, Encoding));
142-
}
140+
141+
WriteObject(BytesToString(byteArray, Encoding));
142+
143143
}
144144
});
145145
}
146146
else if (ParameterSetName.Equals(HeadRowParameterSetName, StringComparison.OrdinalIgnoreCase))
147147
{
148-
var encoding = GetEncoding(Encoding);
149-
WriteObject(DataLakeStoreFileSystemClient.GetStreamRows(Path.TransformedPath, Account, Head, encoding), true);
148+
WriteObject(DataLakeStoreFileSystemClient.GetStreamRows(Path.TransformedPath, Account, Head, Encoding), true);
150149
}
151150
else
152151
{
153-
var encoding = GetEncoding(Encoding);
154-
WriteObject(DataLakeStoreFileSystemClient.GetStreamRows(Path.TransformedPath, Account, Tail, encoding, true), true);
152+
WriteObject(DataLakeStoreFileSystemClient.GetStreamRows(Path.TransformedPath, Account, Tail, Encoding, true), true);
155153
}
156154
}
157155
}

‎src/ResourceManager/DataLakeStore/Commands.DataLakeStore/DataPlaneCommands/NewAzureRmDataLakeStoreItem.cs

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,8 @@
1717
using Microsoft.Rest.Azure;
1818
using System.Management.Automation;
1919
using Microsoft.Azure.DataLake.Store;
20+
using System.Text;
21+
using Microsoft.Azure.Commands.ResourceManager.Common.ArgumentCompleters;
2022

2123
namespace Microsoft.Azure.Commands.DataLakeStore
2224
{
@@ -45,7 +47,9 @@ public class NewAzureDataLakeStoreItem : DataLakeStoreFileSystemCmdletBase
4547
[Parameter(ValueFromPipelineByPropertyName = true, Position = 3, Mandatory = false,
4648
HelpMessage =
4749
"Optionally indicates the encoding for the content being uploaded as part of 'Value'. Default is UTF8")]
48-
public FileSystemCmdletProviderEncoding Encoding { get; set; } = FileSystemCmdletProviderEncoding.UTF8;
50+
[ArgumentToEncodingTransformation]
51+
[PSArgumentCompleter(EncodingUtils.Unknown, EncodingUtils.String, EncodingUtils.Unicode, EncodingUtils.BigEndianUnicode, EncodingUtils.Ascii, EncodingUtils.Utf8, EncodingUtils.Utf7, EncodingUtils.Utf32, EncodingUtils.Default, EncodingUtils.Oem, EncodingUtils.BigEndianUtf32)]
52+
public Encoding Encoding { get; set; } = Encoding.UTF8;
4953

5054
[Parameter(ValueFromPipelineByPropertyName = true, Position = 3, Mandatory = false,
5155
HelpMessage = "Optionally indicates that this new item is a folder and not a file.")]

‎src/ResourceManager/DataLakeStore/Commands.DataLakeStore/DataPlaneModels/DataLakeStoreFileSystemCmdletBase.cs

Lines changed: 5 additions & 106 deletions
Original file line numberDiff line numberDiff line change
@@ -78,78 +78,13 @@ protected override void StopProcessing()
7878
/// ArgumentException if type is null, empty, or does not represent one
7979
/// of the known encoding types.
8080
/// </throws>
81-
private static byte[] GetBytes(string content, FileSystemCmdletProviderEncoding type)
81+
private static byte[] GetBytes(string content, Encoding encoding)
8282
{
83-
switch (type)
84-
{
85-
case FileSystemCmdletProviderEncoding.String:
86-
return Encoding.UTF8.GetBytes(content);
87-
case FileSystemCmdletProviderEncoding.Unicode:
88-
return Encoding.Unicode.GetBytes(content);
89-
case FileSystemCmdletProviderEncoding.BigEndianUnicode:
90-
return Encoding.BigEndianUnicode.GetBytes(content);
91-
case FileSystemCmdletProviderEncoding.UTF8:
92-
return Encoding.UTF8.GetBytes(content);
93-
case FileSystemCmdletProviderEncoding.UTF7:
94-
return Encoding.UTF7.GetBytes(content);
95-
case FileSystemCmdletProviderEncoding.UTF32:
96-
return Encoding.UTF32.GetBytes(content);
97-
case FileSystemCmdletProviderEncoding.Ascii:
98-
return Encoding.ASCII.GetBytes(content);
99-
case FileSystemCmdletProviderEncoding.Default:
100-
return Encoding.UTF8.GetBytes(content);
101-
case FileSystemCmdletProviderEncoding.Oem:
102-
{
103-
return Encoding.GetEncoding((int)NativeMethods.GetOEMCP()).GetBytes(content);
104-
}
105-
default:
106-
// Default to unicode encoding
107-
return Encoding.UTF8.GetBytes(content);
108-
}
83+
return encoding.GetBytes(content);
10984
}
11085

111-
/// <summary>
112-
/// Gets the Byte Encoding status of the StreamType parameter. Returns true
113-
/// if the stream was opened with "Byte" encoding, false otherwise.
114-
/// </summary>
115-
internal static bool UsingByteEncoding(FileSystemCmdletProviderEncoding encoding)
116-
{
117-
return encoding == FileSystemCmdletProviderEncoding.Byte;
118-
} // UsingByteEncoding
119-
120-
internal static byte[] GetBytes(object content, FileSystemCmdletProviderEncoding encoding)
86+
internal static byte[] GetBytes(object content, Encoding encoding)
12187
{
122-
if (UsingByteEncoding(encoding))
123-
{
124-
// first attempt to convert it directly into a byte array
125-
var byteArray = content as byte[];
126-
if (byteArray != null)
127-
{
128-
return byteArray;
129-
}
130-
131-
// attempt to convert the object into an object array
132-
var contentArray = content as object[];
133-
if (contentArray == null)
134-
{
135-
throw new CloudException(Resources.InvalidEncoding);
136-
}
137-
138-
// now, for each element in the content array, ensure it is of type byte
139-
var byteList = new List<byte>();
140-
foreach (var entry in contentArray)
141-
{
142-
if (!(entry is byte))
143-
{
144-
throw new CloudException(Resources.InvalidEncoding);
145-
}
146-
147-
byteList.Add((byte)entry);
148-
}
149-
150-
return byteList.ToArray();
151-
}
152-
15388
var contentString = content as string;
15489
if (contentString == null)
15590
{
@@ -159,40 +94,9 @@ internal static byte[] GetBytes(object content, FileSystemCmdletProviderEncoding
15994
return GetBytes(contentString, encoding);
16095
}
16196

162-
internal static Encoding GetEncoding(FileSystemCmdletProviderEncoding type)
163-
{
164-
switch (type)
165-
{
166-
case FileSystemCmdletProviderEncoding.String:
167-
return Encoding.UTF8;
168-
case FileSystemCmdletProviderEncoding.Unicode:
169-
return Encoding.Unicode;
170-
case FileSystemCmdletProviderEncoding.BigEndianUnicode:
171-
return Encoding.BigEndianUnicode;
172-
case FileSystemCmdletProviderEncoding.UTF8:
173-
return Encoding.UTF8;
174-
case FileSystemCmdletProviderEncoding.UTF7:
175-
return Encoding.UTF7;
176-
case FileSystemCmdletProviderEncoding.UTF32:
177-
return Encoding.UTF32;
178-
case FileSystemCmdletProviderEncoding.Ascii:
179-
return Encoding.ASCII;
180-
case FileSystemCmdletProviderEncoding.Default:
181-
return Encoding.UTF8;
182-
case FileSystemCmdletProviderEncoding.Oem:
183-
{
184-
var oemCP = NativeMethods.GetOEMCP();
185-
return Encoding.GetEncoding((int)oemCP);
186-
}
187-
default:
188-
// Default to unicode encoding
189-
return Encoding.UTF8;
190-
}
191-
}
192-
193-
internal static string BytesToString(byte[] content, FileSystemCmdletProviderEncoding type)
97+
internal static string BytesToString(byte[] content, Encoding encoding)
19498
{
195-
return GetEncoding(type).GetString(content);
99+
return encoding.GetString(content);
196100
}
197101

198102
/// <summary>
@@ -203,11 +107,6 @@ internal static string BytesToString(byte[] content, FileSystemCmdletProviderEnc
203107
protected CancellationToken CmdletCancellationToken;
204108
}
205109

206-
internal static class NativeMethods
207-
{
208-
[DllImport("api-ms-win-core-localization-l1-2-1.dll", SetLastError = false, CharSet = CharSet.Unicode)]
209-
internal static extern uint GetOEMCP();
210-
}
211110

212111
#endregion
213112
}

0 commit comments

Comments
 (0)