Skip to content

ADLS: Bug Fix to support create and append for any size #4204

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Jun 27, 2017
Merged

ADLS: Bug Fix to support create and append for any size #4204

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
7ea9973
Bug Fix to support create and append for any size
Jun 27, 2017
d435892
Address pre PR comment
Jun 27, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
42 changes: 38 additions & 4 deletions ...ourceManager/DataLakeStore/Commands.DataLakeStore/Models/DataLakeStoreFileSystemClient.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ namespace Microsoft.Azure.Commands.DataLakeStore.Models
public class DataLakeStoreFileSystemClient
{
private const decimal MaximumBytesPerDownloadRequest = 32 * 1024 * 1024; //32MB
private const decimal MaximumBytesPerAppendRequest = 20 * 1024 * 1024; //20MB

/// <summary>
/// The lock object
Expand Down Expand Up @@ -421,9 +422,27 @@ public void ConcatenateFiles(string destinationPath, string accountName, string[
deleteDirectory);
}

public void CreateFile(string filePath, string accountName, Stream contents = null, bool overwrite = false)
public void CreateFile(string filePath, string accountName, MemoryStream contents = null, bool overwrite = false)
{
_client.FileSystem.Create(accountName, filePath, contents, overwrite: overwrite);
if (contents.Length <= MaximumBytesPerAppendRequest)
{
// use content-length header for request
_client.FileSystem.Create(accountName, filePath, contents, overwrite: overwrite);
}
else
{
// use transfer-encoding: chunked header for request
var customHeaders = new Dictionary<string, List<string>>();
customHeaders.Add("Transfer-Encoding", new List<string> { "Chunked" });
_client.FileSystem.CreateWithHttpMessagesAsync(
accountName,
filePath,
contents,
overwrite: overwrite,
customHeaders: customHeaders).GetAwaiter().GetResult();
}


}

public bool CreateDirectory(string dirPath, string accountName)
Expand All @@ -432,9 +451,24 @@ public bool CreateDirectory(string dirPath, string accountName)
return boolean != null && boolean.Value;
}

public void AppendToFile(string filePath, string accountName, Stream contents)
public void AppendToFile(string filePath, string accountName, MemoryStream contents)
{
_client.FileSystem.Append(accountName, filePath, contents);
if (contents.Length <= MaximumBytesPerAppendRequest)
{
// use content-length header for request
_client.FileSystem.Append(accountName, filePath, contents);
}
else
{
// use transfer-encoding: chunked header for request
var customHeaders = new Dictionary<string, List<string>>();
customHeaders.Add("Transfer-Encoding", new List<string> { "Chunked" });
_client.FileSystem.AppendWithHttpMessagesAsync(
accountName,
filePath,
contents,
customHeaders: customHeaders).GetAwaiter().GetResult();
}
}

public void CopyFile(string destinationPath, string accountName, string sourcePath,
Expand Down