Skip to content

Add an option to skip MD5 computation and checking. Also disable MD5 … #988

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Sep 29, 2015
Merged

Add an option to skip MD5 computation and checking. Also disable MD5 … #988

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion src/ServiceManagement/Compute/Commands.ServiceManagement/Model/UploadParameters.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,12 +21,13 @@ namespace Microsoft.WindowsAzure.Commands.ServiceManagement.Model
{
public class UploadParameters
{
public UploadParameters(BlobUri destinationUri, BlobUri baseImageUri, FileInfo localFilePath, bool overWrite, int numberOfUploaderThreads)
public UploadParameters(BlobUri destinationUri, BlobUri baseImageUri, FileInfo localFilePath, bool overWrite, bool skipMd5, int numberOfUploaderThreads)
{
DestinationUri = destinationUri;
BaseImageUri = baseImageUri;
LocalFilePath = localFilePath;
OverWrite = overWrite;
SkipMd5 = skipMd5;
NumberOfUploaderThreads = numberOfUploaderThreads;
}

Expand All @@ -38,6 +39,8 @@ public UploadParameters(BlobUri destinationUri, BlobUri baseImageUri, FileInfo l

public bool OverWrite { get; private set; }

public bool SkipMd5 { get; private set; }

public int NumberOfUploaderThreads { get; private set; }

public AddAzureVhdCommand Cmdlet { get; set; }
Expand Down
2 changes: 1 addition & 1 deletion src/ServiceManagement/Compute/Commands.ServiceManagement/Model/VhdUploaderModel.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ public static VhdUploadContext Upload(UploadParameters uploadParameters)
}
else
{
blobCreator = new BlobCreator(uploadParameters.LocalFilePath, uploadParameters.DestinationUri, uploadParameters.BlobObjectFactory, uploadParameters.OverWrite);
blobCreator = new BlobCreator(uploadParameters.LocalFilePath, uploadParameters.DestinationUri, uploadParameters.BlobObjectFactory, uploadParameters.OverWrite, uploadParameters.SkipMd5);
}

using (var uploadContext = blobCreator.Create())
Expand Down
11 changes: 10 additions & 1 deletion ...erviceManagement/Compute/Commands.ServiceManagement/StorageServices/AddAzureVhdCommand.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,15 @@ public SwitchParameter OverWrite
set;
}

[Parameter(Position = 6, Mandatory = false, ValueFromPipelineByPropertyName = true, ParameterSetName = "Vhd", HelpMessage = "Skip computation of MD5 checksum")]
[ValidateNotNullOrEmpty]
[Alias("sm")]
public SwitchParameter SkipMd5
{
get;
set;
}

public UploadParameters ValidateParameters()
{
BlobUri destinationUri;
Expand Down Expand Up @@ -106,7 +115,7 @@ public UploadParameters ValidateParameters()
PathIntrinsics currentPath = SessionState.Path;
var filePath = new FileInfo(currentPath.GetUnresolvedProviderPathFromPSPath(LocalFilePath.ToString()));

var parameters = new UploadParameters(destinationUri, baseImageUri, filePath, OverWrite.IsPresent, NumberOfUploaderThreads)
var parameters = new UploadParameters(destinationUri, baseImageUri, filePath, OverWrite.IsPresent, SkipMd5.IsPresent, NumberOfUploaderThreads)
{
Cmdlet = this,
BlobObjectFactory = new CloudPageBlobObjectFactory(storageCredentialsFactory, TimeSpan.FromMinutes(1))
Expand Down
4 changes: 2 additions & 2 deletions src/ServiceManagement/Compute/Sync/Upload/BlobCreator.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,8 @@ namespace Microsoft.WindowsAzure.Commands.Sync.Upload
{
public class BlobCreator : BlobCreatorBase
{
public BlobCreator(FileInfo localVhd, BlobUri destination, ICloudPageBlobObjectFactory blobObjectFactory, bool overWrite) :
base(localVhd, destination, blobObjectFactory, overWrite)
public BlobCreator(FileInfo localVhd, BlobUri destination, ICloudPageBlobObjectFactory blobObjectFactory, bool overWrite, bool skipMd5 = false) :
base(localVhd, destination, blobObjectFactory, overWrite, skipMd5)
{
}

Expand Down
6 changes: 4 additions & 2 deletions src/ServiceManagement/Compute/Sync/Upload/BlobCreatorBase.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,19 +50,21 @@ public abstract class BlobCreatorBase
protected CloudPageBlob destinationBlob;
protected BlobRequestOptions requestOptions;
protected bool overWrite;
protected bool skipMd5;
private readonly TimeSpan delayBetweenRetries = TimeSpan.FromSeconds(10);

private const int MegaByte = 1024 * 1024;
private const int PageSizeInBytes = 2 * MegaByte;
private const int MaxBufferSize = 20 * MegaByte;

protected BlobCreatorBase(FileInfo localVhd, BlobUri blobDestination, ICloudPageBlobObjectFactory blobObjectFactory, bool overWrite)
protected BlobCreatorBase(FileInfo localVhd, BlobUri blobDestination, ICloudPageBlobObjectFactory blobObjectFactory, bool overWrite, bool skipMd5 = false)
{
this.localVhd = localVhd;
this.blobObjectFactory = blobObjectFactory;
this.destination = new Uri(blobDestination.BlobPath);
this.blobDestination = blobDestination;
this.overWrite = overWrite;
this.skipMd5 = skipMd5;

this.destinationBlob = blobObjectFactory.Create(blobDestination);
this.requestOptions = this.blobObjectFactory.CreateRequestOptions();
Expand All @@ -79,7 +81,7 @@ public LocalMetaData OperationMetaData
{
this.operationMetaData = new LocalMetaData
{
FileMetaData = FileMetaData.Create(localVhd.FullName),
FileMetaData = FileMetaData.Create(localVhd.FullName, skipMd5),
SystemInformation = SystemInformation.Create()
};
}
Expand Down
16 changes: 13 additions & 3 deletions src/ServiceManagement/Compute/Sync/Upload/BlobSynchronizer.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,10 +56,16 @@ public bool Synchronize()
{
using (dwr)
{
var md5HashOfDataChunk = GetBase64EncodedMd5Hash(dwr.Data, (int) dwr.Range.Length);
using (var stream = new MemoryStream(dwr.Data, 0, (int)dwr.Range.Length))
{
b.Properties.ContentMD5 = md5HashOfDataChunk;
// HTTPS provides transport level security that renders
// MD5 checking redundant
if (blob.Uri.Scheme != Uri.UriSchemeHttps)
{
var md5HashOfDataChunk = GetBase64EncodedMd5Hash(dwr.Data, (int)dwr.Range.Length);
b.Properties.ContentMD5 = md5HashOfDataChunk;
}

b.WritePages(stream, dwr.Range.StartIndex);
}
}
Expand All @@ -78,7 +84,11 @@ public bool Synchronize()
{
using(var bdms = new BlobMetaDataScope(new CloudPageBlob(blob.Uri, blob.ServiceClient.Credentials)))
{
bdms.Current.SetBlobMd5Hash(md5Hash);
if (this.md5Hash != null && this.md5Hash.Length != 0)
{
bdms.Current.SetBlobMd5Hash(md5Hash);
}

bdms.Current.CleanUpUploadMetaData();
bdms.Complete();
}
Expand Down
6 changes: 3 additions & 3 deletions src/ServiceManagement/Compute/Sync/Upload/UploadOperationMetaData.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@ public DateTime LastModifiedDateUtc
set { this.InternalLastModifiedDateUtc = value.ToString(DateTimeFormatInfo.InvariantInfo); }
}

public static FileMetaData Create(string filePath)
public static FileMetaData Create(string filePath, bool skipMd5 = false)
{
var fileInfo = new FileInfo(filePath);
if(!fileInfo.Exists)
Expand All @@ -75,14 +75,14 @@ public static FileMetaData Create(string filePath)
LastModifiedDateUtc = DateTime.SpecifyKind(fileInfo.LastWriteTimeUtc, DateTimeKind.Utc),
Size = fileInfo.Length,
VhdSize = stream.Length,
MD5Hash = CalculateMd5Hash(stream, filePath)
MD5Hash = skipMd5 ? new byte[0] : CalculateMd5Hash(stream, filePath)
};
}
}

private static byte[] CalculateMd5Hash(Stream stream, string filePath)
{
using(var md5 = MD5.Create())
using (var md5 = MD5.Create())
{
using (var swrp = new StreamWithReadProgress(stream, TimeSpan.FromSeconds(1)))
{
Expand Down