Skip to content

Add a test script which is to calculate storage container billable size. #7877

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Nov 26, 2018
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -134,6 +134,7 @@
<None Include="app.config" />
<None Include="MSSharedLibKey.snk" />
<None Include="packages.config" />
<None Include="Scripts\GetBillableSize.ps1" />
</ItemGroup>
<Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
</Project>
283 changes: 283 additions & 0 deletions src/Storage/Commands.Storage.Test/Scripts/GetBillableSize.ps1
Original file line number Diff line number Diff line change
@@ -0,0 +1,283 @@
# this script will show how to get the total size of the blobs in a container
# before running this, you need to create a storage account, create a container,
# and upload some blobs into the container
# note: this retrieves all of the blobs in the container in one command.
# connect Azure with Login-AzureRmAccount before you run the script.
# requests sent as part of this tool will incur transactional costs.
# command line usage: script.ps1 -ResourceGroup {YourResourceGroupName} -StorageAccountName {YourAccountName} -ContainerName {YourContainerName}
#

param(
[Parameter(Mandatory=$true)]
[string]$ResourceGroup,

[Parameter(Mandatory=$true)]
[string]$StorageAccountName,

[Parameter(Mandatory=$true)]
[string]$ContainerName
)

#Set-StrictMode will cause Get-AzureStorageBlob returns result in different data types when there is only one blob
#Set-StrictMode -Version 2

$VerbosePreference = "Continue"

if(((Get-Module -ListAvailable Azure) -eq $null) -or ((Get-Module -ListAvailable Azure.Storage) -eq $null))
{
throw "Azure Powershell not found! Please install from https://docs.microsoft.com/en-us/powershell/azure/install-azurerm-ps"
}

# function Retry-OnRequest
function Retry-OnRequest
{
param(
[Parameter(Mandatory=$true)]
$Action)

# It could encounter various of temporary errors, like network errors, or storage server busy errors.
# Should retry the request on transient errors

# Retry on storage server timeout errors
$clientTimeOut = New-TimeSpan -Minutes 15
$retryPolicy = New-Object -TypeName Microsoft.WindowsAzure.Storage.RetryPolicies.ExponentialRetry -ArgumentList @($clientTimeOut, 10)
$requestOption = @{}
$requestOption.RetryPolicy = $retryPolicy

# Retry on temporary network errors
$shouldRetryOnException = $false
$maxRetryCountOnException = 3

do
{
try
{
return $Action.Invoke($requestOption)
}
catch
{
if ($_.Exception.InnerException -ne $null -And $_.Exception.InnerException.GetType() -Eq [System.TimeoutException] -And $maxRetryCountOnException -gt 0)
{
$shouldRetryOnException = $true
$maxRetryCountOnException--
}
else
{
$shouldRetryOnException = $false
throw
}
}
}
while ($shouldRetryOnException)

}

# function Get-BlobBytes

function Get-BlobBytes
{
param(
[Parameter(Mandatory=$true)]
$Blob,
[Parameter(Mandatory=$false)]
[bool]$IsPremiumAccount = $false)

# Base + blobname
$blobSizeInBytes = 124 + $Blob.Name.Length * 2

# Get size of metadata
$metadataEnumerator=$Blob.ICloudBlob.Metadata.GetEnumerator()
while($metadataEnumerator.MoveNext())
{
$blobSizeInBytes += 3 + $metadataEnumerator.Current.Key.Length + $metadataEnumerator.Current.Value.Length
}

if (!$IsPremiumAccount)
{
if($Blob.BlobType -eq [Microsoft.WindowsAzure.Storage.Blob.BlobType]::BlockBlob)
{
$blobSizeInBytes += 8
# Default is Microsoft.WindowsAzure.Storage.Blob.BlockListingFilter.Committed. Need All
$action = { param($requestOption) return $Blob.ICloudBlob.DownloadBlockList([Microsoft.WindowsAzure.Storage.Blob.BlockListingFilter]::All, $null, $requestOption) }

$blocks=Retry-OnRequest $action

if ($null -eq $blocks)
{
$blobSizeInBytes += $Blob.ICloudBlob.Properties.Length
}
else
{
$blocks | ForEach-Object { $blobSizeInBytes += $_.Length + $_.Name.Length }
}
}
elseif($Blob.BlobType -eq [Microsoft.WindowsAzure.Storage.Blob.BlobType]::PageBlob)
{
# It could cause server time out issue when trying to get page ranges of highly fragmented page blob
# Get page ranges in segment can mitigate chance of meeting such kind of server time out issue
# See https://blogs.msdn.microsoft.com/windowsazurestorage/2012/03/26/getting-the-page-ranges-of-a-large-page-blob-in-segments/ for details.
$pageRangesSegSize = 148 * 1024 * 1024L
$totalSize = $Blob.ICloudBlob.Properties.Length
$pageRangeSegOffset = 0

$pageRangesTemp = New-Object System.Collections.ArrayList

while ($pageRangeSegOffset -lt $totalSize)
{
$action = {param($requestOption) return $Blob.ICloudBlob.GetPageRanges($pageRangeSegOffset, $pageRangesSegSize, $null, $requestOption) }

Retry-OnRequest $action | ForEach-Object { $pageRangesTemp.Add($_) } | Out-Null
$pageRangeSegOffset += $pageRangesSegSize
}

$pageRanges = New-Object System.Collections.ArrayList

foreach ($pageRange in $pageRangesTemp)
{
if($lastRange -eq $Null)
{
$lastRange = New-Object PageRange
$lastRange.StartOffset = $pageRange.StartOffset
$lastRange.EndOffset = $pageRange.EndOffset
}
else
{
if (($lastRange.EndOffset + 1) -eq $pageRange.StartOffset)
{
$lastRange.EndOffset = $pageRange.EndOffset
}
else
{
$pageRanges.Add($lastRange) | Out-Null
$lastRange = New-Object PageRange
$lastRange.StartOffset = $pageRange.StartOffset
$lastRange.EndOffset = $pageRange.EndOffset
}
}
}

$pageRanges.Add($lastRange) | Out-Null
$pageRanges | ForEach-Object {
$blobSizeInBytes += 12 + $_.EndOffset - $_.StartOffset
}
}
else
{
$blobSizeInBytes += $Blob.ICloudBlob.Properties.Length
}
return $blobSizeInBytes
}
else
{
$blobSizeInBytes += $Blob.ICloudBlob.Properties.Length
}
return $blobSizeInBytes
}

# function Get-ContainerBytes

function Get-ContainerBytes
{
param(
[Parameter(Mandatory=$true)]
[Microsoft.WindowsAzure.Storage.Blob.CloudBlobContainer]$Container,
[Parameter(Mandatory=$false)]
[bool]$IsPremiumAccount = $false)

# Base + name of container
$containerSizeInBytes = 48 + $Container.Name.Length*2

# Get size of metadata
$metadataEnumerator = $Container.Metadata.GetEnumerator()
while($metadataEnumerator.MoveNext())
{
$containerSizeInBytes += 3 + $metadataEnumerator.Current.Key.Length + $metadataEnumerator.Current.Value.Length
}

# Get size for SharedAccessPolicies
$containerSizeInBytes += $Container.GetPermissions().SharedAccessPolicies.Count * 512

# Calculate size of all blobs.
$blobCount = 0
$Token = $Null
$MaxReturn = 5000

do {
$Blobs = Get-AzureStorageBlob -Context $storageContext -Container $Container.Name -MaxCount $MaxReturn -ContinuationToken $Token
if($Blobs -eq $Null) { break }

#Set-StrictMode will cause Get-AzureStorageBlob returns result in different data types when there is only one blob
if($Blobs.GetType().Name -eq "AzureStorageBlob")
{
$Token = $Null
}
else
{
$Token = $Blobs[$Blobs.Count - 1].ContinuationToken;
}

$Blobs | ForEach-Object {
$blobSize = Get-BlobBytes $_ $IsPremiumAccount
$containerSizeInBytes += $blobSize
$blobCount++

if(($blobCount % 1000) -eq 0)
{
Write-Verbose("Counting {0} Sizing {1} " -f $blobCount, $containerSizeInBytes)
}
}
}
While ($Token -ne $Null)

return @{ "containerSize" = $containerSizeInBytes; "blobCount" = $blobCount }
}

#Login-AzureRmAccount

$storageAccount = Get-AzureRmStorageAccount -ResourceGroupName $ResourceGroup -Name $StorageAccountName -ErrorAction SilentlyContinue
if($storageAccount -eq $null)
{
throw "The storage account specified does not exist in this subscription."
}

$storageContext = $storageAccount.Context

if (-not ([System.Management.Automation.PSTypeName]'PageRange').Type)
{
$Source = "
public class PageRange
{
public long StartOffset;
public long EndOffset;
}"
Add-Type -TypeDefinition $Source
}

$containers = New-Object System.Collections.ArrayList
if($ContainerName.Length -ne 0)
{
$container = Get-AzureStorageContainer -Context $storageContext -Name $ContainerName -ErrorAction SilentlyContinue |
ForEach-Object { $containers.Add($_) } | Out-Null
}
else
{
Get-AzureStorageContainer -Context $storageContext | ForEach-Object { $containers.Add($_) } | Out-Null
}

$sizeInBytes = 0
$IsPremiumAccount = ($storageAccount.Sku.Tier -eq "Premium")

if($containers.Count -gt 0)
{
$containers | ForEach-Object {
Write-Output("Calculating container {0} ..." -f $_.CloudBlobContainer.Name)
$result = Get-ContainerBytes $_.CloudBlobContainer $IsPremiumAccount
$sizeInBytes += $result.containerSize

Write-Output("Container '{0}' with {1} blobs has a sizeof {2:F2} MB." -f $_.CloudBlobContainer.Name,$result.blobCount,($result.containerSize/1MB))
}
}
else
{
Write-Warning "No containers found to process in storage account '$StorageAccountName'."
}