Skip to content

Commit 51a01e9

Browse files
EmmaZhuEmmaZhu
committed
Refine the script to calculate billable blob size.
1 parent f7b8705 commit 51a01e9

File tree

1 file changed

+139
-14
lines changed

1 file changed

+139
-14
lines changed

‎src/Storage/Commands.Storage.Test/Scripts/GetBillableSize.ps1

Lines changed: 139 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
# and upload some blobs into the container
44
# note: this retrieves all of the blobs in the container in one command.
55
# connect Azure with Login-AzureRmAccount before you run the script.
6+
# requests sent as part of this tool will incur transactional costs.
67
# command line usage: script.ps1 -ResourceGroup {YourResourceGroupName} -StorageAccountName {YourAccountName} -ContainerName {YourContainerName}
78
#
89

@@ -22,20 +23,64 @@ param(
2223

2324
$VerbosePreference = "Continue"
2425

25-
if((Get-Module -ListAvailable Azure) -eq $null)
26+
if(((Get-Module -ListAvailable Azure) -eq $null) -or ((Get-Module -ListAvailable Azure.Storage) -eq $null))
2627
{
27-
throw "Azure Powershell not found! Please install from http://www.windowsazure.com/en-us/downloads/#cmd-line-tools"
28+
throw "Azure Powershell not found! Please install from https://docs.microsoft.com/en-us/powershell/azure/install-azurerm-ps"
2829
}
2930

31+
# function Retry-OnRequest
32+
function Retry-OnRequest
33+
{
34+
param(
35+
[Parameter(Mandatory=$true)]
36+
$Action)
37+
38+
# It could encounter various of temporary errors, like network errors, or storage server busy errors.
39+
# Should retry the request on transient errors
40+
41+
# Retry on storage server timeout errors
42+
$clientTimeOut = New-TimeSpan -Minutes 15
43+
$retryPolicy = New-Object -TypeName Microsoft.WindowsAzure.Storage.RetryPolicies.ExponentialRetry -ArgumentList @($clientTimeOut, 10)
44+
$requestOption = @{}
45+
$requestOption.RetryPolicy = $retryPolicy
46+
47+
# Retry on temporary network errors
48+
$shouldRetryOnException = $false
49+
$maxRetryCountOnException = 3
50+
51+
do
52+
{
53+
try
54+
{
55+
return $Action.Invoke($requestOption)
56+
}
57+
catch
58+
{
59+
if ($_.Exception.InnerException -ne $null -And $_.Exception.InnerException.GetType() -Eq [System.TimeoutException] -And $maxRetryCountOnException -gt 0)
60+
{
61+
$shouldRetryOnException = $true
62+
$maxRetryCountOnException--
63+
}
64+
else
65+
{
66+
$shouldRetryOnException = $false
67+
throw
68+
}
69+
}
70+
}
71+
while ($shouldRetryOnException)
3072

73+
}
3174

3275
# function Get-BlobBytes
3376

3477
function Get-BlobBytes
3578
{
3679
param(
3780
[Parameter(Mandatory=$true)]
38-
$Blob)
81+
$Blob,
82+
[Parameter(Mandatory=$false)]
83+
[bool]$IsPremiumAccount = $false)
3984

4085
# Base + blobname
4186
$blobSizeInBytes = 124 + $Blob.Name.Length * 2
@@ -47,19 +92,85 @@ function Get-BlobBytes
4792
$blobSizeInBytes += 3 + $metadataEnumerator.Current.Key.Length + $metadataEnumerator.Current.Value.Length
4893
}
4994

50-
if($Blob.BlobType -eq [Microsoft.WindowsAzure.Storage.Blob.BlobType]::BlockBlob)
95+
if (!$IsPremiumAccount)
5196
{
52-
$blobSizeInBytes += 8
53-
# Default is Microsoft.WindowsAzure.Storage.Blob.BlockListingFilter.Committed. Need All
54-
$Blob.ICloudBlob.DownloadBlockList([Microsoft.WindowsAzure.Storage.Blob.BlockListingFilter]::All) |
55-
ForEach-Object { $blobSizeInBytes += $_.Length + $_.Name.Length }
97+
if($Blob.BlobType -eq [Microsoft.WindowsAzure.Storage.Blob.BlobType]::BlockBlob)
98+
{
99+
$blobSizeInBytes += 8
100+
# Default is Microsoft.WindowsAzure.Storage.Blob.BlockListingFilter.Committed. Need All
101+
$action = { param($requestOption) return $Blob.ICloudBlob.DownloadBlockList([Microsoft.WindowsAzure.Storage.Blob.BlockListingFilter]::All, $null, $requestOption) }
102+
103+
$blocks=Retry-OnRequest $action
104+
105+
if ($null -eq $blocks)
106+
{
107+
$blobSizeInBytes += $Blob.ICloudBlob.Properties.Length
108+
}
109+
else
110+
{
111+
$blocks | ForEach-Object { $blobSizeInBytes += $_.Length + $_.Name.Length }
112+
}
113+
}
114+
elseif($Blob.BlobType -eq [Microsoft.WindowsAzure.Storage.Blob.BlobType]::PageBlob)
115+
{
116+
# It could cause server time out issue when trying to get page ranges of highly fragmented page blob
117+
# Get page ranges in segment can mitigate chance of meeting such kind of server time out issue
118+
# See https://blogs.msdn.microsoft.com/windowsazurestorage/2012/03/26/getting-the-page-ranges-of-a-large-page-blob-in-segments/ for details.
119+
$pageRangesSegSize = 148 * 1024 * 1024L
120+
$totalSize = $Blob.ICloudBlob.Properties.Length
121+
$pageRangeSegOffset = 0
122+
123+
$pageRangesTemp = New-Object System.Collections.ArrayList
124+
125+
while ($pageRangeSegOffset -lt $totalSize)
126+
{
127+
$action = {param($requestOption) return $Blob.ICloudBlob.GetPageRanges($pageRangeSegOffset, $pageRangesSegSize, $null, $requestOption) }
128+
129+
Retry-OnRequest $action | ForEach-Object { $pageRangesTemp.Add($_) } | Out-Null
130+
$pageRangeSegOffset += $pageRangesSegSize
131+
}
132+
133+
$pageRanges = New-Object System.Collections.ArrayList
134+
135+
foreach ($pageRange in $pageRangesTemp)
136+
{
137+
if($lastRange -eq $Null)
138+
{
139+
$lastRange = New-Object PageRange
140+
$lastRange.StartOffset = $pageRange.StartOffset
141+
$lastRange.EndOffset = $pageRange.EndOffset
142+
}
143+
else
144+
{
145+
if (($lastRange.EndOffset + 1) -eq $pageRange.StartOffset)
146+
{
147+
$lastRange.EndOffset = $pageRange.EndOffset
148+
}
149+
else
150+
{
151+
$pageRanges.Add($lastRange) | Out-Null
152+
$lastRange = New-Object PageRange
153+
$lastRange.StartOffset = $pageRange.StartOffset
154+
$lastRange.EndOffset = $pageRange.EndOffset
155+
}
156+
}
157+
}
158+
159+
$pageRanges.Add($lastRange) | Out-Null
160+
$pageRanges | ForEach-Object {
161+
$blobSizeInBytes += 12 + $_.EndOffset - $_.StartOffset
162+
}
163+
}
164+
else
165+
{
166+
$blobSizeInBytes += $Blob.ICloudBlob.Properties.Length
167+
}
168+
return $blobSizeInBytes
56169
}
57170
else
58171
{
59-
$Blob.ICloudBlob.GetPageRanges() |
60-
ForEach-Object { $blobSizeInBytes += 12 + $_.EndOffset - $_.StartOffset }
172+
$blobSizeInBytes += $Blob.ICloudBlob.Properties.Length
61173
}
62-
63174
return $blobSizeInBytes
64175
}
65176

@@ -69,7 +180,9 @@ function Get-ContainerBytes
69180
{
70181
param(
71182
[Parameter(Mandatory=$true)]
72-
[Microsoft.WindowsAzure.Storage.Blob.CloudBlobContainer]$Container)
183+
[Microsoft.WindowsAzure.Storage.Blob.CloudBlobContainer]$Container,
184+
[Parameter(Mandatory=$false)]
185+
[bool]$IsPremiumAccount = $false)
73186

74187
# Base + name of container
75188
$containerSizeInBytes = 48 + $Container.Name.Length*2
@@ -104,7 +217,7 @@ function Get-ContainerBytes
104217
}
105218

106219
$Blobs | ForEach-Object {
107-
$blobSize = Get-BlobBytes $_
220+
$blobSize = Get-BlobBytes $_ $IsPremiumAccount
108221
$containerSizeInBytes += $blobSize
109222
$blobCount++
110223

@@ -129,6 +242,17 @@ if($storageAccount -eq $null)
129242

130243
$storageContext = $storageAccount.Context
131244

245+
if (-not ([System.Management.Automation.PSTypeName]'PageRange').Type)
246+
{
247+
$Source = "
248+
public class PageRange
249+
{
250+
public long StartOffset;
251+
public long EndOffset;
252+
}"
253+
Add-Type -TypeDefinition $Source
254+
}
255+
132256
$containers = New-Object System.Collections.ArrayList
133257
if($ContainerName.Length -ne 0)
134258
{
@@ -141,12 +265,13 @@ else
141265
}
142266

143267
$sizeInBytes = 0
268+
$IsPremiumAccount = ($storageAccount.Sku.Tier -eq "Premium")
144269

145270
if($containers.Count -gt 0)
146271
{
147272
$containers | ForEach-Object {
148273
Write-Output("Calculating container {0} ..." -f $_.CloudBlobContainer.Name)
149-
$result = Get-ContainerBytes $_.CloudBlobContainer
274+
$result = Get-ContainerBytes $_.CloudBlobContainer $IsPremiumAccount
150275
$sizeInBytes += $result.containerSize
151276

152277
Write-Output("Container '{0}' with {1} blobs has a sizeof {2:F2} MB." -f $_.CloudBlobContainer.Name,$result.blobCount,($result.containerSize/1MB))

0 commit comments

Comments
 (0)