Skip to content

Commit 707f5f9

Browse files
cormacpaynecormacpayne
authored
Merge pull request #7877 from EmmaZhu/preview
Add a test script which is to calculate storage container billable size.
2 parents 2cb2c72 + 51a01e9 commit 707f5f9

File tree

2 files changed

+284
-0
lines changed

2 files changed

+284
-0
lines changed

‎src/Storage/Commands.Storage.Test/Commands.Storage.Test.csproj

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -134,6 +134,7 @@
134134
<None Include="app.config" />
135135
<None Include="MSSharedLibKey.snk" />
136136
<None Include="packages.config" />
137+
<None Include="Scripts\GetBillableSize.ps1" />
137138
</ItemGroup>
138139
<Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
139140
</Project>

‎src/Storage/Commands.Storage.Test/Scripts/GetBillableSize.ps1

Lines changed: 283 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,283 @@
1+
# this script will show how to get the total size of the blobs in a container
2+
# before running this, you need to create a storage account, create a container,
3+
# and upload some blobs into the container
4+
# note: this retrieves all of the blobs in the container in one command.
5+
# connect Azure with Login-AzureRmAccount before you run the script.
6+
# requests sent as part of this tool will incur transactional costs.
7+
# command line usage: script.ps1 -ResourceGroup {YourResourceGroupName} -StorageAccountName {YourAccountName} -ContainerName {YourContainerName}
8+
#
9+
10+
param(
11+
[Parameter(Mandatory=$true)]
12+
[string]$ResourceGroup,
13+
14+
[Parameter(Mandatory=$true)]
15+
[string]$StorageAccountName,
16+
17+
[Parameter(Mandatory=$true)]
18+
[string]$ContainerName
19+
)
20+
21+
#Set-StrictMode will cause Get-AzureStorageBlob returns result in different data types when there is only one blob
22+
#Set-StrictMode -Version 2
23+
24+
$VerbosePreference = "Continue"
25+
26+
if(((Get-Module -ListAvailable Azure) -eq $null) -or ((Get-Module -ListAvailable Azure.Storage) -eq $null))
27+
{
28+
throw "Azure Powershell not found! Please install from https://docs.microsoft.com/en-us/powershell/azure/install-azurerm-ps"
29+
}
30+
31+
# function Retry-OnRequest
32+
function Retry-OnRequest
33+
{
34+
param(
35+
[Parameter(Mandatory=$true)]
36+
$Action)
37+
38+
# It could encounter various of temporary errors, like network errors, or storage server busy errors.
39+
# Should retry the request on transient errors
40+
41+
# Retry on storage server timeout errors
42+
$clientTimeOut = New-TimeSpan -Minutes 15
43+
$retryPolicy = New-Object -TypeName Microsoft.WindowsAzure.Storage.RetryPolicies.ExponentialRetry -ArgumentList @($clientTimeOut, 10)
44+
$requestOption = @{}
45+
$requestOption.RetryPolicy = $retryPolicy
46+
47+
# Retry on temporary network errors
48+
$shouldRetryOnException = $false
49+
$maxRetryCountOnException = 3
50+
51+
do
52+
{
53+
try
54+
{
55+
return $Action.Invoke($requestOption)
56+
}
57+
catch
58+
{
59+
if ($_.Exception.InnerException -ne $null -And $_.Exception.InnerException.GetType() -Eq [System.TimeoutException] -And $maxRetryCountOnException -gt 0)
60+
{
61+
$shouldRetryOnException = $true
62+
$maxRetryCountOnException--
63+
}
64+
else
65+
{
66+
$shouldRetryOnException = $false
67+
throw
68+
}
69+
}
70+
}
71+
while ($shouldRetryOnException)
72+
73+
}
74+
75+
# function Get-BlobBytes
76+
77+
function Get-BlobBytes
78+
{
79+
param(
80+
[Parameter(Mandatory=$true)]
81+
$Blob,
82+
[Parameter(Mandatory=$false)]
83+
[bool]$IsPremiumAccount = $false)
84+
85+
# Base + blobname
86+
$blobSizeInBytes = 124 + $Blob.Name.Length * 2
87+
88+
# Get size of metadata
89+
$metadataEnumerator=$Blob.ICloudBlob.Metadata.GetEnumerator()
90+
while($metadataEnumerator.MoveNext())
91+
{
92+
$blobSizeInBytes += 3 + $metadataEnumerator.Current.Key.Length + $metadataEnumerator.Current.Value.Length
93+
}
94+
95+
if (!$IsPremiumAccount)
96+
{
97+
if($Blob.BlobType -eq [Microsoft.WindowsAzure.Storage.Blob.BlobType]::BlockBlob)
98+
{
99+
$blobSizeInBytes += 8
100+
# Default is Microsoft.WindowsAzure.Storage.Blob.BlockListingFilter.Committed. Need All
101+
$action = { param($requestOption) return $Blob.ICloudBlob.DownloadBlockList([Microsoft.WindowsAzure.Storage.Blob.BlockListingFilter]::All, $null, $requestOption) }
102+
103+
$blocks=Retry-OnRequest $action
104+
105+
if ($null -eq $blocks)
106+
{
107+
$blobSizeInBytes += $Blob.ICloudBlob.Properties.Length
108+
}
109+
else
110+
{
111+
$blocks | ForEach-Object { $blobSizeInBytes += $_.Length + $_.Name.Length }
112+
}
113+
}
114+
elseif($Blob.BlobType -eq [Microsoft.WindowsAzure.Storage.Blob.BlobType]::PageBlob)
115+
{
116+
# It could cause server time out issue when trying to get page ranges of highly fragmented page blob
117+
# Get page ranges in segment can mitigate chance of meeting such kind of server time out issue
118+
# See https://blogs.msdn.microsoft.com/windowsazurestorage/2012/03/26/getting-the-page-ranges-of-a-large-page-blob-in-segments/ for details.
119+
$pageRangesSegSize = 148 * 1024 * 1024L
120+
$totalSize = $Blob.ICloudBlob.Properties.Length
121+
$pageRangeSegOffset = 0
122+
123+
$pageRangesTemp = New-Object System.Collections.ArrayList
124+
125+
while ($pageRangeSegOffset -lt $totalSize)
126+
{
127+
$action = {param($requestOption) return $Blob.ICloudBlob.GetPageRanges($pageRangeSegOffset, $pageRangesSegSize, $null, $requestOption) }
128+
129+
Retry-OnRequest $action | ForEach-Object { $pageRangesTemp.Add($_) } | Out-Null
130+
$pageRangeSegOffset += $pageRangesSegSize
131+
}
132+
133+
$pageRanges = New-Object System.Collections.ArrayList
134+
135+
foreach ($pageRange in $pageRangesTemp)
136+
{
137+
if($lastRange -eq $Null)
138+
{
139+
$lastRange = New-Object PageRange
140+
$lastRange.StartOffset = $pageRange.StartOffset
141+
$lastRange.EndOffset = $pageRange.EndOffset
142+
}
143+
else
144+
{
145+
if (($lastRange.EndOffset + 1) -eq $pageRange.StartOffset)
146+
{
147+
$lastRange.EndOffset = $pageRange.EndOffset
148+
}
149+
else
150+
{
151+
$pageRanges.Add($lastRange) | Out-Null
152+
$lastRange = New-Object PageRange
153+
$lastRange.StartOffset = $pageRange.StartOffset
154+
$lastRange.EndOffset = $pageRange.EndOffset
155+
}
156+
}
157+
}
158+
159+
$pageRanges.Add($lastRange) | Out-Null
160+
$pageRanges | ForEach-Object {
161+
$blobSizeInBytes += 12 + $_.EndOffset - $_.StartOffset
162+
}
163+
}
164+
else
165+
{
166+
$blobSizeInBytes += $Blob.ICloudBlob.Properties.Length
167+
}
168+
return $blobSizeInBytes
169+
}
170+
else
171+
{
172+
$blobSizeInBytes += $Blob.ICloudBlob.Properties.Length
173+
}
174+
return $blobSizeInBytes
175+
}
176+
177+
# function Get-ContainerBytes
178+
179+
function Get-ContainerBytes
180+
{
181+
param(
182+
[Parameter(Mandatory=$true)]
183+
[Microsoft.WindowsAzure.Storage.Blob.CloudBlobContainer]$Container,
184+
[Parameter(Mandatory=$false)]
185+
[bool]$IsPremiumAccount = $false)
186+
187+
# Base + name of container
188+
$containerSizeInBytes = 48 + $Container.Name.Length*2
189+
190+
# Get size of metadata
191+
$metadataEnumerator = $Container.Metadata.GetEnumerator()
192+
while($metadataEnumerator.MoveNext())
193+
{
194+
$containerSizeInBytes += 3 + $metadataEnumerator.Current.Key.Length + $metadataEnumerator.Current.Value.Length
195+
}
196+
197+
# Get size for SharedAccessPolicies
198+
$containerSizeInBytes += $Container.GetPermissions().SharedAccessPolicies.Count * 512
199+
200+
# Calculate size of all blobs.
201+
$blobCount = 0
202+
$Token = $Null
203+
$MaxReturn = 5000
204+
205+
do {
206+
$Blobs = Get-AzureStorageBlob -Context $storageContext -Container $Container.Name -MaxCount $MaxReturn -ContinuationToken $Token
207+
if($Blobs -eq $Null) { break }
208+
209+
#Set-StrictMode will cause Get-AzureStorageBlob returns result in different data types when there is only one blob
210+
if($Blobs.GetType().Name -eq "AzureStorageBlob")
211+
{
212+
$Token = $Null
213+
}
214+
else
215+
{
216+
$Token = $Blobs[$Blobs.Count - 1].ContinuationToken;
217+
}
218+
219+
$Blobs | ForEach-Object {
220+
$blobSize = Get-BlobBytes $_ $IsPremiumAccount
221+
$containerSizeInBytes += $blobSize
222+
$blobCount++
223+
224+
if(($blobCount % 1000) -eq 0)
225+
{
226+
Write-Verbose("Counting {0} Sizing {1} " -f $blobCount, $containerSizeInBytes)
227+
}
228+
}
229+
}
230+
While ($Token -ne $Null)
231+
232+
return @{ "containerSize" = $containerSizeInBytes; "blobCount" = $blobCount }
233+
}
234+
235+
#Login-AzureRmAccount
236+
237+
$storageAccount = Get-AzureRmStorageAccount -ResourceGroupName $ResourceGroup -Name $StorageAccountName -ErrorAction SilentlyContinue
238+
if($storageAccount -eq $null)
239+
{
240+
throw "The storage account specified does not exist in this subscription."
241+
}
242+
243+
$storageContext = $storageAccount.Context
244+
245+
if (-not ([System.Management.Automation.PSTypeName]'PageRange').Type)
246+
{
247+
$Source = "
248+
public class PageRange
249+
{
250+
public long StartOffset;
251+
public long EndOffset;
252+
}"
253+
Add-Type -TypeDefinition $Source
254+
}
255+
256+
$containers = New-Object System.Collections.ArrayList
257+
if($ContainerName.Length -ne 0)
258+
{
259+
$container = Get-AzureStorageContainer -Context $storageContext -Name $ContainerName -ErrorAction SilentlyContinue |
260+
ForEach-Object { $containers.Add($_) } | Out-Null
261+
}
262+
else
263+
{
264+
Get-AzureStorageContainer -Context $storageContext | ForEach-Object { $containers.Add($_) } | Out-Null
265+
}
266+
267+
$sizeInBytes = 0
268+
$IsPremiumAccount = ($storageAccount.Sku.Tier -eq "Premium")
269+
270+
if($containers.Count -gt 0)
271+
{
272+
$containers | ForEach-Object {
273+
Write-Output("Calculating container {0} ..." -f $_.CloudBlobContainer.Name)
274+
$result = Get-ContainerBytes $_.CloudBlobContainer $IsPremiumAccount
275+
$sizeInBytes += $result.containerSize
276+
277+
Write-Output("Container '{0}' with {1} blobs has a sizeof {2:F2} MB." -f $_.CloudBlobContainer.Name,$result.blobCount,($result.containerSize/1MB))
278+
}
279+
}
280+
else
281+
{
282+
Write-Warning "No containers found to process in storage account '$StorageAccountName'."
283+
}

0 commit comments

Comments
 (0)