Skip to content

Commit 862d3c4

Browse files
bluewwblueww
authored
[Storage] Support Parquet in Quick query (Azure#15539)
* [Storage] Support Parquet in Quick query * fix staticanalysis failure
1 parent 19267e1 commit 862d3c4

File tree

8 files changed

+93
-19
lines changed

8 files changed

+93
-19
lines changed

‎src/Storage/Storage.Management.Test/Storage.Management.Test.csproj

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -11,10 +11,10 @@
1111
</PropertyGroup>
1212

1313
<ItemGroup>
14-
<PackageReference Include="Azure.Storage.Blobs" Version="12.8.0" />
15-
<PackageReference Include="Azure.Storage.Files.DataLake" Version="12.6.0" />
16-
<PackageReference Include="Azure.Storage.Files.Shares" Version="12.6.0" />
17-
<PackageReference Include="Azure.Storage.Queues" Version="12.6.0" />
14+
<PackageReference Include="Azure.Storage.Blobs" Version="12.10.0-beta.2" />
15+
<PackageReference Include="Azure.Storage.Files.DataLake" Version="12.8.0-beta.2" />
16+
<PackageReference Include="Azure.Storage.Files.Shares" Version="12.8.0-beta.2" />
17+
<PackageReference Include="Azure.Storage.Queues" Version="12.8.0-beta.2" />
1818
<PackageReference Include="Microsoft.Azure.Management.Storage" Version="22.0.0" />
1919
</ItemGroup>
2020

‎src/Storage/Storage.Management/ChangeLog.md

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,9 @@
2222
- `Enable-AzStorageBlobLastAccessTimeTracking`
2323
- `Disable-AzStorageBlobLastAccessTimeTracking`
2424
- `Add-AzStorageAccountManagementPolicyAction`
25+
* Supported Parquest as input text config in blob quick query
26+
- `New-AzStorageBlobQueryConfig`
27+
- `Get-AzStorageBlobQueryResult`
2528
* Made `Get-AzDataLakeGen2ChildItem` list all datalake gen2 items by default, instead of needing user to list chunk by chunk.
2629
* Fixed BlobProperties is empty issue when using sas without prefix '?' [#15460]
2730

‎src/Storage/Storage.Management/help/Get-AzStorageBlobQueryResult.md

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -51,9 +51,9 @@ The **Get-AzStorageBlobQueryResult** cmdlet applies a simple Structured Query La
5151

5252
### Example 1: Query a blob
5353
```powershell
54-
PS C:\> $inputconfig = New-AzStorageBlobQueryConfig -AsCsv -HasHeader
54+
PS C:\> $inputconfig = New-AzStorageBlobQueryConfig -AsParquet
5555
56-
PS C:\> $outputconfig = New-AzStorageBlobQueryConfig -AsJson
56+
PS C:\> $outputconfig = New-AzStorageBlobQueryConfig -AsCsv -HasHeader
5757
5858
PS C:\> $queryString = "SELECT * FROM BlobStorage WHERE Name = 'a'"
5959
@@ -387,7 +387,7 @@ Accept wildcard characters: False
387387
```
388388
389389
### CommonParameters
390-
This cmdlet supports the common parameters: -Debug, -ErrorAction, -ErrorVariable, -InformationAction, -InformationVariable, -OutVariable, -OutBuffer, -PipelineVariable, -Verbose, -WarningAction, and -WarningVariable. For more information, see about_CommonParameters (http://go.microsoft.com/fwlink/?LinkID=113216).
390+
This cmdlet supports the common parameters: -Debug, -ErrorAction, -ErrorVariable, -InformationAction, -InformationVariable, -OutVariable, -OutBuffer, -PipelineVariable, -Verbose, -WarningAction, and -WarningVariable. For more information, see [about_CommonParameters](http://go.microsoft.com/fwlink/?LinkID=113216).
391391
392392
## INPUTS
393393

‎src/Storage/Storage.Management/help/New-AzStorageBlobQueryConfig.md

Lines changed: 22 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,11 @@ New-AzStorageBlobQueryConfig [-AsCsv] [-RecordSeparator <String>] [-ColumnSepara
2323
New-AzStorageBlobQueryConfig [-AsJson] [-RecordSeparator <String>] [-AsJob] [<CommonParameters>]
2424
```
2525

26+
### Parquet
27+
```
28+
New-AzStorageBlobQueryConfig [-AsParquet] [-AsJob] [<CommonParameters>]
29+
```
30+
2631
## DESCRIPTION
2732
The **New-AzStorageBlobQueryConfig** cmdlet creates a blob query configuration object, which can be used in Get-AzStorageBlobQueryResult.
2833

@@ -94,6 +99,21 @@ Accept pipeline input: False
9499
Accept wildcard characters: False
95100
```
96101
102+
### -AsParquet
103+
Indicate to create a Blob Query Configuration for Parquet.
104+
105+
```yaml
106+
Type: System.Management.Automation.SwitchParameter
107+
Parameter Sets: Parquet
108+
Aliases:
109+
110+
Required: True
111+
Position: 0
112+
Default value: None
113+
Accept pipeline input: False
114+
Accept wildcard characters: False
115+
```
116+
97117
### -ColumnSeparator
98118
Optional.
99119
The string used to separate columns.
@@ -164,7 +184,7 @@ The string used to separate records.
164184
165185
```yaml
166186
Type: System.String
167-
Parameter Sets: (All)
187+
Parameter Sets: Csv, Json
168188
Aliases:
169189

170190
Required: False
@@ -175,7 +195,7 @@ Accept wildcard characters: False
175195
```
176196
177197
### CommonParameters
178-
This cmdlet supports the common parameters: -Debug, -ErrorAction, -ErrorVariable, -InformationAction, -InformationVariable, -OutVariable, -OutBuffer, -PipelineVariable, -Verbose, -WarningAction, and -WarningVariable. For more information, see about_CommonParameters (http://go.microsoft.com/fwlink/?LinkID=113216).
198+
This cmdlet supports the common parameters: -Debug, -ErrorAction, -ErrorVariable, -InformationAction, -InformationVariable, -OutVariable, -OutBuffer, -PipelineVariable, -Verbose, -WarningAction, and -WarningVariable. For more information, see [about_CommonParameters](http://go.microsoft.com/fwlink/?LinkID=113216).
179199
180200
## INPUTS
181201

‎src/Storage/Storage/Blob/Cmdlet/NewAzStorageBlobQueryConfig .cs

Lines changed: 23 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,12 @@ public class NewAzStorageBlobQueryConfigCommand : AzureDataCmdlet
3232
/// </summary>
3333
private const string JsonParameterSet = "Json";
3434

35-
[Parameter(Mandatory = true,
35+
/// <summary>
36+
/// Parquet parameter set name
37+
/// </summary>
38+
private const string ParquetParameterSet = "Parquet";
39+
40+
[Parameter(Mandatory = true,
3641
Position = 0,
3742
HelpMessage = "Indicate to create a Blob Query Configuration for CSV.",
3843
ParameterSetName = CsvParameterSet)]
@@ -41,11 +46,18 @@ public class NewAzStorageBlobQueryConfigCommand : AzureDataCmdlet
4146

4247
[Parameter(Mandatory = true,
4348
Position = 0,
44-
HelpMessage = "Indicate to create a Blob Query Configuration for Json.",
49+
HelpMessage = "Indicate to create a Blob Query Configuration for Json.",
4550
ParameterSetName = JsonParameterSet)]
4651
[ValidateNotNullOrEmpty]
4752
public SwitchParameter AsJson { get; set; }
4853

54+
[Parameter(Mandatory = true,
55+
Position = 0,
56+
HelpMessage = "Indicate to create a Blob Query Configuration for Parquet.",
57+
ParameterSetName = ParquetParameterSet)]
58+
[ValidateNotNullOrEmpty]
59+
public SwitchParameter AsParquet { get; set; }
60+
4961
[Parameter(Mandatory = false, HelpMessage = "Optional. The string used to separate records.", ParameterSetName = CsvParameterSet)]
5062
[Parameter(Mandatory = false, HelpMessage = "Optional. The string used to separate records.", ParameterSetName = JsonParameterSet)]
5163
public string RecordSeparator { get; set; }
@@ -114,7 +126,7 @@ public override void ExecuteCmdlet()
114126
};
115127
WriteObject(queryConfig);
116128
}
117-
else // Csv
129+
else if (this.AsCsv.IsPresent) // Csv
118130
{
119131
PSBlobQueryTextConfiguration queryConfig = new PSBlobQueryCsvTextConfiguration()
120132
{
@@ -127,6 +139,14 @@ public override void ExecuteCmdlet()
127139
};
128140
WriteObject(queryConfig);
129141
}
142+
else // Parquet
143+
{
144+
PSBlobQueryParquetTextOptions queryConfig = new PSBlobQueryParquetTextOptions()
145+
{
146+
Type = BlobQueryConfigType.Parquet
147+
};
148+
WriteObject(queryConfig);
149+
}
130150
}
131151
}
132152
}

‎src/Storage/Storage/Common/BlobQueryTextConfiguration.cs

Lines changed: 31 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,8 @@ namespace Microsoft.WindowsAzure.Commands.Common.Storage.ResourceModel
2323
public enum BlobQueryConfigType
2424
{
2525
Csv,
26-
Json
26+
Json,
27+
Parquet
2728
}
2829

2930
/// <summary>
@@ -32,7 +33,6 @@ public enum BlobQueryConfigType
3233
public abstract class PSBlobQueryTextConfiguration
3334
{
3435
public BlobQueryConfigType Type { get; set; }
35-
public string RecordSeparator { get; set; }
3636

3737
public BlobQueryTextOptions ParseBlobQueryTextConfiguration()
3838
{
@@ -48,14 +48,18 @@ public BlobQueryTextOptions ParseBlobQueryTextConfiguration()
4848
HasHeaders = csvconfig.HasHeaders
4949
};
5050
}
51-
else //json
51+
else if (this.Type == BlobQueryConfigType.Json) //json
5252
{
5353
PSBlobQueryJsonTextConfiguration jsonconfig = (PSBlobQueryJsonTextConfiguration)this;
5454
return new BlobQueryJsonTextOptions()
5555
{
5656
RecordSeparator = jsonconfig.RecordSeparator
5757
};
5858
}
59+
else //Parquet
60+
{
61+
return new BlobQueryParquetTextOptions();
62+
}
5963
}
6064
}
6165

@@ -64,6 +68,7 @@ public BlobQueryTextOptions ParseBlobQueryTextConfiguration()
6468
/// </summary>
6569
public class PSBlobQueryJsonTextConfiguration : PSBlobQueryTextConfiguration
6670
{
71+
public string RecordSeparator { get; set; }
6772
public PSBlobQueryJsonTextConfiguration() { }
6873

6974
public PSBlobQueryJsonTextConfiguration(BlobQueryJsonTextOptions config)
@@ -86,6 +91,7 @@ public BlobQueryJsonTextOptions ParseBlobQueryJsonTextConfiguration()
8691
/// </summary>
8792
public class PSBlobQueryCsvTextConfiguration : PSBlobQueryTextConfiguration
8893
{
94+
public string RecordSeparator { get; set; }
8995
public string ColumnSeparator { get; set; }
9096
public char? QuotationCharacter { get; set; }
9197
public char? EscapeCharacter { get; set; }
@@ -117,6 +123,28 @@ public BlobQueryCsvTextOptions ParseBlobQueryCsvTextConfiguration()
117123
}
118124
}
119125

126+
127+
/// <summary>
128+
/// Wrapper Class for BlobQueryParquetTextOptions
129+
/// </summary>
130+
public class PSBlobQueryParquetTextOptions : PSBlobQueryTextConfiguration
131+
{
132+
public PSBlobQueryParquetTextOptions()
133+
{
134+
this.Type = BlobQueryConfigType.Parquet;
135+
}
136+
137+
public PSBlobQueryParquetTextOptions(BlobQueryParquetTextOptions config)
138+
{
139+
this.Type = BlobQueryConfigType.Parquet;
140+
}
141+
142+
public BlobQueryParquetTextOptions ParseBlobQueryParquetTextOptions()
143+
{
144+
return new BlobQueryParquetTextOptions();
145+
}
146+
}
147+
120148
/// <summary>
121149
/// Wrapper of BlobQueryError
122150
/// </summary>

‎src/Storage/Storage/Storage.csproj

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -12,10 +12,10 @@
1212
</PropertyGroup>
1313

1414
<ItemGroup>
15-
<PackageReference Include="Azure.Storage.Blobs" Version="12.8.0" />
16-
<PackageReference Include="Azure.Storage.Files.DataLake" Version="12.6.0" />
17-
<PackageReference Include="Azure.Storage.Files.Shares" Version="12.6.0" />
18-
<PackageReference Include="Azure.Storage.Queues" Version="12.6.0" />
15+
<PackageReference Include="Azure.Storage.Blobs" Version="12.10.0-beta.2" />
16+
<PackageReference Include="Azure.Storage.Files.DataLake" Version="12.8.0-beta.2" />
17+
<PackageReference Include="Azure.Storage.Files.Shares" Version="12.8.0-beta.2" />
18+
<PackageReference Include="Azure.Storage.Queues" Version="12.8.0-beta.2" />
1919
<PackageReference Include="Microsoft.Azure.Cosmos.Table" Version="1.0.8" />
2020
<PackageReference Include="System.Security.Permissions" Version="4.5.0" />
2121
<PackageReference Include="Microsoft.Azure.Storage.Blob" Version="11.2.2" />

‎tools/StaticAnalysis/Exceptions/Az.Storage/BreakingChangeIssues.csv

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,3 +12,6 @@
1212
"Az.Storage","Microsoft.WindowsAzure.Commands.Storage.File.Cmdlet.SetAzureStorageFileContent","Set-AzStorageFileContent","0","1050","The parameter set '__AllParameterSets' for cmdlet 'Set-AzStorageFileContent' has been removed.","Add parameter set '__AllParameterSets' back to cmdlet 'Set-AzStorageFileContent'."
1313
"Az.Storage","Microsoft.Azure.Commands.Management.Storage.NewAzureStorageAccountCommand","New-AzStorageAccount","0","2040","The validation set for parameter 'DefaultSharePermission' for cmdlet 'New-AzStorageAccount' no longer contains the value 'StorageFileDataSmbShareOwner'.","Add 'StorageFileDataSmbShareOwner' back to the validation set for 'DefaultSharePermission'."
1414
"Az.Storage","Microsoft.Azure.Commands.Management.Storage.SetAzureStorageAccountCommand","Set-AzStorageAccount","0","2040","The validation set for parameter 'DefaultSharePermission' for cmdlet 'Set-AzStorageAccount' no longer contains the value 'StorageFileDataSmbShareOwner'.","Add 'StorageFileDataSmbShareOwner' back to the validation set for 'DefaultSharePermission'."
15+
"Az.Storage","Microsoft.WindowsAzure.Commands.Storage.Blob.GetStorageAzureBlobQueryResultCommand","Get-AzStorageBlobQueryResult","0","3010","The property 'RecordSeparator' of type 'Microsoft.WindowsAzure.Commands.Common.Storage.ResourceModel.PSBlobQueryTextConfiguration' has been removed.","Add the property 'RecordSeparator' back to type 'Microsoft.WindowsAzure.Commands.Common.Storage.ResourceModel.PSBlobQueryTextConfiguration'."
16+
"Az.Storage","Microsoft.WindowsAzure.Commands.Storage.Blob.GetStorageAzureBlobQueryResultCommand","Get-AzStorageBlobQueryResult","0","3010","The property 'RecordSeparator' of type 'Microsoft.WindowsAzure.Commands.Common.Storage.ResourceModel.PSBlobQueryTextConfiguration' has been removed.","Add the property 'RecordSeparator' back to type 'Microsoft.WindowsAzure.Commands.Common.Storage.ResourceModel.PSBlobQueryTextConfiguration'."
17+
"Az.Storage","Microsoft.Azure.Commands.Management.Storage.NewAzStorageBlobQueryConfigCommand","New-AzStorageBlobQueryConfig","0","3010","The property 'RecordSeparator' of type 'Microsoft.WindowsAzure.Commands.Common.Storage.ResourceModel.PSBlobQueryTextConfiguration' has been removed.","Add the property 'RecordSeparator' back to type 'Microsoft.WindowsAzure.Commands.Common.Storage.ResourceModel.PSBlobQueryTextConfiguration'."

0 commit comments

Comments
 (0)