Skip to content

Commit c885305

Browse files
yanjungao718yanjungao718
committed
Updated and to support for setting spark pool configuration artifact by
1 parent 537fc6a commit c885305

File tree

7 files changed

+96
-21
lines changed

7 files changed

+96
-21
lines changed

‎src/Synapse/Synapse/ChangeLog.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
-->
2020

2121
## Upcoming Release
22+
* Updated `New-AzSynapseSparkPool` and `Update-AzSynapseSparkPool` to support for setting spark pool configuration artifact by `-SparkCongifuration`, parameter `-SparkConfigFilePath` is deprecated now
2223

2324
## Version 2.1.0
2425
* Updated `Update-AzSynaspeWorkSpace` and `New-AzSynpaseWorkspace` to support for Workspace Encrytion Managed Identity setting

‎src/Synapse/Synapse/Commands/ManagementCommands/SparkPool/NewAzureSynapseSparkPool.cs

Lines changed: 17 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,6 @@
2121
using Microsoft.Azure.Management.Internal.Resources.Utilities.Models;
2222
using Microsoft.Azure.Management.Synapse.Models;
2323
using Microsoft.WindowsAzure.Commands.Utilities.Common;
24-
using System;
2524
using System.Collections;
2625
using System.IO;
2726
using System.Management.Automation;
@@ -131,6 +130,10 @@ public class NewAzureSynapseSparkPool : SynapseManagementCmdletBase
131130
[ValidateNotNullOrEmpty]
132131
public string SparkConfigFilePath { get; set; }
133132

133+
[Parameter(ValueFromPipelineByPropertyName = false, Mandatory = false,
134+
HelpMessage = HelpMessages.SparkConfigurationResource)]
135+
public PSSparkConfigurationResource SparkConfiguration { get; set; }
136+
134137
[Parameter(Mandatory = false, HelpMessage = HelpMessages.AsJob)]
135138
public SwitchParameter AsJob { get; set; }
136139

@@ -197,6 +200,19 @@ public override void ExecuteCmdlet()
197200
};
198201
}
199202

203+
if (this.IsParameterBound(c => c.SparkConfiguration))
204+
{
205+
if (this.SparkConfiguration != null)
206+
{
207+
sparkConfigProperties = new SparkConfigProperties()
208+
{
209+
Filename = SparkConfiguration.Name,
210+
ConfigurationType = ConfigurationType.Artifact,
211+
Content = Newtonsoft.Json.JsonConvert.SerializeObject(SparkConfiguration)
212+
};
213+
}
214+
}
215+
200216
var createParams = new BigDataPoolResourceInfo
201217
{
202218
Location = existingWorkspace.Location,

‎src/Synapse/Synapse/Commands/ManagementCommands/SparkPool/UpdateAzureSynapseSparkPool.cs

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -142,6 +142,10 @@ public class UpdateAzureSynapseSparkPool : SynapseManagementCmdletBase
142142
HelpMessage = HelpMessages.SparkConfigPropertiesFilePath)]
143143
public string SparkConfigFilePath { get; set; }
144144

145+
[Parameter(ValueFromPipelineByPropertyName = false, Mandatory = false,
146+
HelpMessage = HelpMessages.SparkConfigurationResource)]
147+
public PSSparkConfigurationResource SparkConfiguration { get; set; }
148+
145149
[Parameter(ValueFromPipelineByPropertyName = false, Mandatory = false,
146150
HelpMessage = HelpMessages.PackageAction)]
147151
public SynapseConstants.PackageActionType PackageAction { get; set; }
@@ -278,6 +282,21 @@ public override void ExecuteCmdlet()
278282
}
279283
}
280284

285+
if (this.IsParameterBound(c => c.SparkConfiguration))
286+
{
287+
if (this.SparkConfiguration != null)
288+
{
289+
existingSparkPool.SparkConfigProperties = new SparkConfigProperties()
290+
{
291+
Filename = SparkConfiguration.Name,
292+
ConfigurationType = ConfigurationType.Artifact,
293+
Content = Newtonsoft.Json.JsonConvert.SerializeObject(SparkConfiguration)
294+
};
295+
}
296+
else
297+
existingSparkPool.SparkConfigProperties = null;
298+
}
299+
281300
bool? isForceApplySetting = false;
282301
if (ForceApplySetting.IsPresent)
283302
{

‎src/Synapse/Synapse/Common/HelpMessages.cs

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -114,7 +114,9 @@ public static class HelpMessages
114114

115115
public const string LibraryRequirementsFilePath = "Environment configuration file (\"PIP freeze\" output).";
116116

117-
public const string SparkConfigPropertiesFilePath = "Spark pool properties configuration file.";
117+
public const string SparkConfigPropertiesFilePath = "[Deprecated] Spark pool properties configuration file. This parameter is deprecated, please use \"-SparkConfiguration\" instead.";
118+
119+
public const string SparkConfigurationResource = "Apache Spark configuration. When a job is submitted to the pool, the properties specified in the selected configuration will be referenced.";
118120

119121
public const string Batch = "Indicates Spark batch.";
120122

‎src/Synapse/Synapse/Models/DataPlaneModels/Artifact/SparkConfiguration/PSSparkConfigurationResource.cs

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
// ----------------------------------------------------------------------------------
1414

1515
using Azure.Analytics.Synapse.Artifacts.Models;
16+
using Newtonsoft.Json;
1617

1718
namespace Microsoft.Azure.Commands.Synapse.Models
1819
{
@@ -27,9 +28,10 @@ public PSSparkConfigurationResource(SparkConfigurationResource sparkConfiguratio
2728
this.WorkspaceName = workspaceName;
2829
this.Properties = sparkConfiguration?.Properties != null? new PSSparkConfiguration(sparkConfiguration.Properties) : null;
2930
}
30-
31+
3132
public string WorkspaceName { get; set; }
3233

34+
[JsonProperty(PropertyName = "properties")]
3335
public PSSparkConfiguration Properties { get; set; }
3436
}
3537
}

‎src/Synapse/Synapse/help/New-AzSynapseSparkPool.md

Lines changed: 29 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -17,35 +17,37 @@ Creates a Synapse Analytics Spark pool.
1717
New-AzSynapseSparkPool [-ResourceGroupName <String>] -WorkspaceName <String> -Name <String> [-Tag <Hashtable>]
1818
-NodeSize <String> -AutoScaleMinNodeCount <Int32> -AutoScaleMaxNodeCount <Int32> [-EnableAutoPause]
1919
[-AutoPauseDelayInMinute <Int32>] [-EnableDynamicExecutorAllocation] [-MinExecutorCount <Int32>]
20-
[-MaxExecutorCount <Int32>] -SparkVersion <String> [-SparkConfigFilePath <String>] [-AsJob]
21-
[-DefaultProfile <IAzureContextContainer>] [-WhatIf] [-Confirm] [<CommonParameters>]
20+
[-MaxExecutorCount <Int32>] -SparkVersion <String> [-SparkConfigFilePath <String>]
21+
[-SparkConfiguration <PSSparkConfigurationResource>] [-AsJob] [-DefaultProfile <IAzureContextContainer>]
22+
[-WhatIf] [-Confirm] [<CommonParameters>]
2223
```
2324

2425
### CreateByNameAndDisableAutoScaleParameterSet
2526
```
2627
New-AzSynapseSparkPool [-ResourceGroupName <String>] -WorkspaceName <String> -Name <String> [-Tag <Hashtable>]
2728
-NodeCount <Int32> -NodeSize <String> [-EnableAutoPause] [-AutoPauseDelayInMinute <Int32>]
2829
[-EnableDynamicExecutorAllocation] [-MinExecutorCount <Int32>] [-MaxExecutorCount <Int32>]
29-
-SparkVersion <String> [-SparkConfigFilePath <String>] [-AsJob] [-DefaultProfile <IAzureContextContainer>]
30-
[-WhatIf] [-Confirm] [<CommonParameters>]
30+
-SparkVersion <String> [-SparkConfigFilePath <String>] [-SparkConfiguration <PSSparkConfigurationResource>]
31+
[-AsJob] [-DefaultProfile <IAzureContextContainer>] [-WhatIf] [-Confirm] [<CommonParameters>]
3132
```
3233

3334
### CreateByParentObjectAndEnableAutoScaleParameterSet
3435
```
3536
New-AzSynapseSparkPool -WorkspaceObject <PSSynapseWorkspace> -Name <String> [-Tag <Hashtable>]
3637
-NodeSize <String> -AutoScaleMinNodeCount <Int32> -AutoScaleMaxNodeCount <Int32> [-EnableAutoPause]
3738
[-AutoPauseDelayInMinute <Int32>] [-EnableDynamicExecutorAllocation] [-MinExecutorCount <Int32>]
38-
[-MaxExecutorCount <Int32>] -SparkVersion <String> [-SparkConfigFilePath <String>] [-AsJob]
39-
[-DefaultProfile <IAzureContextContainer>] [-WhatIf] [-Confirm] [<CommonParameters>]
39+
[-MaxExecutorCount <Int32>] -SparkVersion <String> [-SparkConfigFilePath <String>]
40+
[-SparkConfiguration <PSSparkConfigurationResource>] [-AsJob] [-DefaultProfile <IAzureContextContainer>]
41+
[-WhatIf] [-Confirm] [<CommonParameters>]
4042
```
4143

4244
### CreateByParentObjectAndDisableAutoScaleParameterSet
4345
```
4446
New-AzSynapseSparkPool -WorkspaceObject <PSSynapseWorkspace> -Name <String> [-Tag <Hashtable>]
4547
-NodeCount <Int32> -NodeSize <String> [-EnableAutoPause] [-AutoPauseDelayInMinute <Int32>]
4648
[-EnableDynamicExecutorAllocation] [-MinExecutorCount <Int32>] [-MaxExecutorCount <Int32>]
47-
-SparkVersion <String> [-SparkConfigFilePath <String>] [-AsJob] [-DefaultProfile <IAzureContextContainer>]
48-
[-WhatIf] [-Confirm] [<CommonParameters>]
49+
-SparkVersion <String> [-SparkConfigFilePath <String>] [-SparkConfiguration <PSSparkConfigurationResource>]
50+
[-AsJob] [-DefaultProfile <IAzureContextContainer>] [-WhatIf] [-Confirm] [<CommonParameters>]
4951
```
5052

5153
## DESCRIPTION
@@ -76,10 +78,11 @@ This command creates an Azure Synapse Analytics Spark pool with dynamic executor
7678

7779
### Example 4
7880
```powershell
79-
New-AzSynapseSparkPool -WorkspaceName ContosoWorkspace -Name ContosoSparkPool -NodeCount 3 -SparkVersion 2.4 -NodeSize Small -SparkConfigFilePath "c:\sparkproperties.txt"
81+
$config = Get-AzSynapseSparkConfiguration -WorkspaceName ContosoWorkspace -Name ContosoSparkConfig1
82+
New-AzSynapseSparkPool -WorkspaceName ContosoWorkspace -Name ContosoSparkPool -NodeCount 3 -SparkVersion 2.4 -NodeSize Small -SparkConfiguration $config
8083
```
8184

82-
This command creates an Azure Synapse Analytics Spark pool and upload a spark configuration file.
85+
This command creates an Azure Synapse Analytics Spark pool and specify a Spark configuration for Spark pool.
8386

8487
### Example 5
8588
```powershell
@@ -299,7 +302,7 @@ Accept wildcard characters: False
299302
```
300303
301304
### -SparkConfigFilePath
302-
Spark pool properties configuration file.
305+
[Deprecated] Spark pool properties configuration file. This parameter is deprecated, please use "-SparkConfiguration" instead.
303306
304307
```yaml
305308
Type: System.String
@@ -313,6 +316,21 @@ Accept pipeline input: False
313316
Accept wildcard characters: False
314317
```
315318
319+
### -SparkConfiguration
320+
Apache Spark configuration. When a job is submitted to the pool, the properties specified in the selected configuration will be referenced.
321+
322+
```yaml
323+
Type: Microsoft.Azure.Commands.Synapse.Models.PSSparkConfigurationResource
324+
Parameter Sets: (All)
325+
Aliases:
326+
327+
Required: False
328+
Position: Named
329+
Default value: None
330+
Accept pipeline input: False
331+
Accept wildcard characters: False
332+
```
333+
316334
### -SparkVersion
317335
Apache Spark version.
318336
Allowed values: 2.4

‎src/Synapse/Synapse/help/Update-AzSynapseSparkPool.md

Lines changed: 24 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,8 @@ Update-AzSynapseSparkPool [-ResourceGroupName <String>] -WorkspaceName <String>
1919
[-AutoScaleMaxNodeCount <Int32>] [-EnableAutoPause <Boolean>] [-AutoPauseDelayInMinute <Int32>]
2020
[-NodeCount <Int32>] [-NodeSize <String>] [-EnableDynamicExecutorAllocation <Boolean>]
2121
[-MinExecutorCount <Int32>] [-MaxExecutorCount <Int32>] [-SparkVersion <String>]
22-
[-LibraryRequirementsFilePath <String>] [-SparkConfigFilePath <String>] [-PackageAction <PackageActionType>]
22+
[-LibraryRequirementsFilePath <String>] [-SparkConfigFilePath <String>]
23+
[-SparkConfiguration <PSSparkConfigurationResource>] [-PackageAction <PackageActionType>]
2324
[-Package <System.Collections.Generic.List`1[Microsoft.Azure.Commands.Synapse.Models.WorkspacePackages.PSSynapseWorkspacePackage]>]
2425
[-ForceApplySetting] [-AsJob] [-DefaultProfile <IAzureContextContainer>] [-WhatIf] [-Confirm]
2526
[<CommonParameters>]
@@ -32,7 +33,7 @@ Update-AzSynapseSparkPool -Name <String> -WorkspaceObject <PSSynapseWorkspace> [
3233
[-EnableAutoPause <Boolean>] [-AutoPauseDelayInMinute <Int32>] [-NodeCount <Int32>] [-NodeSize <String>]
3334
[-EnableDynamicExecutorAllocation <Boolean>] [-MinExecutorCount <Int32>] [-MaxExecutorCount <Int32>]
3435
[-SparkVersion <String>] [-LibraryRequirementsFilePath <String>] [-SparkConfigFilePath <String>]
35-
[-PackageAction <PackageActionType>]
36+
[-SparkConfiguration <PSSparkConfigurationResource>] [-PackageAction <PackageActionType>]
3637
[-Package <System.Collections.Generic.List`1[Microsoft.Azure.Commands.Synapse.Models.WorkspacePackages.PSSynapseWorkspacePackage]>]
3738
[-ForceApplySetting] [-AsJob] [-DefaultProfile <IAzureContextContainer>] [-WhatIf] [-Confirm]
3839
[<CommonParameters>]
@@ -45,7 +46,7 @@ Update-AzSynapseSparkPool -InputObject <PSSynapseSparkPool> [-Tag <Hashtable>] [
4546
[-AutoPauseDelayInMinute <Int32>] [-NodeCount <Int32>] [-NodeSize <String>]
4647
[-EnableDynamicExecutorAllocation <Boolean>] [-MinExecutorCount <Int32>] [-MaxExecutorCount <Int32>]
4748
[-SparkVersion <String>] [-LibraryRequirementsFilePath <String>] [-SparkConfigFilePath <String>]
48-
[-PackageAction <PackageActionType>]
49+
[-SparkConfiguration <PSSparkConfigurationResource>] [-PackageAction <PackageActionType>]
4950
[-Package <System.Collections.Generic.List`1[Microsoft.Azure.Commands.Synapse.Models.WorkspacePackages.PSSynapseWorkspacePackage]>]
5051
[-ForceApplySetting] [-AsJob] [-DefaultProfile <IAzureContextContainer>] [-WhatIf] [-Confirm]
5152
[<CommonParameters>]
@@ -58,7 +59,7 @@ Update-AzSynapseSparkPool -ResourceId <String> [-Tag <Hashtable>] [-EnableAutoSc
5859
[-AutoPauseDelayInMinute <Int32>] [-NodeCount <Int32>] [-NodeSize <String>]
5960
[-EnableDynamicExecutorAllocation <Boolean>] [-MinExecutorCount <Int32>] [-MaxExecutorCount <Int32>]
6061
[-SparkVersion <String>] [-LibraryRequirementsFilePath <String>] [-SparkConfigFilePath <String>]
61-
[-PackageAction <PackageActionType>]
62+
[-SparkConfiguration <PSSparkConfigurationResource>] [-PackageAction <PackageActionType>]
6263
[-Package <System.Collections.Generic.List`1[Microsoft.Azure.Commands.Synapse.Models.WorkspacePackages.PSSynapseWorkspacePackage]>]
6364
[-ForceApplySetting] [-AsJob] [-DefaultProfile <IAzureContextContainer>] [-WhatIf] [-Confirm]
6465
[<CommonParameters>]
@@ -167,10 +168,11 @@ The first command retrieves an Apache Spark pool in Azure Synapse Analytics. The
167168

168169
### Example 14
169170
```powershell
170-
Update-AzSynapseSparkPool -WorkspaceName ContosoWorkspace -Name ContosoSparkPool -Tag @{"key" = "value"} -NodeCount 5 -NodeSize Medium -SparkConfigFilePath "c:\sparkconfig.txt"
171+
$config = Get-AzSynapseSparkConfiguration -WorkspaceName ContosoWorkspace -Name ContosoSparkConfig1
172+
Update-AzSynapseSparkPool -WorkspaceName ContosoWorkspace -Name ContosoSparkPool -Tag @{"key" = "value"} -NodeCount 5 -NodeSize Medium -SparkConfiguration $configs
171173
```
172174

173-
This command updates an Apache Spark pool in Azure Synapse Analytics and upload a spark configuration file for the spark pool.
175+
This command updates an Apache Spark pool in Azure Synapse Analytics and specify a Spark configuration for the Spark pool.
174176

175177
### Example 15
176178
```powershell
@@ -495,7 +497,7 @@ Accept wildcard characters: False
495497
```
496498
497499
### -SparkConfigFilePath
498-
Spark pool properties configuration file.
500+
[Deprecated] Spark pool properties configuration file. This parameter is deprecated, please use "-SparkConfiguration" instead.
499501
500502
```yaml
501503
Type: System.String
@@ -509,6 +511,21 @@ Accept pipeline input: False
509511
Accept wildcard characters: False
510512
```
511513
514+
### -SparkConfiguration
515+
Apache Spark configuration. When a job is submitted to the pool, the properties specified in the selected configuration will be referenced.
516+
517+
```yaml
518+
Type: Microsoft.Azure.Commands.Synapse.Models.PSSparkConfigurationResource
519+
Parameter Sets: (All)
520+
Aliases:
521+
522+
Required: False
523+
Position: Named
524+
Default value: None
525+
Accept pipeline input: False
526+
Accept wildcard characters: False
527+
```
528+
512529
### -SparkVersion
513530
Apache Spark version.
514531
Allowed values: 2.3,2.4,3.1,3.2

0 commit comments

Comments
 (0)