Skip to content

Commit 26d8c9a

Browse files
yanjungao718yanjungao718
authored
[Synapse] Spark Configuration cmdlets (Azure#16156)
* [Synapse] Spark Configuration cmdlets modify signature issue info modify SignatureIssues.csv modify help doc add change log for spark configuration add help doc for spark configuration remove unnecessary parameter set add/remove blank lines revert local test endpoint add export command, made some ohter changes add export command & refine code revert PSActivityRun edited PSActivityRun by mistake, revert it use lowercase for parameter initials refine code for spark configuration spark configuration related command * remove unnecessary release note
1 parent ec24657 commit 26d8c9a

19 files changed

+1276
-3
lines changed

‎src/Synapse/Synapse/Az.Synapse.psd1

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -220,7 +220,11 @@ CmdletsToExport = 'Get-AzSynapseSparkJob', 'Stop-AzSynapseSparkJob',
220220
'Stop-AzSynapseDataFlowDebugSession',
221221
'Start-AzSynapseDataFlowDebugSession', 'Get-AzSynapseSqlScript',
222222
'Remove-AzSynapseSqlScript', 'Export-AzSynapseSqlScript',
223-
'Set-AzSynapseSqlScript'
223+
'Set-AzSynapseSqlScript',
224+
'Get-AzSynapseSparkConfiguration',
225+
'New-AzSynapseSparkConfiguration',
226+
'Export-AzSynapseSparkConfiguration',
227+
'Remove-AzSynapseSparkConfiguration'
224228

225229
# Variables to export from this module
226230
# VariablesToExport = @()
@@ -241,7 +245,9 @@ AliasesToExport = 'New-AzSynapsePipeline', 'New-AzSynapseLinkedService',
241245
'Disable-AzSynapseSqlAdvancedThreatProtection',
242246
'New-AzSynapseSparkJobDefinition',
243247
'Set-AzSynapseManagedPrivateEndpoint', 'New-AzSynapseSqlScript',
244-
'Import-AzSynapseSqlScript'
248+
'Import-AzSynapseSqlScript',
249+
'Set-AzSynapseSparkConfiguration',
250+
'Import-AzSynapseSparkConfiguration'
245251

246252
# DSC resources to export from this module
247253
# DscResourcesToExport = @()

‎src/Synapse/Synapse/ChangeLog.md

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,11 @@
2121
## Upcoming Release
2222
* Renamed parameter FolderName in `Set-AzSynapseSqlScript` to FolderPath and keeped FolderName as alias
2323
* Updated `Set-AzSynapseNoteBook` and `Set-AzSynapseSparkJobDefinition` to support new parameter [-FolderPath]
24+
* Added cmdlets for Synapse Spark Configuration
25+
- Added `Get-AzSynapseSparkConfiguration` cmdlet
26+
- Added `New-AzSynapseSparkConfiguration` cmdlet
27+
- Added `Export-AzSynapseSparkConfiguration` cmdlet
28+
- Added `Remove-AzSynapseSparkConfiguration` cmdlet
2429

2530
## Version 0.18.0
2631
* Added cmdlets for Synapse Kusto pool

‎src/Synapse/Synapse/Commands/DataPlaneCommands/Artifact/SparkConfiguration/ExportAzureSynapseSparkConfiguration.cs

Lines changed: 109 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,109 @@
1+
// ----------------------------------------------------------------------------------
2+
//
3+
// Copyright Microsoft Corporation
4+
// Licensed under the Apache License, Version 2.0 (the "License");
5+
// you may not use this file except in compliance with the License.
6+
// You may obtain a copy of the License at
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
// Unless required by applicable law or agreed to in writing, software
9+
// distributed under the License is distributed on an "AS IS" BASIS,
10+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11+
// See the License for the specific language governing permissions and
12+
// limitations under the License.
13+
// ----------------------------------------------------------------------------------
14+
15+
using Microsoft.Azure.Commands.ResourceManager.Common.ArgumentCompleters;
16+
using Microsoft.Azure.Commands.Synapse.Common;
17+
using Microsoft.Azure.Commands.Synapse.Models;
18+
using Microsoft.WindowsAzure.Commands.Utilities.Common;
19+
using Newtonsoft.Json;
20+
using System.Collections.Generic;
21+
using System.IO;
22+
using System.Linq;
23+
using System.Management.Automation;
24+
25+
namespace Microsoft.Azure.Commands.Synapse
26+
{
27+
[Cmdlet(VerbsData.Export, ResourceManager.Common.AzureRMConstants.AzureRMPrefix + SynapseConstants.SynapsePrefix + SynapseConstants.SparkConfiguration,
28+
DefaultParameterSetName = ExportByName)]
29+
[OutputType(typeof(FileInfo))]
30+
public class ExportAzureSynapseSparkConfiguration : SynapseArtifactsCmdletBase
31+
{
32+
private const string ExportByName = "ExportByName";
33+
private const string ExportByObject = "ExportByObject";
34+
private const string ExportByInputObject = "ExportByInputObject";
35+
36+
[Parameter(ValueFromPipelineByPropertyName = false, ParameterSetName = ExportByName,
37+
Mandatory = true, HelpMessage = HelpMessages.WorkspaceName)]
38+
[ResourceNameCompleter(ResourceTypes.Workspace, "ResourceGroupName")]
39+
[ValidateNotNullOrEmpty]
40+
public override string WorkspaceName { get; set; }
41+
42+
[Parameter(ValueFromPipeline = true, ParameterSetName = ExportByObject,
43+
Mandatory = true, HelpMessage = HelpMessages.WorkspaceObject)]
44+
[ValidateNotNull]
45+
public PSSynapseWorkspace WorkspaceObject { get; set; }
46+
47+
[Parameter(ValueFromPipelineByPropertyName = false, Mandatory = false, ParameterSetName = ExportByName,
48+
HelpMessage = HelpMessages.SparkConfigurationName)]
49+
[Parameter(ValueFromPipelineByPropertyName = false, Mandatory = false, ParameterSetName = ExportByObject,
50+
HelpMessage = HelpMessages.SparkConfigurationName)]
51+
[ValidateNotNullOrEmpty]
52+
[Alias("SparkConfigurationName")]
53+
public string Name { get; set; }
54+
55+
[Parameter(ValueFromPipeline = true, ParameterSetName = ExportByInputObject,
56+
Mandatory = true, HelpMessage = HelpMessages.SparkConfigurationObject)]
57+
[ValidateNotNull]
58+
public PSSparkConfigurationResource InputObject { get; set; }
59+
60+
[Parameter(ValueFromPipelineByPropertyName = false, Mandatory = true, HelpMessage = HelpMessages.OutputFolder)]
61+
[ValidateNotNullOrEmpty]
62+
public string OutputFolder { get; set; }
63+
64+
[Parameter(Mandatory = false, HelpMessage = HelpMessages.AsJob)]
65+
public SwitchParameter AsJob { get; set; }
66+
67+
public override void ExecuteCmdlet()
68+
{
69+
if (this.IsParameterBound(c => c.WorkspaceObject))
70+
{
71+
this.WorkspaceName = this.WorkspaceObject.Name;
72+
}
73+
74+
var fileExtension = ".json";
75+
if (this.IsParameterBound(c => c.InputObject))
76+
{
77+
WriteToFile(this.InputObject);
78+
WriteObject(new FileInfo(Path.Combine(this.OutputFolder, this.InputObject.Name + fileExtension)));
79+
}
80+
else
81+
{
82+
if (this.IsParameterBound(c => c.Name))
83+
{
84+
var sparkConfiguration = new PSSparkConfigurationResource(SynapseAnalyticsClient.GetSparkConfiguration(this.Name), this.WorkspaceName);
85+
WriteToFile(sparkConfiguration);
86+
WriteObject(new FileInfo(Path.Combine(this.OutputFolder, sparkConfiguration.Name + fileExtension)));
87+
}
88+
else
89+
{
90+
var infoList = new List<FileInfo>();
91+
var sparkConfigurations = SynapseAnalyticsClient.GetSparkConfigurationByWorkspace()
92+
.Select(element => new PSSparkConfigurationResource(element, this.WorkspaceName));
93+
foreach (var sparkConfiguration in sparkConfigurations)
94+
{
95+
WriteToFile(sparkConfiguration);
96+
infoList.Add(new FileInfo(Path.Combine(this.OutputFolder, sparkConfiguration.Name + fileExtension)));
97+
}
98+
WriteObject(infoList, true);
99+
}
100+
}
101+
}
102+
103+
private void WriteToFile(PSSparkConfigurationResource sparkConfigurationResource)
104+
{
105+
string json = Newtonsoft.Json.JsonConvert.SerializeObject(sparkConfigurationResource.Properties, Formatting.Indented);
106+
File.WriteAllText(Path.Combine(this.OutputFolder, sparkConfigurationResource.Name + ".json"), json);
107+
}
108+
}
109+
}

‎src/Synapse/Synapse/Commands/DataPlaneCommands/Artifact/SparkConfiguration/GetAzureSynapseSparkConfiguration.cs

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
// ----------------------------------------------------------------------------------
2+
//
3+
// Copyright Microsoft Corporation
4+
// Licensed under the Apache License, Version 2.0 (the "License");
5+
// you may not use this file except in compliance with the License.
6+
// You may obtain a copy of the License at
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
// Unless required by applicable law or agreed to in writing, software
9+
// distributed under the License is distributed on an "AS IS" BASIS,
10+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11+
// See the License for the specific language governing permissions and
12+
// limitations under the License.
13+
// ----------------------------------------------------------------------------------
14+
15+
using Microsoft.Azure.Commands.ResourceManager.Common.ArgumentCompleters;
16+
using Microsoft.Azure.Commands.Synapse.Common;
17+
using Microsoft.Azure.Commands.Synapse.Models;
18+
using Microsoft.WindowsAzure.Commands.Utilities.Common;
19+
using System.Linq;
20+
using System.Management.Automation;
21+
22+
namespace Microsoft.Azure.Commands.Synapse
23+
{
24+
[Cmdlet(VerbsCommon.Get, ResourceManager.Common.AzureRMConstants.AzureRMPrefix + SynapseConstants.SynapsePrefix + SynapseConstants.SparkConfiguration,
25+
DefaultParameterSetName = GetByName)]
26+
[OutputType(typeof(PSSparkConfigurationResource))]
27+
public class GetAzureSynapseSparkConfiguration : SynapseArtifactsCmdletBase
28+
{
29+
private const string GetByName = "GetByName";
30+
private const string GetByObject = "GetByObject";
31+
32+
[Parameter(ValueFromPipelineByPropertyName = false, ParameterSetName = GetByName,
33+
Mandatory = true, HelpMessage = HelpMessages.WorkspaceName)]
34+
[ResourceNameCompleter(ResourceTypes.Workspace, "ResourceGroupName")]
35+
[ValidateNotNullOrEmpty]
36+
public override string WorkspaceName { get; set; }
37+
38+
[Parameter(ValueFromPipeline = true, ParameterSetName = GetByObject,
39+
Mandatory = true, HelpMessage = HelpMessages.WorkspaceObject)]
40+
[ValidateNotNull]
41+
public PSSynapseWorkspace WorkspaceObject { get; set; }
42+
43+
[Parameter(ValueFromPipelineByPropertyName = false, Mandatory = false, HelpMessage = HelpMessages.SparkConfigurationName)]
44+
[ValidateNotNullOrEmpty]
45+
[Alias("SparkConfigurationName")]
46+
public string Name { get; set; }
47+
48+
public override void ExecuteCmdlet()
49+
{
50+
if (this.IsParameterBound(c => c.WorkspaceObject))
51+
{
52+
this.WorkspaceName = this.WorkspaceObject.Name;
53+
}
54+
55+
if (this.IsParameterBound(c => c.Name))
56+
{
57+
WriteObject(new PSSparkConfigurationResource(SynapseAnalyticsClient.GetSparkConfiguration(this.Name), this.WorkspaceName));
58+
}
59+
else
60+
{
61+
var sparkConfigurations = SynapseAnalyticsClient.GetSparkConfigurationByWorkspace()
62+
.Select(element => new PSSparkConfigurationResource(element, this.WorkspaceName));
63+
WriteObject(sparkConfigurations, true);
64+
}
65+
}
66+
}
67+
}

‎src/Synapse/Synapse/Commands/DataPlaneCommands/Artifact/SparkConfiguration/NewAzureSynapseSparkConfiguration.cs

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
// ----------------------------------------------------------------------------------
2+
//
3+
// Copyright Microsoft Corporation
4+
// Licensed under the Apache License, Version 2.0 (the "License");
5+
// you may not use this file except in compliance with the License.
6+
// You may obtain a copy of the License at
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
// Unless required by applicable law or agreed to in writing, software
9+
// distributed under the License is distributed on an "AS IS" BASIS,
10+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11+
// See the License for the specific language governing permissions and
12+
// limitations under the License.
13+
// ----------------------------------------------------------------------------------
14+
15+
using Microsoft.Azure.Commands.ResourceManager.Common.ArgumentCompleters;
16+
using Microsoft.Azure.Commands.Synapse.Common;
17+
using Microsoft.Azure.Commands.Synapse.Models;
18+
using Microsoft.Azure.Commands.Synapse.Properties;
19+
using Microsoft.WindowsAzure.Commands.Utilities.Common;
20+
using System;
21+
using System.IO;
22+
using System.Management.Automation;
23+
24+
namespace Microsoft.Azure.Commands.Synapse
25+
{
26+
[Cmdlet(VerbsCommon.New, ResourceManager.Common.AzureRMConstants.AzureRMPrefix + SynapseConstants.SynapsePrefix + SynapseConstants.SparkConfiguration,
27+
DefaultParameterSetName = CreateByName, SupportsShouldProcess = true)]
28+
[Alias("Set-" + ResourceManager.Common.AzureRMConstants.AzureRMPrefix + SynapseConstants.SynapsePrefix + SynapseConstants.SparkConfiguration,
29+
"Import-" + ResourceManager.Common.AzureRMConstants.AzureRMPrefix + SynapseConstants.SynapsePrefix + SynapseConstants.SparkConfiguration)]
30+
[OutputType(typeof(PSSparkConfigurationResource))]
31+
public class NewAzureSynapseSparkConfiguration : SynapseArtifactsCmdletBase
32+
{
33+
private const string CreateByName = "CreateByName";
34+
private const string CreateByObject = "CreateByObject";
35+
36+
[Parameter(ValueFromPipelineByPropertyName = false, ParameterSetName = CreateByName,
37+
Mandatory = true, HelpMessage = HelpMessages.WorkspaceName)]
38+
[ResourceNameCompleter(ResourceTypes.Workspace, "ResourceGroupName")]
39+
[ValidateNotNullOrEmpty]
40+
public override string WorkspaceName { get; set; }
41+
42+
[Parameter(ValueFromPipeline = true, ParameterSetName = CreateByObject,
43+
Mandatory = true, HelpMessage = HelpMessages.WorkspaceObject)]
44+
[ValidateNotNull]
45+
public PSSynapseWorkspace WorkspaceObject { get; set; }
46+
47+
[Parameter(ValueFromPipelineByPropertyName = false, Mandatory = false, HelpMessage = HelpMessages.SparkConfigurationName)]
48+
[ValidateNotNullOrEmpty]
49+
[Alias("SparkConfigurationName")]
50+
public string Name { get; set; }
51+
52+
[Parameter(ValueFromPipelineByPropertyName = false, Mandatory = true, HelpMessage = HelpMessages.JsonFilePath)]
53+
[ValidateNotNullOrEmpty]
54+
[Alias("File")]
55+
public string DefinitionFile { get; set; }
56+
57+
[Parameter(Mandatory = false, HelpMessage = HelpMessages.AsJob)]
58+
public SwitchParameter AsJob { get; set; }
59+
60+
public override void ExecuteCmdlet()
61+
{
62+
if (this.IsParameterBound(c => c.WorkspaceObject))
63+
{
64+
this.WorkspaceName = this.WorkspaceObject.Name;
65+
}
66+
67+
if(!this.IsParameterBound(c => c.Name))
68+
{
69+
string path = this.TryResolvePath(DefinitionFile);
70+
this.Name = Path.GetFileNameWithoutExtension(path);
71+
}
72+
73+
if (this.ShouldProcess(this.WorkspaceName, String.Format(Resources.SettingSynapseSparkConfiguration, this.Name, this.WorkspaceName)))
74+
{
75+
string rawJsonContent = SynapseAnalyticsClient.ReadJsonFileContent(this.TryResolvePath(DefinitionFile));
76+
WriteObject(new PSSparkConfigurationResource(SynapseAnalyticsClient.CreateOrUpdateSparkConfiguration(this.Name, rawJsonContent),this.WorkspaceName));
77+
}
78+
}
79+
}
80+
}

‎src/Synapse/Synapse/Commands/DataPlaneCommands/Artifact/SparkConfiguration/RemoveAzureSynapseSparkConfiguration.cs

Lines changed: 92 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,92 @@
1+
// ----------------------------------------------------------------------------------
2+
//
3+
// Copyright Microsoft Corporation
4+
// Licensed under the Apache License, Version 2.0 (the "License");
5+
// you may not use this file except in compliance with the License.
6+
// You may obtain a copy of the License at
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
// Unless required by applicable law or agreed to in writing, software
9+
// distributed under the License is distributed on an "AS IS" BASIS,
10+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11+
// See the License for the specific language governing permissions and
12+
// limitations under the License.
13+
// ----------------------------------------------------------------------------------
14+
15+
using Microsoft.Azure.Commands.ResourceManager.Common.ArgumentCompleters;
16+
using Microsoft.Azure.Commands.Synapse.Common;
17+
using Microsoft.Azure.Commands.Synapse.Models;
18+
using Microsoft.Azure.Commands.Synapse.Properties;
19+
using Microsoft.WindowsAzure.Commands.Utilities.Common;
20+
using System.Management.Automation;
21+
22+
namespace Microsoft.Azure.Commands.Synapse
23+
{
24+
[Cmdlet(VerbsCommon.Remove, ResourceManager.Common.AzureRMConstants.AzureRMPrefix + SynapseConstants.SynapsePrefix + SynapseConstants.SparkConfiguration,
25+
DefaultParameterSetName = RemoveByName, SupportsShouldProcess = true)]
26+
[OutputType(typeof(bool))]
27+
public class RemoveAzureSynapseSparkConfiguration : SynapseArtifactsCmdletBase
28+
{
29+
private const string RemoveByName = "RemoveByName";
30+
private const string RemoveByObject = "RemoveByObject";
31+
private const string RemoveByInputObject = "RemoveByInputObject";
32+
33+
[Parameter(ValueFromPipelineByPropertyName = false, ParameterSetName = RemoveByName,
34+
Mandatory = true, HelpMessage = HelpMessages.WorkspaceName)]
35+
[ResourceNameCompleter(ResourceTypes.Workspace, "ResourceGroupName")]
36+
[ValidateNotNullOrEmpty]
37+
public override string WorkspaceName { get; set; }
38+
39+
[Parameter(ValueFromPipeline = true, ParameterSetName = RemoveByObject,
40+
Mandatory = true, HelpMessage = HelpMessages.WorkspaceObject)]
41+
[ValidateNotNull]
42+
public PSSynapseWorkspace WorkspaceObject { get; set; }
43+
44+
[Parameter(ValueFromPipelineByPropertyName = false, ParameterSetName = RemoveByName, Mandatory = true, HelpMessage = HelpMessages.SparkConfigurationName)]
45+
[Parameter(ValueFromPipelineByPropertyName = false, ParameterSetName = RemoveByObject, Mandatory = true, HelpMessage = HelpMessages.SparkConfigurationName)]
46+
[ValidateNotNullOrEmpty]
47+
[Alias("SparkConfigurationName")]
48+
public string Name { get; set; }
49+
50+
[Parameter(ValueFromPipeline = true, ParameterSetName = RemoveByInputObject,
51+
Mandatory = true, HelpMessage = HelpMessages.SparkConfigurationObject)]
52+
[ValidateNotNull]
53+
public PSSparkConfigurationResource InputObject { get; set; }
54+
55+
[Parameter(Mandatory = false, HelpMessage = HelpMessages.PassThru)]
56+
public SwitchParameter PassThru { get; set; }
57+
58+
[Parameter(Mandatory = false, HelpMessage = HelpMessages.AsJob)]
59+
public SwitchParameter AsJob { get; set; }
60+
61+
[Parameter(Mandatory = false, HelpMessage = HelpMessages.Force)]
62+
public SwitchParameter Force { get; set; }
63+
64+
public override void ExecuteCmdlet()
65+
{
66+
if (this.IsParameterBound(c => c.WorkspaceObject))
67+
{
68+
this.WorkspaceName = this.WorkspaceObject.Name;
69+
}
70+
71+
if (this.IsParameterBound(c => c.InputObject))
72+
{
73+
this.WorkspaceName = this.InputObject.WorkspaceName;
74+
this.Name = this.InputObject.Name;
75+
}
76+
77+
ConfirmAction(
78+
Force.IsPresent,
79+
string.Format(Resources.RemoveSynapseSparkConfiguration, this.Name),
80+
string.Format(Resources.RemovingSynapseSparkConfiguration, this.Name, this.WorkspaceName),
81+
Name,
82+
() =>
83+
{
84+
SynapseAnalyticsClient.DeleteSparkConfiguration(this.Name);
85+
if (PassThru)
86+
{
87+
WriteObject(true);
88+
}
89+
});
90+
}
91+
}
92+
}

‎src/Synapse/Synapse/Common/HelpMessages.cs

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -544,5 +544,9 @@ SELECT on dbo.myTable by public
544544
public const string SparkConfigurationFolderPath = "The folder that this Spark job definition is in. If specify a multi-level path such as [rootFolder/subFolder], the Spark job definition will appear at the bottom level. If not specified, this Spark job definition will appear at the root level.";
545545

546546
public const string NoteBookFolderPath = "The folder that this notebook is in. If specify a multi-level path such as [rootFolder/subFolder], the notebook will appear at the bottom level. If not specified, this notebook will appear at the root level.";
547+
548+
public const string SparkConfigurationName = "The Spark Configuration name.";
549+
550+
public const string SparkConfigurationObject = "The Spark configuration object.";
547551
}
548552
}

0 commit comments

Comments
 (0)