Skip to content

Commit e47633d

Browse files
yanjungao718yanjungao718
committed
[Synapse] Spark Configuration cmdlets
modify signature issue info modify SignatureIssues.csv modify help doc add change log for spark configuration add help doc for spark configuration remove unnecessary parameter set add/remove blank lines revert local test endpoint add export command, made some ohter changes add export command & refine code revert PSActivityRun edited PSActivityRun by mistake, revert it use lowercase for parameter initials refine code for spark configuration spark configuration related command
1 parent ec24657 commit e47633d

19 files changed

+1282
-4
lines changed

‎src/Synapse/Synapse/Az.Synapse.psd1

Lines changed: 14 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -220,7 +220,11 @@ CmdletsToExport = 'Get-AzSynapseSparkJob', 'Stop-AzSynapseSparkJob',
220220
'Stop-AzSynapseDataFlowDebugSession',
221221
'Start-AzSynapseDataFlowDebugSession', 'Get-AzSynapseSqlScript',
222222
'Remove-AzSynapseSqlScript', 'Export-AzSynapseSqlScript',
223-
'Set-AzSynapseSqlScript'
223+
'Set-AzSynapseSqlScript',
224+
'Get-AzSynapseSparkConfiguration',
225+
'New-AzSynapseSparkConfiguration',
226+
'Export-AzSynapseSparkConfiguration',
227+
'Remove-AzSynapseSparkConfiguration'
224228

225229
# Variables to export from this module
226230
# VariablesToExport = @()
@@ -241,7 +245,9 @@ AliasesToExport = 'New-AzSynapsePipeline', 'New-AzSynapseLinkedService',
241245
'Disable-AzSynapseSqlAdvancedThreatProtection',
242246
'New-AzSynapseSparkJobDefinition',
243247
'Set-AzSynapseManagedPrivateEndpoint', 'New-AzSynapseSqlScript',
244-
'Import-AzSynapseSqlScript'
248+
'Import-AzSynapseSqlScript',
249+
'Set-AzSynapseSparkConfiguration',
250+
'Import-AzSynapseSparkConfiguration'
245251

246252
# DSC resources to export from this module
247253
# DscResourcesToExport = @()
@@ -298,7 +304,12 @@ PrivateData = @{
298304
- Added ''Get-AzSynapseSqlScript'' cmdlet
299305
- Added ''Remove-AzSynapseSqlScript'' cmdlet
300306
- Added ''Export-AzSynapseSqlScript'' cmdlet
301-
- Added ''Set-AzSynapseSqlScript'' cmdlet'
307+
- Added ''Set-AzSynapseSqlScript'' cmdlet
308+
* Added support for Synapse Spark Configuration
309+
- Added ''Get-AzSynapseSparkConfiguration'' cmdlet
310+
- Added ''New-AzSynapseSparkConfiguration'' cmdlet
311+
- Added ''Export-AzSynapseSparkConfiguration'' cmdlet
312+
- Added ''Remove-AzSynapseSparkConfiguration'' cmdlet'
302313

303314
# Prerelease string of this module
304315
# Prerelease = ''

‎src/Synapse/Synapse/ChangeLog.md

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,11 @@
2121
## Upcoming Release
2222
* Renamed parameter FolderName in `Set-AzSynapseSqlScript` to FolderPath and keeped FolderName as alias
2323
* Updated `Set-AzSynapseNoteBook` and `Set-AzSynapseSparkJobDefinition` to support new parameter [-FolderPath]
24+
* Added cmdlets for Synapse Spark Configuration
25+
- Added `Get-AzSynapseSparkConfiguration` cmdlet
26+
- Added `New-AzSynapseSparkConfiguration` cmdlet
27+
- Added `Export-AzSynapseSparkConfiguration` cmdlet
28+
- Added `Remove-AzSynapseSparkConfiguration` cmdlet
2429

2530
## Version 0.18.0
2631
* Added cmdlets for Synapse Kusto pool

‎src/Synapse/Synapse/Commands/DataPlaneCommands/Artifact/SparkConfiguration/ExportAzureSynapseSparkConfiguration.cs

Lines changed: 109 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,109 @@
1+
// ----------------------------------------------------------------------------------
2+
//
3+
// Copyright Microsoft Corporation
4+
// Licensed under the Apache License, Version 2.0 (the "License");
5+
// you may not use this file except in compliance with the License.
6+
// You may obtain a copy of the License at
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
// Unless required by applicable law or agreed to in writing, software
9+
// distributed under the License is distributed on an "AS IS" BASIS,
10+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11+
// See the License for the specific language governing permissions and
12+
// limitations under the License.
13+
// ----------------------------------------------------------------------------------
14+
15+
using Microsoft.Azure.Commands.ResourceManager.Common.ArgumentCompleters;
16+
using Microsoft.Azure.Commands.Synapse.Common;
17+
using Microsoft.Azure.Commands.Synapse.Models;
18+
using Microsoft.WindowsAzure.Commands.Utilities.Common;
19+
using Newtonsoft.Json;
20+
using System.Collections.Generic;
21+
using System.IO;
22+
using System.Linq;
23+
using System.Management.Automation;
24+
25+
namespace Microsoft.Azure.Commands.Synapse
26+
{
27+
[Cmdlet(VerbsData.Export, ResourceManager.Common.AzureRMConstants.AzureRMPrefix + SynapseConstants.SynapsePrefix + SynapseConstants.SparkConfiguration,
28+
DefaultParameterSetName = ExportByName)]
29+
[OutputType(typeof(FileInfo))]
30+
public class ExportAzureSynapseSparkConfiguration : SynapseArtifactsCmdletBase
31+
{
32+
private const string ExportByName = "ExportByName";
33+
private const string ExportByObject = "ExportByObject";
34+
private const string ExportByInputObject = "ExportByInputObject";
35+
36+
[Parameter(ValueFromPipelineByPropertyName = false, ParameterSetName = ExportByName,
37+
Mandatory = true, HelpMessage = HelpMessages.WorkspaceName)]
38+
[ResourceNameCompleter(ResourceTypes.Workspace, "ResourceGroupName")]
39+
[ValidateNotNullOrEmpty]
40+
public override string WorkspaceName { get; set; }
41+
42+
[Parameter(ValueFromPipeline = true, ParameterSetName = ExportByObject,
43+
Mandatory = true, HelpMessage = HelpMessages.WorkspaceObject)]
44+
[ValidateNotNull]
45+
public PSSynapseWorkspace WorkspaceObject { get; set; }
46+
47+
[Parameter(ValueFromPipelineByPropertyName = false, Mandatory = false, ParameterSetName = ExportByName,
48+
HelpMessage = HelpMessages.SparkConfigurationName)]
49+
[Parameter(ValueFromPipelineByPropertyName = false, Mandatory = false, ParameterSetName = ExportByObject,
50+
HelpMessage = HelpMessages.SparkConfigurationName)]
51+
[ValidateNotNullOrEmpty]
52+
[Alias("SparkConfigurationName")]
53+
public string Name { get; set; }
54+
55+
[Parameter(ValueFromPipeline = true, ParameterSetName = ExportByInputObject,
56+
Mandatory = true, HelpMessage = HelpMessages.SparkConfigurationObject)]
57+
[ValidateNotNull]
58+
public PSSparkConfigurationResource InputObject { get; set; }
59+
60+
[Parameter(ValueFromPipelineByPropertyName = false, Mandatory = true, HelpMessage = HelpMessages.OutputFolder)]
61+
[ValidateNotNullOrEmpty]
62+
public string OutputFolder { get; set; }
63+
64+
[Parameter(Mandatory = false, HelpMessage = HelpMessages.AsJob)]
65+
public SwitchParameter AsJob { get; set; }
66+
67+
public override void ExecuteCmdlet()
68+
{
69+
if (this.IsParameterBound(c => c.WorkspaceObject))
70+
{
71+
this.WorkspaceName = this.WorkspaceObject.Name;
72+
}
73+
74+
var fileExtension = ".json";
75+
if (this.IsParameterBound(c => c.InputObject))
76+
{
77+
WriteToFile(this.InputObject);
78+
WriteObject(new FileInfo(Path.Combine(this.OutputFolder, this.InputObject.Name + fileExtension)));
79+
}
80+
else
81+
{
82+
if (this.IsParameterBound(c => c.Name))
83+
{
84+
var sparkConfiguration = new PSSparkConfigurationResource(SynapseAnalyticsClient.GetSparkConfiguration(this.Name), this.WorkspaceName);
85+
WriteToFile(sparkConfiguration);
86+
WriteObject(new FileInfo(Path.Combine(this.OutputFolder, sparkConfiguration.Name + fileExtension)));
87+
}
88+
else
89+
{
90+
var infoList = new List<FileInfo>();
91+
var sparkConfigurations = SynapseAnalyticsClient.GetSparkConfigurationByWorkspace()
92+
.Select(element => new PSSparkConfigurationResource(element, this.WorkspaceName));
93+
foreach (var sparkConfiguration in sparkConfigurations)
94+
{
95+
WriteToFile(sparkConfiguration);
96+
infoList.Add(new FileInfo(Path.Combine(this.OutputFolder, sparkConfiguration.Name + fileExtension)));
97+
}
98+
WriteObject(infoList, true);
99+
}
100+
}
101+
}
102+
103+
private void WriteToFile(PSSparkConfigurationResource sparkConfigurationResource)
104+
{
105+
string json = Newtonsoft.Json.JsonConvert.SerializeObject(sparkConfigurationResource.Properties, Formatting.Indented);
106+
File.WriteAllText(Path.Combine(this.OutputFolder, sparkConfigurationResource.Name + ".json"), json);
107+
}
108+
}
109+
}

‎src/Synapse/Synapse/Commands/DataPlaneCommands/Artifact/SparkConfiguration/GetAzureSynapseSparkConfiguration.cs

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
// ----------------------------------------------------------------------------------
2+
//
3+
// Copyright Microsoft Corporation
4+
// Licensed under the Apache License, Version 2.0 (the "License");
5+
// you may not use this file except in compliance with the License.
6+
// You may obtain a copy of the License at
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
// Unless required by applicable law or agreed to in writing, software
9+
// distributed under the License is distributed on an "AS IS" BASIS,
10+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11+
// See the License for the specific language governing permissions and
12+
// limitations under the License.
13+
// ----------------------------------------------------------------------------------
14+
15+
using Microsoft.Azure.Commands.ResourceManager.Common.ArgumentCompleters;
16+
using Microsoft.Azure.Commands.Synapse.Common;
17+
using Microsoft.Azure.Commands.Synapse.Models;
18+
using Microsoft.WindowsAzure.Commands.Utilities.Common;
19+
using System.Linq;
20+
using System.Management.Automation;
21+
22+
namespace Microsoft.Azure.Commands.Synapse
23+
{
24+
[Cmdlet(VerbsCommon.Get, ResourceManager.Common.AzureRMConstants.AzureRMPrefix + SynapseConstants.SynapsePrefix + SynapseConstants.SparkConfiguration,
25+
DefaultParameterSetName = GetByName)]
26+
[OutputType(typeof(PSSparkConfigurationResource))]
27+
public class GetAzureSynapseSparkConfiguration : SynapseArtifactsCmdletBase
28+
{
29+
private const string GetByName = "GetByName";
30+
private const string GetByObject = "GetByObject";
31+
32+
[Parameter(ValueFromPipelineByPropertyName = false, ParameterSetName = GetByName,
33+
Mandatory = true, HelpMessage = HelpMessages.WorkspaceName)]
34+
[ResourceNameCompleter(ResourceTypes.Workspace, "ResourceGroupName")]
35+
[ValidateNotNullOrEmpty]
36+
public override string WorkspaceName { get; set; }
37+
38+
[Parameter(ValueFromPipeline = true, ParameterSetName = GetByObject,
39+
Mandatory = true, HelpMessage = HelpMessages.WorkspaceObject)]
40+
[ValidateNotNull]
41+
public PSSynapseWorkspace WorkspaceObject { get; set; }
42+
43+
[Parameter(ValueFromPipelineByPropertyName = false, Mandatory = false, HelpMessage = HelpMessages.SparkConfigurationName)]
44+
[ValidateNotNullOrEmpty]
45+
[Alias("SparkConfigurationName")]
46+
public string Name { get; set; }
47+
48+
public override void ExecuteCmdlet()
49+
{
50+
if (this.IsParameterBound(c => c.WorkspaceObject))
51+
{
52+
this.WorkspaceName = this.WorkspaceObject.Name;
53+
}
54+
55+
if (this.IsParameterBound(c => c.Name))
56+
{
57+
WriteObject(new PSSparkConfigurationResource(SynapseAnalyticsClient.GetSparkConfiguration(this.Name), this.WorkspaceName));
58+
}
59+
else
60+
{
61+
var sparkConfigurations = SynapseAnalyticsClient.GetSparkConfigurationByWorkspace()
62+
.Select(element => new PSSparkConfigurationResource(element, this.WorkspaceName));
63+
WriteObject(sparkConfigurations, true);
64+
}
65+
}
66+
}
67+
}

‎src/Synapse/Synapse/Commands/DataPlaneCommands/Artifact/SparkConfiguration/NewAzureSynapseSparkConfiguration.cs

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
// ----------------------------------------------------------------------------------
2+
//
3+
// Copyright Microsoft Corporation
4+
// Licensed under the Apache License, Version 2.0 (the "License");
5+
// you may not use this file except in compliance with the License.
6+
// You may obtain a copy of the License at
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
// Unless required by applicable law or agreed to in writing, software
9+
// distributed under the License is distributed on an "AS IS" BASIS,
10+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11+
// See the License for the specific language governing permissions and
12+
// limitations under the License.
13+
// ----------------------------------------------------------------------------------
14+
15+
using Microsoft.Azure.Commands.ResourceManager.Common.ArgumentCompleters;
16+
using Microsoft.Azure.Commands.Synapse.Common;
17+
using Microsoft.Azure.Commands.Synapse.Models;
18+
using Microsoft.Azure.Commands.Synapse.Properties;
19+
using Microsoft.WindowsAzure.Commands.Utilities.Common;
20+
using System;
21+
using System.IO;
22+
using System.Management.Automation;
23+
24+
namespace Microsoft.Azure.Commands.Synapse
25+
{
26+
[Cmdlet(VerbsCommon.New, ResourceManager.Common.AzureRMConstants.AzureRMPrefix + SynapseConstants.SynapsePrefix + SynapseConstants.SparkConfiguration,
27+
DefaultParameterSetName = CreateByName, SupportsShouldProcess = true)]
28+
[Alias("Set-" + ResourceManager.Common.AzureRMConstants.AzureRMPrefix + SynapseConstants.SynapsePrefix + SynapseConstants.SparkConfiguration,
29+
"Import-" + ResourceManager.Common.AzureRMConstants.AzureRMPrefix + SynapseConstants.SynapsePrefix + SynapseConstants.SparkConfiguration)]
30+
[OutputType(typeof(PSSparkConfigurationResource))]
31+
public class NewAzureSynapseSparkConfiguration : SynapseArtifactsCmdletBase
32+
{
33+
private const string CreateByName = "CreateByName";
34+
private const string CreateByObject = "CreateByObject";
35+
36+
[Parameter(ValueFromPipelineByPropertyName = false, ParameterSetName = CreateByName,
37+
Mandatory = true, HelpMessage = HelpMessages.WorkspaceName)]
38+
[ResourceNameCompleter(ResourceTypes.Workspace, "ResourceGroupName")]
39+
[ValidateNotNullOrEmpty]
40+
public override string WorkspaceName { get; set; }
41+
42+
[Parameter(ValueFromPipeline = true, ParameterSetName = CreateByObject,
43+
Mandatory = true, HelpMessage = HelpMessages.WorkspaceObject)]
44+
[ValidateNotNull]
45+
public PSSynapseWorkspace WorkspaceObject { get; set; }
46+
47+
[Parameter(ValueFromPipelineByPropertyName = false, Mandatory = false, HelpMessage = HelpMessages.SparkConfigurationName)]
48+
[ValidateNotNullOrEmpty]
49+
[Alias("SparkConfigurationName")]
50+
public string Name { get; set; }
51+
52+
[Parameter(ValueFromPipelineByPropertyName = false, Mandatory = true, HelpMessage = HelpMessages.JsonFilePath)]
53+
[ValidateNotNullOrEmpty]
54+
[Alias("File")]
55+
public string DefinitionFile { get; set; }
56+
57+
[Parameter(Mandatory = false, HelpMessage = HelpMessages.AsJob)]
58+
public SwitchParameter AsJob { get; set; }
59+
60+
public override void ExecuteCmdlet()
61+
{
62+
if (this.IsParameterBound(c => c.WorkspaceObject))
63+
{
64+
this.WorkspaceName = this.WorkspaceObject.Name;
65+
}
66+
67+
if(!this.IsParameterBound(c => c.Name))
68+
{
69+
string path = this.TryResolvePath(DefinitionFile);
70+
this.Name = Path.GetFileNameWithoutExtension(path);
71+
}
72+
73+
if (this.ShouldProcess(this.WorkspaceName, String.Format(Resources.SettingSynapseSparkConfiguration, this.Name, this.WorkspaceName)))
74+
{
75+
string rawJsonContent = SynapseAnalyticsClient.ReadJsonFileContent(this.TryResolvePath(DefinitionFile));
76+
WriteObject(new PSSparkConfigurationResource(SynapseAnalyticsClient.CreateOrUpdateSparkConfiguration(this.Name, rawJsonContent),this.WorkspaceName));
77+
}
78+
}
79+
}
80+
}

0 commit comments

Comments
 (0)