[Synapse] Spark Configuration cmdlets

src/Synapse/Synapse/Az.Synapse.psd1

@@ -220,7 +220,11 @@ CmdletsToExport = 'Get-AzSynapseSparkJob', 'Stop-AzSynapseSparkJob',
                'Stop-AzSynapseDataFlowDebugSession', 
                'Start-AzSynapseDataFlowDebugSession', 'Get-AzSynapseSqlScript', 
                'Remove-AzSynapseSqlScript', 'Export-AzSynapseSqlScript', 
-               'Set-AzSynapseSqlScript'
+               'Set-AzSynapseSqlScript',
+               'Get-AzSynapseSparkConfiguration',
+               'New-AzSynapseSparkConfiguration',
+               'Export-AzSynapseSparkConfiguration',
+               'Remove-AzSynapseSparkConfiguration'
 
 # Variables to export from this module
 # VariablesToExport = @()
@@ -241,7 +245,9 @@ AliasesToExport = 'New-AzSynapsePipeline', 'New-AzSynapseLinkedService',
                'Disable-AzSynapseSqlAdvancedThreatProtection', 
                'New-AzSynapseSparkJobDefinition', 
                'Set-AzSynapseManagedPrivateEndpoint', 'New-AzSynapseSqlScript', 
-               'Import-AzSynapseSqlScript'
+               'Import-AzSynapseSqlScript',
+               'Set-AzSynapseSparkConfiguration',
+               'Import-AzSynapseSparkConfiguration'
 
 # DSC resources to export from this module
 # DscResourcesToExport = @()

src/Synapse/Synapse/ChangeLog.md

@@ -21,6 +21,11 @@
 ## Upcoming Release
 * Renamed parameter FolderName in `Set-AzSynapseSqlScript` to FolderPath and keeped FolderName as alias
 * Updated `Set-AzSynapseNoteBook` and `Set-AzSynapseSparkJobDefinition` to support new parameter [-FolderPath]
+* Added cmdlets for Synapse Spark Configuration
+    - Added `Get-AzSynapseSparkConfiguration` cmdlet
+    - Added `New-AzSynapseSparkConfiguration` cmdlet
+    - Added `Export-AzSynapseSparkConfiguration` cmdlet
+    - Added `Remove-AzSynapseSparkConfiguration` cmdlet
 
 ## Version 0.18.0
 * Added cmdlets for Synapse Kusto pool

src/Synapse/Synapse/Commands/DataPlaneCommands/Artifact/SparkConfiguration/ExportAzureSynapseSparkConfiguration.cs

@@ -0,0 +1,109 @@
+// ----------------------------------------------------------------------------------
+//
+// Copyright Microsoft Corporation
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+// ----------------------------------------------------------------------------------
+
+using Microsoft.Azure.Commands.ResourceManager.Common.ArgumentCompleters;
+using Microsoft.Azure.Commands.Synapse.Common;
+using Microsoft.Azure.Commands.Synapse.Models;
+using Microsoft.WindowsAzure.Commands.Utilities.Common;
+using Newtonsoft.Json;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Management.Automation;
+
+namespace Microsoft.Azure.Commands.Synapse
+{
+    [Cmdlet(VerbsData.Export, ResourceManager.Common.AzureRMConstants.AzureRMPrefix + SynapseConstants.SynapsePrefix + SynapseConstants.SparkConfiguration,
+        DefaultParameterSetName = ExportByName)]
+    [OutputType(typeof(FileInfo))]
+    public class ExportAzureSynapseSparkConfiguration : SynapseArtifactsCmdletBase
+    {
+        private const string ExportByName = "ExportByName";
+        private const string ExportByObject = "ExportByObject";
+        private const string ExportByInputObject = "ExportByInputObject";
+
+        [Parameter(ValueFromPipelineByPropertyName = false, ParameterSetName = ExportByName,
+            Mandatory = true, HelpMessage = HelpMessages.WorkspaceName)]
+        [ResourceNameCompleter(ResourceTypes.Workspace, "ResourceGroupName")]
+        [ValidateNotNullOrEmpty]
+        public override string WorkspaceName { get; set; }
+
+        [Parameter(ValueFromPipeline = true, ParameterSetName = ExportByObject,
+            Mandatory = true, HelpMessage = HelpMessages.WorkspaceObject)]
+        [ValidateNotNull]
+        public PSSynapseWorkspace WorkspaceObject { get; set; }
+
+        [Parameter(ValueFromPipelineByPropertyName = false, Mandatory = false, ParameterSetName = ExportByName,
+            HelpMessage = HelpMessages.SparkConfigurationName)]
+        [Parameter(ValueFromPipelineByPropertyName = false, Mandatory = false, ParameterSetName = ExportByObject,
+            HelpMessage = HelpMessages.SparkConfigurationName)]
+        [ValidateNotNullOrEmpty]
+        [Alias("SparkConfigurationName")]
+        public string Name { get; set; }
+
+        [Parameter(ValueFromPipeline = true, ParameterSetName = ExportByInputObject,
+            Mandatory = true, HelpMessage = HelpMessages.SparkConfigurationObject)]
+        [ValidateNotNull]
+        public PSSparkConfigurationResource InputObject { get; set; }
+
+        [Parameter(ValueFromPipelineByPropertyName = false, Mandatory = true, HelpMessage = HelpMessages.OutputFolder)]
+        [ValidateNotNullOrEmpty]
+        public string OutputFolder { get; set; }
+
+        [Parameter(Mandatory = false, HelpMessage = HelpMessages.AsJob)]
+        public SwitchParameter AsJob { get; set; }
+
+        public override void ExecuteCmdlet()
+        {
+            if (this.IsParameterBound(c => c.WorkspaceObject))
+            {
+                this.WorkspaceName = this.WorkspaceObject.Name;
+            }
+
+            var fileExtension = ".json";
+            if (this.IsParameterBound(c => c.InputObject))
+            {
+                WriteToFile(this.InputObject);
+                WriteObject(new FileInfo(Path.Combine(this.OutputFolder, this.InputObject.Name + fileExtension)));
+            }
+            else
+            {
+                if (this.IsParameterBound(c => c.Name))
+                {
+                    var sparkConfiguration = new PSSparkConfigurationResource(SynapseAnalyticsClient.GetSparkConfiguration(this.Name), this.WorkspaceName);
+                    WriteToFile(sparkConfiguration);
+                    WriteObject(new FileInfo(Path.Combine(this.OutputFolder, sparkConfiguration.Name + fileExtension)));
+                }
+                else
+                {
+                    var infoList = new List<FileInfo>();
+                    var sparkConfigurations = SynapseAnalyticsClient.GetSparkConfigurationByWorkspace()
+                        .Select(element => new PSSparkConfigurationResource(element, this.WorkspaceName));
+                    foreach (var sparkConfiguration in sparkConfigurations)
+                    {
+                        WriteToFile(sparkConfiguration);
+                        infoList.Add(new FileInfo(Path.Combine(this.OutputFolder, sparkConfiguration.Name + fileExtension)));
+                    }
+                    WriteObject(infoList, true);
+                }
+            }
+        }
+
+        private void WriteToFile(PSSparkConfigurationResource sparkConfigurationResource)
+        {
+            string json = Newtonsoft.Json.JsonConvert.SerializeObject(sparkConfigurationResource.Properties, Formatting.Indented);
+            File.WriteAllText(Path.Combine(this.OutputFolder, sparkConfigurationResource.Name + ".json"), json);
+        }
+    }
+}

src/Synapse/Synapse/Commands/DataPlaneCommands/Artifact/SparkConfiguration/GetAzureSynapseSparkConfiguration.cs

@@ -0,0 +1,67 @@
+// ----------------------------------------------------------------------------------
+//
+// Copyright Microsoft Corporation
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+// ----------------------------------------------------------------------------------
+
+using Microsoft.Azure.Commands.ResourceManager.Common.ArgumentCompleters;
+using Microsoft.Azure.Commands.Synapse.Common;
+using Microsoft.Azure.Commands.Synapse.Models;
+using Microsoft.WindowsAzure.Commands.Utilities.Common;
+using System.Linq;
+using System.Management.Automation;
+
+namespace Microsoft.Azure.Commands.Synapse
+{
+    [Cmdlet(VerbsCommon.Get, ResourceManager.Common.AzureRMConstants.AzureRMPrefix + SynapseConstants.SynapsePrefix + SynapseConstants.SparkConfiguration,
+        DefaultParameterSetName = GetByName)]
+    [OutputType(typeof(PSSparkConfigurationResource))]
+    public class GetAzureSynapseSparkConfiguration : SynapseArtifactsCmdletBase
+    {
+        private const string GetByName = "GetByName";
+        private const string GetByObject = "GetByObject";
+
+        [Parameter(ValueFromPipelineByPropertyName = false, ParameterSetName = GetByName,
+            Mandatory = true, HelpMessage = HelpMessages.WorkspaceName)]
+        [ResourceNameCompleter(ResourceTypes.Workspace, "ResourceGroupName")]
+        [ValidateNotNullOrEmpty]
+        public override string WorkspaceName { get; set; }
+
+        [Parameter(ValueFromPipeline = true, ParameterSetName = GetByObject,
+            Mandatory = true, HelpMessage = HelpMessages.WorkspaceObject)]
+        [ValidateNotNull]
+        public PSSynapseWorkspace WorkspaceObject { get; set; }
+
+        [Parameter(ValueFromPipelineByPropertyName = false, Mandatory = false, HelpMessage = HelpMessages.SparkConfigurationName)]
+        [ValidateNotNullOrEmpty]
+        [Alias("SparkConfigurationName")]
+        public string Name { get; set; }
+
+        public override void ExecuteCmdlet()
+        {
+            if (this.IsParameterBound(c => c.WorkspaceObject))
+            {
+                this.WorkspaceName = this.WorkspaceObject.Name;
+            }
+
+            if (this.IsParameterBound(c => c.Name))
+            {
+                WriteObject(new PSSparkConfigurationResource(SynapseAnalyticsClient.GetSparkConfiguration(this.Name), this.WorkspaceName));
+            }
+            else
+            {
+                var sparkConfigurations = SynapseAnalyticsClient.GetSparkConfigurationByWorkspace()
+                    .Select(element => new PSSparkConfigurationResource(element, this.WorkspaceName));
+                WriteObject(sparkConfigurations, true);
+            }
+        }
+    }
+}

src/Synapse/Synapse/Commands/DataPlaneCommands/Artifact/SparkConfiguration/NewAzureSynapseSparkConfiguration.cs

@@ -0,0 +1,80 @@
+// ----------------------------------------------------------------------------------
+//
+// Copyright Microsoft Corporation
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+// ----------------------------------------------------------------------------------
+
+using Microsoft.Azure.Commands.ResourceManager.Common.ArgumentCompleters;
+using Microsoft.Azure.Commands.Synapse.Common;
+using Microsoft.Azure.Commands.Synapse.Models;
+using Microsoft.Azure.Commands.Synapse.Properties;
+using Microsoft.WindowsAzure.Commands.Utilities.Common;
+using System;
+using System.IO;
+using System.Management.Automation;
+
+namespace Microsoft.Azure.Commands.Synapse
+{
+    [Cmdlet(VerbsCommon.New, ResourceManager.Common.AzureRMConstants.AzureRMPrefix + SynapseConstants.SynapsePrefix + SynapseConstants.SparkConfiguration,
+        DefaultParameterSetName = CreateByName, SupportsShouldProcess = true)]
+    [Alias("Set-" + ResourceManager.Common.AzureRMConstants.AzureRMPrefix + SynapseConstants.SynapsePrefix + SynapseConstants.SparkConfiguration,
+        "Import-" + ResourceManager.Common.AzureRMConstants.AzureRMPrefix + SynapseConstants.SynapsePrefix + SynapseConstants.SparkConfiguration)]
+    [OutputType(typeof(PSSparkConfigurationResource))]
+    public class NewAzureSynapseSparkConfiguration : SynapseArtifactsCmdletBase
+    {
+        private const string CreateByName = "CreateByName";
+        private const string CreateByObject = "CreateByObject";
+
+        [Parameter(ValueFromPipelineByPropertyName = false, ParameterSetName = CreateByName,
+            Mandatory = true, HelpMessage = HelpMessages.WorkspaceName)]
+        [ResourceNameCompleter(ResourceTypes.Workspace, "ResourceGroupName")]
+        [ValidateNotNullOrEmpty]
+        public override string WorkspaceName { get; set; }
+
+        [Parameter(ValueFromPipeline = true, ParameterSetName = CreateByObject,
+            Mandatory = true, HelpMessage = HelpMessages.WorkspaceObject)]
+        [ValidateNotNull]
+        public PSSynapseWorkspace WorkspaceObject { get; set; }
+
+        [Parameter(ValueFromPipelineByPropertyName = false, Mandatory = false, HelpMessage = HelpMessages.SparkConfigurationName)]
+        [ValidateNotNullOrEmpty]
+        [Alias("SparkConfigurationName")]
+        public string Name { get; set; }
+
+        [Parameter(ValueFromPipelineByPropertyName = false, Mandatory = true, HelpMessage = HelpMessages.JsonFilePath)]
+        [ValidateNotNullOrEmpty]
+        [Alias("File")]
+        public string DefinitionFile { get; set; }
+
+        [Parameter(Mandatory = false, HelpMessage = HelpMessages.AsJob)]
+        public SwitchParameter AsJob { get; set; }
+
+        public override void ExecuteCmdlet()
+        {
+            if (this.IsParameterBound(c => c.WorkspaceObject))
+            {
+                this.WorkspaceName = this.WorkspaceObject.Name;
+            }
+
+            if(!this.IsParameterBound(c => c.Name))
+            {
+                string path = this.TryResolvePath(DefinitionFile);
+                this.Name = Path.GetFileNameWithoutExtension(path);
+            }
+
+            if (this.ShouldProcess(this.WorkspaceName, String.Format(Resources.SettingSynapseSparkConfiguration, this.Name, this.WorkspaceName)))
+            {
+                string rawJsonContent = SynapseAnalyticsClient.ReadJsonFileContent(this.TryResolvePath(DefinitionFile));
+                WriteObject(new PSSparkConfigurationResource(SynapseAnalyticsClient.CreateOrUpdateSparkConfiguration(this.Name, rawJsonContent),this.WorkspaceName));
+            }
+        }
+    }
+}

src/Synapse/Synapse/Commands/DataPlaneCommands/Artifact/SparkConfiguration/RemoveAzureSynapseSparkConfiguration.cs

@@ -0,0 +1,92 @@
+// ----------------------------------------------------------------------------------
+//
+// Copyright Microsoft Corporation
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+// ----------------------------------------------------------------------------------
+
+using Microsoft.Azure.Commands.ResourceManager.Common.ArgumentCompleters;
+using Microsoft.Azure.Commands.Synapse.Common;
+using Microsoft.Azure.Commands.Synapse.Models;
+using Microsoft.Azure.Commands.Synapse.Properties;
+using Microsoft.WindowsAzure.Commands.Utilities.Common;
+using System.Management.Automation;
+
+namespace Microsoft.Azure.Commands.Synapse
+{
+    [Cmdlet(VerbsCommon.Remove, ResourceManager.Common.AzureRMConstants.AzureRMPrefix + SynapseConstants.SynapsePrefix + SynapseConstants.SparkConfiguration,
+        DefaultParameterSetName = RemoveByName, SupportsShouldProcess = true)]
+    [OutputType(typeof(bool))]
+    public class RemoveAzureSynapseSparkConfiguration : SynapseArtifactsCmdletBase
+    {
+        private const string RemoveByName = "RemoveByName";
+        private const string RemoveByObject = "RemoveByObject";
+        private const string RemoveByInputObject = "RemoveByInputObject";
+
+        [Parameter(ValueFromPipelineByPropertyName = false, ParameterSetName = RemoveByName,
+            Mandatory = true, HelpMessage = HelpMessages.WorkspaceName)]
+        [ResourceNameCompleter(ResourceTypes.Workspace, "ResourceGroupName")]
+        [ValidateNotNullOrEmpty]
+        public override string WorkspaceName { get; set; }
+
+        [Parameter(ValueFromPipeline = true, ParameterSetName = RemoveByObject,
+            Mandatory = true, HelpMessage = HelpMessages.WorkspaceObject)]
+        [ValidateNotNull]
+        public PSSynapseWorkspace WorkspaceObject { get; set; }
+
+        [Parameter(ValueFromPipelineByPropertyName = false, ParameterSetName = RemoveByName, Mandatory = true, HelpMessage = HelpMessages.SparkConfigurationName)]
+        [Parameter(ValueFromPipelineByPropertyName = false, ParameterSetName = RemoveByObject, Mandatory = true, HelpMessage = HelpMessages.SparkConfigurationName)]
+        [ValidateNotNullOrEmpty]
+        [Alias("SparkConfigurationName")]
+        public string Name { get; set; }
+
+        [Parameter(ValueFromPipeline = true, ParameterSetName = RemoveByInputObject,
+            Mandatory = true, HelpMessage = HelpMessages.SparkConfigurationObject)]
+        [ValidateNotNull]
+        public PSSparkConfigurationResource InputObject { get; set; }
+
+        [Parameter(Mandatory = false, HelpMessage = HelpMessages.PassThru)]
+        public SwitchParameter PassThru { get; set; }
+
+        [Parameter(Mandatory = false, HelpMessage = HelpMessages.AsJob)]
+        public SwitchParameter AsJob { get; set; }
+
+        [Parameter(Mandatory = false, HelpMessage = HelpMessages.Force)]
+        public SwitchParameter Force { get; set; }
+
+        public override void ExecuteCmdlet()
+        {
+            if (this.IsParameterBound(c => c.WorkspaceObject))
+            {
+                this.WorkspaceName = this.WorkspaceObject.Name;
+            }
+
+            if (this.IsParameterBound(c => c.InputObject))
+            {
+                this.WorkspaceName = this.InputObject.WorkspaceName;
+                this.Name = this.InputObject.Name;
+            }
+
+            ConfirmAction(
+                Force.IsPresent,
+                string.Format(Resources.RemoveSynapseSparkConfiguration, this.Name),
+                string.Format(Resources.RemovingSynapseSparkConfiguration, this.Name, this.WorkspaceName),
+                Name,
+                () =>
+                {
+                    SynapseAnalyticsClient.DeleteSparkConfiguration(this.Name);
+                    if (PassThru)
+                    {
+                        WriteObject(true);
+                    }
+                });
+        }
+    }
+}

src/Synapse/Synapse/Common/HelpMessages.cs

@@ -544,5 +544,9 @@ SELECT on dbo.myTable by public
         public const string SparkConfigurationFolderPath = "The folder that this Spark job definition is in. If specify a multi-level path such as [rootFolder/subFolder], the Spark job definition will appear at the bottom level. If not specified, this Spark job definition will appear at the root level.";
 
         public const string NoteBookFolderPath = "The folder that this notebook is in. If specify a multi-level path such as [rootFolder/subFolder], the notebook will appear at the bottom level. If not specified, this notebook will appear at the root level.";
+
+        public const string SparkConfigurationName = "The Spark Configuration name.";
+
+        public const string SparkConfigurationObject = "The Spark configuration object.";
     }
 }