Skip to content

Commit a3a5e56

Browse files
wonnerWan Yangisra-fel
authored
[Synapse] Add pipeline, linked service, trigger, notebook management cmdlets (#12616)
* add support for pipeline * add support for linked service * add support for trigger * fix new notebook endpoint * fix csproj * add support for IR * create or update notebook by definition file * fix using package * delete start/stop IR cmdlet * mark executors as required * new/set notebook by .ipynb file * add support for Import/Export notebook * fix HelpMessage * fix CmdletsToExport * modify import/export notebook cmdlets * resolve LRO * use setProperties * update PSNotebook * delete IR * delete IR * add help doc and changelog * update help doc * update remove cmdlets * update help doc * update help doc * update output in table format * update output type * support extension parameters * fix help docs * fix for static analysis * use release package * change invoke pipeline output * Revert "delete IR" This reverts commit 51a98ed5845dd0f99005927c4981c9b0bd7bc59d. * Revert "delete IR" This reverts commit ad3fed743322596a7105c57b6bcaf9becec0df90. * rebase master * update artifacts package version * reorganize changelog to prevent merging issue Co-authored-by: Wan Yang <[email protected]> Co-authored-by: Yeming Liu <[email protected]>
1 parent f65c1d3 commit a3a5e56

File tree

228 files changed

+23934
-4
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

228 files changed

+23934
-4
lines changed

‎src/Synapse/Synapse/Az.Synapse.psd1

Lines changed: 17 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -58,7 +58,8 @@ RequiredModules = @(@{ModuleName = 'Az.Accounts'; ModuleVersion = '1.9.3'; })
5858
# Assemblies that must be loaded prior to importing this module
5959
RequiredAssemblies = 'Microsoft.Azure.Management.Synapse.dll',
6060
'Azure.Analytics.Synapse.Spark.dll',
61-
'Azure.Analytics.Synapse.AccessControl.dll'
61+
'Azure.Analytics.Synapse.AccessControl.dll',
62+
'Azure.Analytics.Synapse.Artifacts.dll'
6263

6364
# Script files (.ps1) that are run in the caller's environment prior to importing this module.
6465
# ScriptsToProcess = @()
@@ -106,13 +107,26 @@ CmdletsToExport = 'Get-AzSynapseSparkJob', 'Stop-AzSynapseSparkJob',
106107
'Set-AzSynapseIntegrationRuntime',
107108
'Sync-AzSynapseIntegrationRuntimeCredential',
108109
'Update-AzSynapseIntegrationRuntime',
109-
'Update-AzSynapseIntegrationRuntimeNode'
110+
'Update-AzSynapseIntegrationRuntimeNode',
111+
'Get-AzSynapsePipeline', 'Remove-AzSynapsePipeline',
112+
'Set-AzSynapsePipeline', 'Invoke-AzSynapsePipeline',
113+
'Get-AzSynapsePipelineRun', 'Stop-AzSynapsePipelineRun',
114+
'Get-AzSynapseActivityRun', 'Get-AzSynapseLinkedService',
115+
'Set-AzSynapseLinkedService', 'Remove-AzSynapseLinkedService',
116+
'Get-AzSynapseNotebook', 'Set-AzSynapseNotebook',
117+
'Remove-AzSynapseNotebook', 'Export-AzSynapseNotebook',
118+
'Set-AzSynapseTrigger', 'Get-AzSynapseTrigger', 'Remove-AzSynapseTrigger',
119+
'Get-AzSynapseTriggerSubscriptionStatus', 'Add-AzSynapseTriggerSubscription',
120+
'Remove-AzSynapseTriggerSubscription', 'Start-AzSynapseTrigger',
121+
'Stop-AzSynapseTrigger', 'Get-AzSynapseTriggerRun'
110122

111123
# Variables to export from this module
112124
# VariablesToExport = @()
113125

114126
# Aliases to export from this module, for best performance, do not use wildcards and do not delete the entry, use an empty array if there are no aliases to export.
115-
AliasesToExport = @()
127+
AliasesToExport = 'New-AzSynapsePipeline', 'New-AzSynapseLinkedService',
128+
'New-AzSynapseNotebook', 'Import-AzSynapseNotebook',
129+
'New-AzSynapseTrigger'
116130

117131
# DSC resources to export from this module
118132
# DscResourcesToExport = @()

‎src/Synapse/Synapse/ChangeLog.md

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,38 @@
1818
- Additional information about change #1
1919
-->
2020
## Upcoming Release
21+
* Added support for operation of Synapse LinkedService
22+
- Add `Get-AzSynapseLinkedService` cmdlet
23+
- Add `Remove-AzSynapseLinkedService` cmdlet
24+
- Add `Set-AzSynapseLinkedService` cmdlet
25+
- Add `New-AzSynapseLinkedService` cmdlet
26+
* Added support for operation of Synapse Notebook
27+
- Add `Get-AzSynapseNotebook` cmdlet
28+
- Add `Export-AzSynapseNotebook` cmdlet
29+
- Add `Remove-AzSynapseNotebook` cmdlet
30+
- Add `Set-AzSynapseNotebook` cmdlet
31+
- Add `New-AzSynapseNotebook` cmdlet
32+
- Add `Import-AzSynapseNotebook` cmdlet
33+
* Added support for operation of Synapse Pipeline
34+
- Add `Get-AzSynapsePipeline` cmdlet
35+
- Add `Remove-AzSynapsePipeline` cmdlet
36+
- Add `Set-AzSynapsePipeline` cmdlet
37+
- Add `New-AzSynapsePipeline` cmdlet
38+
- Add `Get-AzSynapseActivityRun` cmdlet
39+
- Add `Get-AzSynapsePipelineRun` cmdlet
40+
- Add `Invoke-AzSynapsePipeline` cmdlet
41+
- Add `Stop-AzSynapsePipelineRun` cmdlet
42+
* Added support for operation of Synapse Trigger
43+
- Add `Get-AzSynapseTrigger` cmdlet
44+
- Add `Remove-AzSynapseTrigger` cmdlet
45+
- Add `Set-AzSynapseTrigger` cmdlet
46+
- Add `New-AzSynapseTrigger` cmdlet
47+
- Add `Add-AzSynapseTriggerSubscription` cmdlet
48+
- Add `Get-AzSynapseTriggerSubscriptionStatus` cmdlet
49+
- Add `Remove-AzSynapseTriggerSubscription` cmdlet
50+
- Add `Start-AzSynapseTrigger` cmdlet
51+
- Add `Stop-AzSynapseTrigger` cmdlet
52+
- Add `Get-AzSynapseTriggerRun` cmdlet
2153

2254
## Version 0.2.0
2355

‎src/Synapse/Synapse/Commands/DataPlaneCommands/Artifact/LinkedServices/GetAzureSynapseLinkedService.cs

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
using Microsoft.Azure.Commands.ResourceManager.Common.ArgumentCompleters;
2+
using Microsoft.Azure.Commands.Synapse.Common;
3+
using Microsoft.Azure.Commands.Synapse.Models;
4+
using Microsoft.WindowsAzure.Commands.Utilities.Common;
5+
using System.Linq;
6+
using System.Management.Automation;
7+
8+
namespace Microsoft.Azure.Commands.Synapse
9+
{
10+
[Cmdlet(VerbsCommon.Get, ResourceManager.Common.AzureRMConstants.AzureRMPrefix + SynapseConstants.SynapsePrefix + SynapseConstants.LinkedService,
11+
DefaultParameterSetName = GetByName)]
12+
[OutputType(typeof(PSLinkedServiceResource))]
13+
public class GetAzureSynapseLinkedService : SynapseArtifactsCmdletBase
14+
{
15+
private const string GetByName = "GetByName";
16+
private const string GetByObject = "GetByObject";
17+
18+
[Parameter(ValueFromPipelineByPropertyName = false, ParameterSetName = GetByName,
19+
Mandatory = true, HelpMessage = HelpMessages.WorkspaceName)]
20+
[ResourceNameCompleter(ResourceTypes.Workspace, "ResourceGroupName")]
21+
[ValidateNotNullOrEmpty]
22+
public override string WorkspaceName { get; set; }
23+
24+
[Parameter(ValueFromPipeline = true, ParameterSetName = GetByObject,
25+
Mandatory = true, HelpMessage = HelpMessages.WorkspaceObject)]
26+
[ValidateNotNull]
27+
public PSSynapseWorkspace WorkspaceObject { get; set; }
28+
29+
[Parameter(ValueFromPipelineByPropertyName = false, Mandatory = false, HelpMessage = HelpMessages.LinkedServiceName)]
30+
[ValidateNotNullOrEmpty]
31+
[Alias("LinkedServiceName")]
32+
public string Name { get; set; }
33+
34+
public override void ExecuteCmdlet()
35+
{
36+
if (this.IsParameterBound(c => c.WorkspaceObject))
37+
{
38+
this.WorkspaceName = this.WorkspaceObject.Name;
39+
}
40+
41+
if (this.IsParameterBound(c => c.Name))
42+
{
43+
WriteObject(new PSLinkedServiceResource(SynapseAnalyticsClient.GetLinkedService(this.Name),this.WorkspaceName));
44+
}
45+
else
46+
{
47+
var linkedservices = SynapseAnalyticsClient.GetLinkedServicesByWorkspace()
48+
.Select(element => new PSLinkedServiceResource(element,this.WorkspaceName));
49+
WriteObject(linkedservices, true);
50+
}
51+
}
52+
}
53+
}

‎src/Synapse/Synapse/Commands/DataPlaneCommands/Artifact/LinkedServices/RemoveAzureSynapseLinkedService.cs

Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,73 @@
1+
using Microsoft.Azure.Commands.ResourceManager.Common.ArgumentCompleters;
2+
using Microsoft.Azure.Commands.Synapse.Common;
3+
using Microsoft.Azure.Commands.Synapse.Models;
4+
using Microsoft.Azure.Commands.Synapse.Properties;
5+
using Microsoft.WindowsAzure.Commands.Utilities.Common;
6+
using System;
7+
using System.Management.Automation;
8+
9+
namespace Microsoft.Azure.Commands.Synapse
10+
{
11+
[Cmdlet(VerbsCommon.Remove, ResourceManager.Common.AzureRMConstants.AzureRMPrefix + SynapseConstants.SynapsePrefix + SynapseConstants.LinkedService,
12+
DefaultParameterSetName = RemoveByName, SupportsShouldProcess = true)]
13+
[OutputType(typeof(bool))]
14+
public class RemoveAzureSynapseLinkedService : SynapseArtifactsCmdletBase
15+
{
16+
private const string RemoveByName = "RemoveByName";
17+
private const string RemoveByObject = "RemoveByObject";
18+
private const string RemoveByInputObject = "RemoveByInputObject";
19+
20+
[Parameter(ValueFromPipelineByPropertyName = false, ParameterSetName = RemoveByName,
21+
Mandatory = true, HelpMessage = HelpMessages.WorkspaceName)]
22+
[ResourceNameCompleter(ResourceTypes.Workspace, "ResourceGroupName")]
23+
[ValidateNotNullOrEmpty]
24+
public override string WorkspaceName { get; set; }
25+
26+
[Parameter(ValueFromPipeline = true, ParameterSetName = RemoveByObject,
27+
Mandatory = true, HelpMessage = HelpMessages.WorkspaceObject)]
28+
[ValidateNotNull]
29+
public PSSynapseWorkspace WorkspaceObject { get; set; }
30+
31+
[Parameter(ValueFromPipelineByPropertyName = false, ParameterSetName = RemoveByName,
32+
Mandatory = true, HelpMessage = HelpMessages.LinkedServiceName)]
33+
[Parameter(ValueFromPipelineByPropertyName = false, ParameterSetName = RemoveByObject,
34+
Mandatory = true, HelpMessage = HelpMessages.LinkedServiceName)]
35+
[ValidateNotNullOrEmpty]
36+
[Alias("LinkedServiceName")]
37+
public string Name { get; set; }
38+
39+
[Parameter(ValueFromPipeline = true, ParameterSetName = RemoveByInputObject,
40+
Mandatory = true, HelpMessage = HelpMessages.LinkedServiceObject)]
41+
[ValidateNotNull]
42+
public PSLinkedServiceResource InputObject { get; set; }
43+
44+
[Parameter(Mandatory = false, HelpMessage = HelpMessages.PassThru)]
45+
public SwitchParameter PassThru { get; set; }
46+
47+
[Parameter(Mandatory = false, HelpMessage = HelpMessages.AsJob)]
48+
public SwitchParameter AsJob { get; set; }
49+
50+
public override void ExecuteCmdlet()
51+
{
52+
if (this.IsParameterBound(c => c.WorkspaceObject))
53+
{
54+
this.WorkspaceName = this.WorkspaceObject.Name;
55+
}
56+
57+
if (this.IsParameterBound(c => c.InputObject))
58+
{
59+
this.WorkspaceName = this.InputObject.WorkspaceName;
60+
this.Name = this.InputObject.Name;
61+
}
62+
63+
if (this.ShouldProcess(this.WorkspaceName, String.Format(Resources.RemovingSynapseLinkedService, this.Name, this.WorkspaceName)))
64+
{
65+
SynapseAnalyticsClient.DeleteLinkedService(this.Name);
66+
if (PassThru)
67+
{
68+
WriteObject(true);
69+
}
70+
}
71+
}
72+
}
73+
}

‎src/Synapse/Synapse/Commands/DataPlaneCommands/Artifact/LinkedServices/SetAzureSynapseLinkedService.cs

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
using Microsoft.Azure.Commands.ResourceManager.Common.ArgumentCompleters;
2+
using Microsoft.Azure.Commands.Synapse.Common;
3+
using Microsoft.Azure.Commands.Synapse.Models;
4+
using Microsoft.Azure.Commands.Synapse.Properties;
5+
using Microsoft.WindowsAzure.Commands.Utilities.Common;
6+
using System;
7+
using System.Management.Automation;
8+
9+
namespace Microsoft.Azure.Commands.Synapse
10+
{
11+
[Cmdlet(VerbsCommon.Set, ResourceManager.Common.AzureRMConstants.AzureRMPrefix + SynapseConstants.SynapsePrefix + SynapseConstants.LinkedService,
12+
DefaultParameterSetName = SetByName, SupportsShouldProcess = true)]
13+
[Alias("New-" + ResourceManager.Common.AzureRMConstants.AzureRMPrefix + SynapseConstants.SynapsePrefix + SynapseConstants.LinkedService)]
14+
[OutputType(typeof(PSLinkedServiceResource))]
15+
public class SetAzureSynapseLinkedService : SynapseArtifactsCmdletBase
16+
{
17+
private const string SetByName = "SetByName";
18+
private const string SetByObject = "SetByObject";
19+
20+
[Parameter(ValueFromPipelineByPropertyName = false, ParameterSetName = SetByName,
21+
Mandatory = true, HelpMessage = HelpMessages.WorkspaceName)]
22+
[ResourceNameCompleter(ResourceTypes.Workspace, "ResourceGroupName")]
23+
[ValidateNotNullOrEmpty]
24+
public override string WorkspaceName { get; set; }
25+
26+
[Parameter(ValueFromPipeline = true, ParameterSetName = SetByObject,
27+
Mandatory = true, HelpMessage = HelpMessages.WorkspaceObject)]
28+
[ValidateNotNull]
29+
public PSSynapseWorkspace WorkspaceObject { get; set; }
30+
31+
[Parameter(ValueFromPipelineByPropertyName = false, Mandatory = true, HelpMessage = HelpMessages.LinkedServiceName)]
32+
[ValidateNotNullOrEmpty]
33+
[Alias("LinkedServiceName")]
34+
public string Name { get; set; }
35+
36+
[Parameter(ValueFromPipelineByPropertyName = false, Mandatory = true, HelpMessage = HelpMessages.JsonFilePath)]
37+
[ValidateNotNullOrEmpty]
38+
[Alias("File")]
39+
public string DefinitionFile { get; set; }
40+
41+
[Parameter(Mandatory = false, HelpMessage = HelpMessages.AsJob)]
42+
public SwitchParameter AsJob { get; set; }
43+
44+
public override void ExecuteCmdlet()
45+
{
46+
if (this.IsParameterBound(c => c.WorkspaceObject))
47+
{
48+
this.WorkspaceName = this.WorkspaceObject.Name;
49+
}
50+
51+
if (this.ShouldProcess(this.WorkspaceName, String.Format(Resources.SettingSynapseLinkedService, this.Name, this.WorkspaceName)))
52+
{
53+
string rawJsonContent = SynapseAnalyticsClient.ReadJsonFileContent(this.TryResolvePath(DefinitionFile));
54+
WriteObject(new PSLinkedServiceResource(SynapseAnalyticsClient.CreateOrUpdateLinkedService(this.Name, rawJsonContent), this.WorkspaceName));
55+
}
56+
}
57+
}
58+
}

‎src/Synapse/Synapse/Commands/DataPlaneCommands/Artifact/Notebooks/ExportAzureSynapseNotebook.cs

Lines changed: 95 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,95 @@
1+
using Microsoft.Azure.Commands.ResourceManager.Common.ArgumentCompleters;
2+
using Microsoft.Azure.Commands.Synapse.Common;
3+
using Microsoft.Azure.Commands.Synapse.Models;
4+
using Microsoft.WindowsAzure.Commands.Utilities.Common;
5+
using Newtonsoft.Json;
6+
using System.Collections.Generic;
7+
using System.IO;
8+
using System.Linq;
9+
using System.Management.Automation;
10+
11+
namespace Microsoft.Azure.Commands.Synapse
12+
{
13+
[Cmdlet(VerbsData.Export, ResourceManager.Common.AzureRMConstants.AzureRMPrefix + SynapseConstants.SynapsePrefix + SynapseConstants.Notebook,
14+
DefaultParameterSetName = ExportByName)]
15+
[OutputType(typeof(FileInfo))]
16+
public class ExportAzureSynapseNotebook : SynapseArtifactsCmdletBase
17+
{
18+
private const string ExportByName = "ExportByName";
19+
private const string ExportByObject = "ExportByObject";
20+
private const string ExportByInputObject = "ExportByInputObject";
21+
22+
[Parameter(ValueFromPipelineByPropertyName = false, ParameterSetName = ExportByName,
23+
Mandatory = true, HelpMessage = HelpMessages.WorkspaceName)]
24+
[ResourceNameCompleter(ResourceTypes.Workspace, "ResourceGroupName")]
25+
[ValidateNotNullOrEmpty]
26+
public override string WorkspaceName { get; set; }
27+
28+
[Parameter(ValueFromPipeline = true, ParameterSetName = ExportByObject,
29+
Mandatory = true, HelpMessage = HelpMessages.WorkspaceObject)]
30+
[ValidateNotNull]
31+
public PSSynapseWorkspace WorkspaceObject { get; set; }
32+
33+
[Parameter(ValueFromPipelineByPropertyName = false, Mandatory = false, ParameterSetName = ExportByName,
34+
HelpMessage = HelpMessages.NotebookName)]
35+
[Parameter(ValueFromPipelineByPropertyName = false, Mandatory = false, ParameterSetName = ExportByObject,
36+
HelpMessage = HelpMessages.NotebookName)]
37+
[ValidateNotNullOrEmpty]
38+
[Alias("NotebookName")]
39+
public string Name { get; set; }
40+
41+
[Parameter(ValueFromPipeline = true, ParameterSetName = ExportByInputObject,
42+
Mandatory = true, HelpMessage = HelpMessages.NotebookObject)]
43+
[ValidateNotNull]
44+
public PSNotebookResource InputObject { get; set; }
45+
46+
[Parameter(ValueFromPipelineByPropertyName = false, Mandatory = true, HelpMessage = HelpMessages.OutputFolder)]
47+
[ValidateNotNullOrEmpty]
48+
public string OutputFolder { get; set; }
49+
50+
[Parameter(Mandatory = false, HelpMessage = HelpMessages.AsJob)]
51+
public SwitchParameter AsJob { get; set; }
52+
53+
public override void ExecuteCmdlet()
54+
{
55+
if (this.IsParameterBound(c => c.WorkspaceObject))
56+
{
57+
this.WorkspaceName = this.WorkspaceObject.Name;
58+
}
59+
60+
var fileExtension = ".ipynb";
61+
if (this.IsParameterBound(c => c.InputObject))
62+
{
63+
WriteToFile(this.InputObject);
64+
WriteObject(new FileInfo(Path.Combine(this.OutputFolder, this.InputObject.Name + fileExtension)));
65+
}
66+
else
67+
{
68+
if (this.IsParameterBound(c => c.Name))
69+
{
70+
var notebook = new PSNotebookResource(SynapseAnalyticsClient.GetNotebook(this.Name), this.WorkspaceName);
71+
WriteToFile(notebook);
72+
WriteObject(new FileInfo(Path.Combine(this.OutputFolder, notebook.Name + fileExtension)));
73+
}
74+
else
75+
{
76+
var infoList = new List<FileInfo>();
77+
var notebooks = SynapseAnalyticsClient.GetNotebooksByWorkspace()
78+
.Select(element => new PSNotebookResource(element, this.WorkspaceName));
79+
foreach (var notebook in notebooks)
80+
{
81+
WriteToFile(notebook);
82+
infoList.Add(new FileInfo(Path.Combine(this.OutputFolder, notebook.Name + fileExtension)));
83+
}
84+
WriteObject(infoList, true);
85+
}
86+
}
87+
}
88+
89+
private void WriteToFile(PSNotebookResource notebook)
90+
{
91+
string json = JsonConvert.SerializeObject(notebook.Properties, Formatting.Indented);
92+
File.WriteAllText(Path.Combine(this.OutputFolder, notebook.Name + ".ipynb"), json);
93+
}
94+
}
95+
}

0 commit comments

Comments
 (0)