Skip to content

Commit b2d1d8b

Browse files
wonnerWan Yang
wonner
and
Wan Yang
authored
[Synapse] Fix deserialization issue for creating Pipeline/Dataset/Trigger (Azure#13643)
* fix deserialization error for artifacts * change return type for artifacts * solve null reference * solve null reference * add internal properties * fix for CI Co-authored-by: Wan Yang <[email protected]>
1 parent 93608d3 commit b2d1d8b

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

49 files changed

+522
-109
lines changed

‎src/Synapse/Synapse/ChangeLog.md

Lines changed: 18 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,22 @@
1818
- Additional information about change #1
1919
-->
2020
## Upcoming Release
21+
* Added support for operation of Advanced Threat Protection settings in SqlPool-level
22+
- Add `Update-AzSynapseSqlPoolAdvancedThreatProtectionSetting` cmdlet
23+
- Add `Get-AzSynapseSqlPoolAdvancedThreatProtectionSetting` cmdlet
24+
- Add `Reset-AzSynapseSqlPoolAdvancedThreatProtectionSetting` cmdlet
25+
* Added support for operation of Vulnerability Assessment settings in SqlPool-level
26+
- Add `Update-AzSynapseSqlPoolVulnerabilityAssessmentSetting` cmdlet
27+
- Add `Get-AzSynapseSqlPoolVulnerabilityAssessmentSetting` cmdlet
28+
- Add `Reset-AzSynapseSqlPoolVulnerabilityAssessmentSetting` cmdlet
29+
* Added support for operation of SQL Advanced Data Security
30+
- Add `Enable-AzSynapseSqlAdvancedDataSecurity` cmdlet
31+
- Add `Disable-AzSynapseSqlAdvancedDataSecurity` cmdlet
32+
- Add `Get-AzSynapseSqlAdvancedDataSecurityPolicy` cmdlet
33+
* Added support for operation of Transparent Data Encryption in SqlPool-level
34+
- Add `Get-AzSynapseSqlPoolTransparentDataEncryption` cmdlet
35+
- Add `Set-AzSynapseSqlPoolTransparentDataEncryption` cmdlet
36+
* Fixed deserialization error when create Pipeline/Dataset/Trigger through DefinitionFile
2137

2238
## Version 0.5.0
2339
* Added support for operation of Synapse SQL Pool Restore Point
@@ -30,32 +46,19 @@
3046
- Add `Set-AzSynapseSqlPoolAuditSetting` cmdlet
3147
- Add `Get-AzSynapseSqlPoolAuditSetting` cmdlet
3248
- Add `Reset-AzSynapseSqlPoolAuditSetting` cmdlet
33-
* Added support for operation of Advanced Threat Protection settings in Workspace-level and SqlPool-level
49+
* Added support for operation of Advanced Threat Protection settings in Workspace-level
3450
- Add `Update-AzSynapseSqlAdvancedThreatProtectionSetting` cmdlet
3551
- Add `Get-AzSynapseSqlAdvancedThreatProtectionSetting` cmdlet
3652
- Add `Reset-AzSynapseSqlAdvancedThreatProtectionSetting` cmdlet
37-
- Add `Update-AzSynapseSqlPoolAdvancedThreatProtectionSetting` cmdlet
38-
- Add `Get-AzSynapseSqlPoolAdvancedThreatProtectionSetting` cmdlet
39-
- Add `Reset-AzSynapseSqlPoolAdvancedThreatProtectionSetting` cmdlet
40-
* Added support for operation of Vulnerability Assessment settings in Workspace-level and SqlPool-level
53+
* Added support for operation of Vulnerability Assessment settings in Workspace-level
4154
- Add `Update-AzSynapseSqlVulnerabilityAssessmentSetting` cmdlet
4255
- Add `Get-AzSynapseSqlVulnerabilityAssessmentSetting` cmdlet
4356
- Add `Reset-AzSynapseSqlVulnerabilityAssessmentSetting` cmdlet
44-
- Add `Update-AzSynapseSqlPoolVulnerabilityAssessmentSetting` cmdlet
45-
- Add `Get-AzSynapseSqlPoolVulnerabilityAssessmentSetting` cmdlet
46-
- Add `Reset-AzSynapseSqlPoolVulnerabilityAssessmentSetting` cmdlet
4757
* Added support for operation of SQL Active Directory admin
4858
- Add `Set-AzSynapseSqlActiveDirectoryAdministrator` cmdlet
4959
- Add `Get-AzSynapseSqlActiveDirectoryAdministrator` cmdlet
5060
- Add `Remove-AzSynapseSqlActiveDirectoryAdministrator` cmdlet
5161
* Fixed Null Reference Exception when submit spark job.
52-
* Added support for operation of SQL Advanced Data Security
53-
- Add `Enable-AzSynapseSqlAdvancedDataSecurity` cmdlet
54-
- Add `Disable-AzSynapseSqlAdvancedDataSecurity` cmdlet
55-
- Add `Get-AzSynapseSqlAdvancedDataSecurityPolicy` cmdlet
56-
* Added support for operation of Transparent Data Encryption in SqlPool-level
57-
- Add `Get-AzSynapseSqlPoolTransparentDataEncryption` cmdlet
58-
- Add `Set-AzSynapseSqlPoolTransparentDataEncryption` cmdlet
5962

6063
## Version 0.4.0
6164
* Add `-Force` to all Remove cmdlets

‎src/Synapse/Synapse/Models/Activity/PSActivityPolicy.cs

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
using Azure.Analytics.Synapse.Artifacts.Models;
2+
using Microsoft.WindowsAzure.Commands.Utilities.Common;
3+
using Newtonsoft.Json;
4+
using System.Collections.Generic;
5+
6+
namespace Microsoft.Azure.Commands.Synapse.Models
7+
{
8+
public class PSActivityPolicy
9+
{
10+
public PSActivityPolicy() { }
11+
12+
[JsonProperty(PropertyName = "timeout")]
13+
public object Timeout { get; set; }
14+
15+
[JsonProperty(PropertyName = "retry")]
16+
public object Retry { get; set; }
17+
18+
[JsonProperty(PropertyName = "retryIntervalInSeconds")]
19+
public int? RetryIntervalInSeconds { get; set; }
20+
21+
[JsonProperty(PropertyName = "secureInput")]
22+
public bool? SecureInput { get; set; }
23+
24+
[JsonProperty(PropertyName = "secureOutput")]
25+
public bool? SecureOutput { get; set; }
26+
27+
[JsonExtensionData]
28+
public IDictionary<string, object> AdditionalProperties { get; set; }
29+
30+
public ActivityPolicy ToSdkObject()
31+
{
32+
var policy = new ActivityPolicy()
33+
{
34+
Timeout = this.Timeout,
35+
Retry = this.Retry,
36+
RetryIntervalInSeconds = this.RetryIntervalInSeconds,
37+
SecureInput = this.SecureInput,
38+
SecureOutput = this.SecureOutput
39+
};
40+
this.AdditionalProperties?.ForEach(item => policy.Add(item.Key, item.Value));
41+
return policy;
42+
}
43+
}
44+
}

‎src/Synapse/Synapse/Models/Activity/PSAzureDataExplorerCommandActivity.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -75,7 +75,7 @@ public override Activity ToSdkObject()
7575
var activity = new AzureDataExplorerCommandActivity(this.Name, this.Command);
7676
activity.CommandTimeout = this.CommandTimeout;
7777
activity.LinkedServiceName = this.LinkedServiceName;
78-
activity.Policy = this.Policy;
78+
activity.Policy = this.Policy?.ToSdkObject();
7979
SetProperties(activity);
8080
return activity;
8181
}

‎src/Synapse/Synapse/Models/Activity/PSAzureFunctionActivity.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -95,7 +95,7 @@ public override Activity ToSdkObject()
9595
activity.Headers = this.Headers;
9696
activity.Body = this.Body;
9797
activity.LinkedServiceName = this.LinkedServiceName;
98-
activity.Policy = this.Policy;
98+
activity.Policy = this.Policy?.ToSdkObject();
9999
SetProperties(activity);
100100
return activity;
101101
}

‎src/Synapse/Synapse/Models/Activity/PSAzureMLBatchExecutionActivity.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -88,7 +88,7 @@ public override Activity ToSdkObject()
8888
this.WebServiceOutputs?.ForEach(item => activity.WebServiceOutputs.Add(item));
8989
this.WebServiceInputs?.ForEach(item => activity.WebServiceInputs.Add(item));
9090
activity.LinkedServiceName = this.LinkedServiceName;
91-
activity.Policy = this.Policy;
91+
activity.Policy = this.Policy?.ToSdkObject();
9292
SetProperties(activity);
9393
return activity;
9494
}

‎src/Synapse/Synapse/Models/Activity/PSAzureMLExecutePipelineActivity.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -107,7 +107,7 @@ public override Activity ToSdkObject()
107107
activity.MlParentRunId = this.MlParentRunId;
108108
activity.ContinueOnStepFailure = this.ContinueOnStepFailure;
109109
activity.LinkedServiceName = this.LinkedServiceName;
110-
activity.Policy = this.Policy;
110+
activity.Policy = this.Policy?.ToSdkObject();
111111
SetProperties(activity);
112112
return activity;
113113
}

‎src/Synapse/Synapse/Models/Activity/PSAzureMLUpdateResourceActivity.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -89,7 +89,7 @@ public override Activity ToSdkObject()
8989
{
9090
var activity = new AzureMLUpdateResourceActivity(this.Name, this.TrainedModelName, this.TrainedModelLinkedServiceName, this.TrainedModelFilePath);
9191
activity.LinkedServiceName = this.LinkedServiceName;
92-
activity.Policy = this.Policy;
92+
activity.Policy = this.Policy?.ToSdkObject();
9393
SetProperties(activity);
9494
return activity;
9595
}

‎src/Synapse/Synapse/Models/Activity/PSCopyActivity.cs

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -43,13 +43,13 @@ public PSCopyActivity()
4343
/// Gets or sets copy activity source.
4444
/// </summary>
4545
[JsonProperty(PropertyName = "typeProperties.source")]
46-
public CopySource Source { get; set; }
46+
public PSCopySource Source { get; set; }
4747

4848
/// <summary>
4949
/// Gets or sets copy activity sink.
5050
/// </summary>
5151
[JsonProperty(PropertyName = "typeProperties.sink")]
52-
public CopySink Sink { get; set; }
52+
public PSCopySink Sink { get; set; }
5353

5454
/// <summary>
5555
/// Gets or sets copy activity translator. If not specified, tabular
@@ -71,7 +71,7 @@ public PSCopyActivity()
7171
/// is true.
7272
/// </summary>
7373
[JsonProperty(PropertyName = "typeProperties.stagingSettings")]
74-
public StagingSettings StagingSettings { get; set; }
74+
public PSStagingSettings StagingSettings { get; set; }
7575

7676
/// <summary>
7777
/// Gets or sets maximum number of concurrent sessions opened on the
@@ -101,7 +101,7 @@ public PSCopyActivity()
101101
/// EnableSkipIncompatibleRow is true.
102102
/// </summary>
103103
[JsonProperty(PropertyName = "typeProperties.redirectIncompatibleRowSettings")]
104-
public RedirectIncompatibleRowSettings RedirectIncompatibleRowSettings { get; set; }
104+
public PSRedirectIncompatibleRowSettings RedirectIncompatibleRowSettings { get; set; }
105105

106106
/// <summary>
107107
/// Gets or sets preserve Rules.
@@ -148,20 +148,20 @@ public override void Validate()
148148

149149
public override Activity ToSdkObject()
150150
{
151-
var activity = new CopyActivity(this.Name, this.Source, this.Sink);
151+
var activity = new CopyActivity(this.Name, this.Source?.ToSdkObject(), this.Sink?.ToSdkObject());
152152
this.Inputs?.ForEach(item => activity.Inputs.Add(item));
153153
this.Outputs?.ForEach(item => activity.Outputs.Add(item));
154154
activity.Translator = this.Translator;
155155
activity.EnableStaging = this.EnableStaging;
156-
activity.StagingSettings = this.StagingSettings;
156+
activity.StagingSettings = this.StagingSettings?.ToSdkObject();
157157
activity.ParallelCopies = this.ParallelCopies;
158158
activity.DataIntegrationUnits = this.DataIntegrationUnits;
159159
activity.EnableSkipIncompatibleRow = this.EnableSkipIncompatibleRow;
160-
activity.RedirectIncompatibleRowSettings = this.RedirectIncompatibleRowSettings;
160+
activity.RedirectIncompatibleRowSettings = this.RedirectIncompatibleRowSettings?.ToSdkObject();
161161
this.PreserveRules?.ForEach(item => activity.PreserveRules.Add(item));
162162
this.Preserve?.ForEach(item => activity.Preserve.Add(item));
163163
activity.LinkedServiceName = this.LinkedServiceName;
164-
activity.Policy = this.Policy;
164+
activity.Policy = this.Policy?.ToSdkObject();
165165
SetProperties(activity);
166166
return activity;
167167
}

‎src/Synapse/Synapse/Models/Activity/PSCopySink.cs

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
using Azure.Analytics.Synapse.Artifacts.Models;
2+
using Microsoft.WindowsAzure.Commands.Utilities.Common;
3+
using Newtonsoft.Json;
4+
using System.Collections.Generic;
5+
6+
namespace Microsoft.Azure.Commands.Synapse.Models
7+
{
8+
public class PSCopySink
9+
{
10+
public PSCopySink() { }
11+
12+
[JsonProperty(PropertyName = "writeBatchSize")]
13+
public object WriteBatchSize { get; set; }
14+
15+
[JsonProperty(PropertyName = "writeBatchTimeout")]
16+
public object WriteBatchTimeout { get; set; }
17+
18+
[JsonProperty(PropertyName = "sinkRetryCount")]
19+
public object SinkRetryCount { get; set; }
20+
21+
[JsonProperty(PropertyName = "sinkRetryWait")]
22+
public object SinkRetryWait { get; set; }
23+
24+
[JsonProperty(PropertyName = "maxConcurrentConnections")]
25+
public object MaxConcurrentConnections { get; set; }
26+
27+
[JsonExtensionData]
28+
public IDictionary<string, object> AdditionalProperties { get; set; }
29+
30+
public CopySink ToSdkObject()
31+
{
32+
var copySink = new CopySink()
33+
{
34+
WriteBatchSize = this.WriteBatchSize,
35+
WriteBatchTimeout = this.WriteBatchTimeout,
36+
SinkRetryCount = this.SinkRetryCount,
37+
SinkRetryWait = this.SinkRetryWait,
38+
MaxConcurrentConnections = this.MaxConcurrentConnections
39+
};
40+
this.AdditionalProperties?.ForEach(item => copySink.Add(item.Key, item.Value));
41+
return copySink;
42+
}
43+
}
44+
}

‎src/Synapse/Synapse/Models/Activity/PSCopySource.cs

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
using Azure.Analytics.Synapse.Artifacts.Models;
2+
using Microsoft.WindowsAzure.Commands.Utilities.Common;
3+
using Newtonsoft.Json;
4+
using System.Collections.Generic;
5+
6+
namespace Microsoft.Azure.Commands.Synapse.Models
7+
{
8+
public class PSCopySource
9+
{
10+
public PSCopySource() { }
11+
12+
[JsonProperty(PropertyName = "sourceRetryCount")]
13+
public object SourceRetryCount { get; set; }
14+
15+
[JsonProperty(PropertyName = "sourceRetryWait")]
16+
public object SourceRetryWait { get; set; }
17+
18+
[JsonProperty(PropertyName = "maxConcurrentConnections")]
19+
public object MaxConcurrentConnections { get; set; }
20+
21+
[JsonExtensionData]
22+
public IDictionary<string, object> AdditionalProperties { get; set; }
23+
24+
public CopySource ToSdkObject()
25+
{
26+
var copySource = new CopySource()
27+
{
28+
SourceRetryCount = this.SourceRetryCount,
29+
SourceRetryWait = this.SourceRetryWait,
30+
MaxConcurrentConnections = this.MaxConcurrentConnections
31+
};
32+
this.AdditionalProperties?.ForEach(item => copySource.Add(item.Key, item.Value));
33+
return copySource;
34+
}
35+
}
36+
}

‎src/Synapse/Synapse/Models/Activity/PSCustomActivity.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -105,7 +105,7 @@ public override Activity ToSdkObject()
105105
this.ExtendedProperties?.ForEach(item => activity.ExtendedProperties.Add(item));
106106
activity.RetentionTimeInDays = this.RetentionTimeInDays;
107107
activity.LinkedServiceName = this.LinkedServiceName;
108-
activity.Policy = this.Policy;
108+
activity.Policy = this.Policy?.ToSdkObject();
109109
SetProperties(activity);
110110
return activity;
111111
}

‎src/Synapse/Synapse/Models/Activity/PSDataFlowReference.cs

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
using Azure.Analytics.Synapse.Artifacts.Models;
2+
using Microsoft.WindowsAzure.Commands.Utilities.Common;
3+
using Newtonsoft.Json;
4+
using System.Collections.Generic;
5+
6+
namespace Microsoft.Azure.Commands.Synapse.Models
7+
{
8+
public class PSDataFlowReference
9+
{
10+
public PSDataFlowReference() { }
11+
12+
[JsonProperty(PropertyName = "type")]
13+
public DataFlowReferenceType? Type { get; set; }
14+
15+
[JsonProperty(PropertyName = "referenceName")]
16+
public string ReferenceName { get; set; }
17+
18+
[JsonProperty(PropertyName = "datasetParameters")]
19+
public object DatasetParameters { get; set; }
20+
21+
[JsonExtensionData]
22+
public IDictionary<string, object> AdditionalProperties { get; set; }
23+
24+
public DataFlowReference ToSdkObject()
25+
{
26+
var dataFlowReference = new DataFlowReference(this.Type.GetValueOrDefault(), this.ReferenceName)
27+
{
28+
DatasetParameters = this.DatasetParameters
29+
};
30+
this.AdditionalProperties?.ForEach(item => dataFlowReference.Add(item.Key, item.Value));
31+
return dataFlowReference;
32+
}
33+
}
34+
}

‎src/Synapse/Synapse/Models/Activity/PSDataLakeAnalyticsUsqlActivity.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -119,7 +119,7 @@ public override Activity ToSdkObject()
119119
activity.RuntimeVersion = this.RuntimeVersion;
120120
activity.CompilationMode = this.CompilationMode;
121121
activity.LinkedServiceName = this.LinkedServiceName;
122-
activity.Policy = this.Policy;
122+
activity.Policy = this.Policy?.ToSdkObject();
123123
SetProperties(activity);
124124
return activity;
125125
}

‎src/Synapse/Synapse/Models/Activity/PSDatabricksNotebookActivity.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -84,7 +84,7 @@ public override Activity ToSdkObject()
8484
this.BaseParameters?.ForEach(item => activity.BaseParameters.Add(item));
8585
this.Libraries?.ForEach(item => activity.Libraries.Add(item));
8686
activity.LinkedServiceName = this.LinkedServiceName;
87-
activity.Policy = this.Policy;
87+
activity.Policy = this.Policy?.ToSdkObject();
8888
SetProperties(activity);
8989
return activity;
9090
}

‎src/Synapse/Synapse/Models/Activity/PSDatabricksSparkJarActivity.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -82,7 +82,7 @@ public override Activity ToSdkObject()
8282
this.Parameters?.ForEach(item => activity.Parameters.Add(item));
8383
this.Libraries?.ForEach(item => activity.Libraries.Add(item));
8484
activity.LinkedServiceName = this.LinkedServiceName;
85-
activity.Policy = this.Policy;
85+
activity.Policy = this.Policy?.ToSdkObject();
8686
SetProperties(activity);
8787
return activity;
8888
}

‎src/Synapse/Synapse/Models/Activity/PSDatabricksSparkPythonActivity.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -82,7 +82,7 @@ public override Activity ToSdkObject()
8282
this.Parameters?.ForEach(item => activity.Parameters.Add(item));
8383
this.Libraries?.ForEach(item => activity.Libraries.Add(item));
8484
activity.LinkedServiceName = this.LinkedServiceName;
85-
activity.Policy = this.Policy;
85+
activity.Policy = this.Policy?.ToSdkObject();
8686
SetProperties(activity);
8787
return activity;
8888
}

‎src/Synapse/Synapse/Models/Activity/PSDeleteActivity.cs

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -66,7 +66,7 @@ public PSDeleteActivity()
6666
/// enableLogging is true.
6767
/// </summary>
6868
[JsonProperty(PropertyName = "typeProperties.logStorageSettings")]
69-
public LogStorageSettings LogStorageSettings { get; set; }
69+
public PSLogStorageSettings LogStorageSettings { get; set; }
7070

7171
/// <summary>
7272
/// Gets or sets delete activity dataset reference.
@@ -99,9 +99,9 @@ public override Activity ToSdkObject()
9999
activity.Recursive = this.Recursive;
100100
activity.MaxConcurrentConnections = this.MaxConcurrentConnections;
101101
activity.EnableLogging = this.EnableLogging;
102-
activity.LogStorageSettings = this.LogStorageSettings;
102+
activity.LogStorageSettings = this.LogStorageSettings?.ToSdkObject();
103103
activity.LinkedServiceName = this.LinkedServiceName;
104-
activity.Policy = this.Policy;
104+
activity.Policy = this.Policy?.ToSdkObject();
105105
SetProperties(activity);
106106
return activity;
107107
}

0 commit comments

Comments
 (0)