Skip to content

Commit a349712

Browse files
bashaheebashahee
authored
Enhance pefrormance of Data Classification Cmdlets. (#11629)
* Enhance pefrormance of Data Classification Cmdlets. * Update ChangeLog.md
1 parent d626e74 commit a349712

10 files changed

+1615
-2030
lines changed

‎src/Sql/Sql.Test/SessionRecords/Microsoft.Azure.Commands.Sql.Test.ScenarioTests.DataClassificationTests/TestDataClassificationOnSqlDatabase.json

Lines changed: 737 additions & 1253 deletions
Large diffs are not rendered by default.

‎src/Sql/Sql.Test/SessionRecords/Microsoft.Azure.Commands.Sql.Test.ScenarioTests.DataClassificationTests/TestEnableDisableRecommendationsOnSqlDatabase.json

Lines changed: 570 additions & 690 deletions
Large diffs are not rendered by default.

‎src/Sql/Sql/ChangeLog.md

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,15 @@
1818
- Additional information about change #1
1919
-->
2020
## Upcoming Release
21+
* Enhance performance of:
22+
- Set-AzSqlDatabaseSensitivityClassification.
23+
- Set-AzSqlInstanceDatabaseSensitivityClassification.
24+
- Remove-AzSqlDatabaseSensitivityClassification.
25+
- Remove-AzSqlInstanceDatabaseSensitivityClassification.
26+
- Enable-AzSqlDatabaseSensitivityRecommendation.
27+
- Enable-AzSqlInstanceDatabaseSensitivityRecommendation.
28+
- Disable-AzSqlDatabaseSensitivityRecommendation.
29+
- Disable-AzSqlInstanceDatabaseSensitivityRecommendation.
2130

2231
## Version 2.6.0
2332
* Added cmdlets `Get-AzSqlInstanceOperation` and `Stop-AzSqlInstanceOperation`

‎src/Sql/Sql/DataClassification/Cmdlet/DisableAzSqlInstanceDatabaseSensitivityRecommendation.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,7 @@ protected override ManagedDatabaseSensitivityClassificationModel GetEntity()
5757

5858
protected override ManagedDatabaseSensitivityClassificationModel PersistChanges(ManagedDatabaseSensitivityClassificationModel entity)
5959
{
60-
ModelAdapter.DisableManagedDatabaseSensitivityRecommendations(entity);
60+
ModelAdapter.DisableSensitivityRecommendations(entity);
6161
return null;
6262
}
6363
}

‎src/Sql/Sql/DataClassification/Cmdlet/EnableAzSqlInstanceDatabaseSensitivityRecommendation.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,7 @@ protected override ManagedDatabaseSensitivityClassificationModel GetEntity()
5757

5858
protected override ManagedDatabaseSensitivityClassificationModel PersistChanges(ManagedDatabaseSensitivityClassificationModel entity)
5959
{
60-
ModelAdapter.EnableManagedDatabaseSensitivityRecommendations(entity);
60+
ModelAdapter.EnableSensitivityRecommendations(entity);
6161
return null;
6262
}
6363
}

‎src/Sql/Sql/DataClassification/Cmdlet/RemoveAzSqlInstanceDatabaseSensitivityClassification.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,7 @@ protected override ManagedDatabaseSensitivityClassificationModel GetEntity()
5757

5858
protected override ManagedDatabaseSensitivityClassificationModel PersistChanges(ManagedDatabaseSensitivityClassificationModel entity)
5959
{
60-
ModelAdapter.RemoveManagedDatabaseSensitivityLabels(entity);
60+
ModelAdapter.RemoveSensitivityLabels(entity);
6161
return null;
6262
}
6363
}

‎src/Sql/Sql/DataClassification/Services/DataClassificationAdapter.cs

Lines changed: 153 additions & 82 deletions
Original file line numberDiff line numberDiff line change
@@ -38,38 +38,167 @@ public DataClassificationAdapter(IAzureContext context)
3838

3939
internal void SetSensitivityLabels(SqlDatabaseSensitivityClassificationModel model)
4040
{
41-
ModifySensitivityLabels(model, sensitivityLabelModel => Communicator.SetSensitivityLabel(
41+
SplitSensitivityLabelsIntoListsAndPatch(model.SensitivityLabels,
42+
list => Communicator.PatchSensitivityLabels(
4243
model.ResourceGroupName,
4344
model.ServerName,
4445
model.DatabaseName,
45-
sensitivityLabelModel.SchemaName,
46-
sensitivityLabelModel.TableName,
47-
sensitivityLabelModel.ColumnName,
48-
ToSensitivityLabel(sensitivityLabelModel)));
46+
new PatchOperations
47+
{
48+
Operations = list.Select(sensitivityLabelModel => new PatchOperation
49+
{
50+
OperationKind = PatchOperationKind.Set,
51+
Schema = sensitivityLabelModel.SchemaName,
52+
Table = sensitivityLabelModel.TableName,
53+
Column = sensitivityLabelModel.ColumnName,
54+
SensitivityLabel = ToSensitivityLabel(sensitivityLabelModel)
55+
}).ToList()
56+
}));
4957
}
5058

5159
internal void SetSensitivityLabels(ManagedDatabaseSensitivityClassificationModel model)
5260
{
53-
ModifySensitivityLabels(model, sensitivityLabelModel => Communicator.SetManagedDatabaseSensitivityLabel(
61+
SplitSensitivityLabelsIntoListsAndPatch(model.SensitivityLabels,
62+
list => Communicator.PatchManagedDatabaseSensitivityLabels(
63+
model.ResourceGroupName,
64+
model.InstanceName,
65+
model.DatabaseName,
66+
new PatchOperations
67+
{
68+
Operations = list.Select(sensitivityLabelModel => new PatchOperation
69+
{
70+
OperationKind = PatchOperationKind.Set,
71+
Schema = sensitivityLabelModel.SchemaName,
72+
Table = sensitivityLabelModel.TableName,
73+
Column = sensitivityLabelModel.ColumnName,
74+
SensitivityLabel = ToSensitivityLabel(sensitivityLabelModel)
75+
}).ToList()
76+
}));
77+
}
78+
79+
internal void RemoveSensitivityLabels(SqlDatabaseSensitivityClassificationModel model)
80+
{
81+
SplitSensitivityLabelsIntoListsAndPatch(model.SensitivityLabels,
82+
list => Communicator.PatchSensitivityLabels(
83+
model.ResourceGroupName,
84+
model.ServerName,
85+
model.DatabaseName,
86+
new PatchOperations
87+
{
88+
Operations = list.Select(sensitivityLabelModel => new PatchOperation
89+
{
90+
OperationKind = PatchOperationKind.Remove,
91+
Schema = sensitivityLabelModel.SchemaName,
92+
Table = sensitivityLabelModel.TableName,
93+
Column = sensitivityLabelModel.ColumnName,
94+
}).ToList()
95+
}));
96+
}
97+
98+
internal void RemoveSensitivityLabels(ManagedDatabaseSensitivityClassificationModel model)
99+
{
100+
SplitSensitivityLabelsIntoListsAndPatch(model.SensitivityLabels,
101+
list => Communicator.PatchManagedDatabaseSensitivityLabels(
102+
model.ResourceGroupName,
103+
model.InstanceName,
104+
model.DatabaseName,
105+
new PatchOperations
106+
{
107+
Operations = list.Select(sensitivityLabelModel => new PatchOperation
108+
{
109+
OperationKind = PatchOperationKind.Remove,
110+
Schema = sensitivityLabelModel.SchemaName,
111+
Table = sensitivityLabelModel.TableName,
112+
Column = sensitivityLabelModel.ColumnName,
113+
}).ToList()
114+
}));
115+
}
116+
117+
internal void EnableSensitivityRecommendations(SqlDatabaseSensitivityClassificationModel model)
118+
{
119+
SplitSensitivityLabelsIntoListsAndPatch(model.SensitivityLabels,
120+
list => Communicator.PatchSensitivityRecommendations(
121+
model.ResourceGroupName,
122+
model.ServerName,
123+
model.DatabaseName,
124+
new PatchOperations
125+
{
126+
Operations = list.Select(sensitivityLabelModel => new PatchOperation
127+
{
128+
OperationKind = PatchOperationKind.Enable,
129+
Schema = sensitivityLabelModel.SchemaName,
130+
Table = sensitivityLabelModel.TableName,
131+
Column = sensitivityLabelModel.ColumnName,
132+
}).ToList()
133+
}));
134+
}
135+
136+
internal void EnableSensitivityRecommendations(ManagedDatabaseSensitivityClassificationModel model)
137+
{
138+
SplitSensitivityLabelsIntoListsAndPatch(model.SensitivityLabels,
139+
list => Communicator.PatchManagedDatabaseSensitivityRecommendations(
140+
model.ResourceGroupName,
141+
model.InstanceName,
142+
model.DatabaseName,
143+
new PatchOperations
144+
{
145+
Operations = list.Select(sensitivityLabelModel => new PatchOperation
146+
{
147+
OperationKind = PatchOperationKind.Enable,
148+
Schema = sensitivityLabelModel.SchemaName,
149+
Table = sensitivityLabelModel.TableName,
150+
Column = sensitivityLabelModel.ColumnName,
151+
}).ToList()
152+
}));
153+
}
154+
155+
internal void DisableSensitivityRecommendations(SqlDatabaseSensitivityClassificationModel model)
156+
{
157+
SplitSensitivityLabelsIntoListsAndPatch(model.SensitivityLabels,
158+
list => Communicator.PatchSensitivityRecommendations(
159+
model.ResourceGroupName,
160+
model.ServerName,
161+
model.DatabaseName,
162+
new PatchOperations
163+
{
164+
Operations = list.Select(sensitivityLabelModel => new PatchOperation
165+
{
166+
OperationKind = PatchOperationKind.Disable,
167+
Schema = sensitivityLabelModel.SchemaName,
168+
Table = sensitivityLabelModel.TableName,
169+
Column = sensitivityLabelModel.ColumnName,
170+
}).ToList()
171+
}));
172+
}
173+
internal void DisableSensitivityRecommendations(ManagedDatabaseSensitivityClassificationModel model)
174+
{
175+
SplitSensitivityLabelsIntoListsAndPatch(model.SensitivityLabels,
176+
list => Communicator.PatchManagedDatabaseSensitivityRecommendations(
54177
model.ResourceGroupName,
55178
model.InstanceName,
56179
model.DatabaseName,
57-
sensitivityLabelModel.SchemaName,
58-
sensitivityLabelModel.TableName,
59-
sensitivityLabelModel.ColumnName,
60-
ToSensitivityLabel(sensitivityLabelModel)));
180+
new PatchOperations
181+
{
182+
Operations = list.Select(sensitivityLabelModel => new PatchOperation
183+
{
184+
OperationKind = PatchOperationKind.Disable,
185+
Schema = sensitivityLabelModel.SchemaName,
186+
Table = sensitivityLabelModel.TableName,
187+
Column = sensitivityLabelModel.ColumnName,
188+
}).ToList()
189+
}));
61190
}
62191

63-
internal void ModifySensitivityLabels(SensitivityClassificationModel model,
64-
Action<SensitivityLabelModel> modifySensitivityLabel)
192+
internal void SplitSensitivityLabelsIntoListsAndPatch(List<SensitivityLabelModel> sensitivityLabelsToModify,
193+
Action<List<SensitivityLabelModel>> patchSensitivityLabels)
65194
{
66195
ConcurrentQueue<Exception> exceptions = new ConcurrentQueue<Exception>();
67-
Parallel.ForEach<SensitivityLabelModel>(model.SensitivityLabels,
68-
sensitivityLabelModel =>
196+
Parallel.ForEach<List<SensitivityLabelModel>>(SplitList(sensitivityLabelsToModify),
197+
sensitivityLabelsList =>
69198
{
70199
try
71200
{
72-
modifySensitivityLabel(sensitivityLabelModel);
201+
patchSensitivityLabels(sensitivityLabelsList);
73202
}
74203
catch (Exception e)
75204
{
@@ -86,28 +215,6 @@ internal void ModifySensitivityLabels(SensitivityClassificationModel model,
86215
}
87216
}
88217

89-
internal void RemoveSensitivityLabels(SqlDatabaseSensitivityClassificationModel model)
90-
{
91-
ModifySensitivityLabels(model, sensitivityLabelModel => Communicator.DeleteSensitivityLabel(
92-
model.ResourceGroupName,
93-
model.ServerName,
94-
model.DatabaseName,
95-
sensitivityLabelModel.SchemaName,
96-
sensitivityLabelModel.TableName,
97-
sensitivityLabelModel.ColumnName));
98-
}
99-
100-
internal void RemoveManagedDatabaseSensitivityLabels(ManagedDatabaseSensitivityClassificationModel model)
101-
{
102-
ModifySensitivityLabels(model, sensitivityLabelModel => Communicator.DeleteManagedDatabaseSensitivityLabel(
103-
model.ResourceGroupName,
104-
model.InstanceName,
105-
model.DatabaseName,
106-
sensitivityLabelModel.SchemaName,
107-
sensitivityLabelModel.TableName,
108-
sensitivityLabelModel.ColumnName));
109-
}
110-
111218
internal List<SensitivityLabelModel> GetCurrentSensitivityLabel(
112219
string resourceGroupName, string serverName, string databaseName,
113220
string schemaName, string tableName, string columnName)
@@ -119,7 +226,6 @@ internal List<SensitivityLabelModel> GetCurrentSensitivityLabel(
119226
internal List<SensitivityLabelModel> GetCurrentSensitivityLabels(
120227
string resourceGroupName, string serverName, string databaseName)
121228
{
122-
var n = Context.Environment.Name;
123229
return ToSensitivityLabelModelList(Communicator.GetCurrentSensitivityLabels(resourceGroupName, serverName, databaseName));
124230
}
125231

@@ -160,50 +266,6 @@ internal async Task<InformationProtectionPolicy> RetrieveInformationProtectionPo
160266
: InformationProtectionPolicy.DefaultInformationProtectionPolicy;
161267
}
162268

163-
internal void EnableSensitivityRecommendations(SqlDatabaseSensitivityClassificationModel model)
164-
{
165-
ModifySensitivityLabels(model, sensitivityLabelModel => Communicator.EnableSensitivityRecommendation(
166-
model.ResourceGroupName,
167-
model.ServerName,
168-
model.DatabaseName,
169-
sensitivityLabelModel.SchemaName,
170-
sensitivityLabelModel.TableName,
171-
sensitivityLabelModel.ColumnName));
172-
}
173-
174-
internal void DisableSensitivityRecommendations(SqlDatabaseSensitivityClassificationModel model)
175-
{
176-
ModifySensitivityLabels(model, sensitivityLabelModel => Communicator.DisableSensitivityRecommendation(
177-
model.ResourceGroupName,
178-
model.ServerName,
179-
model.DatabaseName,
180-
sensitivityLabelModel.SchemaName,
181-
sensitivityLabelModel.TableName,
182-
sensitivityLabelModel.ColumnName));
183-
}
184-
185-
internal void EnableManagedDatabaseSensitivityRecommendations(ManagedDatabaseSensitivityClassificationModel model)
186-
{
187-
ModifySensitivityLabels(model, sensitivityLabelModel => Communicator.EnableManagedDatabaseSensitivityRecommendation(
188-
model.ResourceGroupName,
189-
model.InstanceName,
190-
model.DatabaseName,
191-
sensitivityLabelModel.SchemaName,
192-
sensitivityLabelModel.TableName,
193-
sensitivityLabelModel.ColumnName));
194-
}
195-
196-
internal void DisableManagedDatabaseSensitivityRecommendations(ManagedDatabaseSensitivityClassificationModel model)
197-
{
198-
ModifySensitivityLabels(model, sensitivityLabelModel => Communicator.DisableManagedDatabaseSensitivityRecommendation(
199-
model.ResourceGroupName,
200-
model.InstanceName,
201-
model.DatabaseName,
202-
sensitivityLabelModel.SchemaName,
203-
sensitivityLabelModel.TableName,
204-
sensitivityLabelModel.ColumnName));
205-
}
206-
207269
private List<SensitivityLabelModel> ToSensitivityLabelModelList(List<SensitivityLabel> sensitivityLabels)
208270
{
209271
return sensitivityLabels.Select(l => ToSensitivityLabelModel(l)).ToList();
@@ -297,5 +359,14 @@ private static string NullifyStringIfEmpty(string s)
297359

298360
return sensitivityLabelRank;
299361
}
362+
363+
private static IEnumerable<List<T>> SplitList<T>(List<T> elements)
364+
{
365+
const int ListSize = 5000;
366+
for (int i = 0; i < elements.Count; i += ListSize)
367+
{
368+
yield return elements.GetRange(i, Math.Min(ListSize, elements.Count - i));
369+
}
370+
}
300371
}
301372
}

0 commit comments

Comments
 (0)