Skip to content

Commit 3b131a3

Browse files
markcowlmarkcowl
authored
Merge pull request Azure#10177 from jackmagic313/jikma/adddataflow
[Data Factory] Add data flow and debug session cmdlets
2 parents d12dfeb + bf48ca8 commit 3b131a3

File tree

39 files changed

+6405
-4
lines changed

39 files changed

+6405
-4
lines changed

‎src/DataFactory/DataFactoryV2.Test/DataFactoryV2.Test.csproj

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@
1111
</PropertyGroup>
1212

1313
<ItemGroup>
14-
<PackageReference Include="Microsoft.Azure.Management.DataFactory" Version="4.1.3" />
14+
<PackageReference Include="Microsoft.Azure.Management.DataFactory" Version="4.2.0" />
1515
<PackageReference Include="Microsoft.DataTransfer.Gateway.Encryption" Version="3.11.6898.1" />
1616
</ItemGroup>
1717

‎src/DataFactory/DataFactoryV2.Test/Resources/dataFlow.json

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
{
2+
"properties": {
3+
"type": "MappingDataFlow",
4+
"typeProperties": {
5+
"sources": [
6+
{
7+
"dataset": {
8+
"referenceName": "DelimitedTextInput",
9+
"type": "DatasetReference"
10+
},
11+
"name": "source1"
12+
}
13+
],
14+
"sinks": [
15+
{
16+
"dataset": {
17+
"referenceName": "DelimitedTextOutput",
18+
"type": "DatasetReference"
19+
},
20+
"name": "sink1"
21+
}
22+
],
23+
"transformations": [],
24+
"script": "\n\nsource(allowSchemaDrift: true,\n\tvalidateSchema: false,\n\trowUrlColumn: 'source',\n\twildcardPaths:['Resources-Sample-211UTF*.csv'],\n\tpartitionBy('roundRobin', 2)) ~> source1\nsource1 sink(allowSchemaDrift: true,\n\tvalidateSchema: false,\n\tpartitionFileNames:['test.csv'],\n\tpartitionBy('hash', 1)) ~> sink1"
25+
}
26+
}
27+
}

‎src/DataFactory/DataFactoryV2.Test/Resources/dataFlowDebugPackage.json

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
{
2+
"dataFlow": {
3+
"name": "dataflow5",
4+
"properties": {
5+
"type": "MappingDataFlow",
6+
"typeProperties": {
7+
"sources": [
8+
{
9+
"dataset": {
10+
"referenceName": "DelimitedTextInput",
11+
"type": "DatasetReference"
12+
},
13+
"name": "source1",
14+
"typeProperties": {}
15+
}
16+
],
17+
"sinks": [],
18+
"transformations": [],
19+
"script": "\n\nsource(output(\n\t\tResourceAgencyNum as string,\n\t\tPublicName as string\n\t),\n\tallowSchemaDrift: true,\n\tvalidateSchema: false) ~> source1"
20+
}
21+
}
22+
},
23+
"datasets": [
24+
{
25+
"name": "DelimitedTextInput",
26+
"properties": {
27+
"linkedServiceName": {
28+
"referenceName": "AzureBlobStorage1",
29+
"type": "LinkedServiceReference"
30+
},
31+
"annotations": [],
32+
"type": "DelimitedText",
33+
"typeProperties": {
34+
"location": {
35+
"type": "AzureBlobStorageLocation",
36+
"container": "20192019"
37+
},
38+
"columnDelimiter": ",",
39+
"escapeChar": "\\",
40+
"firstRowAsHeader": true,
41+
"quoteChar": "\""
42+
},
43+
"schema": [
44+
{
45+
"name": "ResourceAgencyNum",
46+
"type": "String"
47+
},
48+
{
49+
"name": "PublicName",
50+
"type": "String"
51+
}
52+
]
53+
},
54+
"type": "Microsoft.DataFactory/factories/datasets"
55+
}
56+
],
57+
"linkedServices": [
58+
{
59+
"name": "AzureBlobStorage1",
60+
"type": "Microsoft.DataFactory/factories/linkedservices",
61+
"properties": {
62+
"annotations": [],
63+
"type": "AzureBlobStorage",
64+
"typeProperties": {
65+
"connectionString": "DefaultEndpointsProtocol=https;AccountName=name;AccountKey=key;EndpointSuffix=core.windows.net"
66+
}
67+
}
68+
}
69+
],
70+
"debugSettings": {
71+
"sourceSettings": [
72+
{
73+
"sourceName": "source1",
74+
"rowLimit": 1000
75+
}
76+
]
77+
},
78+
"sessionId": "4f988caf-e765-47d2-82cd-430334a6b135",
79+
"userSessionId": "82de11be-9e33-4371-a841-fabe8f43db03",
80+
"incrementalDebug": false
81+
}

‎src/DataFactory/DataFactoryV2.Test/ScenarioTests/DataFlowDebugSessionTests.cs

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
// ----------------------------------------------------------------------------------
2+
//
3+
// Copyright Microsoft Corporation
4+
// Licensed under the Apache License, Version 2.0 (the "License");
5+
// you may not use this file except in compliance with the License.
6+
// You may obtain a copy of the License at
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
// Unless required by applicable law or agreed to in writing, software
9+
// distributed under the License is distributed on an "AS IS" BASIS,
10+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11+
// See the License for the specific language governing permissions and
12+
// limitations under the License.
13+
// ----------------------------------------------------------------------------------
14+
15+
using Microsoft.Azure.ServiceManagement.Common.Models;
16+
using Microsoft.WindowsAzure.Commands.ScenarioTest;
17+
using Xunit;
18+
19+
namespace Microsoft.Azure.Commands.DataFactoryV2.Test
20+
{
21+
public class DataFlowDebugSessionTests : DataFactoriesScenarioTestsBase
22+
{
23+
public XunitTracingInterceptor _logger;
24+
25+
public DataFlowDebugSessionTests(Xunit.Abstractions.ITestOutputHelper output)
26+
{
27+
_logger = new XunitTracingInterceptor(output);
28+
XunitTracingInterceptor.AddToContext(_logger);
29+
}
30+
31+
[Fact]
32+
[Trait(Category.AcceptanceType, Category.CheckIn)]
33+
public void TestDataFlowDebugScenario()
34+
{
35+
RunPowerShellTest(_logger, "Test-DataFlowDebugScenario");
36+
}
37+
}
38+
}

‎src/DataFactory/DataFactoryV2.Test/ScenarioTests/DataFlowDebugSessionTests.ps1

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
2+
3+
# ----------------------------------------------------------------------------------
4+
#
5+
# Copyright Microsoft Corporation
6+
# Licensed under the Apache License, Version 2.0 (the "License");
7+
# you may not use this file except in compliance with the License.
8+
# You may obtain a copy of the License at
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
# ----------------------------------------------------------------------------------
16+
17+
<#
18+
.SYNOPSIS
19+
E2E scenario for data flow debug workflow.
20+
#>
21+
function Test-DataFlowDebugScenario
22+
{
23+
$dfname = Get-DataFactoryName
24+
$rgname = Get-ResourceGroupName
25+
$rglocation = Get-ProviderLocation ResourceManagement
26+
$dflocation = Get-ProviderLocation DataFactoryManagement
27+
28+
New-AzResourceGroup -Name $rgname -Location $rglocation -Force
29+
30+
try
31+
{
32+
$df = Set-AzDataFactoryV2 -ResourceGroupName $rgname -Name $dfname -Location $dflocation -Force
33+
34+
# Start one debug session
35+
$session = Start-AzDataFactoryV2DataFlowDebugSession -DataFactory $df
36+
37+
# Get all debug sessions and check the count
38+
$list = Get-AzDataFactoryV2DataFlowDebugSession -DataFactory $df
39+
Assert-AreEqual 1 $list.Count
40+
41+
# Add data flow debug package
42+
Add-AzDataFactoryV2DataFlowDebugSessionPackage -DataFactory $df -PackageFile .\Resources\dataFlowDebugPackage.json -SessionId $session.SessionId
43+
44+
# Execute debug command
45+
$result = Invoke-AzDataFactoryV2DataFlowDebugSessionCommand -DataFactory $df -Command executePreviewQuery -SessionId $session.SessionId -StreamName source1 -RowLimit 100
46+
Assert-AreEqual 'Succeeded' $result.Status
47+
Assert-NotNull $result.Data
48+
49+
# Remove session eventually
50+
Stop-AzDataFactoryV2DataFlowDebugSession -DataFactory $df -SessionId $session.SessionId -Force
51+
$newList = Get-AzDataFactoryV2DataFlowDebugSession -DataFactory $df
52+
Assert-AreEqual 0 $newList.Count
53+
}
54+
finally
55+
{
56+
CleanUp $rgname $dfname
57+
}
58+
}

‎src/DataFactory/DataFactoryV2.Test/ScenarioTests/DataFlowTests.cs

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
// ----------------------------------------------------------------------------------
2+
//
3+
// Copyright Microsoft Corporation
4+
// Licensed under the Apache License, Version 2.0 (the "License");
5+
// you may not use this file except in compliance with the License.
6+
// You may obtain a copy of the License at
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
// Unless required by applicable law or agreed to in writing, software
9+
// distributed under the License is distributed on an "AS IS" BASIS,
10+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11+
// See the License for the specific language governing permissions and
12+
// limitations under the License.
13+
// ----------------------------------------------------------------------------------
14+
15+
using Microsoft.Azure.ServiceManagement.Common.Models;
16+
using Microsoft.WindowsAzure.Commands.ScenarioTest;
17+
using Xunit;
18+
19+
namespace Microsoft.Azure.Commands.DataFactoryV2.Test
20+
{
21+
public class DataFlowTests : DataFactoriesScenarioTestsBase
22+
{
23+
public XunitTracingInterceptor _logger;
24+
25+
public DataFlowTests(Xunit.Abstractions.ITestOutputHelper output)
26+
{
27+
_logger = new XunitTracingInterceptor(output);
28+
XunitTracingInterceptor.AddToContext(_logger);
29+
}
30+
31+
[Fact]
32+
[Trait(Category.AcceptanceType, Category.CheckIn)]
33+
public void TestDataFlow()
34+
{
35+
RunPowerShellTest(_logger, "Test-DataFlow");
36+
}
37+
}
38+
}

‎src/DataFactory/DataFactoryV2.Test/ScenarioTests/DataFlowTests.ps1

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
2+
3+
# ----------------------------------------------------------------------------------
4+
#
5+
# Copyright Microsoft Corporation
6+
# Licensed under the Apache License, Version 2.0 (the "License");
7+
# you may not use this file except in compliance with the License.
8+
# You may obtain a copy of the License at
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
# ----------------------------------------------------------------------------------
16+
17+
<#
18+
.SYNOPSIS
19+
Creates a sample data flow with all of its dependencies. Then deletes the data flow with piping.
20+
#>
21+
function Test-DataFlow
22+
{
23+
$dfname = Get-DataFactoryName
24+
$rgname = Get-ResourceGroupName
25+
$rglocation = Get-ProviderLocation ResourceManagement
26+
$dflocation = Get-ProviderLocation DataFactoryManagement
27+
28+
New-AzResourceGroup -Name $rgname -Location $rglocation -Force
29+
30+
try
31+
{
32+
$df = Set-AzDataFactoryV2 -ResourceGroupName $rgname -Name $dfname -Location $dflocation -Force
33+
34+
$lsName = "foo1"
35+
Set-AzDataFactoryV2LinkedService -ResourceGroupName $rgname -DataFactoryName $dfname -File .\Resources\linkedService.json -Name $lsName -Force
36+
37+
Set-AzDataFactoryV2Dataset -ResourceGroupName $rgname -DataFactoryName $dfname -Name "DelimitedTextInput" -File .\Resources\dataset-dsIn.json -Force
38+
Set-AzDataFactoryV2Dataset -ResourceGroupName $rgname -DataFactoryName $dfname -Name "DelimitedTextOutput" -File .\Resources\dataset-dsIn.json -Force
39+
40+
$dataFlowName = "sample"
41+
$expected = Set-AzDataFactoryV2DataFlow -ResourceGroupName $rgname -Name $dataFlowName -DataFactoryName $dfname -File ".\Resources\dataFlow.json" -Force
42+
$actual = Get-AzDataFactoryV2DataFlow -ResourceGroupName $rgname -Name $dataFlowName -DataFactoryName $dfname
43+
44+
Verify-AdfSubResource $expected $actual $rgname $dfname $dataFlowName
45+
46+
#remove the pipeline through piping
47+
Get-AzDataFactoryV2DataFlow -DataFactory $df -Name $dataFlowName | Remove-AzDataFactoryV2DataFlow -Force
48+
49+
#test the pipeline no longer exists
50+
Assert-ThrowsContains { Get-AzDataFactoryV2DataFlow -DataFactory $df -Name $dataFlowName } "NotFound"
51+
52+
#remove the pipeline again should not throw
53+
Remove-AzDataFactoryV2DataFlow -ResourceGroupName $rgname -DataFactoryName $dfname -Name $dataFlowName -Force
54+
}
55+
finally
56+
{
57+
CleanUp $rgname $dfname
58+
}
59+
}

0 commit comments

Comments
 (0)