[Data Factory] Add data flow and debug session cmdlets

src/DataFactory/DataFactoryV2.Test/DataFactoryV2.Test.csproj

@@ -11,7 +11,7 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="Microsoft.Azure.Management.DataFactory" Version="4.1.3" />
+    <PackageReference Include="Microsoft.Azure.Management.DataFactory" Version="4.2.0" />
     <PackageReference Include="Microsoft.DataTransfer.Gateway.Encryption" Version="3.11.6898.1" />
   </ItemGroup>
 

src/DataFactory/DataFactoryV2.Test/Resources/dataFlow.json

@@ -0,0 +1,27 @@
+{
+  "properties": {
+    "type": "MappingDataFlow",
+    "typeProperties": {
+      "sources": [
+        {
+          "dataset": {
+            "referenceName": "DelimitedTextInput",
+            "type": "DatasetReference"
+          },
+          "name": "source1"
+        }
+      ],
+      "sinks": [
+        {
+          "dataset": {
+            "referenceName": "DelimitedTextOutput",
+            "type": "DatasetReference"
+          },
+          "name": "sink1"
+        }
+      ],
+      "transformations": [],
+      "script": "\n\nsource(allowSchemaDrift: true,\n\tvalidateSchema: false,\n\trowUrlColumn: 'source',\n\twildcardPaths:['Resources-Sample-211UTF*.csv'],\n\tpartitionBy('roundRobin', 2)) ~> source1\nsource1 sink(allowSchemaDrift: true,\n\tvalidateSchema: false,\n\tpartitionFileNames:['test.csv'],\n\tpartitionBy('hash', 1)) ~> sink1"
+    }
+  }
+}

src/DataFactory/DataFactoryV2.Test/Resources/dataFlowDebugPackage.json

@@ -0,0 +1,81 @@
+{
+  "dataFlow": {
+    "name": "dataflow5",
+    "properties": {
+      "type": "MappingDataFlow",
+      "typeProperties": {
+        "sources": [
+          {
+            "dataset": {
+              "referenceName": "DelimitedTextInput",
+              "type": "DatasetReference"
+            },
+            "name": "source1",
+            "typeProperties": {}
+          }
+        ],
+        "sinks": [],
+        "transformations": [],
+        "script": "\n\nsource(output(\n\t\tResourceAgencyNum as string,\n\t\tPublicName as string\n\t),\n\tallowSchemaDrift: true,\n\tvalidateSchema: false) ~> source1"
+      }
+    }
+  },
+  "datasets": [
+    {
+      "name": "DelimitedTextInput",
+      "properties": {
+        "linkedServiceName": {
+          "referenceName": "AzureBlobStorage1",
+          "type": "LinkedServiceReference"
+        },
+        "annotations": [],
+        "type": "DelimitedText",
+        "typeProperties": {
+          "location": {
+            "type": "AzureBlobStorageLocation",
+            "container": "20192019"
+          },
+          "columnDelimiter": ",",
+          "escapeChar": "\\",
+          "firstRowAsHeader": true,
+          "quoteChar": "\""
+        },
+        "schema": [
+          {
+            "name": "ResourceAgencyNum",
+            "type": "String"
+          },
+          {
+            "name": "PublicName",
+            "type": "String"
+          }
+        ]
+      },
+      "type": "Microsoft.DataFactory/factories/datasets"
+    }
+  ],
+  "linkedServices": [
+    {
+      "name": "AzureBlobStorage1",
+      "type": "Microsoft.DataFactory/factories/linkedservices",
+      "properties": {
+        "annotations": [],
+        "type": "AzureBlobStorage",
+        "typeProperties": {
+          "connectionString": "DefaultEndpointsProtocol=https;AccountName=name;AccountKey=key;EndpointSuffix=core.windows.net"
+        }
+      }
+    }
+  ],
+  "debugSettings": {
+    "sourceSettings": [
+      {
+        "sourceName": "source1",
+        "rowLimit": 1000
+      }
+    ]
+  },
+  "sessionId": "4f988caf-e765-47d2-82cd-430334a6b135",
+  "userSessionId": "82de11be-9e33-4371-a841-fabe8f43db03",
+  "incrementalDebug": false
+}

src/DataFactory/DataFactoryV2.Test/ScenarioTests/DataFlowDebugSessionTests.cs

@@ -0,0 +1,38 @@
+// ----------------------------------------------------------------------------------
+//
+// Copyright Microsoft Corporation
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+// ----------------------------------------------------------------------------------
+
+using Microsoft.Azure.ServiceManagement.Common.Models;
+using Microsoft.WindowsAzure.Commands.ScenarioTest;
+using Xunit;
+
+namespace Microsoft.Azure.Commands.DataFactoryV2.Test
+{
+    public class DataFlowDebugSessionTests : DataFactoriesScenarioTestsBase
+    {
+        public XunitTracingInterceptor _logger;
+
+        public DataFlowDebugSessionTests(Xunit.Abstractions.ITestOutputHelper output)
+        {
+            _logger = new XunitTracingInterceptor(output);
+            XunitTracingInterceptor.AddToContext(_logger);
+        }
+
+        [Fact]
+        [Trait(Category.AcceptanceType, Category.CheckIn)]
+        public void TestDataFlowDebugScenario()
+        {
+            RunPowerShellTest(_logger, "Test-DataFlowDebugScenario");
+        }
+    }
+}

src/DataFactory/DataFactoryV2.Test/ScenarioTests/DataFlowDebugSessionTests.ps1

@@ -0,0 +1,58 @@
+
+
+# ----------------------------------------------------------------------------------
+#
+# Copyright Microsoft Corporation
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ----------------------------------------------------------------------------------
+
+<#
+.SYNOPSIS
+E2E scenario for data flow debug workflow.
+#>
+function Test-DataFlowDebugScenario
+{
+    $dfname = Get-DataFactoryName
+    $rgname = Get-ResourceGroupName
+    $rglocation = Get-ProviderLocation ResourceManagement
+    $dflocation = Get-ProviderLocation DataFactoryManagement
+
+    New-AzResourceGroup -Name $rgname -Location $rglocation -Force
+
+    try
+    {
+        $df = Set-AzDataFactoryV2 -ResourceGroupName $rgname -Name $dfname -Location $dflocation -Force
+
+		# Start one debug session
+        $session = Start-AzDataFactoryV2DataFlowDebugSession -DataFactory $df
+
+		# Get all debug sessions and check the count
+		$list = Get-AzDataFactoryV2DataFlowDebugSession -DataFactory $df
+		Assert-AreEqual 1 $list.Count
+
+		# Add data flow debug package
+		Add-AzDataFactoryV2DataFlowDebugSessionPackage -DataFactory $df -PackageFile .\Resources\dataFlowDebugPackage.json -SessionId $session.SessionId
+
+		# Execute debug command 
+        $result = Invoke-AzDataFactoryV2DataFlowDebugSessionCommand -DataFactory $df -Command executePreviewQuery -SessionId $session.SessionId -StreamName source1 -RowLimits 100
+		Assert-AreEqual 'Succeeded' $result.Status
+		Assert-NotNull $result.Data
+
+        # Remove session eventually
+		Stop-AzDataFactoryV2DataFlowDebugSession -DataFactory $df -SessionId $session.SessionId -Force
+	    $newList = Get-AzDataFactoryV2DataFlowDebugSession -DataFactory $df
+		Assert-AreEqual 0 $newList.Count
+    }
+    finally
+    {
+        CleanUp $rgname $dfname
+    }
+}

src/DataFactory/DataFactoryV2.Test/ScenarioTests/DataFlowTests.cs

@@ -0,0 +1,38 @@
+// ----------------------------------------------------------------------------------
+//
+// Copyright Microsoft Corporation
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+// ----------------------------------------------------------------------------------
+
+using Microsoft.Azure.ServiceManagement.Common.Models;
+using Microsoft.WindowsAzure.Commands.ScenarioTest;
+using Xunit;
+
+namespace Microsoft.Azure.Commands.DataFactoryV2.Test
+{
+    public class DataFlowTests : DataFactoriesScenarioTestsBase
+    {
+        public XunitTracingInterceptor _logger;
+
+        public DataFlowTests(Xunit.Abstractions.ITestOutputHelper output)
+        {
+            _logger = new XunitTracingInterceptor(output);
+            XunitTracingInterceptor.AddToContext(_logger);
+        }
+
+        [Fact]
+        [Trait(Category.AcceptanceType, Category.CheckIn)]
+        public void TestDataFlow()
+        {
+            RunPowerShellTest(_logger, "Test-DataFlow");
+        }
+    }
+}

src/DataFactory/DataFactoryV2.Test/ScenarioTests/DataFlowTests.ps1

@@ -0,0 +1,59 @@
+
+
+# ----------------------------------------------------------------------------------
+#
+# Copyright Microsoft Corporation
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ----------------------------------------------------------------------------------
+
+<#
+.SYNOPSIS
+Creates a sample data flow with all of its dependencies. Then deletes the data flow with piping.
+#>
+function Test-DataFlow
+{
+    $dfname = Get-DataFactoryName
+    $rgname = Get-ResourceGroupName
+    $rglocation = Get-ProviderLocation ResourceManagement
+    $dflocation = Get-ProviderLocation DataFactoryManagement
+
+    New-AzResourceGroup -Name $rgname -Location $rglocation -Force
+
+    try
+    {
+        $df = Set-AzDataFactoryV2 -ResourceGroupName $rgname -Name $dfname -Location $dflocation -Force
+
+        $lsName = "foo1"
+        Set-AzDataFactoryV2LinkedService -ResourceGroupName $rgname -DataFactoryName $dfname -File .\Resources\linkedService.json -Name $lsName -Force
+
+        Set-AzDataFactoryV2Dataset -ResourceGroupName $rgname -DataFactoryName $dfname -Name "DelimitedTextInput" -File .\Resources\dataset-dsIn.json -Force
+		Set-AzDataFactoryV2Dataset -ResourceGroupName $rgname -DataFactoryName $dfname -Name "DelimitedTextOutput" -File .\Resources\dataset-dsIn.json -Force
+
+        $dataFlowName = "sample"   
+        $expected = Set-AzDataFactoryV2DataFlow -ResourceGroupName $rgname -Name $dataFlowName -DataFactoryName $dfname -File ".\Resources\dataFlow.json" -Force
+        $actual = Get-AzDataFactoryV2DataFlow -ResourceGroupName $rgname -Name $dataFlowName -DataFactoryName $dfname
+
+        Verify-AdfSubResource $expected $actual $rgname $dfname $dataFlowName
+
+        #remove the pipeline through piping
+        Get-AzDataFactoryV2DataFlow -DataFactory $df -Name $dataFlowName | Remove-AzDataFactoryV2DataFlow -Force
+
+        #test the pipeline no longer exists
+        Assert-ThrowsContains { Get-AzDataFactoryV2DataFlow -DataFactory $df -Name $dataFlowName } "NotFound" 
+
+        #remove the pipeline again should not throw
+        Remove-AzDataFactoryV2DataFlow -ResourceGroupName $rgname -DataFactoryName $dfname -Name $dataFlowName -Force
+    }
+    finally
+    {
+        CleanUp $rgname $dfname
+    }
+}