Skip to content

Commit 721c7bf

Browse files
erich-wangerich-wang
authored
Merge branch 'release-2019-10-15' into october
2 parents f710fd2 + 958bc48 commit 721c7bf

File tree

412 files changed

+1181973
-217620
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

412 files changed

+1181973
-217620
lines changed

‎.azure-pipelines/powershell-core.yml

Lines changed: 0 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -18,9 +18,6 @@ jobs:
1818
windows:
1919
OSName: ${{ variables.WindowsName }}
2020
ImageName: ${{ variables.WindowsImage }}
21-
linux:
22-
OSName: ${{ variables.LinuxName }}
23-
ImageName: ${{ variables.LinuxImage }}
2421
macOS:
2522
OSName: ${{ variables.MacOSName }}
2623
ImageName: ${{ variables.MacOSImage }}
@@ -43,9 +40,6 @@ jobs:
4340
windows:
4441
OSName: ${{ variables.WindowsName }}
4542
ImageName: ${{ variables.WindowsImage }}
46-
linux:
47-
OSName: ${{ variables.LinuxName }}
48-
ImageName: ${{ variables.LinuxImage }}
4943
macOS:
5044
OSName: ${{ variables.MacOSName }}
5145
ImageName: ${{ variables.MacOSImage }}
@@ -68,9 +62,6 @@ jobs:
6862
windows:
6963
OSName: ${{ variables.WindowsName }}
7064
ImageName: ${{ variables.WindowsImage }}
71-
linux:
72-
OSName: ${{ variables.LinuxName }}
73-
ImageName: ${{ variables.LinuxImage }}
7465
macOS:
7566
OSName: ${{ variables.MacOSName }}
7667
ImageName: ${{ variables.MacOSImage }}

‎src/CognitiveServices/CognitiveServices.Test/CognitiveServices.Test.csproj

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@
1212

1313
<ItemGroup>
1414
<PackageReference Include="Microsoft.Azure.Management.CognitiveServices" Version="6.0.0-preview" />
15-
<PackageReference Include="Microsoft.Azure.Management.Network" Version="19.15.0-preview" />
15+
<PackageReference Include="Microsoft.Azure.Management.Network" Version="19.16.0-preview" />
1616
</ItemGroup>
1717

1818
</Project>

‎src/Compute/Compute.Test/Compute.Test.csproj

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@
1414
<PackageReference Include="Microsoft.Azure.Graph.RBAC" Version="3.4.0-preview" />
1515
<PackageReference Include="Microsoft.Azure.Management.Compute" Version="29.1.0" />
1616
<PackageReference Include="Microsoft.Azure.Management.KeyVault" Version="2.4.2" />
17-
<PackageReference Include="Microsoft.Azure.Management.Network" Version="19.15.0-preview" />
17+
<PackageReference Include="Microsoft.Azure.Management.Network" Version="19.16.0-preview" />
1818
</ItemGroup>
1919

2020
<ItemGroup>

‎src/DataFactory/DataFactoryV2.Test/DataFactoryV2.Test.csproj

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@
1111
</PropertyGroup>
1212

1313
<ItemGroup>
14-
<PackageReference Include="Microsoft.Azure.Management.DataFactory" Version="4.1.3" />
14+
<PackageReference Include="Microsoft.Azure.Management.DataFactory" Version="4.2.0" />
1515
<PackageReference Include="Microsoft.DataTransfer.Gateway.Encryption" Version="3.11.6898.1" />
1616
</ItemGroup>
1717

‎src/DataFactory/DataFactoryV2.Test/Resources/dataFlow.json

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
{
2+
"properties": {
3+
"type": "MappingDataFlow",
4+
"typeProperties": {
5+
"sources": [
6+
{
7+
"dataset": {
8+
"referenceName": "DelimitedTextInput",
9+
"type": "DatasetReference"
10+
},
11+
"name": "source1"
12+
}
13+
],
14+
"sinks": [
15+
{
16+
"dataset": {
17+
"referenceName": "DelimitedTextOutput",
18+
"type": "DatasetReference"
19+
},
20+
"name": "sink1"
21+
}
22+
],
23+
"transformations": [],
24+
"script": "\n\nsource(allowSchemaDrift: true,\n\tvalidateSchema: false,\n\trowUrlColumn: 'source',\n\twildcardPaths:['Resources-Sample-211UTF*.csv'],\n\tpartitionBy('roundRobin', 2)) ~> source1\nsource1 sink(allowSchemaDrift: true,\n\tvalidateSchema: false,\n\tpartitionFileNames:['test.csv'],\n\tpartitionBy('hash', 1)) ~> sink1"
25+
}
26+
}
27+
}

‎src/DataFactory/DataFactoryV2.Test/Resources/dataFlowDebugPackage.json

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
{
2+
"dataFlow": {
3+
"name": "dataflow5",
4+
"properties": {
5+
"type": "MappingDataFlow",
6+
"typeProperties": {
7+
"sources": [
8+
{
9+
"dataset": {
10+
"referenceName": "DelimitedTextInput",
11+
"type": "DatasetReference"
12+
},
13+
"name": "source1",
14+
"typeProperties": {}
15+
}
16+
],
17+
"sinks": [],
18+
"transformations": [],
19+
"script": "\n\nsource(output(\n\t\tResourceAgencyNum as string,\n\t\tPublicName as string\n\t),\n\tallowSchemaDrift: true,\n\tvalidateSchema: false) ~> source1"
20+
}
21+
}
22+
},
23+
"datasets": [
24+
{
25+
"name": "DelimitedTextInput",
26+
"properties": {
27+
"linkedServiceName": {
28+
"referenceName": "AzureBlobStorage1",
29+
"type": "LinkedServiceReference"
30+
},
31+
"annotations": [],
32+
"type": "DelimitedText",
33+
"typeProperties": {
34+
"location": {
35+
"type": "AzureBlobStorageLocation",
36+
"container": "20192019"
37+
},
38+
"columnDelimiter": ",",
39+
"escapeChar": "\\",
40+
"firstRowAsHeader": true,
41+
"quoteChar": "\""
42+
},
43+
"schema": [
44+
{
45+
"name": "ResourceAgencyNum",
46+
"type": "String"
47+
},
48+
{
49+
"name": "PublicName",
50+
"type": "String"
51+
}
52+
]
53+
},
54+
"type": "Microsoft.DataFactory/factories/datasets"
55+
}
56+
],
57+
"linkedServices": [
58+
{
59+
"name": "AzureBlobStorage1",
60+
"type": "Microsoft.DataFactory/factories/linkedservices",
61+
"properties": {
62+
"annotations": [],
63+
"type": "AzureBlobStorage",
64+
"typeProperties": {
65+
"connectionString": "DefaultEndpointsProtocol=https;AccountName=name;AccountKey=key;EndpointSuffix=core.windows.net"
66+
}
67+
}
68+
}
69+
],
70+
"debugSettings": {
71+
"sourceSettings": [
72+
{
73+
"sourceName": "source1",
74+
"rowLimit": 1000
75+
}
76+
]
77+
},
78+
"sessionId": "4f988caf-e765-47d2-82cd-430334a6b135",
79+
"userSessionId": "82de11be-9e33-4371-a841-fabe8f43db03",
80+
"incrementalDebug": false
81+
}

‎src/DataFactory/DataFactoryV2.Test/ScenarioTests/DataFlowDebugSessionTests.cs

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
// ----------------------------------------------------------------------------------
2+
//
3+
// Copyright Microsoft Corporation
4+
// Licensed under the Apache License, Version 2.0 (the "License");
5+
// you may not use this file except in compliance with the License.
6+
// You may obtain a copy of the License at
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
// Unless required by applicable law or agreed to in writing, software
9+
// distributed under the License is distributed on an "AS IS" BASIS,
10+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11+
// See the License for the specific language governing permissions and
12+
// limitations under the License.
13+
// ----------------------------------------------------------------------------------
14+
15+
using Microsoft.Azure.ServiceManagement.Common.Models;
16+
using Microsoft.WindowsAzure.Commands.ScenarioTest;
17+
using Xunit;
18+
19+
namespace Microsoft.Azure.Commands.DataFactoryV2.Test
20+
{
21+
public class DataFlowDebugSessionTests : DataFactoriesScenarioTestsBase
22+
{
23+
public XunitTracingInterceptor _logger;
24+
25+
public DataFlowDebugSessionTests(Xunit.Abstractions.ITestOutputHelper output)
26+
{
27+
_logger = new XunitTracingInterceptor(output);
28+
XunitTracingInterceptor.AddToContext(_logger);
29+
}
30+
31+
[Fact]
32+
[Trait(Category.AcceptanceType, Category.CheckIn)]
33+
public void TestDataFlowDebugScenario()
34+
{
35+
RunPowerShellTest(_logger, "Test-DataFlowDebugScenario");
36+
}
37+
}
38+
}

‎src/DataFactory/DataFactoryV2.Test/ScenarioTests/DataFlowDebugSessionTests.ps1

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
2+
3+
# ----------------------------------------------------------------------------------
4+
#
5+
# Copyright Microsoft Corporation
6+
# Licensed under the Apache License, Version 2.0 (the "License");
7+
# you may not use this file except in compliance with the License.
8+
# You may obtain a copy of the License at
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
# ----------------------------------------------------------------------------------
16+
17+
<#
18+
.SYNOPSIS
19+
E2E scenario for data flow debug workflow.
20+
#>
21+
function Test-DataFlowDebugScenario
22+
{
23+
$dfname = Get-DataFactoryName
24+
$rgname = Get-ResourceGroupName
25+
$rglocation = Get-ProviderLocation ResourceManagement
26+
$dflocation = Get-ProviderLocation DataFactoryManagement
27+
28+
New-AzResourceGroup -Name $rgname -Location $rglocation -Force
29+
30+
try
31+
{
32+
$df = Set-AzDataFactoryV2 -ResourceGroupName $rgname -Name $dfname -Location $dflocation -Force
33+
34+
# Start one debug session
35+
$session = Start-AzDataFactoryV2DataFlowDebugSession -DataFactory $df
36+
37+
# Get all debug sessions and check the count
38+
$list = Get-AzDataFactoryV2DataFlowDebugSession -DataFactory $df
39+
Assert-AreEqual 1 $list.Count
40+
41+
# Add data flow debug package
42+
Add-AzDataFactoryV2DataFlowDebugSessionPackage -DataFactory $df -PackageFile .\Resources\dataFlowDebugPackage.json -SessionId $session.SessionId
43+
44+
# Execute debug command
45+
$result = Invoke-AzDataFactoryV2DataFlowDebugSessionCommand -DataFactory $df -Command executePreviewQuery -SessionId $session.SessionId -StreamName source1 -RowLimit 100
46+
Assert-AreEqual 'Succeeded' $result.Status
47+
Assert-NotNull $result.Data
48+
49+
# Remove session eventually
50+
Stop-AzDataFactoryV2DataFlowDebugSession -DataFactory $df -SessionId $session.SessionId -Force
51+
$newList = Get-AzDataFactoryV2DataFlowDebugSession -DataFactory $df
52+
Assert-AreEqual 0 $newList.Count
53+
}
54+
finally
55+
{
56+
CleanUp $rgname $dfname
57+
}
58+
}

‎src/DataFactory/DataFactoryV2.Test/ScenarioTests/DataFlowTests.cs

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
// ----------------------------------------------------------------------------------
2+
//
3+
// Copyright Microsoft Corporation
4+
// Licensed under the Apache License, Version 2.0 (the "License");
5+
// you may not use this file except in compliance with the License.
6+
// You may obtain a copy of the License at
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
// Unless required by applicable law or agreed to in writing, software
9+
// distributed under the License is distributed on an "AS IS" BASIS,
10+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11+
// See the License for the specific language governing permissions and
12+
// limitations under the License.
13+
// ----------------------------------------------------------------------------------
14+
15+
using Microsoft.Azure.ServiceManagement.Common.Models;
16+
using Microsoft.WindowsAzure.Commands.ScenarioTest;
17+
using Xunit;
18+
19+
namespace Microsoft.Azure.Commands.DataFactoryV2.Test
20+
{
21+
public class DataFlowTests : DataFactoriesScenarioTestsBase
22+
{
23+
public XunitTracingInterceptor _logger;
24+
25+
public DataFlowTests(Xunit.Abstractions.ITestOutputHelper output)
26+
{
27+
_logger = new XunitTracingInterceptor(output);
28+
XunitTracingInterceptor.AddToContext(_logger);
29+
}
30+
31+
[Fact]
32+
[Trait(Category.AcceptanceType, Category.CheckIn)]
33+
public void TestDataFlow()
34+
{
35+
RunPowerShellTest(_logger, "Test-DataFlow");
36+
}
37+
}
38+
}

‎src/DataFactory/DataFactoryV2.Test/ScenarioTests/DataFlowTests.ps1

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
2+
3+
# ----------------------------------------------------------------------------------
4+
#
5+
# Copyright Microsoft Corporation
6+
# Licensed under the Apache License, Version 2.0 (the "License");
7+
# you may not use this file except in compliance with the License.
8+
# You may obtain a copy of the License at
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
# ----------------------------------------------------------------------------------
16+
17+
<#
18+
.SYNOPSIS
19+
Creates a sample data flow with all of its dependencies. Then deletes the data flow with piping.
20+
#>
21+
function Test-DataFlow
22+
{
23+
$dfname = Get-DataFactoryName
24+
$rgname = Get-ResourceGroupName
25+
$rglocation = Get-ProviderLocation ResourceManagement
26+
$dflocation = Get-ProviderLocation DataFactoryManagement
27+
28+
New-AzResourceGroup -Name $rgname -Location $rglocation -Force
29+
30+
try
31+
{
32+
$df = Set-AzDataFactoryV2 -ResourceGroupName $rgname -Name $dfname -Location $dflocation -Force
33+
34+
$lsName = "foo1"
35+
Set-AzDataFactoryV2LinkedService -ResourceGroupName $rgname -DataFactoryName $dfname -File .\Resources\linkedService.json -Name $lsName -Force
36+
37+
Set-AzDataFactoryV2Dataset -ResourceGroupName $rgname -DataFactoryName $dfname -Name "DelimitedTextInput" -File .\Resources\dataset-dsIn.json -Force
38+
Set-AzDataFactoryV2Dataset -ResourceGroupName $rgname -DataFactoryName $dfname -Name "DelimitedTextOutput" -File .\Resources\dataset-dsIn.json -Force
39+
40+
$dataFlowName = "sample"
41+
$expected = Set-AzDataFactoryV2DataFlow -ResourceGroupName $rgname -Name $dataFlowName -DataFactoryName $dfname -File ".\Resources\dataFlow.json" -Force
42+
$actual = Get-AzDataFactoryV2DataFlow -ResourceGroupName $rgname -Name $dataFlowName -DataFactoryName $dfname
43+
44+
Verify-AdfSubResource $expected $actual $rgname $dfname $dataFlowName
45+
46+
#remove the pipeline through piping
47+
Get-AzDataFactoryV2DataFlow -DataFactory $df -Name $dataFlowName | Remove-AzDataFactoryV2DataFlow -Force
48+
49+
#test the pipeline no longer exists
50+
Assert-ThrowsContains { Get-AzDataFactoryV2DataFlow -DataFactory $df -Name $dataFlowName } "NotFound"
51+
52+
#remove the pipeline again should not throw
53+
Remove-AzDataFactoryV2DataFlow -ResourceGroupName $rgname -DataFactoryName $dfname -Name $dataFlowName -Force
54+
}
55+
finally
56+
{
57+
CleanUp $rgname $dfname
58+
}
59+
}

0 commit comments

Comments
 (0)