Skip to content

Commit 6451e93

Browse files
committed
Re-record Batch tests
- Label tests as "CheckIn" category. - Have a large timeout when in replay mode to deal with possibility of thread starvation in massively parallel run.
1 parent e9dbc00 commit 6451e93

File tree

48 files changed

+4277
-6599
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

48 files changed

+4277
-6599
lines changed

src/ResourceManager/AzureBatch/Commands.Batch.Test/ScenarioTests/BatchAccountTests.cs

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -24,16 +24,15 @@ public BatchAccountTests(Xunit.Abstractions.ITestOutputHelper output)
2424
ServiceManagemenet.Common.Models.XunitTracingInterceptor.AddToContext(new ServiceManagemenet.Common.Models.XunitTracingInterceptor(output));
2525
}
2626

27-
[Fact(Skip = "Need service team to re-record test after changes to the ClientRuntime.")]
27+
[Fact]
2828
[Trait(Category.AcceptanceType, Category.CheckIn)]
29-
[Trait("Re-record", "ClientRuntime changes")]
3029
public void TestBatchAccountEndToEnd()
3130
{
3231
BatchController.NewInstance.RunPsTest("Test-BatchAccountEndToEnd");
3332
}
3433

3534
[Fact]
36-
[Trait(Category.AcceptanceType, Category.Flaky)]
35+
[Trait(Category.AcceptanceType, Category.CheckIn)]
3736
public void TestListNodeAgentSkus()
3837
{
3938
BatchController.NewInstance.RunPsTest("Test-GetBatchNodeAgentSkus");

src/ResourceManager/AzureBatch/Commands.Batch.Test/ScenarioTests/BatchApplicationPackageTests.cs

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ public class BatchApplicationPackageTests : WindowsAzure.Commands.Test.Utilities
2626
private const string version = "foo";
2727

2828
[Fact]
29-
[Trait(Category.AcceptanceType, Category.Flaky)]
29+
[Trait(Category.AcceptanceType, Category.CheckIn)]
3030
public void TestUploadApplicationPackage()
3131
{
3232
string id = "newApplicationPackage";
@@ -56,7 +56,7 @@ public void TestUploadApplicationPackage()
5656
}
5757

5858
[Fact]
59-
[Trait(Category.AcceptanceType, Category.Flaky)]
59+
[Trait(Category.AcceptanceType, Category.CheckIn)]
6060
public void TestUpdateApplicationPackage()
6161
{
6262
string id = "updateApplicationPackage";
@@ -85,8 +85,8 @@ public void TestUpdateApplicationPackage()
8585
TestUtilities.GetCurrentMethodName());
8686
}
8787

88-
[Fact(Skip = "Need Batch team to re-record failing test. See the following issue: https://github.com/Azure/azure-powershell/issues/5006")]
89-
[Trait(Category.AcceptanceType, Category.Flaky)]
88+
[Fact]
89+
[Trait(Category.AcceptanceType, Category.CheckIn)]
9090
public void TestCreatePoolWithApplicationPackage()
9191
{
9292
string id = "createPoolWithApplicationPackage";
@@ -114,8 +114,8 @@ public void TestCreatePoolWithApplicationPackage()
114114
TestUtilities.GetCurrentMethodName());
115115
}
116116

117-
[Fact(Skip = "Need Batch team to re-record failing test. See the following issue: https://github.com/Azure/azure-powershell/issues/5006")]
118-
[Trait(Category.AcceptanceType, Category.Flaky)]
117+
[Fact]
118+
[Trait(Category.AcceptanceType, Category.CheckIn)]
119119
public void TestUpdatePoolWithApplicationPackage()
120120
{
121121
string id = "updatePoolWithApplicationPackage";

src/ResourceManager/AzureBatch/Commands.Batch.Test/ScenarioTests/CertificateTests.cs

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -27,15 +27,15 @@ public CertificateTests(Xunit.Abstractions.ITestOutputHelper output)
2727
ServiceManagemenet.Common.Models.XunitTracingInterceptor.AddToContext(new ServiceManagemenet.Common.Models.XunitTracingInterceptor(output));
2828
}
2929

30-
[Fact(Skip = "Need Batch team to re-record failing test. See the following issue: https://github.com/Azure/azure-powershell/issues/5006")]
31-
[Trait(Category.AcceptanceType, Category.Flaky)]
30+
[Fact]
31+
[Trait(Category.AcceptanceType, Category.CheckIn)]
3232
public void TestCertificateCrudOperations()
3333
{
3434
BatchController.NewInstance.RunPsTest("Test-CertificateCrudOperations");
3535
}
3636

37-
[Fact(Skip = "Need Batch team to re-record failing test. See the following issue: https://github.com/Azure/azure-powershell/issues/5006")]
38-
[Trait(Category.AcceptanceType, Category.Flaky)]
37+
[Fact]
38+
[Trait(Category.AcceptanceType, Category.CheckIn)]
3939
public void TestCancelCertificateDelete()
4040
{
4141
BatchController controller = BatchController.NewInstance;

src/ResourceManager/AzureBatch/Commands.Batch.Test/ScenarioTests/CertificateTests.ps1

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,8 +22,11 @@ function Test-CertificateCrudOperations
2222
$thumbprintAlgorithm = "sha1"
2323

2424
$localDir = ($pwd).Path # Use $pwd to get the local directory. If $pwd is not used, paths are relative to [Environment]::CurrentDirectory, which can be different
25-
$certPath = $localDir + "\Resources\BatchTestCert01.cer"
26-
$x509cert = New-Object System.Security.Cryptography.X509Certificates.X509Certificate2 -ArgumentList $certPath
25+
26+
$certPathVs = $localDir + "/Resources/BatchTestCert01.cer"
27+
# $certPathConsole = [System.IO.Path]::GetDirectoryName($PSScriptRoot) + "/Resources/BatchTestCert01.cer"
28+
# $certPath = if (Test-Path $certPathVs -PathType Leaf) { $certPathVs } Else { $certPathConsole }
29+
$x509cert = New-Object System.Security.Cryptography.X509Certificates.X509Certificate2 -ArgumentList $certPathVs
2730

2831
# Add the cert
2932
$x509cert | New-AzureBatchCertificate -BatchContext $context

src/ResourceManager/AzureBatch/Commands.Batch.Test/ScenarioTests/Common.ps1

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,19 @@ function Start-TestSleep($milliseconds)
2424
}
2525
}
2626

27+
function Compute-TestTimeout($seconds)
28+
{
29+
if ([Microsoft.Azure.Test.HttpRecorder.HttpMockServer]::Mode -eq [Microsoft.Azure.Test.HttpRecorder.HttpRecorderMode]::Playback)
30+
{
31+
# Timeout of 3 hours, to deal with long delays in recording framework
32+
return 60 * 24 * 3
33+
}
34+
else
35+
{
36+
return $seconds
37+
}
38+
}
39+
2740
<#
2841
.SYNOPSIS
2942
Gets a Batch account name for testing.

src/ResourceManager/AzureBatch/Commands.Batch.Test/ScenarioTests/ComputeNodeTests.cs

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -28,8 +28,8 @@ public ComputeNodeTests(Xunit.Abstractions.ITestOutputHelper output)
2828
ServiceManagemenet.Common.Models.XunitTracingInterceptor.AddToContext(new ServiceManagemenet.Common.Models.XunitTracingInterceptor(output));
2929
}
3030

31-
[Fact(Skip = "Need Batch team to re-record failing test. See the following issue: https://github.com/Azure/azure-powershell/issues/5006")]
32-
[Trait(Category.AcceptanceType, Category.Flaky)]
31+
[Fact]
32+
[Trait(Category.AcceptanceType, Category.CheckIn)]
3333
public void TestRemoveComputeNodes()
3434
{
3535
BatchController controller = BatchController.NewInstance;
@@ -51,8 +51,8 @@ public void TestRemoveComputeNodes()
5151
TestUtilities.GetCurrentMethodName());
5252
}
5353

54-
[Fact(Skip = "Need Batch team to re-record failing test. See the following issue: https://github.com/Azure/azure-powershell/issues/5006")]
55-
[Trait(Category.AcceptanceType, Category.Flaky)]
54+
[Fact]
55+
[Trait(Category.AcceptanceType, Category.CheckIn)]
5656
public void TestRebootAndReimageComputeNode()
5757
{
5858
BatchController controller = BatchController.NewInstance;
@@ -74,8 +74,8 @@ public void TestRebootAndReimageComputeNode()
7474
TestUtilities.GetCurrentMethodName());
7575
}
7676

77-
[Fact(Skip = "Need Batch team to re-record failing test. See the following issue: https://github.com/Azure/azure-powershell/issues/5006")]
78-
[Trait(Category.AcceptanceType, Category.Flaky)]
77+
[Fact]
78+
[Trait(Category.AcceptanceType, Category.CheckIn)]
7979
public void TestDisableAndEnableComputeNodeScheduling()
8080
{
8181
BatchController controller = BatchController.NewInstance;
@@ -94,8 +94,8 @@ public void TestDisableAndEnableComputeNodeScheduling()
9494
TestUtilities.GetCurrentMethodName());
9595
}
9696

97-
[Fact(Skip = "Need Batch team to re-record failing test. See the following issue: https://github.com/Azure/azure-powershell/issues/5006")]
98-
[Trait(Category.AcceptanceType, Category.Flaky)]
97+
[Fact]
98+
[Trait(Category.AcceptanceType, Category.CheckIn)]
9999
public void TestGetComputeNodeRemoteLoginSettings()
100100
{
101101
BatchController controller = BatchController.NewInstance;

src/ResourceManager/AzureBatch/Commands.Batch.Test/ScenarioTests/ComputeNodeTests.ps1

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,8 @@ function Test-RemoveComputeNodes
3535
$select = "id,state"
3636
$computeNodes = Get-AzureBatchComputeNode -PoolId $poolId -Select $select -BatchContext $context
3737
$start = [DateTime]::Now
38-
$end = $start.AddSeconds(30)
38+
$timeout = Compute-TestTimeout 30
39+
$end = $start.AddSeconds($timeout)
3940
while ($computeNodes[0].State -ne 'LeavingPool' -and $computeNodes[1].State -ne 'LeavingPool')
4041
{
4142
if ([DateTime]::Now -gt $end)

src/ResourceManager/AzureBatch/Commands.Batch.Test/ScenarioTests/ComputeNodeUserTests.cs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -27,8 +27,8 @@ public ComputeNodeUserTests(Xunit.Abstractions.ITestOutputHelper output)
2727
ServiceManagemenet.Common.Models.XunitTracingInterceptor.AddToContext(new ServiceManagemenet.Common.Models.XunitTracingInterceptor(output));
2828
}
2929

30-
[Fact(Skip = "Need Batch team to re-record failing test. See the following issue: https://github.com/Azure/azure-powershell/issues/5006")]
31-
[Trait(Category.AcceptanceType, Category.Flaky)]
30+
[Fact]
31+
[Trait(Category.AcceptanceType, Category.CheckIn)]
3232
public void TestComputeNodeUserEndToEnd()
3333
{
3434
BatchController controller = BatchController.NewInstance;

src/ResourceManager/AzureBatch/Commands.Batch.Test/ScenarioTests/FileTests.cs

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -29,8 +29,8 @@ public FileTests(Xunit.Abstractions.ITestOutputHelper output)
2929
ServiceManagemenet.Common.Models.XunitTracingInterceptor.AddToContext(new ServiceManagemenet.Common.Models.XunitTracingInterceptor(output));
3030
}
3131

32-
[Fact(Skip = "Need Batch team to re-record failing test. See the following issue: https://github.com/Azure/azure-powershell/issues/5006")]
33-
[Trait(Category.AcceptanceType, Category.Flaky)]
32+
[Fact]
33+
[Trait(Category.AcceptanceType, Category.CheckIn)]
3434
public void TestGetNodeFileContentByTask()
3535
{
3636
BatchController controller = BatchController.NewInstance;
@@ -57,8 +57,8 @@ public void TestGetNodeFileContentByTask()
5757
TestUtilities.GetCurrentMethodName());
5858
}
5959

60-
[Fact(Skip = "Need Batch team to re-record failing test. See the following issue: https://github.com/Azure/azure-powershell/issues/5006")]
61-
[Trait(Category.AcceptanceType, Category.Flaky)]
60+
[Fact]
61+
[Trait(Category.AcceptanceType, Category.CheckIn)]
6262
public void TestGetNodeFileContentByComputeNode()
6363
{
6464
BatchController controller = BatchController.NewInstance;
@@ -76,8 +76,8 @@ public void TestGetNodeFileContentByComputeNode()
7676
TestUtilities.GetCurrentMethodName());
7777
}
7878

79-
[Fact(Skip = "Need Batch team to re-record failing test. See the following issue: https://github.com/Azure/azure-powershell/issues/5006")]
80-
[Trait(Category.AcceptanceType, Category.Flaky)]
79+
[Fact]
80+
[Trait(Category.AcceptanceType, Category.CheckIn)]
8181
public void TestGetRemoteDesktopProtocolFile()
8282
{
8383
BatchController controller = BatchController.NewInstance;
@@ -95,8 +95,8 @@ public void TestGetRemoteDesktopProtocolFile()
9595
TestUtilities.GetCurrentMethodName());
9696
}
9797

98-
[Fact(Skip = "Need Batch team to re-record failing test. See the following issue: https://github.com/Azure/azure-powershell/issues/5006")]
99-
[Trait(Category.AcceptanceType, Category.Flaky)]
98+
[Fact]
99+
[Trait(Category.AcceptanceType, Category.CheckIn)]
100100
public void TestDeleteNodeFileByTask()
101101
{
102102
BatchController controller = BatchController.NewInstance;
@@ -122,8 +122,8 @@ public void TestDeleteNodeFileByTask()
122122
TestUtilities.GetCurrentMethodName());
123123
}
124124

125-
[Fact(Skip = "Need Batch team to re-record failing test. See the following issue: https://github.com/Azure/azure-powershell/issues/5006")]
126-
[Trait(Category.AcceptanceType, Category.Flaky)]
125+
[Fact]
126+
[Trait(Category.AcceptanceType, Category.CheckIn)]
127127
public void TestDeleteNodeFileByComputeNode()
128128
{
129129
BatchController controller = BatchController.NewInstance;

src/ResourceManager/AzureBatch/Commands.Batch.Test/ScenarioTests/JobScheduleTests.cs

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -25,16 +25,16 @@ public JobScheduleTests(Xunit.Abstractions.ITestOutputHelper output)
2525
ServiceManagemenet.Common.Models.XunitTracingInterceptor.AddToContext(new ServiceManagemenet.Common.Models.XunitTracingInterceptor(output));
2626
}
2727

28-
[Fact(Skip = "Need Batch team to re-record failing test. See the following issue: https://github.com/Azure/azure-powershell/issues/5006")]
29-
[Trait(Category.AcceptanceType, Category.Flaky)]
28+
[Fact]
29+
[Trait(Category.AcceptanceType, Category.CheckIn)]
3030
public void TestJobScheduleCRUD()
3131
{
3232
BatchController.NewInstance.RunPsTest("Test-JobScheduleCRUD");
3333
}
3434

3535

36-
[Fact(Skip = "Need Batch team to re-record failing test. See the following issue: https://github.com/Azure/azure-powershell/issues/5006")]
37-
[Trait(Category.AcceptanceType, Category.Flaky)]
36+
[Fact]
37+
[Trait(Category.AcceptanceType, Category.CheckIn)]
3838
public void TestDisableEnableTerminateJobSchedule()
3939
{
4040
BatchController controller = BatchController.NewInstance;

src/ResourceManager/AzureBatch/Commands.Batch.Test/ScenarioTests/JobTests.cs

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -27,16 +27,16 @@ public JobTests(Xunit.Abstractions.ITestOutputHelper output)
2727
ServiceManagemenet.Common.Models.XunitTracingInterceptor.AddToContext(new ServiceManagemenet.Common.Models.XunitTracingInterceptor(output));
2828
}
2929

30-
[Fact(Skip = "Need Batch team to re-record failing test. See the following issue: https://github.com/Azure/azure-powershell/issues/5006")]
31-
[Trait(Category.AcceptanceType, Category.Flaky)]
30+
[Fact]
31+
[Trait(Category.AcceptanceType, Category.CheckIn)]
3232
public void TestJobCRUD()
3333
{
3434
BatchController.NewInstance.RunPsTest("Test-JobCRUD");
3535
}
3636

3737

38-
[Fact(Skip = "Need Batch team to re-record failing test. See the following issue: https://github.com/Azure/azure-powershell/issues/5006")]
39-
[Trait(Category.AcceptanceType, Category.Flaky)]
38+
[Fact]
39+
[Trait(Category.AcceptanceType, Category.CheckIn)]
4040
public void TestDisableEnableTerminateJob()
4141
{
4242
BatchController controller = BatchController.NewInstance;
@@ -58,8 +58,8 @@ public void TestDisableEnableTerminateJob()
5858
TestUtilities.GetCurrentMethodName());
5959
}
6060

61-
[Fact(Skip = "Need Batch team to re-record failing test. See the following issue: https://github.com/Azure/azure-powershell/issues/5006")]
62-
[Trait(Category.AcceptanceType, Category.Flaky)]
61+
[Fact]
62+
[Trait(Category.AcceptanceType, Category.CheckIn)]
6363
public void IfJobSetsAutoFailure_ItCompletesWhenAnyTaskFails()
6464
{
6565
BatchController controller = BatchController.NewInstance;

src/ResourceManager/AzureBatch/Commands.Batch.Test/ScenarioTests/PoolTests.cs

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -30,15 +30,15 @@ public PoolTests(Xunit.Abstractions.ITestOutputHelper output)
3030
ServiceManagemenet.Common.Models.XunitTracingInterceptor.AddToContext(new ServiceManagemenet.Common.Models.XunitTracingInterceptor(output));
3131
}
3232

33-
[Fact(Skip = "Need Batch team to re-record failing test. See the following issue: https://github.com/Azure/azure-powershell/issues/5006")]
34-
[Trait(Category.AcceptanceType, Category.Flaky)]
33+
[Fact]
34+
[Trait(Category.AcceptanceType, Category.CheckIn)]
3535
public void TestPoolCRUD()
3636
{
3737
BatchController.NewInstance.RunPsTest("Test-PoolCRUD");
3838
}
3939

40-
[Fact(Skip = "Need Batch team to re-record failing test. See the following issue: https://github.com/Azure/azure-powershell/issues/5006")]
41-
[Trait(Category.AcceptanceType, Category.Flaky)]
40+
[Fact]
41+
[Trait(Category.AcceptanceType, Category.CheckIn)]
4242
public void TestResizeAndStopResizePool()
4343
{
4444
BatchController controller = BatchController.NewInstance;
@@ -59,8 +59,8 @@ public void TestResizeAndStopResizePool()
5959
TestUtilities.GetCurrentMethodName());
6060
}
6161

62-
[Fact(Skip = "Need Batch team to re-record failing test. See the following issue: https://github.com/Azure/azure-powershell/issues/5006")]
63-
[Trait(Category.AcceptanceType, Category.Flaky)]
62+
[Fact]
63+
[Trait(Category.AcceptanceType, Category.CheckIn)]
6464
public void TestAutoScaleActions()
6565
{
6666
BatchController controller = BatchController.NewInstance;
@@ -81,8 +81,8 @@ public void TestAutoScaleActions()
8181
TestUtilities.GetCurrentMethodName());
8282
}
8383

84-
[Fact(Skip = "Need Batch team to re-record failing test. See the following issue: https://github.com/Azure/azure-powershell/issues/5006")]
85-
[Trait(Category.AcceptanceType, Category.Flaky)]
84+
[Fact]
85+
[Trait(Category.AcceptanceType, Category.CheckIn)]
8686
public void TestChangeOSVersion()
8787
{
8888
BatchController controller = BatchController.NewInstance;

src/ResourceManager/AzureBatch/Commands.Batch.Test/ScenarioTests/ScenarioTestHelpers.cs

Lines changed: 16 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -96,7 +96,8 @@ public static string AddTestCertificate(BatchController controller, BatchAccount
9696
{
9797
PSCertificate existingCert = client.ListCertificates(getParameters).FirstOrDefault();
9898
DateTime start = DateTime.Now;
99-
DateTime end = start.AddMinutes(5);
99+
TimeSpan timeout = GetTimeout(TimeSpan.FromMinutes(5));
100+
DateTime end = start.Add(timeout);
100101

101102
// Cert might still be deleting from other tests, so we wait for the delete to finish.
102103
while (existingCert != null && existingCert.State == CertificateState.Deleting)
@@ -153,7 +154,7 @@ public static void WaitForCertificateToFailDeletion(BatchController controller,
153154

154155
PSCertificate cert = client.ListCertificates(parameters).First();
155156

156-
DateTime timeout = DateTime.Now.AddMinutes(2);
157+
DateTime timeout = DateTime.Now.Add(GetTimeout(TimeSpan.FromMinutes(2)));
157158
while (cert.State != CertificateState.DeleteFailed)
158159
{
159160
if (DateTime.Now > timeout)
@@ -261,7 +262,7 @@ public static void WaitForSteadyPoolAllocation(BatchController controller, Batch
261262
PoolId = poolId
262263
};
263264

264-
DateTime timeout = DateTime.Now.AddMinutes(5);
265+
DateTime timeout = DateTime.Now.Add(GetTimeout(TimeSpan.FromMinutes(5)));
265266
PSCloudPool pool = client.ListPools(options).First();
266267
while (pool.AllocationState != AllocationState.Steady)
267268
{
@@ -346,7 +347,7 @@ public static void CreateTestJob(BatchController controller, BatchAccountContext
346347
/// </summary>
347348
public static string WaitForRecentJob(BatchController controller, BatchAccountContext context, string jobScheduleId, string previousJob = null)
348349
{
349-
DateTime timeout = DateTime.Now.AddMinutes(2);
350+
DateTime timeout = DateTime.Now.Add(GetTimeout(TimeSpan.FromMinutes(2)));
350351
BatchClient client = new BatchClient(controller.BatchManagementClient, controller.ResourceManagementClient);
351352

352353
ListJobScheduleOptions options = new ListJobScheduleOptions(context)
@@ -507,7 +508,7 @@ public static void WaitForIdleComputeNode(BatchController controller, BatchAccou
507508
Select = "id,state"
508509
};
509510

510-
DateTime timeout = DateTime.Now.AddMinutes(5);
511+
DateTime timeout = DateTime.Now.Add(GetTimeout(TimeSpan.FromMinutes(10)));
511512
PSComputeNode computeNode = client.ListComputeNodes(options).First();
512513
while (computeNode.State != ComputeNodeState.Idle)
513514
{
@@ -601,5 +602,15 @@ private static void Sleep(int milliseconds)
601602
Thread.Sleep(milliseconds);
602603
}
603604
}
605+
606+
private static TimeSpan GetTimeout(TimeSpan timeout)
607+
{
608+
if (HttpMockServer.Mode == HttpRecorderMode.Playback)
609+
{
610+
return TimeSpan.FromHours(3);
611+
}
612+
613+
return timeout;
614+
}
604615
}
605616
}

0 commit comments

Comments
 (0)