Fixing flaky ServiceManagement tests

Author: markcowl

src/ServiceManagement/Common/Commands.Common.Test/Commands.Common.Test.csproj

@@ -209,6 +209,7 @@
     </Compile>
     <Compile Include="Common\PSTestTracingInterceptor.cs" />
     <Compile Include="Resources\ResourceLocator.cs" />
+    <Compile Include="TestExecutionHelpers.cs" />
   </ItemGroup>
   <ItemGroup>
     <ProjectReference Include="..\..\..\Common\Commands.Common.Authentication\Commands.Common.Authentication.csproj">

src/ServiceManagement/Common/Commands.Common.Test/TestExecutionHelpers.cs

@@ -0,0 +1,47 @@
+// ----------------------------------------------------------------------------------
+//
+// Copyright Microsoft Corporation
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+// ----------------------------------------------------------------------------------
+
+using System;
+
+namespace Microsoft.WindowsAzure.Commands.Common.Test
+{
+    public static class TestExecutionHelpers
+    {
+        /// <summary>
+        /// Retry an action until it succeeds - use for flaky tests
+        /// </summary>
+        /// <param name="testAction">Action tor etry</param>
+        /// <param name="maxTries">The maximum number of times to try the action</param>
+        public static void RetryAction(Action testAction, int maxTries = 3)
+        {
+            var tries = 0;
+            var succeeded = false;
+            do
+            {
+                try
+                {
+                    testAction();
+                    succeeded = true;
+                }
+                catch (Exception)
+                {
+                    if (++tries >= maxTries)
+                    {
+                        throw;
+                    }
+                }
+            } while (!succeeded);
+        }
+    }
+}

src/ServiceManagement/Services/Commands.Test/WAPackIaaS/Operations/JobOperationsTests.cs

@@ -13,6 +13,7 @@
 // ----------------------------------------------------------------------------------
 
 using System;
+using Microsoft.WindowsAzure.Commands.Common.Test;
 using Microsoft.WindowsAzure.Commands.ScenarioTest;
 using Xunit;
 using Microsoft.WindowsAzure.Commands.Test.WAPackIaaS.Mocks;
@@ -35,17 +36,22 @@ public class JobOperationsTests
         [Trait("Type", "WAPackIaaS-Unit")]
         public void WaitOnJobCompletesImmediately()
         {
-            Guid jobId = Guid.NewGuid();
+            // Fix test flakiness
+            TestExecutionHelpers.RetryAction(
+                () =>
+                {
+                    Guid jobId = Guid.NewGuid();
 
-            MockRequestChannel mockChannel = MockRequestChannel.Create();
-            mockChannel.AddReturnObject(new Job {Name = "TestJob", ID = jobId, IsCompleted = true});
+                    MockRequestChannel mockChannel = MockRequestChannel.Create();
+                    mockChannel.AddReturnObject(new Job {Name = "TestJob", ID = jobId, IsCompleted = true});
 
-            var jobOperations = new JobOperations(new WebClientFactory(
-                                                      new Subscription(),
-                                                      mockChannel));
-            DateTime start = DateTime.Now;
-            jobOperations.WaitOnJob(jobId);
-            Assert.True((DateTime.Now - start).TotalMilliseconds < 500);
+                    var jobOperations = new JobOperations(new WebClientFactory(
+                        new Subscription(),
+                        mockChannel));
+                    DateTime start = DateTime.Now;
+                    jobOperations.WaitOnJob(jobId);
+                    Assert.True((DateTime.Now - start).TotalMilliseconds < 500);
+                });
         }
 
         /// <summary>