Adding infrastructure to manage flaky check-in tests and applying to one flaky test

Author: markcowl

src/ResourceManager/Common/Commands.ScenarioTests.ResourceManager.Common/Commands.ScenarioTests.ResourceManager.Common.csproj

@@ -163,6 +163,7 @@
     <Compile Include="PowerShellExtensions.cs" />
     <Compile Include="Properties\AssemblyInfo.cs" />
     <Compile Include="RMTestBase.cs" />
+    <Compile Include="TestExecutionHelpers.cs" />
     <Compile Include="XunitTracingInterceptor.cs" />
   </ItemGroup>
   <ItemGroup>

src/ResourceManager/Common/Commands.ScenarioTests.ResourceManager.Common/TestExecutionHelpers.cs

@@ -0,0 +1,52 @@
+// ----------------------------------------------------------------------------------
+//
+// Copyright Microsoft Corporation
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+// ----------------------------------------------------------------------------------
+
+using System;
+using Microsoft.WindowsAzure.Commands.Utilities.Common;
+
+namespace Microsoft.Azure.Commands.ScenarioTest
+{
+    public static class TestExecutionHelpers
+    {
+        /// <summary>
+        /// Retry an action until it succeeds - use for flaky tests
+        /// </summary>
+        /// <param name="testAction">Action tor etry</param>
+        /// <param name="maxTries">The maximum number of times to try the action</param>
+        public static void RetryAction(Action testAction, int maxTries = 3)
+        {
+            var tries = 0;
+            var succeeded = false;
+            do
+            {
+                try
+                {
+                    testAction();
+                    succeeded = true;
+                }
+                catch (Exception)
+                {
+                    if (++tries < maxTries)
+                    {
+                        TestMockSupport.Delay(TimeSpan.FromSeconds(1));
+                    }
+                    else
+                    {
+                        throw;
+                    }
+                }
+            } while (!succeeded);
+        }
+    }
+}

src/ResourceManager/Resources/Commands.Resources.Test/Models.ResourceGroups/GalleryTemplatesClientTests.cs

@@ -26,6 +26,7 @@
 using System.Linq;
 using System.Management.Automation;
 using System.Security;
+using Microsoft.Azure.Commands.ScenarioTest;
 using Newtonsoft.Json.Linq;
 using Xunit;
 using Xunit.Abstractions;
@@ -498,10 +499,14 @@ public void HandlesInvalidTemplateFiles()
         [Trait(Category.AcceptanceType, Category.CheckIn)]
         public void ParseTemplateParameterFileContents_DeserializeWithCorrectType()
         {
-            Dictionary<string, TemplateFileParameterV1> result =
-                TemplateUtility.ParseTemplateParameterFileContents(@"Resources\WebSite.param.dev.json");
-            Assert.Equal(true, result["isWorker"].Value);
-            Assert.Equal((System.Int64)1, result["numberOfWorker"].Value);
+            // Add up to 3 retries for flaky test
+            TestExecutionHelpers.RetryAction(() =>
+            {
+                Dictionary<string, TemplateFileParameterV1> result =
+                    TemplateUtility.ParseTemplateParameterFileContents(@"Resources\WebSite.param.dev.json");
+                Assert.Equal(true, result["isWorker"].Value);
+                Assert.Equal((System.Int64) 1, result["numberOfWorker"].Value);
+            });
         }
     }
 }

src/ResourceManager/Resources/Resources.sln

@@ -1,6 +1,6 @@
 Microsoft Visual Studio Solution File, Format Version 12.00
-# Visual Studio 2013
-VisualStudioVersion = 12.0.31101.0
+# Visual Studio 14
+VisualStudioVersion = 14.0.25123.0
 MinimumVisualStudioVersion = 10.0.40219.1
 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Tests", "Tests", "{95C16AED-FD57-42A0-86C3-2CF4300A4817}"
 EndProject