Add ability to skip tests on specific helix queues (#8231)

Author: HaoK

eng/helix/vstest/runtests.cmd

@@ -1,6 +1,7 @@
 set target=%1
 set sdkVersion=%2
 set runtimeVersion=%3
+set helixQueue=%4
 
 set DOTNET_HOME=%HELIX_CORRELATION_PAYLOAD%\sdk
 set DOTNET_ROOT=%DOTNET_HOME%\x64
@@ -13,7 +14,7 @@ set PATH=%DOTNET_ROOT%;%PATH%
 powershell.exe -NoProfile -ExecutionPolicy unrestricted -Command "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12; &([scriptblock]::Create((Invoke-WebRequest -useb 'https://dot.net/v1/dotnet-install.ps1'))) -Architecture x64 -Version %sdkVersion% -InstallDir %DOTNET_ROOT%"
 powershell.exe -NoProfile -ExecutionPolicy unrestricted -Command "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12; &([scriptblock]::Create((Invoke-WebRequest -useb 'https://dot.net/v1/dotnet-install.ps1'))) -Architecture x64 -Runtime dotnet -Version %runtimeVersion% -InstallDir %DOTNET_ROOT%"
 
-set HELIX=true
+set HELIX=%helixQueue%
 
 %DOTNET_ROOT%\dotnet vstest %target% -lt >discovered.txt
 find /c "Exception thrown" discovered.txt

eng/helix/vstest/runtests.sh

@@ -58,7 +58,7 @@ export DOTNET_MULTILEVEL_LOOKUP=0
 # Avoid contaminating userprofiles
 export DOTNET_CLI_HOME="$HELIX_CORRELATION_PAYLOAD/home"
 
-export helix="true"
+export helix="$4"
 
 $DOTNET_ROOT/dotnet vstest $1 -lt >discovered.txt
 if grep -q "Exception thrown" discovered.txt; then

eng/helix/xunit/runtests.cmd

@@ -1,3 +1,3 @@
 set target=%1
-set helix=true
+set helix=%4
 xunit.console.exe %target% -xml testResults.xml

eng/targets/Helix.targets

@@ -64,8 +64,8 @@
         <TestAssembly>$(TargetFileName)</TestAssembly>
         <PreCommands>@(HelixPreCommand)</PreCommands>
         <PostCommands>@(HelixPostCommand)</PostCommands>
-        <Command Condition="$(IsWindowsHelixQueue)">call runtests.cmd $(TargetFileName) $(NETCoreSdkVersion) $(MicrosoftNETCoreAppPackageVersion)</Command>
-        <Command Condition="!$(IsWindowsHelixQueue)">./runtests.sh $(TargetFileName) $(NETCoreSdkVersion) $(MicrosoftNETCoreAppPackageVersion)</Command>
+        <Command Condition="$(IsWindowsHelixQueue)">call runtests.cmd $(TargetFileName) $(NETCoreSdkVersion) $(MicrosoftNETCoreAppPackageVersion) $(HelixTargetQueue)</Command>
+        <Command Condition="!$(IsWindowsHelixQueue)">./runtests.sh $(TargetFileName) $(NETCoreSdkVersion) $(MicrosoftNETCoreAppPackageVersion) $(HelixTargetQueue)</Command>
         <Timeout>$(HelixTimeout)</Timeout>
       </HelixWorkItem>
     </ItemGroup>

src/Shared/test/SkipOnHelixAttribute.cs

@@ -2,12 +2,12 @@
 // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
 
 using System;
+using System.Linq;
 
 namespace Microsoft.AspNetCore.Testing.xunit
 {
     /// <summary>
-    /// Skip test if a given environment variable is not enabled. To enable the test, set environment variable
-    /// to "true" for the test process.
+    /// Skip test if running on helix (or a particular helix queue).
     /// </summary>
     [AttributeUsage(AttributeTargets.Method | AttributeTargets.Class, AllowMultiple = false)]
     public class SkipOnHelixAttribute : Attribute, ITestCondition
@@ -16,10 +16,14 @@ public bool IsMet
         {
             get
             {
-                return !OnHelix();
+                var skip = OnHelix() && (Queues == null || Queues.ToLowerInvariant().Split(";").Contains(GetTargetHelixQueue().ToLowerInvariant()));
+                return !skip;
             }
         }
 
+        // Queues that should be skipped on, i.e. "Windows.10.Amd64.ClientRS4.VS2017.Open;OSX.1012.Amd64.Open"
+        public string Queues { get; set; }
+
         public string SkipReason
         {
             get
@@ -28,6 +32,8 @@ public string SkipReason
             }
         }
 
-        public static bool OnHelix() => string.Equals(Environment.GetEnvironmentVariable("helix"), "true", StringComparison.OrdinalIgnoreCase);
+        public static bool OnHelix() => !string.IsNullOrEmpty(GetTargetHelixQueue());
+        
+        public static string GetTargetHelixQueue() => Environment.GetEnvironmentVariable("helix");
     }
 }