Skip to content

Add ability to skip tests on specific helix queues #8231

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 11 commits into from
Mar 7, 2019
Merged

Add ability to skip tests on specific helix queues #8231

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
d96e638
Pass helix target queue to runtests
HaoK Mar 6, 2019
7645368
Use the helixQueue instead of true in the helix env
HaoK Mar 6, 2019
61e73e5
Set helix queue
HaoK Mar 6, 2019
a4c350a
Set helix queue to env
HaoK Mar 6, 2019
519af90
SkipOnHelix is when helix env is set to anything
HaoK Mar 6, 2019
458a42f
Add queues to SkipOnHelix
HaoK Mar 6, 2019
42003ab
See if target queue skipping works
HaoK Mar 6, 2019
17d5614
More robust contains
HaoK Mar 6, 2019
53bd41b
Normalize strings to lower
HaoK Mar 6, 2019
7736f74
Update SkipOnHelixAttribute.cs
HaoK Mar 6, 2019
ac70b84
Revert test skips
HaoK Mar 6, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion eng/helix/vstest/runtests.cmd
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
set target=%1
set sdkVersion=%2
set runtimeVersion=%3
set helixQueue=%4

set DOTNET_HOME=%HELIX_CORRELATION_PAYLOAD%\sdk
set DOTNET_ROOT=%DOTNET_HOME%\x64
Expand All @@ -13,7 +14,7 @@ set PATH=%DOTNET_ROOT%;%PATH%
powershell.exe -NoProfile -ExecutionPolicy unrestricted -Command "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12; &([scriptblock]::Create((Invoke-WebRequest -useb 'https://dot.net/v1/dotnet-install.ps1'))) -Architecture x64 -Version %sdkVersion% -InstallDir %DOTNET_ROOT%"
powershell.exe -NoProfile -ExecutionPolicy unrestricted -Command "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12; &([scriptblock]::Create((Invoke-WebRequest -useb 'https://dot.net/v1/dotnet-install.ps1'))) -Architecture x64 -Runtime dotnet -Version %runtimeVersion% -InstallDir %DOTNET_ROOT%"

set HELIX=true
set HELIX=%helixQueue%

%DOTNET_ROOT%\dotnet vstest %target% -lt >discovered.txt
find /c "Exception thrown" discovered.txt
Expand Down
2 changes: 1 addition & 1 deletion eng/helix/vstest/runtests.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@ export DOTNET_MULTILEVEL_LOOKUP=0
# Avoid contaminating userprofiles
export DOTNET_CLI_HOME="$HELIX_CORRELATION_PAYLOAD/home"

export helix="true"
export helix="$4"

$DOTNET_ROOT/dotnet vstest $1 -lt >discovered.txt
if grep -q "Exception thrown" discovered.txt; then
Expand Down
2 changes: 1 addition & 1 deletion eng/helix/xunit/runtests.cmd
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
set target=%1
set helix=true
set helix=%4
xunit.console.exe %target% -xml testResults.xml
4 changes: 2 additions & 2 deletions eng/targets/Helix.targets
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -64,8 +64,8 @@
<TestAssembly>$(TargetFileName)</TestAssembly>
<PreCommands>@(HelixPreCommand)</PreCommands>
<PostCommands>@(HelixPostCommand)</PostCommands>
<Command Condition="$(IsWindowsHelixQueue)">call runtests.cmd $(TargetFileName) $(NETCoreSdkVersion) $(MicrosoftNETCoreAppPackageVersion)</Command>
<Command Condition="!$(IsWindowsHelixQueue)">./runtests.sh $(TargetFileName) $(NETCoreSdkVersion) $(MicrosoftNETCoreAppPackageVersion)</Command>
<Command Condition="$(IsWindowsHelixQueue)">call runtests.cmd $(TargetFileName) $(NETCoreSdkVersion) $(MicrosoftNETCoreAppPackageVersion) $(HelixTargetQueue)</Command>
<Command Condition="!$(IsWindowsHelixQueue)">./runtests.sh $(TargetFileName) $(NETCoreSdkVersion) $(MicrosoftNETCoreAppPackageVersion) $(HelixTargetQueue)</Command>
<Timeout>$(HelixTimeout)</Timeout>
</HelixWorkItem>
</ItemGroup>
Expand Down
14 changes: 10 additions & 4 deletions src/Shared/test/SkipOnHelixAttribute.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,12 +2,12 @@
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.

using System;
using System.Linq;

namespace Microsoft.AspNetCore.Testing.xunit
{
/// <summary>
/// Skip test if a given environment variable is not enabled. To enable the test, set environment variable
/// to "true" for the test process.
/// Skip test if running on helix (or a particular helix queue).
/// </summary>
[AttributeUsage(AttributeTargets.Method | AttributeTargets.Class, AllowMultiple = false)]
public class SkipOnHelixAttribute : Attribute, ITestCondition
Expand All @@ -16,10 +16,14 @@ public bool IsMet
{
get
{
return !OnHelix();
var skip = OnHelix() && (Queues == null || Queues.ToLowerInvariant().Split(";").Contains(GetTargetHelixQueue().ToLowerInvariant()));
return !skip;
}
}

// Queues that should be skipped on, i.e. "Windows.10.Amd64.ClientRS4.VS2017.Open;OSX.1012.Amd64.Open"
public string Queues { get; set; }

public string SkipReason
{
get
Expand All @@ -28,6 +32,8 @@ public string SkipReason
}
}

public static bool OnHelix() => string.Equals(Environment.GetEnvironmentVariable("helix"), "true", StringComparison.OrdinalIgnoreCase);
public static bool OnHelix() => !string.IsNullOrEmpty(GetTargetHelixQueue());

public static string GetTargetHelixQueue() => Environment.GetEnvironmentVariable("helix");
}
}