Skip to content

Optimization of CI pipeline #19799

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Oct 20, 2022
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion .ci-config.json
Original file line number Diff line number Diff line change
Expand Up @@ -299,5 +299,6 @@
"Resources",
"Storage",
"Websites"
]
],
"artifactPipelineInfoFolder": "artifacts/PipelineResult"
}
23 changes: 11 additions & 12 deletions build.proj
Original file line number Diff line number Diff line change
Expand Up @@ -236,35 +236,34 @@
<!-- Everything except Publish -->
<Target Name="Full" DependsOnTargets="Clean;Build;OnPremChecks;CopyAboutTopics;GenerateHelp;StaticAnalysis;Test" />

<Target Name="StaticAnalysisBreakingChange" Condition="'$(RunStaticAnalysis)' == 'true'" DependsOnTargets="Build">
<Target Name="StaticAnalysisBreakingChange" Condition="'$(RunStaticAnalysis)' == 'true'">
<Message Importance="high" Text="Running static analysis for breaking change..." />
<MakeDir Directories="$(StaticAnalysisOutputDirectory)" />

<Exec Command="dotnet $(RepoArtifacts)StaticAnalysis/StaticAnalysis.Netcore.dll -p $(RepoArtifacts)$(Configuration) -r $(StaticAnalysisOutputDirectory) --analyzers breaking-change -u -m '%(FilterTaskResult.breaking-change)'" />
<Exec Command="$(PowerShellCoreCommandPrefix) &quot;.\tools\ExecuteCIStep.ps1 -StaticAnalysisBreakingChange -RepoArtifacts $(RepoArtifacts) -Configuration $(Configuration) -StaticAnalysisOutputDirectory $(StaticAnalysisOutputDirectory)&quot;" />
</Target>

<Target Name="StaticAnalysisDependency" Condition="'$(RunStaticAnalysis)' == 'true'" DependsOnTargets="Build">
<Target Name="StaticAnalysisDependency" Condition="'$(RunStaticAnalysis)' == 'true'">
<Message Importance="high" Text="Running static analysis for dependency..." />
<MakeDir Directories="$(StaticAnalysisOutputDirectory)" />

<Exec Command="dotnet $(RepoArtifacts)StaticAnalysis/StaticAnalysis.Netcore.dll -p $(RepoArtifacts)$(Configuration) -r $(StaticAnalysisOutputDirectory) --analyzers dependency -u -m '%(FilterTaskResult.breaking-change)'" />
<Exec Command="$(PowerShellCoreCommandPrefix) &quot;.\tools\ExecuteCIStep.ps1 -StaticAnalysisDependency -RepoArtifacts $(RepoArtifacts) -Configuration $(Configuration) -StaticAnalysisOutputDirectory $(StaticAnalysisOutputDirectory)&quot;" />
</Target>

<Target Name="StaticAnalysisSignature" Condition="'$(RunStaticAnalysis)' == 'true'" DependsOnTargets="Build">
<Target Name="StaticAnalysisSignature" Condition="'$(RunStaticAnalysis)' == 'true'">
<Message Importance="high" Text="Running static analysis for signature..." />
<MakeDir Directories="$(StaticAnalysisOutputDirectory)" />

<Exec Command="dotnet $(RepoArtifacts)StaticAnalysis/StaticAnalysis.Netcore.dll -p $(RepoArtifacts)$(Configuration) -r $(StaticAnalysisOutputDirectory) --analyzers signature -u -m '%(FilterTaskResult.signature)'" />
<Exec Command="$(PowerShellCoreCommandPrefix) &quot;.\tools\ExecuteCIStep.ps1 -StaticAnalysisSignature -RepoArtifacts $(RepoArtifacts) -Configuration $(Configuration) -StaticAnalysisOutputDirectory $(StaticAnalysisOutputDirectory)&quot;" />
</Target>

<Target Name="StaticAnalysisHelp" Condition="'$(RunStaticAnalysis)' == 'true'" DependsOnTargets="Build">
<Target Name="StaticAnalysisHelp" Condition="'$(RunStaticAnalysis)' == 'true'">
<Message Importance="high" Text="Running static analysis for help..." />
<MakeDir Directories="$(StaticAnalysisOutputDirectory)" />

<Exec Command="dotnet $(RepoArtifacts)StaticAnalysis/StaticAnalysis.Netcore.dll -p $(RepoArtifacts)$(Configuration) -r $(StaticAnalysisOutputDirectory) --analyzers help -u -m '%(FilterTaskResult.help)'" />
<Exec Command="$(PowerShellCoreCommandPrefix) &quot;.\tools\ExecuteCIStep.ps1 -StaticAnalysisHelp -RepoArtifacts $(RepoArtifacts) -Configuration $(Configuration) -StaticAnalysisOutputDirectory $(StaticAnalysisOutputDirectory)&quot;" />
</Target>

<Target Name="StaticAnalysisExample" Condition="'$(RunStaticAnalysis)' == 'true'" DependsOnTargets="Build" AfterTargets="StaticAnalysisHelp">
<Target Name="StaticAnalysisExample" Condition="'$(RunStaticAnalysis)' == 'true'" AfterTargets="StaticAnalysisHelp">
<Message Importance="high" Text="Running static analysis for PowerShell examples..." />
<Exec Command="$(PowerShellCoreCommandPrefix) &quot;. $(RepoTools)/StaticAnalysis/ExampleAnalyzer/Measure-MarkdownOrScript.ps1 -MarkdownPaths $(RepoArtifacts)/FilesChanged.txt -RulePaths $(RepoTools)/StaticAnalysis/ExampleAnalyzer/AnalyzeRules/*.psm1 &quot;"/>
</Target>
Expand Down Expand Up @@ -318,12 +317,12 @@
<Exec Command="$(PowerShellCoreCommandPrefix) &quot;. $(DockerTools)/CleanArtifacts.ps1 -Docker $(DockerRoot) &quot;" />
</Target>

<Target Name="Test" DependsOnTargets="FilterBuild" Condition="'$(IsGenerateBased)' == 'false'">
<Target Name="Test" Condition="'$(IsGenerateBased)' == 'false'">
<Message Importance="high" Text="Running check in tests..." />
<MakeDir Directories="$(TestOutputDirectory)" ContinueOnError="false" />
<!-- https://github.com/Microsoft/msbuild/issues/3157#issuecomment-378002971 -->
<!-- https://github.com/xunit/xunit/issues/1154#issuecomment-405366373 -->
<Exec Command="dotnet test $(RepoArtifacts)Azure.PowerShell.sln --filter &quot;AcceptanceType=CheckIn%26RunType!=DesktopOnly&quot; --configuration $(Configuration) --framework $(TestFramework) --logger trx --results-directory &quot;$(TestOutputDirectory)&quot;" Condition="'$(TestCsprojList)' != ''" />
<Exec Command="$(PowerShellCoreCommandPrefix) &quot;.\tools\ExecuteCIStep.ps1 -Test -RepoArtifacts $(RepoArtifacts) -Configuration $(Configuration) -TestFramework $(TestFramework) -TestOutputDirectory $(TestOutputDirectory)&quot;" />
<Message Importance="high" Text="Finish running tests..." />
<OnError ExecuteTargets="TestFailureErrorMessage" />
</Target>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -337,7 +337,7 @@ private Dictionary<string, HashSet<string>> CalculateCsprojForBuildAndTest(Dicti
}
if (csprojSet.Count != 0)
{
foreach (string filename in Directory.GetFiles(@"src/Accounts", "*.csproj", SearchOption.AllDirectories).Where(x => !x.Contains("Test")))
foreach (string filename in Directory.GetFiles(@"src\Accounts", "*.csproj", SearchOption.AllDirectories).Where(x => !x.Contains("Test")))
{
csprojSet.Add(filename);
}
Expand All @@ -353,7 +353,7 @@ private Dictionary<string, HashSet<string>> CalculateCsprojForBuildAndTest(Dicti
}
if (csprojSet.Count != 0)
{
csprojSet.Add("tools/TestFx/TestFx.csproj");
csprojSet.Add(@"tools\TestFx\TestFx.csproj");
}
influencedModuleInfo[phaseName] = csprojSet;
}
Expand Down Expand Up @@ -397,9 +397,26 @@ private bool ProcessFileChanged(Dictionary<string, string[]> csprojMap)

FilterTaskResult.PhaseInfo = influencedModuleInfo;

influencedModuleInfo[string.Format("{0}-module", BUILD_PHASE)] = new HashSet<string>(influencedModuleInfo[BUILD_PHASE].Select(GetModuleNameFromPath).Where(x => x != null));
influencedModuleInfo[string.Format("{0}-module", TEST_PHASE)] = new HashSet<string>(influencedModuleInfo[TEST_PHASE].Select(GetModuleNameFromPath).Where(x => x != null));
if (!Directory.Exists(config.ArtifactPipelineInfoFolder))
{
Directory.CreateDirectory(config.ArtifactPipelineInfoFolder);
}
File.WriteAllText(Path.Combine(config.ArtifactPipelineInfoFolder, "CIPlan.json"), JsonConvert.SerializeObject(influencedModuleInfo, Formatting.Indented));

return true;
}

private string GetModuleNameFromPath(string path)
{
if (path.IndexOf("src") == -1)
{
return null;
}
return path.Replace("\\", "/").Split("src/")[1].Split('/')[0];
}

/// <summary>
/// Executes the task to generate a list of test assemblies
/// based on file changes from a specified Pull Request.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ class CIPhaseFilterConfig
{
public List<Rule> Rules { get; set; }
public List<string> SelectModuleList { get; set; }
public string ArtifactPipelineInfoFolder { get; set; }
}

class Rule
Expand Down
2 changes: 1 addition & 1 deletion tools/CreateFilterMappings.ps1
Original file line number Diff line number Diff line change
Expand Up @@ -100,7 +100,7 @@ function Create-ProjectToFullPathMappings
{
throw ($CsprojFile.FullName + " is conflicts with " + $Mappings[$CsprojFile.BaseName])
}
$Mappings[$CsprojFile.BaseName] = $CsprojFile.FullName
$Mappings[$CsprojFile.BaseName] = [IO.Path]::GetRelativePath([IO.Path]::Combine($PSScriptRoot, ".."), $CsprojFile.FullName)
}
}

Expand Down
112 changes: 112 additions & 0 deletions tools/ExecuteCIStep.ps1
Original file line number Diff line number Diff line change
@@ -0,0 +1,112 @@
# ----------------------------------------------------------------------------------
# Copyright (c) Microsoft Corporation. All rights reserved.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# http://www.apache.org/licenses/LICENSE-2.0
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Code generated by Microsoft (R) AutoRest Code Generator.Changes may cause incorrect behavior and will be lost If the code
# is regenerated.
# ----------------------------------------------------------------------------------

Param(
[Switch]
$Test,

[Switch]
$StaticAnalysisBreakingChange,

[Switch]
$StaticAnalysisDependency,

[Switch]
$StaticAnalysisSignature,

[Switch]
$StaticAnalysisHelp,

[String]
$RepoArtifacts='artifacts',

[String]
$Configuration='Debug',

[String]
$TestFramework='netcoreapp2.2',

[String]
$TestOutputDirectory='artifacts\TestResults',

[String]
$StaticAnalysisOutputDirectory='artifacts\StaticAnalysisResults',

[String]
$ModuleList
)
$ErrorActionPreference = 'Stop'

If (-Not $PSBoundParameters.ContainsKey("ModuleList"))
{
$CIPlan = Get-Content $RepoArtifacts/PipelineResult/CIPlan.json | ConvertFrom-Json
}

If ($Test -And $CIPlan.test.Length -Ne 0)
{
dotnet test $RepoArtifacts/Azure.PowerShell.sln --filter "AcceptanceType=CheckIn&RunType!=DesktopOnly" --configuration $Configuration --framework $TestFramework --logger trx --results-directory $TestOutputDirectory
}

If ($StaticAnalysisBreakingChange)
{
If ($PSBoundParameters.ContainsKey("ModuleList"))
{
$BreakingChangeCheckModuleList = $ModuleList
}
Else
{
$BreakingChangeCheckModuleList = Join-String -Separator ';' -InputObject $CIPlan.'breaking-change'
}
dotnet $RepoArtifacts/StaticAnalysis/StaticAnalysis.Netcore.dll -p $RepoArtifacts/$Configuration -r $StaticAnalysisOutputDirectory --analyzers breaking-change -u -m $BreakingChangeCheckModuleList
}

If ($StaticAnalysisDependency)
{
If ($PSBoundParameters.ContainsKey("ModuleList"))
{
$DependencyCheckModuleList = $ModuleList
}
Else
{
$DependencyCheckModuleList = Join-String -Separator ';' -InputObject $CIPlan.dependency
}
dotnet $RepoArtifacts/StaticAnalysis/StaticAnalysis.Netcore.dll -p $RepoArtifacts/$Configuration -r $StaticAnalysisOutputDirectory --analyzers dependency -u -m $DependencyCheckModuleList
}

If ($StaticAnalysisSignature)
{
If ($PSBoundParameters.ContainsKey("ModuleList"))
{
$SignatureCheckModuleList = $ModuleList
}
Else
{
$SignatureCheckModuleList = Join-String -Separator ';' -InputObject $CIPlan.signature
}
dotnet $RepoArtifacts/StaticAnalysis/StaticAnalysis.Netcore.dll -p $RepoArtifacts/$Configuration -r $StaticAnalysisOutputDirectory --analyzers signature -u -m $SignatureCheckModuleList
}

If ($StaticAnalysisHelp)
{
If ($PSBoundParameters.ContainsKey("ModuleList"))
{
$HelpCheckModuleList = $ModuleList
}
Else
{
$HelpCheckModuleList = Join-String -Separator ';' -InputObject $CIPlan.help
}
dotnet $RepoArtifacts/StaticAnalysis/StaticAnalysis.Netcore.dll -p $RepoArtifacts/$Configuration -r $StaticAnalysisOutputDirectory --analyzers help -u -m $HelpCheckModuleList
}