Skip to content

Commit 87067f8

Browse files
wyunchi-mswyunchi-ms
authored
[Tools] Update tools to compatible CI pipeline for generation branch (Azure#15276)
* Update tools to compatible CI pipeline for generation branch * Add Support for hybrid module Co-authored-by: wyunchi-ms <[email protected]>
1 parent 585b83c commit 87067f8

File tree

9 files changed

+181
-18
lines changed

9 files changed

+181
-18
lines changed

‎.ci-config.json

Lines changed: 20 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,18 @@
11
{
22
"rules": [
3+
{
4+
"patterns": [
5+
"src/{ModuleName}/readme.md",
6+
"src/{ModuleName}[a-zA-Z`.`/]*/readme.md"
7+
],
8+
"phases": [
9+
"build:related-module",
10+
"breaking-change:module",
11+
"help:module",
12+
"signature:module",
13+
"test:dependence-module"
14+
]
15+
},
316
{
417
"patterns": [
518
".azure-pipeline/*",
@@ -96,10 +109,10 @@
96109
},
97110
{
98111
"patterns": [
99-
"tools/StaticAnalysis/Exceptions/{ModuleName}/MissingAssemblies.csv",
100-
"tools/StaticAnalysis/Exceptions/{ModuleName}/AssemblyVersionConflict.csv",
101-
"tools/StaticAnalysis/Exceptions/{ModuleName}/ExtraAssemblies.csv",
102-
"tools/StaticAnalysis/Exceptions/{ModuleName}/SharedAssemblyConflict.csv"
112+
"tools/StaticAnalysis/Exceptions/Az.{ModuleName}/MissingAssemblies.csv",
113+
"tools/StaticAnalysis/Exceptions/Az.{ModuleName}/AssemblyVersionConflict.csv",
114+
"tools/StaticAnalysis/Exceptions/Az.{ModuleName}/ExtraAssemblies.csv",
115+
"tools/StaticAnalysis/Exceptions/Az.{ModuleName}/SharedAssemblyConflict.csv"
103116
],
104117
"phases": [
105118
"build:module",
@@ -108,7 +121,7 @@
108121
},
109122
{
110123
"patterns": [
111-
"tools/StaticAnalysis/Exceptions/{ModuleName}/BreakingChangeIssues.csv"
124+
"tools/StaticAnalysis/Exceptions/Az.{ModuleName}/BreakingChangeIssues.csv"
112125
],
113126
"phases": [
114127
"build:module",
@@ -117,7 +130,7 @@
117130
},
118131
{
119132
"patterns": [
120-
"tools/StaticAnalysis/Exceptions/{ModuleName}/HelpIssues.csv"
133+
"tools/StaticAnalysis/Exceptions/Az.{ModuleName}/HelpIssues.csv"
121134
],
122135
"phases": [
123136
"build:module",
@@ -126,7 +139,7 @@
126139
},
127140
{
128141
"patterns": [
129-
"tools/StaticAnalysis/Exceptions/{ModuleName}/SignatureIssues.csv"
142+
"tools/StaticAnalysis/Exceptions/Az.{ModuleName}/SignatureIssues.csv"
130143
],
131144
"phases": [
132145
"build:module",

‎build.proj

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@
3232
<Configuration Condition="'$(Configuration)' != 'Release'">Debug</Configuration>
3333
<Scope Condition="$(Scope) == ''">All</Scope>
3434
<TestFramework Condition="'$(TestFramework)' == ''">netcoreapp2.2</TestFramework>
35+
<IsGenerateBased Condition="'$(IsGenerateBased)' != 'true'">false</IsGenerateBased>
3536

3637
<!-- Flags -->
3738
<CodeSign Condition ="'$(CodeSign)' == ''">false</CodeSign>
@@ -108,8 +109,8 @@
108109

109110
<Target Name="FilterBuild" Condition="$(PullRequestNumber) != '' OR $(TargetModule) != ''" DependsOnTargets="BuildTools">
110111
<Message Importance="high" Text="Filtering projects and modules..." />
111-
<Exec Command="$(PowerShellCoreCommandPrefix) &quot;. $(RepoTools)/CreateFilterMappings.ps1&quot;" />
112112

113+
<Message Text="$(IsGenerateBased)" />
113114
<!-- Build the Microsoft.Azure.Build.Tasks project -->
114115
<Exec Command="dotnet publish $(RepoTools)BuildPackagesTask/Microsoft.Azure.Build.Tasks/Microsoft.Azure.Build.Tasks.csproj -c $(Configuration)" />
115116

@@ -118,6 +119,9 @@
118119
<Output TaskParameter="FilesChanged" ItemName="FilesChanged" />
119120
</FilesChangedTask>
120121

122+
<Exec Command="$(PowerShellCoreCommandPrefix) &quot;$(RepoTools)PrepareAutorestModule.ps1&quot;" Condition="'$(IsGenerateBased)' == 'true'"/>
123+
<Exec Command="$(PowerShellCoreCommandPrefix) &quot;. $(RepoTools)/CreateFilterMappings.ps1&quot;" />
124+
121125
<CIFilterTask FilesChanged="@(FilesChanged)" TargetModule="$(TargetModule)" Mode="$(Configuration)" CsprojMapFilePath="./CsprojMappings.json">
122126
<Output TaskParameter="FilterTaskResult" ItemName="FilterTaskResult" />
123127
</CIFilterTask>

‎tools/BuildPackagesTask/Microsoft.Azure.Build.Tasks/FilesChangedTask.cs

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
namespace Microsoft.WindowsAzure.Build.Tasks
1616
{
1717
using System;
18+
using System.IO;
1819
using System.Collections.Generic;
1920
using System.Runtime.InteropServices;
2021
using Microsoft.Build.Framework;
@@ -142,7 +143,15 @@ public override bool Execute()
142143
FilesChanged = new string[] { };
143144
}
144145

146+
SerializeChangedFilesToFile(FilesChanged);
147+
145148
return true;
146149
}
150+
151+
// This method will record the changed files into FilesChanged.txt under root folder for other task to consum.
152+
private void SerializeChangedFilesToFile(string[] FilesChanged)
153+
{
154+
File.WriteAllLines("FilesChanged.txt", FilesChanged);
155+
}
147156
}
148157
}

‎tools/CreateFilterMappings.ps1

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -128,7 +128,7 @@ function Create-SolutionToProjectMappings
128128
$Mappings = [ordered]@{}
129129
foreach ($ServiceFolder in $Script:ServiceFolders)
130130
{
131-
$SolutionFiles = Get-ChildItem -Path $ServiceFolder.FullName -Filter "*.sln"
131+
$SolutionFiles = Get-ChildItem -Path $ServiceFolder.FullName -Filter "*.sln" -Recurse
132132
foreach ($SolutionFile in $SolutionFiles)
133133
{
134134
$Mappings = Add-ProjectDependencies -Mappings $Mappings -SolutionPath $SolutionFile.FullName

‎tools/Gen2Master/MoveFromGeneration2Master.ps1

Lines changed: 51 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -7,12 +7,25 @@ Function Move-Generation2Master {
77
)
88

99
process {
10-
$ModuleName = ($SourcePath.Trim("\").Split("\"))[-1]
10+
#Region Handle the hybrid module whoes folder is a subfolder of the module folder.
11+
$ModuleName = $SourcePath.Replace('/', '\').Split('\src\')[1].Split('\')[0]
12+
$SourcePsd1Path = Get-ChildItem -path $ModuleFolder -filter Az.$ModuleName.psd1 -Recurse
13+
$FolderPathRelativeToSrc = $SourcePsd1Path.Directory.FullName.Replace('/', '\').Split('\src\')[1]
14+
if ($FolderPathRelativeToSrc -eq $ModuleName)
15+
{
16+
$IsHybridModule = $False
17+
}
18+
else
19+
{
20+
$IsHybridModule = $True
21+
}
22+
$SourcePath = $SourcePsd1Path.Directory.FullName
1123
If (-not ($DestPath.Trim("\").Split("\")[-1] -eq $ModuleName)) {
12-
$DestPath = Join-Path -Path $DestPath -ChildPath $ModuleName
24+
$DestPath = Join-Path -Path $DestPath -ChildPath $FolderPathRelativeToSrc
1325
}
26+
#EndRegion
1427
If (-not (Test-Path $DestPath)) {
15-
New-Item -ItemType Directory -Path $DestPath
28+
New-Item -ItemType Directory -Path $DestPath -Force
1629
}
1730
$Dir2Copy = @('custom', 'examples', 'exports', 'generated', 'internal', 'test', 'utils')
1831
Foreach($Dir in $Dir2Copy) {
@@ -64,7 +77,8 @@ Function Move-Generation2Master {
6477
$Psd1Version = $Psd1Metadata.ModuleVersion
6578
}
6679
$Psd1Metadata = Import-LocalizedData -BaseDirectory $SourcePath -FileName "Az.$ModuleName.psd1"
67-
if ($Null -ne $Psd1Version) {
80+
If ($Null -ne $Psd1Version)
81+
{
6882
$Psd1Metadata.ModuleVersion = $Psd1Version
6983
}
7084
If ($Null -ne $ModuleGuid) {
@@ -76,7 +90,10 @@ Function Move-Generation2Master {
7690
$AccountsMetadata = Import-LocalizedData -BaseDirectory $AccountsModulePath -FileName "Az.Accounts.psd1"
7791
$RequiredModule = @(@{ModuleName = 'Az.Accounts'; ModuleVersion = $AccountsMetadata.ModuleVersion; })
7892
}
79-
$Psd1Metadata.RequiredModules = $RequiredModule
93+
If ($Null -ne $RequiredModule)
94+
{
95+
$Psd1Metadata.RequiredModules = $RequiredModule
96+
}
8097
If ($Psd1Metadata.FunctionsToExport -Contains "*") {
8198
$Psd1Metadata.FunctionsToExport = ($Psd1Metadata.FunctionsToExport | Where-Object {$_ -ne "*"})
8299
}
@@ -143,9 +160,36 @@ Function Move-Generation2Master {
143160
Set-Content -Path $GeneratedModuleListPath -Value $NewModules
144161
#EndRegion
145162

146-
Copy-Template -SourceName Az.ModuleName.csproj -DestPath $DestPath -DestName "Az.$ModuleName.csproj"
163+
if ($IsHybridModule)
164+
{
165+
Copy-Template -SourceName Az.ModuleName.hybrid.csproj -DestPath $DestPath -DestName "Az.$ModuleName.csproj"
166+
}
167+
else
168+
{
169+
Copy-Template -SourceName Az.ModuleName.csproj -DestPath $DestPath -DestName "Az.$ModuleName.csproj"
170+
}
147171
Copy-Template -SourceName Changelog.md -DestPath $DestPath -DestName Changelog.md
148-
Copy-Template -SourceName ModuleName.sln -DestPath $DestPath -DestName "$ModuleName.sln"
172+
#Region create a solution file for module and add the related csproj files to this solution.
173+
dotnet new sln -n $ModuleName -o $DestPath
174+
$DestParentPath = $DestPath
175+
While ("" -eq $DestParentPath)
176+
{
177+
$DestAccountsPath = Get-ChildItem -Path $DestParentPath -Filter Accounts
178+
if ($Null -eq $DestAccountsPath)
179+
{
180+
$DestParentPath = Split-Path -path $DestParentPath -Parent
181+
}
182+
else
183+
{
184+
Break
185+
}
186+
}
187+
$SolutionPath = Join-Path -Path $DestPath -ChildPath $ModuleName.sln
188+
foreach ($DependenceCsproj in (Get-ChildItem -path $DestAccountsPath -Recurse -Filter *.csproj -Exclude *test*))
189+
{
190+
dotnet sln $SolutionPath add $DependenceCsproj
191+
}
192+
#EndRegion
149193

150194
$PropertiesPath = Join-Path -Path $DestPath -ChildPath "Properties"
151195
If (-not (Test-Path $PropertiesPath)) {

‎tools/Gen2Master/Templates/Az.ModuleName.csproj

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,5 +3,5 @@
33
<PsModuleName>{ModuleNamePlaceHolder}</PsModuleName>
44
</PropertyGroup>
55

6-
<Import Project="$(MSBuildThisFileDirectory)..\Az.autorest.props" />
6+
<Import Project="$([MSBuild]::GetDirectoryNameOfFileAbove($(MSBuildThisFileDirectory).., build.proj))\src\Az.autorest.props" />
77
</Project>

‎tools/Gen2Master/Templates/Az.ModuleName.hybrid.csproj

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
<Project Sdk="Microsoft.NET.Sdk">
2+
<PropertyGroup>
3+
<PsModuleName>{ModuleNamePlaceHolder}</PsModuleName>
4+
</PropertyGroup>
5+
6+
<Import Project="$([MSBuild]::GetDirectoryNameOfFileAbove($(MSBuildThisFileDirectory).., build.proj))\src\Az.autorest.hybrid.props" />
7+
</Project>

‎tools/PrepareAutorestModule.ps1

Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,86 @@
1+
# ----------------------------------------------------------------------------------
2+
#
3+
# Copyright Microsoft Corporation
4+
# Licensed under the Apache License, Version 2.0 (the "License");
5+
# you may not use this file except in compliance with the License.
6+
# You may obtain a copy of the License at
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
# Unless required by applicable law or agreed to in writing, software
9+
# distributed under the License is distributed on an "AS IS" BASIS,
10+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11+
# See the License for the specific language governing permissions and
12+
# limitations under the License.
13+
# ----------------------------------------------------------------------------------
14+
15+
#This script will pack build artifacts under temporary folder "artifacts/tmp" and output Az.*.nupkg to "artifacts"
16+
17+
18+
param(
19+
)
20+
$ChangedFiles = Get-Content -Path "$PSScriptRoot\..\FilesChanged.txt"
21+
22+
$ALL_MODULE = "ALL_MODULE"
23+
24+
#Region Detect which module should be processed
25+
$ModuleSet = New-Object System.Collections.Generic.HashSet[string]
26+
foreach ($file in $ChangedFiles)
27+
{
28+
$ParentFolder = Split-Path -Path $file -Parent
29+
if ($ParentFolder.StartsWith("src"))
30+
{
31+
if ($ParentFolder -eq "src")
32+
{
33+
$NUll = $ModuleSet.Add($ALL_MODULE)
34+
}
35+
else
36+
{
37+
$NUll = $ModuleSet.Add($ParentFolder.Replace("/", "\").Split('\')[1])
38+
}
39+
}
40+
else
41+
{
42+
$NUll = $ModuleSet.Add($ALL_MODULE)
43+
}
44+
}
45+
if ($ModuleSet.Contains($ALL_MODULE))
46+
{
47+
$ModuleList = (Get-ChildItem "$PSScriptRoot\..\src\" -Directory).Name
48+
}
49+
else
50+
{
51+
$ModuleList = $ModuleSet | Where-Object { $_ }
52+
}
53+
#EndRegion
54+
55+
Import-Module "$PSScriptRoot\..\tools\Gen2Master\MoveFromGeneration2Master.ps1" -Force
56+
$TmpFolder = "$PSScriptRoot\..\tmp"
57+
New-Item -ItemType Directory -Force -Path $TmpFolder
58+
Remove-Item -Path "$TmpFolder\*" -Recurse -Force
59+
60+
#Region Clone latest Az.Accounts code
61+
Set-Location -Path $TmpFolder
62+
git init
63+
git remote add -f origin https://github.com/Azure/azure-powershell.git
64+
git config core.sparseCheckout true
65+
Add-Content -Path .git/info/sparse-checkout -Value "src/Accounts/"
66+
git pull origin main
67+
Move-Item -Path "$TmpFolder\src\Accounts" -Destination "$TmpFolder\Accounts"
68+
#EndRegion
69+
70+
#Region generate the code and make the struture same with main branch.
71+
foreach ($Module in $ModuleList)
72+
{
73+
$ModuleFolder = "$PSScriptRoot\..\src\$Module\"
74+
$ModuleFolder = (Get-ChildItem -path $ModuleFolder -filter Az.$Module.psd1 -Recurse).Directory
75+
if ($Null -eq $ModuleFolder)
76+
{
77+
Throw "Cannot find Az.$Module.psd1 in $ModuleFolder."
78+
}
79+
Set-Location -Path $ModuleFolder
80+
autorest
81+
./build-module.ps1
82+
Move-Generation2Master -SourcePath "$PSScriptRoot\..\src\$Module\" -DestPath $TmpFolder
83+
Remove-Item "$ModuleFolder\*" -Recurse
84+
}
85+
#EndRegion
86+
Copy-Item "$TmpFolder\*" "$PSScriptRoot\..\src" -Recurse -Force

‎tools/UpdateModules.psm1

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -58,7 +58,7 @@ function New-ModulePsm1 {
5858
PROCESS {
5959
$manifestDir = Get-Item -Path $ModulePath
6060
$moduleName = $manifestDir.Name + ".psd1"
61-
$manifestPath = Join-Path -Path $ModulePath -ChildPath $moduleName
61+
$manifestPath = Get-ChildItem -Path $manifestDir -Filter $moduleName -Recurse
6262
$file = Get-Item $manifestPath
6363
Import-LocalizedData -BindingVariable ModuleMetadata -BaseDirectory $file.DirectoryName -FileName $file.Name
6464

0 commit comments

Comments
 (0)