Sync back pipeline tools from generation branch. (#15386)

wyunchi-ms · web-flow · commit 0c0bcc37fc14 · 2021-07-02T15:02:04.000+08:00
Co-authored-by: wyunchi-ms &lt;yunwang@microsoft.com&gt;
diff --git a/.azure-pipelines/powershell-core.yml b/.azure-pipelines/powershell-core.yml
@@ -9,6 +9,9 @@ variables:
   TestTarget: Test
   Configuration: Debug
   DebugLocalBuildTasks: true
+  IsGenerateBased: $[eq(variables['system.pullRequest.targetBranch'], 'generation')]
+  BuildTimeoutInMinutes: 120
+  AnalysisTimeoutInMinutes: 120
 
 trigger: none
 
@@ -23,6 +26,7 @@ jobs:
 - job: Build
   displayName: Build
   condition: succeeded()
+  timeoutInMinutes: ${{ variables.BuildTimeoutInMinutes }}
   strategy:
     matrix:
       windows:
@@ -31,8 +35,8 @@ jobs:
       linux:
         OSName: ${{ variables.LinuxName }}
         ImageName: ${{ variables.LinuxImage }}
-      macOS:	
-        OSName: ${{ variables.MacOSName }}	
+      macOS:
+        OSName: ${{ variables.MacOSName }}
         ImageName: ${{ variables.MacOSImage }}
   pool:
     vmImage: $(ImageName)
@@ -48,7 +52,7 @@ jobs:
   displayName: Analyze
   dependsOn: Build
   condition: succeeded()
-  timeoutInMinutes: 90
+  timeoutInMinutes: ${{ variables.AnalysisTimeoutInMinutes }}
   strategy:
     matrix:
       windows:
diff --git a/.azure-pipelines/util/analyze-steps.yml b/.azure-pipelines/util/analyze-steps.yml
@@ -6,6 +6,21 @@ steps:
 - template: download-build-steps.yml
   parameters:
     osName: ${{ parameters.osName }}
+- task: NodeTool@0
+  condition: eq(variables.IsGenerateBased, true)
+  displayName: Install autorest
+  inputs:
+    versionSpec: '14.17.1'
+    command: custom
+    verbose: false
+    customCommand: install autorest@latest
+- task: PowerShell@2
+  condition: eq(variables.IsGenerateBased, true)
+  displayName: Setup environment for autorest
+  inputs:
+    targetType: inline
+    script: "$env:NODE_OPTIONS=\"--max-old-space-size=65536\""
+    pwsh: true
 
 - task: UseDotNet@2
   displayName: 'Use .NET Core sdk'
diff --git a/.azure-pipelines/util/build-steps.yml b/.azure-pipelines/util/build-steps.yml
@@ -4,6 +4,21 @@ parameters:
   configuration: ''
 
 steps:
+- task: NodeTool@0
+  condition: eq(variables.IsGenerateBased, true)
+  displayName: Install autorest
+  inputs:
+    versionSpec: '14.17.1'
+    command: custom
+    verbose: false
+    customCommand: install autorest@latest
+- task: PowerShell@2
+  condition: eq(variables.IsGenerateBased, true)
+  displayName: Setup environment for autorest
+  inputs:
+    targetType: inline
+    script: "$env:NODE_OPTIONS=\"--max-old-space-size=65536\""
+    pwsh: true
 - task: PowerShell@2
   displayName: 'Check Ingored File'
   inputs:
diff --git a/.azure-pipelines/util/test-steps.yml b/.azure-pipelines/util/test-steps.yml
@@ -8,6 +8,21 @@ steps:
 - template: download-build-steps.yml
   parameters:
     osName: ${{ parameters.osName }}
+- task: NodeTool@0
+  condition: eq(variables.IsGenerateBased, true)
+  displayName: Install autorest
+  inputs:
+    versionSpec: '14.17.1'
+    command: custom
+    verbose: false
+    customCommand: install autorest@latest
+- task: PowerShell@2
+  condition: eq(variables.IsGenerateBased, true)
+  displayName: Setup environment for autorest
+  inputs:
+    targetType: inline
+    script: "$env:NODE_OPTIONS=\"--max-old-space-size=65536\""
+    pwsh: true
 
 - task: UseDotNet@2
   displayName: 'Use .NET Core sdk'
diff --git a/.azure-pipelines/windows-powershell.yml b/.azure-pipelines/windows-powershell.yml
@@ -5,6 +5,7 @@ variables:
   TestTarget: Test
   Configuration: Debug
   DebugLocalBuildTasks: true
+  IsGenerateBased: $[eq(variables['system.pullRequest.targetBranch'], 'generation')]
 
 trigger: none
 
@@ -18,7 +19,7 @@ pr:
 jobs:
 - job: Build
   displayName: Build
-  condition: succeeded()
+  condition: and(eq(variables.IsGenerateBased, false), succeeded())
   pool:
     vmImage: ${{ variables.WindowsImage }}
 
@@ -35,7 +36,7 @@ jobs:
 - job: Analyze
   displayName: Analyze
   dependsOn: Build
-  condition: succeeded()
+  condition: and(eq(variables.IsGenerateBased, false), succeeded())
   timeoutInMinutes: 90
   pool:
     vmImage: ${{ variables.WindowsImage }}
@@ -66,7 +67,7 @@ jobs:
 - job: Test
   displayName: Test
   dependsOn: Build
-  condition: succeeded()
+  condition: and(eq(variables.IsGenerateBased, false), succeeded())
   timeoutInMinutes: 180
   pool:
     vmImage: ${{ variables.WindowsImage }}
diff --git a/tools/Gen2Master/MoveFromGeneration2Master.ps1 b/tools/Gen2Master/MoveFromGeneration2Master.ps1
@@ -9,7 +9,7 @@ Function Move-Generation2Master {
     process {
         #Region Handle the hybrid module whoes folder is a subfolder of the module folder.
         $ModuleName = $SourcePath.Replace('/', '\').Split('\src\')[1].Split('\')[0]
-        $SourcePsd1Path = Get-ChildItem -path $ModuleFolder -filter Az.$ModuleName.psd1 -Recurse
+        $SourcePsd1Path = Get-ChildItem -path $SourcePath -filter Az.$ModuleName.psd1 -Recurse
         $FolderPathRelativeToSrc = $SourcePsd1Path.Directory.FullName.Replace('/', '\').Split('\src\')[1]
         if ($FolderPathRelativeToSrc -eq $ModuleName)
         {
@@ -23,6 +23,20 @@ Function Move-Generation2Master {
         If (-not ($DestPath.Trim("\").Split("\")[-1] -eq $ModuleName)) {
             $DestPath = Join-Path -Path $DestPath -ChildPath $FolderPathRelativeToSrc
         }
+        
+        $DestParentPath = $DestPath
+        While ("" -ne $DestParentPath)
+        {
+            $DestAccountsPath = Join-Path -Path $DestParentPath -ChildPath Accounts
+            if (-not (Test-Path -Path $DestAccountsPath))
+            {
+                $DestParentPath = Split-Path -path $DestParentPath -Parent
+            }
+            else
+            {
+                Break
+            }
+        }
         #EndRegion
         If (-not (Test-Path $DestPath)) {
             New-Item -ItemType Directory -Path $DestPath -Force
@@ -35,13 +49,16 @@ Function Move-Generation2Master {
                 Remove-Item -Path $DestItem -Recurse
             }
             Write-Host "Copying folder: $SourceItem." -ForegroundColor Yellow
-            Copy-Item -Recurse -Path $SourceItem -Destination $DestItem
+            if (Test-Path -Path $SourceItem)
+            {
+                Copy-Item -Recurse -Path $SourceItem -Destination $DestItem
+            }
         }
         #Region Clean Local Modules
         $LocalModulesPath = Join-Path -Path (Join-Path -Path $DestPath -ChildPath 'generated') -ChildPath 'modules'
         If (Test-Path $LocalModulesPath) {
             Write-Host "Removing local modules: $LocalModulesPath." -ForegroundColor Yellow
-            Remove-Item -Path $LocalModulesPath -Recurse
+            Remove-Item -Path $LocalModulesPath -Recurse -Force
         }
         #EndRegion
         #Region copy docs
@@ -85,8 +102,7 @@ Function Move-Generation2Master {
             $Psd1Metadata.GUID = $ModuleGuid
         }
         If ($Null -eq $RequiredModule) {
-            $FullDestPath = Resolve-Path -path $DestPath
-            $AccountsModulePath = [System.IO.Path]::Combine($FullDestPath, '..', 'Accounts', 'Accounts')
+            $AccountsModulePath = [System.IO.Path]::Combine($DestParentPath, 'Accounts', 'Accounts')
             $AccountsMetadata = Import-LocalizedData -BaseDirectory $AccountsModulePath -FileName "Az.Accounts.psd1"
             $RequiredModule = @(@{ModuleName = 'Az.Accounts'; ModuleVersion = $AccountsMetadata.ModuleVersion; })
         }
@@ -97,7 +113,7 @@ Function Move-Generation2Master {
         If ($Psd1Metadata.FunctionsToExport -Contains "*") {
             $Psd1Metadata.FunctionsToExport = ($Psd1Metadata.FunctionsToExport | Where-Object {$_ -ne "*"})
         }
-        Update-ModuleManifest -Path (Join-Path -Path $SourcePath -ChildPath "Az.$ModuleName.psd1") @Psd1Metadata
+        Update-ModuleManifest -Path $SourcePsd1Path @Psd1Metadata
         Copy-Item -Path $SourcePsd1Path -Destination $DestPsd1Path
         #EndRegion
 
@@ -128,6 +144,10 @@ Function Move-Generation2Master {
         $generateInfo.Add("autorest", ($autorest_info[$autorest_info.count - 2]).trim())
         $extensions = ls ~/.autorest
         ForEach ($ex in $extensions) {
+            if ($Null -eq $ex.Name)
+            {
+                continue
+            }
             $info = $ex.Name.Split('@')
             $packageName = $info[1]
             $version = $info[2]
@@ -170,25 +190,13 @@ Function Move-Generation2Master {
         }
         Copy-Template -SourceName Changelog.md -DestPath $DestPath -DestName Changelog.md
         #Region create a solution file for module and add the related csproj files to this solution.
-        dotnet new sln -n $ModuleName -o $DestPath
-        $DestParentPath = $DestPath
-        While ("" -eq $DestParentPath)
-        {
-            $DestAccountsPath = Get-ChildItem -Path $DestParentPath -Filter Accounts
-            if ($Null -eq $DestAccountsPath)
-            {
-                $DestParentPath = Split-Path -path $DestParentPath -Parent
-            }
-            else
-            {
-                Break
-            }
-        }
+        dotnet new sln -n $ModuleName -o $DestPath --force
         $SolutionPath = Join-Path -Path $DestPath -ChildPath $ModuleName.sln
         foreach ($DependenceCsproj in (Get-ChildItem -path $DestAccountsPath -Recurse -Filter *.csproj -Exclude *test*))
         {
             dotnet sln $SolutionPath add $DependenceCsproj
         }
+        dotnet sln $SolutionPath add (Join-Path -Path $DestPath -ChildPath Az.$ModuleName.csproj)
         #EndRegion
 
         $PropertiesPath = Join-Path -Path $DestPath -ChildPath "Properties"
diff --git a/tools/GenerateHelp.ps1 b/tools/GenerateHelp.ps1
@@ -11,7 +11,7 @@ Param(
     [string]$FilteredModules
 )
 
-$ResourceManagerFolders = Get-ChildItem -Directory -Path "$PSScriptRoot\..\src" | Where-Object { $_.Name -ne 'lib' -and $_.Name -ne 'Package' -and $_.Name -ne 'packages' }
+$ResourceManagerFolders = Get-ChildItem -Directory -Path "$PSScriptRoot\..\src" | Where-Object { $_.Name -ne 'lib' -and $_.Name -ne 'Package' -and $_.Name -ne 'packages' } | Where-Object { (Get-ChildItem -Directory -Path $_ -Filter *.psd1).Count -ne 0 }
 Import-Module "$PSScriptRoot\HelpGeneration\HelpGeneration.psm1"
 
 .($PSScriptRoot + "\PreloadToolDll.ps1")
diff --git a/tools/HelpGeneration/Exceptions/ValidateHelpIssues.csv b/tools/HelpGeneration/Exceptions/ValidateHelpIssues.csv
@@ -137,4 +137,6 @@ Reset-AzWebAppPublishingProfile.md,No synopsis found
 Reset-AzWebAppSlotPublishingProfile.md,No synopsis found
 Restart-AzWebAppSlot.md,No synopsis found
 Restore-AzWebAppBackup.md,No synopsis found
-Add-AzHDInsightSecurityProfile.md,No examples found
+Add-AzHDInsightSecurityProfile.md,No examples found
+Find-AzDataProtectionRestorableTimeRange.md,No synopsis found
+Find-AzDataProtectionRestorableTimeRange.md,No description found
diff --git a/tools/PrepareAutorestModule.ps1 b/tools/PrepareAutorestModule.ps1
@@ -21,6 +21,8 @@ $ChangedFiles = Get-Content -Path "$PSScriptRoot\..\FilesChanged.txt"
 
 $ALL_MODULE = "ALL_MODULE"
 
+$SKIP_MODULES = @("Aks", "AppService", "Billing", "Compute", "ContainerInstance", "ConnectedMachine", "ContainerRegistry", "Dns", "DnsResolver", "KeyVault", "Media", "Monitor", "Network", "Resources", "ServiceBus", "Storage")
+
 #Region Detect which module should be processed
 $ModuleSet = New-Object System.Collections.Generic.HashSet[string]
 foreach ($file in $ChangedFiles)
@@ -44,11 +46,11 @@ foreach ($file in $ChangedFiles)
 }
 if ($ModuleSet.Contains($ALL_MODULE))
 {
-    $ModuleList = (Get-ChildItem "$PSScriptRoot\..\src\" -Directory).Name
+    $ModuleList = (Get-ChildItem "$PSScriptRoot\..\src\" -Directory -Exclude helpers,lib).Name | Where-Object { $SKIP_MODULES -notcontains $_ }
 }
 else
 {
-    $ModuleList = $ModuleSet | Where-Object { $_ }
+    $ModuleList = $ModuleSet | Where-Object { $SKIP_MODULES -notcontains $_ }
 }
 #EndRegion
 
@@ -65,6 +67,11 @@ git config core.sparseCheckout true
 Add-Content -Path .git/info/sparse-checkout -Value "src/Accounts/"
 git pull origin main
 Move-Item -Path "$TmpFolder\src\Accounts" -Destination "$TmpFolder\Accounts"
+Copy-Item "$TmpFolder\Accounts" "$PSScriptRoot\..\src" -Recurse -Force
+Remove-Item -Path "$TmpFolder\src" -Recurse -Force
+Install-Module Az.Accounts -Repository PSGallery -Force
+Import-Module Az.Accounts
+Copy-Item "$PSScriptRoot\..\src\*.props" $TmpFolder
 #EndRegion
 
 #Region generate the code and make the struture same with main branch.
@@ -77,10 +84,18 @@ foreach ($Module in $ModuleList)
         Throw "Cannot find Az.$Module.psd1 in $ModuleFolder."
     }
     Set-Location -Path $ModuleFolder
-    autorest
+    try
+    {
+        npx autorest --max-memory-size=8192
+    }
+    catch
+    {
+        Write-Host "Generating $currentModule with m3"
+        npx autorest --use:@autorest/powershell@2.1.401 --max-memory-size=8192
+    }
     ./build-module.ps1
     Move-Generation2Master -SourcePath "$PSScriptRoot\..\src\$Module\" -DestPath $TmpFolder
-    Remove-Item "$ModuleFolder\*" -Recurse
+    Remove-Item "$ModuleFolder\*" -Recurse -Force
 }
 #EndRegion
-Copy-Item "$TmpFolder\*" "$PSScriptRoot\..\src" -Recurse -Force
+Copy-Item "$TmpFolder\*" "$PSScriptRoot\..\src" -Recurse -Force

Original file line number	Diff line number	Diff line change
`@@ -11,7 +11,7 @@ Param(`
`11`	`11`	`[string]$FilteredModules`
`12`	`12`	`)`
`13`	`13`
`14`		`-$ResourceManagerFolders = Get-ChildItem -Directory -Path "$PSScriptRoot\..\src" \| Where-Object { $_.Name -ne 'lib' -and $_.Name -ne 'Package' -and $_.Name -ne 'packages' }`
	`14`	`+$ResourceManagerFolders = Get-ChildItem -Directory -Path "$PSScriptRoot\..\src" \| Where-Object { $_.Name -ne 'lib' -and $_.Name -ne 'Package' -and $_.Name -ne 'packages' } \| Where-Object { (Get-ChildItem -Directory -Path $_ -Filter *.psd1).Count -ne 0 }`
`15`	`15`	`Import-Module "$PSScriptRoot\HelpGeneration\HelpGeneration.psm1"`
`16`	`16`
`17`	`17`	`.($PSScriptRoot + "\PreloadToolDll.ps1")`