Skip to content

Commit a0d0eb8

Browse files
JimSupliziobenbp
JimSuplizio
and
benbp
authored
Top-level namespace for ToC children entries instead of the assembly name. (#34946)
* Updates for docs ToC generation * Complete an existing partial comment * Temp DocIndex testing change, will be removed before merge * Actually remove the networkservice usage since it was deleted * Change the if that checks for Namespaces to work in strict mode * change order of pip command line, set namespaces to library name if none are found * Add a couple of temp commands to get pip's version and upgrade * remove pip version/update commands from script and add to yml * revert the yml changes and add an equals sign in between the --extra-index-url option and the url * modify the download command * Fix the command line args for PSNativeCommandArgumentPassing Standard * Remove temporary changes to docindex.yml * Finish a sentence fragment in a comment and add comment to Language-Settings to make a function searchable. * Update eng/scripts/Language-Settings.ps1 Co-authored-by: Ben Broderick Phillips <[email protected]> --------- Co-authored-by: Ben Broderick Phillips <[email protected]>
1 parent 28ea606 commit a0d0eb8

File tree

2 files changed

+178
-25
lines changed

2 files changed

+178
-25
lines changed

‎eng/scripts/Language-Settings.ps1

Lines changed: 33 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -128,6 +128,34 @@ function Get-python-PackageInfoFromPackageFile ($pkg, $workingDirectory)
128128
}
129129
}
130130

131+
# This is the GetDocsMsDevLanguageSpecificPackageInfoFn implementation
132+
function Get-python-DocsMsDevLanguageSpecificPackageInfo($packageInfo, $packageSourceOverride) {
133+
# If the default namespace isn't in the package info then it needs to be added
134+
# Can't check if (!$packageInfo.Namespaces) in strict mode because Namespaces won't exist
135+
# at all.
136+
if (!($packageInfo | Get-Member Namespaces)) {
137+
# If the Version is INGORE that means it's a source install and those
138+
# ones need to be done by hand
139+
if ($packageInfo.Version -ine "IGNORE") {
140+
$version = $packageInfo.Version
141+
# If the dev version is set, use that
142+
if ($packageInfo.DevVersion) {
143+
$version = $packageInfo.DevVersion
144+
}
145+
$namespaces = Get-NamespacesFromWhlFile $packageInfo.Name $version $packageSourceOverride
146+
if ($namespaces.Count -gt 0) {
147+
$packageInfo | Add-Member -Type NoteProperty -Name "Namespaces" -Value $namespaces
148+
} else {
149+
LogWarning "Unable to find namespaces for $($packageInfo.Name):$version, using the package name."
150+
$tempNamespaces = @()
151+
$tempNamespaces += $packageInfo.Name
152+
$packageInfo | Add-Member -Type NoteProperty -Name "Namespaces" -Value $tempNamespaces
153+
}
154+
}
155+
}
156+
return $packageInfo
157+
}
158+
131159
# Stage and Upload Docs to blob Storage
132160
function Publish-python-GithubIODocs ($DocLocation, $PublicArtifactLocation)
133161
{
@@ -587,6 +615,8 @@ function GetExistingPackageVersions ($PackageName, $GroupId=$null)
587615
}
588616
}
589617

618+
# Defined in common.ps1 as:
619+
# GetDocsMsMetadataForPackageFn = Get-${Language}-DocsMsMetadataForPackage
590620
function Get-python-DocsMsMetadataForPackage($PackageInfo) {
591621
$readmeName = $PackageInfo.Name.ToLower()
592622
Write-Host "Docs.ms Readme name: $($readmeName)"
@@ -632,8 +662,8 @@ function Validate-Python-DocMsPackages ($PackageInfo, $PackageInfos, $PackageSou
632662

633663
$allSucceeded = $true
634664
foreach ($item in $PackageInfos) {
635-
# Some packages
636-
if ($item.Version -eq 'IGNORE') {
665+
# If the Version is IGNORE that means it's a source install and those aren't run through ValidatePackage
666+
if ($item.Version -eq 'IGNORE') {
637667
continue
638668
}
639669

@@ -670,7 +700,7 @@ function Get-python-DirectoriesForGeneration () {
670700

671701
function Update-python-GeneratedSdks([string]$PackageDirectoriesFile) {
672702
$packageDirectories = Get-Content $PackageDirectoriesFile | ConvertFrom-Json
673-
703+
674704
$directoriesWithErrors = @()
675705

676706
foreach ($directory in $packageDirectories) {

‎eng/scripts/docs/Docs-ToC.ps1

Lines changed: 145 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,88 @@
1-
function GetOnboardingFileForMoniker($docRepoLocation, $moniker) {
1+
# Download the .whl file into the given destination directory.
2+
function Get-WhlFile {
3+
param(
4+
[Parameter(Mandatory=$true)] [string]$Library,
5+
[Parameter(Mandatory=$true)] [string]$Version,
6+
[Parameter(Mandatory=$true)] [string]$Destination,
7+
[Parameter(Mandatory=$false)] [string]$ExtraIndexUrl = ""
8+
)
9+
10+
$LibArg= "$Library==$Version"
11+
if ($ExtraIndexUrl) {
12+
$ExtraIndexArg = "--extra-index-url=$ExtraIndexUrl"
13+
Write-Host "python -m pip download --quiet --only-binary=:all: --dest $Destination --no-deps $ExtraIndexArg $LibArg"
14+
python -m pip download --quiet --only-binary=:all: --dest $Destination --no-deps $ExtraIndexArg $LibArg
15+
} else {
16+
Write-Host "python -m pip download --quiet --only-binary=:all: --dest $Destination --no-deps $LibArg"
17+
python -m pip download --quiet --only-binary=:all: --dest $Destination --no-deps $LibArg
18+
}
19+
if($LASTEXITCODE -ne 0) {
20+
return $false
21+
}
22+
return $true
23+
}
24+
25+
# Given a library and version, download the .whl file and unpack it to get the namespaces.
26+
# A temp directory is used for the download and unpack which is cleaned up afterwards.
27+
function Get-NamespacesFromWhlFile {
28+
param(
29+
[Parameter(Mandatory=$true)] [string]$Library,
30+
[Parameter(Mandatory=$true)] [string]$Version,
31+
[Parameter(Mandatory=$false)] [string]$ExtraIndexUrl = ""
32+
)
33+
34+
$destination = (Join-Path ([System.IO.Path]::GetTempPath()) "$Library$Version")
35+
$namespaces = @()
36+
37+
try {
38+
39+
# Pulling the whl file generates output, make sure it's sent to null so
40+
# it's not returned as part of this function.
41+
$success = Get-WhlFile $Library $Version $destination $ExtraIndexUrl
42+
if ($success) {
43+
44+
# Each library gets its own temporary directory. There should only be one whl
45+
# file in the destination directory
46+
$whlFile = Get-ChildItem -Path $destination -File -Filter "*.whl" | Select-Object -First 1
47+
$unpackDir = Join-Path -Path $destination -ChildPath "$Library-$Version"
48+
Expand-Archive -Path $whlFile -DestinationPath $unpackDir
49+
50+
# Look for any directory that contains __init__.py with the following exceptions:
51+
# 1. *.dist-info directories shouldn't be included in the results.
52+
# 2. If any subdirectory starts with "_" it's internal and needs to be skipped
53+
# 3. If there's a root level directory named "azure" with an __init__.py file then
54+
# needs to be skipped. This doesn't happen with libraries released from the
55+
# azure-sdk-for-python repository but there are older libraries that are in the
56+
# docs directories which are/were released outside of the repository where this
57+
# is true.
58+
$rootLevelAzureDir = Join-Path -Path $unpackDir -ChildPath "azure"
59+
$namespaceDirs = Get-ChildItem -Path $unpackDir -Recurse -Filter "__init__.py" |
60+
Where-Object{$_.DirectoryName -notlike "*.dist-info"} |
61+
Where-Object{$_.DirectoryName -notlike "*$([IO.Path]::DirectorySeparatorChar)_*" } |
62+
Where-Object{$_.DirectoryName -ine $rootLevelAzureDir}
63+
foreach($namespaceDir in $namespaceDirs) {
64+
# Strip off the root directy, everything left will be subDir1/subDir2/../subDirN.
65+
# The directory separators will be replaced with periods to compute the
66+
# namespace
67+
$partialDir = $namespaceDir.DirectoryName.Replace($unpackDir + $([IO.Path]::DirectorySeparatorChar), "")
68+
$namespaces += $partialDir.Replace([IO.Path]::DirectorySeparatorChar, ".")
69+
# Since only the primary namespace is being pulled, break out of the loop after
70+
# the first one.
71+
break
72+
}
73+
}
74+
}
75+
finally {
76+
# Clean up the temp directory if it was created
77+
if (Test-Path $destination) {
78+
Remove-Item $destination -Recurse -Force
79+
}
80+
}
81+
# Make sure this always returns an array
82+
Write-Output -NoEnumerate $namespaces
83+
}
84+
85+
function GetOnboardingFileForMoniker($docRepoLocation, $moniker) {
286
$packageOnboardingFile = 'ci-configs/packages-latest.json'
387
if ($moniker -eq 'preview') {
488
$packageOnboardingFile = 'ci-configs/packages-preview.json'
@@ -21,7 +105,7 @@ function Get-python-OnboardedDocsMsPackagesForMoniker($DocRepoLocation, $moniker
21105
continue
22106
}
23107
$packageName = $spec.package_info.name
24-
108+
25109
$jsonFile = "$DocRepoLocation/metadata/$moniker/$packageName.json"
26110
if (Test-Path $jsonFile) {
27111
$onboardedPackages[$packageName] = ConvertFrom-Json (Get-Content $jsonFile -Raw)
@@ -55,7 +139,7 @@ function Get-python-OnboardedDocsMsPackages($DocRepoLocation) {
55139
return $onboardedPackages
56140
}
57141

58-
function GetPackageLevelReadme($packageMetadata) {
142+
function GetPackageLevelReadme($packageMetadata) {
59143
# Fallback for package name
60144
$packageLevelReadmeName = $packageMetadata.Package
61145
if ($packageLevelReadmeName.StartsWith('azure-')) {
@@ -70,19 +154,70 @@ function GetPackageLevelReadme($packageMetadata) {
70154
}
71155
return $packageLevelReadmeName
72156
}
73-
74-
function Get-python-PackageLevelReadme($packageMetadata) {
157+
158+
# This function is called within a loop. To prevent multiple reads of the same
159+
# file data, this uses a script-scoped cache variable.
160+
$script:PackageMetadataJsonLookup = $null
161+
function GetPackageMetadataJsonLookup($docRepoLocation) {
162+
if ($script:PackageMetadataJsonLookup) {
163+
return $script:PackageMetadataJsonLookup
164+
}
165+
166+
$script:PackageMetadataJsonLookup = @{}
167+
$packageJsonFiles = Get-ChildItem $docRepoLocation/metadata/ -Filter *.json -Recurse
168+
foreach ($packageJsonFile in $packageJsonFiles) {
169+
$packageJson = Get-Content $packageJsonFile -Raw | ConvertFrom-Json -AsHashtable
170+
171+
if (!$script:PackageMetadataJsonLookup.ContainsKey($packageJson.Name)) {
172+
$script:PackageMetadataJsonLookup[$packageJson.Name] = @($packageJson)
173+
} else {
174+
$script:PackageMetadataJsonLookup[$packageJson.Name] += $packageJson
175+
}
176+
}
177+
178+
return $script:PackageMetadataJsonLookup
179+
}
180+
181+
# Grab the namespaces from the json file
182+
function Get-Toc-Children($package, $docRepoLocation) {
183+
$packageTable = GetPackageMetadataJsonLookup $docRepoLocation
184+
185+
$namespaces = @()
186+
if ($packageTable.ContainsKey($package)) {
187+
foreach ($entry in $packageTable[$package]) {
188+
if ($entry.ContainsKey('Namespaces')) {
189+
$namespaces += $entry['Namespaces']
190+
}
191+
}
192+
}
193+
# Sort the array and clean out any dupes (there shouldn't be any but better safe than sorry)
194+
$namespaces = $namespaces | Sort-Object -Unique
195+
# Ensure that this always returns an array, even if there's one item or 0 items
196+
Write-Output -NoEnumerate $namespaces
197+
}
198+
199+
function Get-python-PackageLevelReadme($packageMetadata) {
75200
return GetPackageLevelReadme -packageMetadata $packageMetadata
76201
}
77202

78-
function Get-python-DocsMsTocData($packageMetadata, $docRepoLocation) {
203+
# Defined in common.ps1
204+
# $GetDocsMsTocDataFn = "Get-${Language}-DocsMsTocData"
205+
function Get-python-DocsMsTocData($packageMetadata, $docRepoLocation, $PackageSourceOverride) {
79206
$packageLevelReadmeName = GetPackageLevelReadme -packageMetadata $packageMetadata
80-
81207
$packageTocHeader = GetDocsTocDisplayName $packageMetadata
208+
209+
# Get-Toc-Children always returns an array, even if there's only 1 item or it's empty
210+
$children = Get-Toc-Children `
211+
-package $packageMetadata.Package `
212+
-docRepoLocation $docRepoLocation
213+
if ($children.Count -eq 0) {
214+
LogWarning "Did not find the package namespaces for $($packageMetadata.Package)"
215+
}
216+
82217
$output = [PSCustomObject]@{
83218
PackageLevelReadmeHref = "~/docs-ref-services/{moniker}/$packageLevelReadmeName-readme.md"
84219
PackageTocHeader = $packageTocHeader
85-
TocChildren = @($packageMetadata.Package)
220+
TocChildren = $children
86221
}
87222

88223
return $output
@@ -113,19 +248,7 @@ function Get-python-UpdatedDocsMsToc($toc) {
113248
$functionService = [PSCustomObject]@{
114249
name = 'Functions';
115250
landingPageType = 'Service';
116-
children = @('azure-functions', 'azure-functions-durable')
117-
}
118-
119-
# The network service is not onboarded into the regular Python build because
120-
# it takes many hours to build.
121-
$networkService = [PSCustomObject]@{
122-
name = 'Network';
123-
href = "~/docs-ref-services/{moniker}/network.md";
124-
items = @([PSCustomObject]@{
125-
name = 'Management';
126-
landingPageType = 'Service';
127-
children = @('azure-mgmt-network')
128-
})
251+
children = @('azure.functions', 'azure.durable_functions')
129252
}
130253

131254
# Add new services which are not onboarded in obvious ways in the CI config.
@@ -134,7 +257,7 @@ function Get-python-UpdatedDocsMsToc($toc) {
134257
# (e.g. "Functions"), sorting the resulting list, then re-adding the ultimate
135258
# item to the end. This ensures that the "Other" service is at the bottom as
136259
# intended.
137-
$sortableServices = $services[0..($services.Length - 2)] + $functionService + $networkService
260+
$sortableServices = $services[0..($services.Length - 2)] + $functionService
138261
$toc[0].items = ($sortableServices | Sort-Object -Property name) + $services[-1]
139262

140263
# PowerShell outputs a single object if the output is an array with only one

0 commit comments

Comments
 (0)