CXX-3081 Keep API doc index pages up-to-date with each release (#1180)

* Fix links to MongoDB C++ Driver doc pages

* Add etc/patch-apidocs-index-pages.py

* Run etc/patch-apidocs-index-pages.py during API doc deployment

* Clarify expected contents of build/docs/api before API doc deployment

* Update etc/requirements.txt with beautifulsoup4

Author: eramongodb

README.md

@@ -1,7 +1,7 @@
 # MongoDB C++ Driver 
 [![codecov](https://codecov.io/gh/mongodb/mongo-cxx-driver/branch/master/graph/badge.svg)](https://codecov.io/gh/mongodb/mongo-cxx-driver)
 [![Documentation](https://img.shields.io/badge/docs-doxygen-blue.svg)](https://mongocxx.org/api/mongocxx-v3/)
-[![Documentation](https://img.shields.io/badge/docs-mongocxx-green.svg)](https://www.mongodb.com/docs/languages/cpp/drivers/current/)
+[![Documentation](https://img.shields.io/badge/docs-mongocxx-green.svg)](https://www.mongodb.com/docs/languages/cpp/cpp-driver/current/)
 [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://github.com/mongodb/mongo-cxx-driver/blob/master/LICENSE)
 
 Welcome to the MongoDB C++ Driver!

etc/apidocmenu.md

@@ -67,8 +67,8 @@ Compatibility of each C++ driver version with each MongoDB server is documented
 
 ## Resources
 
-* [MongoDB C++ Driver Quickstart](https://www.mongodb.com/docs/languages/cpp/drivers/current/tutorial/)
-* [MongoDB C++ Driver Manual](https://www.mongodb.com/docs/languages/cpp/drivers/current/)
+* [MongoDB C++ Driver Quickstart](https://www.mongodb.com/docs/languages/cpp/cpp-driver/current/tutorial/)
+* [MongoDB C++ Driver Manual](https://www.mongodb.com/docs/languages/cpp/cpp-driver/current/)
 * [MongoDB C++ Driver Source Code on GitHub](https://github.com/mongodb/mongo-cxx-driver)
 * [MongoDB Database Manual](https://www.mongodb.com/docs/manual/)
 

etc/deploy-to-ghpages.pl

@@ -41,6 +41,8 @@ sub _doxygen_rsync {
         ( map { ; '--filter' => $_ } @filters ),
         "build/docs/api/", "$tmpdir/api/"
     );
+    $ENV{APIDOCSPATH} = "$tmpdir/api";
+    _try_run(qw{etc/patch-apidocs-index-pages.py})
 }
 
 sub main {

etc/patch-apidocs-index-pages.py

@@ -0,0 +1,162 @@
+#!/usr/bin/env python3
+
+# Copyright 2009-present MongoDB, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Patches the root-level index.html file in each API doc site to match the state of the latest API doc site.
+"""
+
+from packaging.version import Version, InvalidVersion
+from typing import List, Tuple
+
+import bs4
+import os
+
+
+def find_api_docs_path() -> str:
+    """
+    Return an absolute path to the directory containing the API docs.
+    """
+    api_docs_path: str | None = os.environ.get('APIDOCSPATH')
+    if not api_docs_path:
+        raise RuntimeError('APIDOCSPATH environment variable is not set!')
+
+    if not os.path.exists(api_docs_path):
+        raise RuntimeError('path to API docs does not exist!')
+
+    return os.path.abspath(api_docs_path)
+
+
+def find_api_docs(api_docs_path: str) -> List[str]:
+    """
+    Return a list of API doc directories by name.
+    """
+    api_docs: List[str] = []
+    for dir in os.scandir(api_docs_path):
+        if dir.is_dir() and not dir.is_symlink():
+            api_docs.append(dir.name)
+
+    # Sort by legacy vs. modern, then by SemVer. Example:
+    #  - legacy-0.1.0
+    #  - legacy-0.2.0
+    #  - legacy-0.10.0
+    #  - mongocxx-3.1.0
+    #  - mongocxx-3.2.0
+    #  - mongocxx-3.10.0
+    # Skip directories with a version suffix, e.g. `mongocxx-1.2.3-rc0`.
+    def by_version(p: str) -> Tuple[bool, Version] | None:
+        is_legacy: bool = p.startswith('legacy-')
+        try:
+            version = p.removeprefix('legacy-') if is_legacy else p.removeprefix('mongocxx-')
+            if version.find('-') != -1:
+                print(f' - Skipping: {p}')
+                return None
+            return (not is_legacy, Version(version))
+        except InvalidVersion:
+            raise RuntimeError(f'unexpected API doc name "{p}": APIDOCSPATH may not be correct!') from None
+
+    api_docs = [doc for doc in api_docs if by_version(doc) is not None]
+    api_docs.sort(key=by_version)
+
+    return api_docs
+
+
+def find_index_page(root: str, doc: str) -> str:
+    """
+    Return an absolute path to the index page of an API doc.
+    """
+    filename = 'index.html'
+    parent = os.path.join(root, doc)
+    index_page = os.path.join(parent, filename)
+    if not os.path.exists(index_page):
+        raise RuntimeError(f' - error: cannot find {filename} in {doc}!')
+    if not os.path.isfile(index_page):
+        raise RuntimeError(f' - error: {index_page} is not a file in {doc}!')
+    return index_page
+
+
+def format_latest_index_page(latest: str) -> None:
+    """
+    Format the latest index page to improve readability of future patch diffs.
+    """
+    with open(latest, "r+") as file:
+        html = bs4.BeautifulSoup(file, 'html.parser')
+        file.seek(0)
+        file.write(html.prettify(formatter="html"))
+        file.truncate()
+
+
+def extract_latest_contents(latest) -> bs4.PageElement:
+    """
+    Return the page element corresponding to the contents of the latest index page.
+    """
+    with open(latest) as file:
+        html = bs4.BeautifulSoup(file, 'html.parser')
+        return html.find("div", class_='contents')
+
+
+def patch_index_page(latest_contents: bs4.PageElement, index_page: str):
+    """
+    Replace the contents of the index page with the latest contents.
+    """
+    with open(index_page, "r+") as file:
+        html = bs4.BeautifulSoup(file, 'html.parser')
+        contents = html.find("div", class_="contents")
+        contents.replace_with(latest_contents)
+
+        file.seek(0)
+        file.write(html.prettify(formatter="html"))
+        file.truncate()
+
+
+def main():
+    api_docs_path: str = find_api_docs_path()
+
+    print(f'Patching API docs in: {api_docs_path}')
+
+    print('Finding API docs...')
+    api_docs = find_api_docs(api_docs_path)
+    if len(api_docs) == 0:
+        raise RuntimeError(f'no API docs found: APIDOCSPATH may not be correct!')
+    print('Finding API docs... done.')
+
+    print(f' - Found {len(api_docs)} API docs: {api_docs[0]} ... {api_docs[-1]}')
+
+    print('Searching for index pages...')
+    index_pages = [find_index_page(api_docs_path, doc) for doc in api_docs]
+    print('Searching for index pages... done.')
+
+    print(f' - Found {len(index_pages)} index pages within {len(api_docs)} API docs.')
+    latest_doc = api_docs[-1]
+    print(f' - Using {latest_doc} as the latest API doc.')
+
+    (latest_index, index_pages) = (index_pages[-1], index_pages[:-1])
+
+    print('Formatting latest index page...')
+    format_latest_index_page(latest_index)
+    print('Formatting latest index page... done.')
+
+    print('Extracting latest index page contents...')
+    latest_contents = extract_latest_contents(latest_index)
+    print('Extracting latest index page contents... done.')
+
+    print(f'Replacing contents of {len(index_pages)} index pages...')
+    for page in index_pages:
+        patch_index_page(latest_contents, page)
+    print(f'Replacing contents of {len(index_pages)} index pages... done.')
+
+
+if __name__ == '__main__':
+    main()

etc/releasing.md

@@ -592,7 +592,12 @@ Test generating the latest versioned Doxygen docs by building the `doxygen-lates
 cmake --build build --target doxygen-latest
 ```
 
-Verify that the `build/docs/api/mongocxx-X.Y.Z` directory is present and populated.
+Verify that the `build/docs/api/mongocxx-X.Y.Z` directory is present and populated. Verify the resulting API doc looks as expected.
+
+Remove all contents of `build/docs/api` before running the next commands.
+
+> [!IMPORTANT]
+> Remove all contents of `build/docs/api` before running the next commands.
 
 Generate and deploy the updated documentation to GitHub pages by building the `hugo-deploy` and `doxygen-deploy` targets:
 

etc/requirements.txt

@@ -1,31 +1,28 @@
-# Required packages for the release script.
-click==8.1.7
-GitPython==3.1.43
-PyGithub==2.3.0
-cryptography==42.0.7
-
-# Pin `jira` to apply fix of https://github.com/pycontribs/jira/commit/010223289eb66663aaafb70447397038efb2d40d.
-# This avoids the `signature_method_rejected` error described in https://github.com/pycontribs/jira/pull/1643.
-# TODO: replace the following line with `jira` once there is a release of `jira` containing the fix.
-jira @ git+https://github.com/pycontribs/jira.git@010223289eb66663aaafb70447397038efb2d40d
+beautifulsoup4==4.12.3 # etc/patch-apidocs-index-pages.py.
+click==8.1.7           # etc/make_release.py
+cryptography==42.0.7   # etc/make_release.py (PyJWT)
+GitPython==3.1.43      # etc/make_release.py
+jira==3.5.2            # etc/make_release.py
+PyGithub==2.3.0        # etc/make_release.py
 
 # Dependencies of required packages above.
-certifi==2024.2.2
+certifi==2024.7.4
 cffi==1.16.0
 charset-normalizer==3.3.2
 defusedxml==0.7.1
 Deprecated==1.2.14
 gitdb==4.0.11
 idna==3.7
 oauthlib==3.2.2
-packaging==24.0
+packaging==24.1
 pycparser==2.22
 PyJWT==2.8.0
 PyNaCl==1.5.0
-requests==2.32.2
+requests==2.32.3
 requests-oauthlib==2.0.0
 requests-toolbelt==1.0.0
 smmap==5.0.1
-typing_extensions==4.11.0
-urllib3==2.2.1
+soupsieve==2.5
+typing_extensions==4.12.2
+urllib3==2.2.2
 wrapt==1.16.0

examples/README.md

@@ -16,6 +16,6 @@
 ## Building project examples
 
 1. Install `libmongoc` and `mongocxx` following the [installation
-   instructions](https://www.mongodb.com/docs/languages/cpp/drivers/current/installation/)
+   instructions](https://www.mongodb.com/docs/languages/cpp/cpp-driver/current/installation/)
 2. Change to one of the project example directories, e.g. `examples/projects/cmake/mongocxx/shared`
 3. Run `./build.sh`

examples/mongocxx/tutorial.cpp

@@ -1,7 +1,7 @@
 // Compile with: c++ --std=c++11 tutorial.cpp $(pkg-config --cflags --libs libmongocxx)
 
 // The following is a formatted copy from the tutorial
-// https://www.mongodb.com/docs/languages/cpp/drivers/current/tutorial/.
+// https://www.mongodb.com/docs/languages/cpp/cpp-driver/current/tutorial/.
 
 #include <cstdint>
 #include <iostream>

src/bsoncxx/include/bsoncxx/v_noabi/bsoncxx/builder/stream/helpers.hpp

@@ -61,7 +61,7 @@ struct open_array_type {
 /// A stream manipulator to open a subarray.
 ///
 /// @see
-/// https://www.mongodb.com/docs/languages/cpp/drivers/current/working-with-bson/#std-label-cpp-bson-builders
+/// https://www.mongodb.com/docs/languages/cpp/cpp-driver/current/working-with-bson/#std-label-cpp-bson-builders
 /// for help building arrays in loops.
 ///
 constexpr open_array_type open_array;

src/mongocxx/include/mongocxx/v_noabi/mongocxx/client.hpp

@@ -57,7 +57,7 @@ namespace v_noabi {
 /// @endcode
 ///
 /// Note that client is not thread-safe. See
-/// https://www.mongodb.com/docs/languages/cpp/drivers/current/thread-safety/ for more details.
+/// https://www.mongodb.com/docs/languages/cpp/cpp-driver/current/thread-safety/ for more details.
 class client {
    public:
     ///

src/mongocxx/include/mongocxx/v_noabi/mongocxx/client_session.hpp

@@ -39,7 +39,7 @@ namespace v_noabi {
 /// or snapshots.
 ///
 /// Note that client_session is not thread-safe. See
-/// https://www.mongodb.com/docs/languages/cpp/drivers/current/thread-safety/ for more details.
+/// https://www.mongodb.com/docs/languages/cpp/cpp-driver/current/thread-safety/ for more details.
 ///
 /// @see
 /// https://www.mongodb.com/docs/manual/core/read-isolation-consistency-recency/#causal-consistency