Adding Catalog files.

Author: rlhagerm

.github/workflows/catalog.yml

@@ -0,0 +1,35 @@
+name: Validate Catalogs
+
+# Controls when the action will run.
+on:  # yamllint disable-line rule:truthy
+  # Triggers the workflow on push or pull request events but only for the main branch
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+
+# A workflow run is made up of one or more jobs that can run sequentially or in parallel
+jobs:
+  # This workflow contains a single job called "build"
+  build:
+    name: Validate Catalogs
+    # The type of runner that the job will run on
+    runs-on: ubuntu-latest
+
+    # Steps represent a sequence of tasks that will be executed as part of the job
+    steps:
+      - name: Checkout repo content
+        uses: actions/checkout@v4  # checkout the repository content to github runner.
+      - name: Setup Python
+        uses: actions/setup-python@v3
+        with:
+          python-version: 3.11  # install the python needed
+      - name: Install dependencies
+        run: >-
+          python3 -m pip install -r .tools/scanners/requirements.txt
+      - name: Check WRITEMEs
+        run: >-
+          python3 .tools/scanners/catalog.py --check --diff

.github/workflows/writeme.yml

@@ -32,4 +32,4 @@ jobs:
           python3 -m pip install -r .tools/readmes/requirements.txt
       - name: Check WRITEMEs
         run: >-
-          python3 .tools/readmes/writeme.py --check --diff
+          python3 .tools/scanners/writeme.py --check --diff

.tools/readmes/__init__.py renamed to .tools/__init__.py

@@ -0,0 +1,26 @@
+#!/usr/bin/env python3
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+import sys
+
+from update import update
+
+# For debugging, engineers can skip update with the --no-update flag. Yes, it's
+# a double negative, but it's quick and early in the startup because of the
+# reliance on the side-effect imports from `update` and needing them to happen
+# before importing runner, which means before importing the runner argparser.
+NO_UPDATE_FLAG = "--no-update"
+
+
+if __name__ == "__main__":
+    if NO_UPDATE_FLAG not in sys.argv:
+        update()
+    else:
+        sys.argv.remove(NO_UPDATE_FLAG)
+
+    # This import must remain in the main, after the update, as it depends on
+    # importing the things that got changed during update.
+    from catalog_runner import main
+
+    sys.exit(main())

.tools/readmes/README.md renamed to .tools/scanners/README.md

@@ -0,0 +1,213 @@
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+import argparse
+import config
+import logging
+import os
+import json
+from collections import defaultdict
+from difflib import unified_diff
+from pathlib import Path
+from typing import Optional
+
+from render import Renderer, MissingMetadataError, RenderStatus
+from scanner import Scanner
+
+from aws_doc_sdk_examples_tools.doc_gen import DocGen, DocGenEncoder
+
+logging.basicConfig(level=os.environ.get("LOGLEVEL", "INFO").upper(), force=True)
+
+
+def prepare_scanner(doc_gen: DocGen) -> Optional[Scanner]:
+    for path in (doc_gen.root / ".doc_gen/metadata").glob("*_metadata.yaml"):
+        doc_gen.process_metadata(path)
+    doc_gen.collect_snippets()
+    doc_gen.validate()
+    if doc_gen.errors:
+        error_strings = [str(error) for error in doc_gen.errors]
+        failed_list = "\n".join(f"DocGen Error: {e}" for e in error_strings)
+        print(f"Metadata errors encountered:\n\t{failed_list}")
+        return None
+
+    scanner = Scanner(doc_gen)
+
+    # Preload cross-content examples
+    scanner.load_crosses()
+
+    return scanner
+
+
+def main():
+    # Load all examples immediately for cross references. Trades correctness for speed.
+    doc_gen = DocGen.from_root(Path(__file__).parent.parent.parent, incremental=True)
+    # To get the complete list, fill the missing fields.
+    doc_gen.fill_missing_fields()
+
+    languages = ['Python:3']  # Currently enabled only for Python version 3.
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--languages",
+        choices=[*languages] + ["all"],
+        nargs="+",
+        help="The languages of the SDK. Choose from: %(choices)s.",
+        default=["all"],
+    )
+
+    parser.add_argument(
+        "--services",
+        choices=[*doc_gen.services.keys()] + ["all"],
+        nargs="+",
+        help="The targeted service. Choose from: %(choices)s.",
+        default=["all"],
+    )
+
+    parser.add_argument(
+        "--verbose",
+        action="store_true",
+        help="When set, output verbose debugging info.",
+    )
+    parser.add_argument(
+        "--dry-run",
+        action="store_true",
+        dest="dry_run",
+        help="In dry run, compare current vs generated and exit with failure if they do not match.",
+        default=False,  # Change this to default false when we're ready to use this generally.
+    )
+    parser.add_argument("--no-dry-run", dest="dry_run", action="store_false")
+    parser.add_argument("--check", dest="dry_run", action="store_true")
+    parser.add_argument("--diff", action="store_true", default=False)
+    args = parser.parse_args()
+
+    if "all" in args.languages:
+        args.languages = [*languages]
+
+    if "all" in args.services:
+        args.services = [*doc_gen.services.keys()]
+
+    if args.verbose:
+        logging.basicConfig(level=logging.DEBUG)
+
+    logging.debug(f"Args configuration: {args}")
+
+    if args.dry_run:
+        print("Dry run, no changes will be made.")
+
+    skipped = []
+    failed = []
+    written = []
+    unchanged = []
+
+
+    scanner = prepare_scanner(doc_gen)
+    if scanner is None:
+        return -1
+
+    renderer = Renderer(scanner)
+
+    for service in args.services:
+        for language_and_version in args.languages:
+            (language, version) = language_and_version.split(":")
+            id = f"{language}:{version}:{service}"
+            try:
+                renderer.set_example(service, language, int(version), False)
+                service_folder_path = renderer.lang_config["service_folder"]
+                logging.debug("Cataloging %s", id)
+                catalog_status = write_catalog_json(doc_gen, service, language, service_folder_path, args.dry_run)
+                logging.debug("Status %s", catalog_status)
+
+                if catalog_status == RenderStatus.UPDATED:
+                    if args.dry_run:
+                        diff = None
+                        failed.append((id, diff))
+                    else:
+                        written.append(id)
+                elif catalog_status == RenderStatus.UNCHANGED:
+                    unchanged.append(id)
+                    pass
+            except FileNotFoundError as fnfe:
+                logging.debug(fnfe, exc_info=True)
+                skipped.append(id)
+            except MissingMetadataError as mme:
+                logging.debug(mme, exc_info=True)
+                failed.append(id)
+            except Exception as e:
+                logging.error(e, exc_info=True)
+                failed.append(id)
+
+    skip_list = "\n".join(f"Skipped {f}" for f in sorted(skipped))
+    logging.debug(skip_list or "(None Skipped)")
+    if unchanged:
+        unchanged_list = "\n".join(f"Unchanged {f}" for f in sorted(unchanged))
+        print(unchanged_list)
+    if not args.dry_run:
+        done_list = "\n".join(f"Wrote {f}" for f in sorted(written))
+        print(done_list or "(None Written)")
+    if failed:
+        if args.diff:
+            failed_list = "\n".join(
+                f"Diff: {f[1]}" for f in sorted(failed, key=lambda f: f[0])
+            )
+        else:
+            failed_list = "\n".join(f"Incorrect: {f[0]}" for f in sorted(failed))
+        print(failed_list)
+        print("Rerun catalog.py to update the example catalog list.")
+    print("Catalog Run completed.")
+    return len(failed)
+
+
+def write_catalog_json(doc_gen, service_name, language_name, folder_path, is_dry_run):
+    filepath = (
+            Path(__file__).parent.parent.parent
+            / folder_path
+            / 'examples_catalog.json'
+    )
+
+    language_examples = []
+    for example in doc_gen.examples.values():
+        for lang_name, language in example.languages.items():
+            for svc_name in example.services:
+                if svc_name == service_name and lang_name == language_name:
+                    language_examples.append(example)
+
+    new_catalog = json.dumps(
+        {"examples": language_examples},
+        cls=DocGenEncoder, indent="\t"
+    )
+
+    # If the file already exists, read it to compare contents.
+    try:
+        with open(filepath, "r", encoding="utf-8") as example_meta:
+            old_catalog = example_meta.read()
+    except FileNotFoundError:
+        old_catalog = ""
+
+    if old_catalog == new_catalog:
+        return RenderStatus.UNCHANGED
+    else:
+        if not is_dry_run:
+            print(f"Writing serialized versions of DocGen to {filepath}")
+            with open(filepath, "w", encoding="utf-8") as example_meta:
+                example_meta.write(new_catalog)
+        return RenderStatus.UPDATED
+
+
+def write_language_json(doc_gen, language_name):
+    # Test creating a file
+    filepath = f"example_json/{language_name}_examples_list.json"
+    filepath = filepath.lower()
+    print("Writing serialized versions of DocGen to %s", filepath)
+
+    language_examples = []
+    for example in doc_gen.examples.values():
+        for lang_name, language in example.languages.items():
+            if lang_name == language_name:
+                language_examples.append(example)
+
+    with open(filepath, "w") as example_meta:
+        example_meta.write(
+            json.dumps(
+                {"examples": language_examples},
+                cls=DocGenEncoder, indent="\t"
+            )
+        )

.tools/scanners/__init__.py

@@ -115,7 +115,7 @@ def main():
     for service in args.services:
         for language_and_version in args.languages:
             (language, version) = language_and_version.split(":")
-            # write_service_json(doc_gen, service, language)
+             write_service_json(doc_gen, service, language)
             id = f"{language}:{version}:{service}"
             try:
                 renderer.set_example(service, language, int(version), args.safe)
@@ -192,7 +192,7 @@ def write_service_json(doc_gen, service_name, language_name):
     filepath = f"example_json/{language_name}_{service_name}_examples_list.json"
     filepath = filepath.lower()
     print("Writing serialized versions of DocGen to %s", filepath)
-
+    doc_gen.fill_missing_fields()
     language_examples = []
     for example in doc_gen.examples.values():
         for lang_name, language in example.languages.items():