Merge branch 'develop' into rpatel/codebase-snapshot

Author: rushilpatel0

.codegen/.gitignore

@@ -8,12 +8,8 @@ jupyter/
 codegen-system-prompt.txt
 
 # Python cache files
-__pycache__/
+**/__pycache__/
 *.py[cod]
 *$py.class
 *.txt
 *.pyc
-
-# Keep codemods
-!codemods/
-!codemods/**

.codegen/codemods/no_link_backticks/no_link_backticks.py

@@ -2,7 +2,7 @@
 from codegen import Codebase
 
 
-@codegen.function("no-link-backticks")
+@codegen.function(name="no-link-backticks", subdirectories=["test/unit"])
 def run(codebase: Codebase):
     import re
 
@@ -12,6 +12,7 @@ def run(codebase: Codebase):
     # Iterate over all .mdx files in the codebase
     for file in codebase.files(extensions=["mdx"]):
         if file.extension == ".mdx":
+            print(f"Processing {file.path}")
             new_content = file.content
 
             # Find all markdown links with backticks in link text

.codegen/codemods/test_language/test_language.py

@@ -0,0 +1,19 @@
+import codegen
+from codegen.sdk.core.codebase import Codebase
+from codegen.shared.enums.programming_language import ProgrammingLanguage
+
+
+@codegen.function("test-language", subdirectories=["src/codegen/cli"], language=ProgrammingLanguage.PYTHON)
+def run(codebase: Codebase):
+    file = codebase.get_file("src/codegen/cli/errors.py")
+    print(f"File: {file.path}")
+    for s in file.symbols:
+        print(s.name)
+
+
+if __name__ == "__main__":
+    print("Parsing codebase...")
+    codebase = Codebase("./")
+
+    print("Running...")
+    run(codebase)

.codegen/codemods/update_loggers/update_loggers.py

@@ -0,0 +1,18 @@
+import codegen
+from codegen.sdk.core.codebase import PyCodebaseType
+
+
+@codegen.function("update-loggers")
+def run(codebase: PyCodebaseType) -> None:
+    """Updates all loggers in src/codegen to use the new get_logger function."""
+    for file in codebase.files:
+        if not str(file.filepath).startswith("src/codegen/"):
+            continue
+
+        if file.get_import("logging") is None:
+            continue
+
+        if (logger := file.get_global_var("logger")) and logger.value.source == "logging.getLogger(__name__)":
+            print(f"Updating logger in {file.filepath}")
+            logger.set_value("get_logger(__name__)")
+            file.add_import_from_import_string("\nfrom codegen.shared.logging.get_logger import get_logger")

.github/actions/report/action.yml

@@ -19,7 +19,7 @@ runs:
       - name: Upload coverage reports to Codecov
         if: (success() || failure()) # always upload coverage reports even if the tests fail
         continue-on-error: true
-        uses: codecov/codecov-action@v5.3.1
+        uses: codecov/codecov-action@v5.4.0
         with:
           token: ${{ inputs.codecov_token }}
           files: coverage.xml

.github/actions/run-ats/ats.sh

@@ -50,7 +50,7 @@ if [ ! -s codecov_ats/tests_to_run.txt ]; then
     echo "No tests to run, collecting from default tests"
     PYTEST_ARGS="${COLLECT_ARGS} ${DEFAULT_TESTS}"
     echo "Using args: ${PYTEST_ARGS}"
-    TESTS_TO_RUN=$(PYTEST_ARGS=${PYTEST_ARGS} ./.circleci/collect.sh)
+    TESTS_TO_RUN=$(PYTEST_ARGS=${PYTEST_ARGS} ./.github/actions/run-ats/collect.sh)
     echo "${TESTS_TO_RUN}" > codecov_ats/tests_to_run.txt
     run_count=1
     echo "Added ${TESTS_TO_RUN} as fallback. New run count: $run_count"

.github/actions/run-ats/collect.sh

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+TESTS_TO_RUN=$(uv run --frozen pytest --collect-only ${PYTEST_ARGS} -q --disable-warnings --no-summary --no-header)
+TESTS_TO_RUN=$(echo "${TESTS_TO_RUN}" | head -n -2)
+echo $TESTS_TO_RUN

.github/workflows/auto-release.yml

@@ -35,6 +35,40 @@ jobs:
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
+  generate-changelog:
+    if: ${{ needs.semantic-version.outputs.release-tag }}
+    needs: semantic-version
+    name: Generate Changelog
+    runs-on: ubuntu-latest
+    environment: auto-release
+    permissions:
+      contents: write
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          token: ${{ secrets.GHA_TOKEN }}
+
+      - name: Setup backend
+        uses: ./.github/actions/setup-environment
+
+      - name: Generate changelog
+        env:
+          OPEN_AI_TOKEN: ${{ secrets.OPEN_AI_TOKEN }}
+        run: uv run python src/codegen/gscli/cli.py generate changelog --openai-key ${{ secrets.OPEN_AI_TOKEN }}
+
+      - name: Commit changes
+        run: |
+          git config --local user.email "github-actions[bot]@users.noreply.github.com"
+          git config --local user.name "github-actions[bot]"
+          git add docs/changelog/changelog.mdx
+          git diff --staged --quiet || git commit -m "update changelog"
+
+      - name: Push changes
+        uses: ad-m/github-push-action@master
+        with:
+          branch: ${{ github.ref }}
+
   release:
     if: ${{ needs.semantic-version.outputs.release-tag }}
     needs: semantic-version

.github/workflows/generate-docs.yml

@@ -0,0 +1,40 @@
+name: Generate Documentation
+
+on:
+  push:
+    branches:
+      - develop
+
+permissions:
+  contents: write  # Need write permission to commit changes
+
+jobs:
+  generate-docs:
+    environment: auto-release
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          token: ${{ secrets.GHA_TOKEN }}
+
+      - name: Setup backend
+        uses: ./.github/actions/setup-environment
+
+      - name: Generate API reference
+        run: uv run python src/codegen/gscli/cli.py generate docs
+
+      - name: Generate System Prompt
+        run: uv run python src/codegen/gscli/cli.py generate system-prompt
+
+      - name: Commit changes
+        run: |
+          git config --local user.email ${{ secrets.DOCS_USER_EMAIL }}
+          git config --local user.name ${{ secrets.DOCS_USER_NAME }}
+          git add docs/ src/codegen/sdk/system-prompt.txt
+          git diff --staged --quiet || git commit -m "docs: updated API reference"
+
+      - name: Push changes
+        uses: ad-m/github-push-action@master
+        with:
+          branch: ${{ github.ref }}

.github/workflows/pre-commit.yml

@@ -12,15 +12,15 @@ jobs:
   pre-commit:
     runs-on: ubuntu-latest
     timeout-minutes: 10
+    env:
+      REPO_SCOPED_TOKEN: ${{ secrets.REPO_SCOPED_TOKEN }}
 
     steps:
       - name: Check out the repo
         uses: actions/checkout@v4
         with:
           fetch-depth: 0
-          repository: ${{ github.event.pull_request.head.repo.full_name }}
-          ref: ${{ github.event.pull_request.head.ref }}
-          token: ${{ secrets.REPO_SCOPED_TOKEN }}
+          token: ${{ env.REPO_SCOPED_TOKEN || github.token }}
 
       - name: Setup environment
         uses: ./.github/actions/setup-environment
@@ -36,12 +36,9 @@ jobs:
 
       - run: uv run --frozen pre-commit run --show-diff-on-failure --color=always --all-files --source ${{ github.event.pull_request.base.sha || github.event.before }} --origin ${{ github.event.pull_request.head.sha || github.event.after }}
         shell: bash
-        env:
-          SKIP: circleci_validate
 
       - uses: stefanzweifel/git-auto-commit-action@v5
-        # Always commit changes even if pre-commit failed
-        if: always() && github.event_name == 'pull_request'
+        if: ${{ always() && env.REPO_SCOPED_TOKEN && github.event_name == 'pull_request' }}
         with:
           commit_message: "Automated pre-commit update"
           push_options: "--no-verify"

.github/workflows/release.yml

@@ -52,6 +52,7 @@ jobs:
         with:
           fetch-depth: 0
           ref: ${{ inputs.release-tag || github.event.pull_request.head.ref || github.ref }}
+          repository: ${{ github.event.pull_request.head.repo.full_name || github.event.repository.full_name }}
 
       - name: Install UV
         uses: astral-sh/[email protected]
@@ -64,6 +65,7 @@ jobs:
           cache-suffix: 3.${{ matrix.python }}
 
       - name: Fetch tags
+        if: ${{ inputs.release-tag || startsWith(github.ref, 'refs/tags/') }}
         run: |
           git branch
           git fetch --depth=1 origin +refs/tags/*:refs/tags/*

.github/workflows/unit-tests.yml renamed to .github/workflows/test.yml

@@ -1,24 +1,37 @@
-name: Unit Tests
+name: Tests
 
 on:
   push:
     branches:
       - "develop"
-  pull_request:
+  pull_request_target:
     types: [ opened, synchronize, reopened, labeled ]
     branches:
       - "develop"
   workflow_dispatch:
 
 jobs:
+  access-check:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions-cool/check-user-permission@v2
+        with:
+          require: write
+          username: ${{ github.triggering_actor }}
+          error-if-missing: true
+
   unit-tests:
+    needs: access-check
     runs-on: ubuntu-latest-8
     steps:
       - uses: actions/checkout@v4
         with:
           fetch-depth: 0
+          ref: ${{ github.event.pull_request.head.sha }}
+
       - name: Setup environment
         uses: ./.github/actions/setup-environment
+
       - name: Run ATS and Tests
         uses: ./.github/actions/run-ats
         timeout-minutes: 15
@@ -30,6 +43,7 @@ jobs:
           codecov_flags: unit-tests
 
   codemod-tests:
+    needs: access-check
     # TODO: re-enable when this check is a develop required check
     if: false
     runs-on: ubuntu-latest-32
@@ -48,10 +62,15 @@ jobs:
     name: "Codemod tests ${{matrix.size}}: Sync Graph=${{matrix.sync_graph}}"
     steps:
       - uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+
       - name: Setup environment
         uses: ./.github/actions/setup-environment
+
       - name: Cache oss-repos
         uses: ./.github/actions/setup-oss-repos
+
       - name: Run ATS and Tests
         uses: ./.github/actions/run-ats
         timeout-minutes: 15
@@ -66,11 +85,14 @@ jobs:
           GITHUB_WORKSPACE: $GITHUB_WORKSPACE
 
   parse-tests:
-    runs-on: ubuntu-latest-32
+    needs: access-check
     if: contains(github.event.pull_request.labels.*.name, 'parse-tests') || github.event_name == 'push' || github.event_name == 'workflow_dispatch'
-    environment: parse-tests
+    runs-on: ubuntu-latest-32
     steps:
       - uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+
       - name: Setup environment
         uses: ./.github/actions/setup-environment
 
@@ -91,10 +113,12 @@ jobs:
             -n auto \
             -o junit_suite_name="${{github.job}}" \
             tests/integration/codemod/test_parse.py
+
       - uses: ./.github/actions/report
         with:
           flag: no-flag
           codecov_token: ${{ secrets.CODECOV_TOKEN }}
+
       - name: Notify parse tests failure
         uses: slackapi/[email protected]
         if: failure() && github.event_name == 'push' && false
@@ -132,11 +156,16 @@ jobs:
             }
 
   integration-tests:
+    needs: access-check
     runs-on: ubuntu-latest-16
     steps:
       - uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+
       - name: Setup environment
         uses: ./.github/actions/setup-environment
+
       - name: Test with pytest
         timeout-minutes: 5
         env:
@@ -147,6 +176,7 @@ jobs:
             -n auto \
             -o junit_suite_name="${{github.job}}" \
             tests/integration/codegen
+
       - uses: ./.github/actions/report
         with:
           flag: integration-tests

CONTRIBUTING.md

@@ -51,6 +51,10 @@ uv run pytest tests/unit -n auto
 uv run pytest tests/integration/codemod/test_codemods.py -n auto
 ```
 
+> [!TIP]
+>
+> - If on Linux the error `OSError: [Errno 24] Too many open files` appears then you might want to increase your _ulimit_
+
 ## Pull Request Process
 
 1. Fork the repository and create your branch from `develop`.

Dockerfile

@@ -3,7 +3,6 @@ ARG CODEGEN_BOT_GHE_TOKEN=""
 FROM ghcr.io/astral-sh/uv:python${PYTHON_VERSION}-bookworm-slim AS base_uv
 ENV UV_COMPILE_BYTECODE=1 UV_LINK_MODE=copy
 ENV GITHUB_WORKSPACE=/workspace
-LABEL com.circleci.preserve-entrypoint=true
 ## Change the working directory to the `codegen-sdk` directory
 FROM base_uv AS install-tools
 RUN apt-get update && apt-get install -y build-essential curl git

README.md

@@ -79,6 +79,16 @@ See [Getting Started](https://docs.codegen.com/introduction/getting-started) for
 from codegen import Codebase
 ```
 
+## Troubleshooting
+
+Having issues? Here are some common problems and their solutions:
+
+- **I'm hitting an UV error related to `[[ packages ]]`**: This means you're likely using an outdated version of UV. Try updating to the latest version with: `uv self update`.
+- **I'm hitting an error about `No module named 'codegen.sdk.extensions.utils'`**: The compiled cython extensions are out of sync. Update them with `uv sync --reinstall-package codegen`.
+- **I'm hitting a `RecursionError: maximum recursion depth exceeded` error while parsing my codebase**: If you are using python 3.12, try upgrading to 3.13. If you are already on 3.13, try upping the recursion limit with `sys.setrecursionlimit(10000)`.
+
+If you run into additional issues not listed here, please [join our slack community](https://community.codegen.com) and we'll help you out!
+
 ## Resources
 
 - [Docs](https://docs.codegen.com)