build docs dynamically

Author: microdev1

.github/workflows/build.yml

@@ -16,6 +16,7 @@ jobs:
   test:
     runs-on: ubuntu-20.04
     outputs:
+      docs-build: ${{ steps.set-matrix.outputs.docs-build }}
       arm-boards: ${{ steps.set-matrix.outputs.arm-boards }}
       riscv-boards: ${{ steps.set-matrix.outputs.riscv-boards }}
       espressif-boards: ${{ steps.set-matrix.outputs.espressif-boards }}
@@ -41,37 +42,14 @@ jobs:
       run: |
         sudo apt-get update
         sudo apt-get install -y eatmydata
-        sudo eatmydata apt-get install -y gettext librsvg2-bin mingw-w64 latexmk texlive-fonts-recommended texlive-latex-recommended texlive-latex-extra gcc-aarch64-linux-gnu
-        pip install -r requirements-dev.txt
+        sudo eatmydata apt-get install -y gettext gcc-aarch64-linux-gnu mingw-w64
+        pip install -r requirements-ci.txt -r requirements-dev.txt
     - name: Versions
       run: |
         gcc --version
         python3 --version
     - name: Duplicate USB VID/PID Check
       run: python3 -u -m tools.ci_check_duplicate_usb_vid_pid
-    - name: Build and Validate Stubs
-      run: make check-stubs -j2
-    - uses: actions/upload-artifact@v2
-      with:
-        name: stubs
-        path: circuitpython-stubs/dist/*
-    - name: Install pypi dependencies
-      run: |
-        python -m pip install --upgrade pip
-        pip install setuptools wheel twine
-    - name: Test Documentation Build (HTML)
-      run: sphinx-build -E -W -b html -D version=${{ env.CP_VERSION }} -D release=${{ env.CP_VERSION }} . _build/html
-    - uses: actions/upload-artifact@v2
-      with:
-        name: docs
-        path: _build/html
-    - name: Test Documentation Build (LaTeX/PDF)
-      run: |
-        make latexpdf
-    - uses: actions/upload-artifact@v2
-      with:
-        name: docs
-        path: _build/latex
     - name: Build mpy-cross
       run: make -C mpy-cross -j2
     - name: Build unix port
@@ -130,39 +108,22 @@ jobs:
         [ -z "$AWS_ACCESS_KEY_ID" ] || aws s3 cp mpy-cross/mpy-cross.static.exe s3://adafruit-circuit-python/bin/mpy-cross/mpy-cross.static-x64-windows-${{ env.CP_VERSION }}.exe --no-progress --region us-east-1
         zip -9r circuitpython-stubs.zip circuitpython-stubs
         [ -z "$AWS_ACCESS_KEY_ID" ] || aws s3 cp circuitpython-stubs/dist/*.tar.gz s3://adafruit-circuit-python/bin/stubs/circuitpython-stubs-${{ env.CP_VERSION }}.zip --no-progress --region us-east-1
-
-    - name: Upload stubs to PyPi
-      if: github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository_owner == 'adafruit'
-      env:
-        TWINE_USERNAME: ${{ secrets.pypi_username }}
-        TWINE_PASSWORD: ${{ secrets.pypi_password }}
-      run: |
-        # setup.py sdist was run by 'make stubs'
-        [ -z "$TWINE_USERNAME" ] || echo "Uploading dev release to PyPi"
-        [ -z "$TWINE_USERNAME" ] || twine upload circuitpython-stubs/dist/*
-    - uses: dorny/paths-filter@v2
+    - name: "Get changes"
+      uses: dorny/paths-filter@v2
       id: filter
       with:
-        # Enable listing of files matching each filter.
-        # Paths to files will be available in `${FILTER_NAME}_files` output variable.
-        # Paths will be formatted as JSON array
         list-files: json
-
-        # Compare against this branch. (Ignored for PRs.)
         base: ${{ github.ref }}
-
-        # In this example all changed files are passed to the following action to do
-        # some custom processing.
         filters: |
           changed:
             - '**'
-    - name: "Set boards to build"
+    - name: "Set matrix"
       id: set-matrix
       working-directory: tools
       env:
         CHANGED_FILES: ${{ steps.filter.outputs.changed_files }}
-      run: |
-        python3 -u ci_changed_board_list.py
+      run: python3 -u ci_set_matrix.py
+
 
   mpy-cross-mac:
     runs-on: macos-10.15
@@ -221,6 +182,62 @@ jobs:
       if: (github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository_owner == 'adafruit') || (github.event_name == 'release' && (github.event.action == 'published' || github.event.action == 'rerequested'))
 
 
+  build-doc:
+    runs-on: ubuntu-20.04
+    needs: test
+    if: ${{ needs.test.outputs.docs-build == 'True' }}
+    steps:
+    - uses: actions/[email protected]
+      with:
+        submodules: false
+        fetch-depth: 0
+    - name: Populate selected submodules
+      run: git submodule update --init extmod/ulab
+    - run: git fetch --recurse-submodules=no https://github.com/adafruit/circuitpython refs/tags/*:refs/tags/*
+    - name: CircuitPython version
+      run: |
+        git describe --dirty --tags
+        echo >>$GITHUB_ENV CP_VERSION=$(git describe --dirty --tags)
+    - name: Set up Python 3.8
+      uses: actions/setup-python@v1
+      with:
+        python-version: 3.8
+    - name: Install deps
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y eatmydata
+        sudo eatmydata apt-get install -y latexmk librsvg2-bin texlive-fonts-recommended texlive-latex-recommended texlive-latex-extra
+        pip install -r requirements-ci.txt -r requirements-doc.txt
+    - name: Build and Validate Stubs
+      run: make check-stubs -j2
+    - uses: actions/upload-artifact@v2
+      with:
+        name: stubs
+        path: circuitpython-stubs/dist/*
+    - name: Test Documentation Build (HTML)
+      run: sphinx-build -E -W -b html -D version=${{ env.CP_VERSION }} -D release=${{ env.CP_VERSION }} . _build/html
+    - uses: actions/upload-artifact@v2
+      with:
+        name: docs
+        path: _build/html
+    - name: Test Documentation Build (LaTeX/PDF)
+      run: |
+        make latexpdf
+    - uses: actions/upload-artifact@v2
+      with:
+        name: docs
+        path: _build/latex
+    - name: Upload stubs to PyPi
+      if: github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository_owner == 'adafruit'
+      env:
+        TWINE_USERNAME: ${{ secrets.pypi_username }}
+        TWINE_PASSWORD: ${{ secrets.pypi_password }}
+      run: |
+        # setup.py sdist was run by 'make stubs'
+        [ -z "$TWINE_USERNAME" ] || echo "Uploading dev release to PyPi"
+        [ -z "$TWINE_USERNAME" ] || twine upload circuitpython-stubs/dist/*
+
+
   build-arm:
     runs-on: ubuntu-20.04
     needs: test
@@ -229,7 +246,6 @@ jobs:
       matrix:
         board: ${{ fromJSON(needs.test.outputs.arm-boards) }}
     if: ${{ needs.test.outputs.arm-boards != '[]' }}
-
     steps:
     - name: Set up Python 3.8
       uses: actions/setup-python@v1
@@ -243,7 +259,7 @@ jobs:
     - name: Install deps
       run: |
         sudo apt-get install -y gettext
-        pip install -r requirements-dev.txt
+        pip install -r requirements-ci.txt -r requirements-dev.txt
         wget --no-verbose https://adafruit-circuit-python.s3.amazonaws.com/gcc-arm-none-eabi-10-2020-q4-major-x86_64-linux.tar.bz2
         sudo tar -C /usr --strip-components=1 -xaf gcc-arm-none-eabi-10-2020-q4-major-x86_64-linux.tar.bz2
     - name: Versions
@@ -272,6 +288,7 @@ jobs:
         AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
       if: (github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository_owner == 'adafruit') || (github.event_name == 'release' && (github.event.action == 'published' || github.event.action == 'rerequested'))
 
+
   build-riscv:
     runs-on: ubuntu-20.04
     needs: test
@@ -280,7 +297,6 @@ jobs:
       matrix:
         board: ${{ fromJSON(needs.test.outputs.riscv-boards) }}
     if: ${{ needs.test.outputs.riscv-boards != '[]' }}
-
     steps:
     - name: Set up Python 3.8
       uses: actions/setup-python@v1
@@ -294,7 +310,7 @@ jobs:
     - name: Install deps
       run: |
         sudo apt-get install -y gettext
-        pip install -r requirements-dev.txt
+        pip install -r requirements-ci.txt -r requirements-dev.txt
         wget https://static.dev.sifive.com/dev-tools/riscv64-unknown-elf-gcc-8.3.0-2019.08.0-x86_64-linux-centos6.tar.gz
         sudo tar -C /usr --strip-components=1 -xaf riscv64-unknown-elf-gcc-8.3.0-2019.08.0-x86_64-linux-centos6.tar.gz
     - name: Versions
@@ -322,6 +338,8 @@ jobs:
         AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
         AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
       if: (github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository_owner == 'adafruit') || (github.event_name == 'release' && (github.event.action == 'published' || github.event.action == 'rerequested'))
+
+
   build-espressif:
     runs-on: ubuntu-20.04
     needs: test
@@ -330,7 +348,6 @@ jobs:
       matrix:
         board: ${{ fromJSON(needs.test.outputs.espressif-boards) }}
     if: ${{ needs.test.outputs.espressif-boards != '[]' }}
-
     steps:
     - name: Set up Python 3.8
       uses: actions/setup-python@v1
@@ -363,11 +380,11 @@ jobs:
       env:
         IDF_PATH: ${{ github.workspace }}/ports/espressif/esp-idf
         IDF_TOOLS_PATH: ${{ github.workspace }}/.idf_tools
-    - name: Install CircuitPython deps
+    - name: Install deps
       run: |
         source $IDF_PATH/export.sh
-        pip install -r requirements-dev.txt
         sudo apt-get install -y gettext ninja-build
+        pip install -r requirements-ci.txt -r requirements-dev.txt
       env:
         IDF_PATH: ${{ github.workspace }}/ports/espressif/esp-idf
         IDF_TOOLS_PATH: ${{ github.workspace }}/.idf_tools

Makefile

@@ -260,6 +260,7 @@ stubs:
 	@$(PYTHON) tools/extract_pyi.py shared-bindings/ $(STUBDIR)
 	@$(PYTHON) tools/extract_pyi.py extmod/ulab/code/ $(STUBDIR)/ulab
 	@$(PYTHON) tools/extract_pyi.py ports/atmel-samd/bindings $(STUBDIR)
+	@$(PYTHON) tools/extract_pyi.py ports/espressif/bindings $(STUBDIR)
 	@$(PYTHON) tools/extract_pyi.py ports/raspberrypi/bindings $(STUBDIR)
 	@cp setup.py-stubs circuitpython-stubs/setup.py
 	@cp README.rst-stubs circuitpython-stubs/README.rst

ports/espressif/bindings/espidf/__init__.c

@@ -110,10 +110,10 @@ const mp_obj_type_t mp_type_espidf_IDFError = {
 };
 
 
-//| class MemoryError(MemoryError):
-//|     """Raised when an ESP IDF memory allocation fails."""
-//|     ...
-//|
+// class MemoryError(MemoryError):
+//     """Raised when an ESP IDF memory allocation fails."""
+//     ...
+//
 NORETURN void mp_raise_espidf_MemoryError(void) {
     nlr_raise(mp_obj_new_exception(&mp_type_espidf_MemoryError));
 }

requirements-ci.txt

@@ -0,0 +1,2 @@
+# For uploading artifacts
+awscli

requirements-dev.txt

@@ -6,33 +6,18 @@ cascadetoml
 jinja2
 typer
 
-requests
-requests-cache
 sh
 click
-setuptools
 cpp-coveralls
-
-# For docs
-Sphinx<4
-sphinx-rtd-theme
-myst-parser
-sphinx-autoapi
-sphinxcontrib-svg2pdfconverter
-readthedocs-sphinx-search
+requests
+requests-cache
 
 # For translate check
 polib
 
 # For pre-commit
 pyyaml
-astroid
-isort
 black
-mypy
-
-# For uploading artifacts
-awscli
 
 # for combining the Nordic SoftDevice with CircuitPython
 intelhex

requirements-doc.txt

@@ -0,0 +1,16 @@
+# For docs
+mypy
+black
+isort
+twine
+wheel
+astroid
+setuptools
+
+# For sphinx
+Sphinx<4
+sphinx-autoapi
+sphinx-rtd-theme
+sphinxcontrib-svg2pdfconverter
+readthedocs-sphinx-search
+myst-parser