conditionally build all languages

Author: microdev1

.github/actions/deps/ports/broadcom/action.yml

@@ -5,8 +5,8 @@ runs:
   steps:
     - name: Get broadcom toolchain
       run: |
-        wget --no-verbose https://adafruit-circuit-python.s3.amazonaws.com/gcc-arm-10.3-2021.07-x86_64-broadcom64-none-elf.tar.xz
-        sudo tar -C /usr --strip-components=1 -xaf gcc-arm-10.3-2021.07-x86_64-broadcom64-none-elf.tar.xz
+        wget --no-verbose https://adafruit-circuit-python.s3.amazonaws.com/gcc-arm-10.3-2021.07-x86_64-aarch64-none-elf.tar.xz
+        sudo tar -C /usr --strip-components=1 -xaf gcc-arm-10.3-2021.07-x86_64-aarch64-none-elf.tar.xz
         sudo apt-get install -y mtools
       shell: bash
     - name: Install mkfs.fat

.github/workflows/build.yml

@@ -18,7 +18,7 @@ concurrency:
 
 jobs:
   scheduler:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     outputs:
       docs: ${{ steps.set-matrix.outputs.docs }}
       boards: ${{ steps.set-matrix.outputs.boards }}
@@ -287,6 +287,7 @@ jobs:
     uses: ./.github/workflows/re-build-boards.yml
     secrets: inherit
     strategy:
+      fail-fast: false
       matrix:
         port: ${{ fromJSON(needs.scheduler.outputs.boards).ports }}
     with:

.github/workflows/re-run-tests.yml

@@ -9,7 +9,7 @@ on:
 
 jobs:
   run:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     strategy:
       fail-fast: false
       matrix:
@@ -31,7 +31,7 @@ jobs:
     - name: Set up python
       uses: actions/setup-python@v4
       with:
-        python-version: 3.x
+        python-version: 3.8
     - name: Set up submodules
       uses: ./.github/actions/deps/submodules
       with:

ports/atmel-samd/Makefile

@@ -367,7 +367,7 @@ $(BUILD)/firmware.elf: $(OBJ) $(GENERATED_LD_FILE)
 	$(STEPECHO) "LINK $@"
 	$(Q)echo $(OBJ) > $(BUILD)/firmware.objs
 	$(Q)$(CC) -o $@ $(LDFLAGS) @$(BUILD)/firmware.objs -Wl,--start-group $(LIBS) -Wl,--end-group
-	$(Q)$(SIZE) $@ | $(PYTHON) $(TOP)/tools/build_memory_info.py $(GENERATED_LD_FILE)
+	$(Q)$(SIZE) $@ | $(PYTHON) $(TOP)/tools/build_memory_info.py $(GENERATED_LD_FILE) $(BUILD)
 
 $(BUILD)/firmware.bin: $(BUILD)/firmware.elf
 	$(STEPECHO) "Create $@"

ports/espressif/Makefile

@@ -434,7 +434,7 @@ $(BUILD)/firmware.elf: $(OBJ) | esp-idf-stamp
 $(BUILD)/circuitpython-firmware.bin: $(BUILD)/firmware.elf | tools/build_memory_info.py
 	$(STEPECHO) "Create $@"
 	$(Q)esptool.py --chip $(IDF_TARGET) elf2image $(FLASH_FLAGS) --elf-sha256-offset 0xb0 -o $@ $^
-	$(Q)$(PYTHON) tools/build_memory_info.py $< $(BUILD)/esp-idf/sdkconfig $@
+	$(Q)$(PYTHON) tools/build_memory_info.py $< $(BUILD)/esp-idf/sdkconfig $@ $(BUILD)
 
 $(BUILD)/firmware.bin: $(BUILD)/circuitpython-firmware.bin | esp-idf-stamp
 	$(Q)$(PYTHON) ../../tools/join_bins.py $@ $(BOOTLOADER_OFFSET) $(BUILD)/esp-idf/bootloader/bootloader.bin $(PARTITION_TABLE_OFFSET) $(BUILD)/esp-idf/partition_table/partition-table.bin $(FIRMWARE_OFFSET) $(BUILD)/circuitpython-firmware.bin

ports/espressif/tools/build_memory_info.py

@@ -6,13 +6,12 @@
 # SPDX-License-Identifier: MIT
 
 import csv
+import json
 import os
-import re
 import sys
 
 from elftools.elf.elffile import ELFFile
 
-print()
 
 internal_memory = {
     "esp32": [
@@ -117,8 +116,12 @@ def find_region(start_address):
 
 # This file is the bin
 used_flash = os.stat(sys.argv[3]).st_size
-
 free_flash = firmware_region - used_flash
+
+with open(f"{sys.argv[4]}/firmware.size.json", "w") as f:
+    json.dump({"used_flash": used_flash, "firmware_region": firmware_region}, f)
+
+print()
 print(
     "{:7} bytes used, {:7} bytes free in flash firmware space out of {} bytes ({}kB).".format(
         used_flash, free_flash, firmware_region, firmware_region / 1024

ports/litex/Makefile

@@ -136,7 +136,7 @@ all: $(BUILD)/firmware.bin $(BUILD)/firmware.dfu
 $(BUILD)/firmware.elf: $(OBJ)
 	$(STEPECHO) "LINK $@"
 	$(Q)$(CC) -o $@ $(LDFLAGS) $^ -Wl,--start-group $(LIBS) -Wl,--end-group
-	$(Q)$(SIZE) $@ | $(PYTHON) $(TOP)/tools/build_memory_info.py $(LD_FILE)
+	$(Q)$(SIZE) $@ | $(PYTHON) $(TOP)/tools/build_memory_info.py $(LD_FILE) $(BUILD)
 
 $(BUILD)/firmware.bin: $(BUILD)/firmware.elf
 	$(STEPECHO) "Create $@"

ports/nrf/Makefile

@@ -212,7 +212,7 @@ $(BUILD)/firmware.elf: $(OBJ) $(GENERATED_LD_FILE)
 	$(STEPECHO) "LINK $@"
 	$(Q)echo $(OBJ) > $(BUILD)/firmware.objs
 	$(Q)$(CC) -o $@ $(LDFLAGS) @$(BUILD)/firmware.objs -Wl,--start-group $(LIBS) -Wl,--end-group
-	$(Q)$(SIZE) $@ | $(PYTHON) $(TOP)/tools/build_memory_info.py $(GENERATED_LD_FILE)
+	$(Q)$(SIZE) $@ | $(PYTHON) $(TOP)/tools/build_memory_info.py $(GENERATED_LD_FILE) $(BUILD)
 
 $(BUILD)/firmware.bin: $(BUILD)/firmware.elf
 	$(STEPECHO) "Create $@"

ports/raspberrypi/Makefile

@@ -417,7 +417,7 @@ $(BUILD)/firmware.elf: $(OBJ) $(LINK_LD)
 	$(Q)echo $(OBJ) > $(BUILD)/firmware.objs
 	$(Q)echo $(PICO_LDFLAGS) > $(BUILD)/firmware.ldflags
 	$(Q)$(CC) -o $@ $(CFLAGS) @$(BUILD)/firmware.ldflags -Wl,-T,$(LINK_LD) -Wl,-Map=$@.map -Wl,-cref -Wl,--gc-sections @$(BUILD)/firmware.objs -Wl,-lc
-	$(Q)$(SIZE) $@ | $(PYTHON) $(TOP)/tools/build_memory_info.py $(LINK_LD)
+	$(Q)$(SIZE) $@ | $(PYTHON) $(TOP)/tools/build_memory_info.py $(LINK_LD) $(BUILD)
 
 $(BUILD)/firmware.bin: $(BUILD)/firmware.elf
 	$(STEPECHO) "Create $@"

ports/stm/Makefile

@@ -268,7 +268,7 @@ $(BUILD)/firmware.elf: $(OBJ)
 	$(STEPECHO) "LINK $@"
 	$(Q)echo $^ > $(BUILD)/firmware.objs
 	$(Q)$(CC) -o $@ $(LDFLAGS) @$(BUILD)/firmware.objs -Wl,--start-group $(LIBS) -Wl,--end-group
-	$(Q)$(SIZE) $@ | $(PYTHON) $(TOP)/tools/build_memory_info.py $(LD_FILE)
+	$(Q)$(SIZE) $@ | $(PYTHON) $(TOP)/tools/build_memory_info.py $(LD_FILE) $(BUILD)
 
 $(BUILD)/firmware.bin: $(BUILD)/firmware.elf
 	$(STEPECHO) "Create $@"

tools/build_memory_info.py

@@ -7,6 +7,8 @@
 
 import re
 import sys
+import json
+
 
 # Handle size constants with K or M suffixes (allowed in .ld but not in Python).
 K_PATTERN = re.compile(r"([0-9]+)[kK]")
@@ -15,11 +17,10 @@
 M_PATTERN = re.compile(r"([0-9]+)[mM]")
 M_REPLACE = r"(\1*1024*1024)"
 
-print()
-
 text = 0
 data = 0
 bss = 0
+
 # stdin is the linker output.
 for line in sys.stdin:
     # Uncomment to see linker output.
@@ -29,6 +30,7 @@
         text, data, bss = map(int, line.split()[:3])
 
 regions = {}
+
 # This file is the linker script.
 with open(sys.argv[1], "r") as f:
     for line in f:
@@ -51,6 +53,11 @@
 free_flash = firmware_region - used_flash
 used_ram = data + bss
 free_ram = ram_region - used_ram
+
+with open(f"{sys.argv[2]}/firmware.size.json", "w") as f:
+    json.dump({"used_flash": used_flash, "firmware_region": firmware_region}, f)
+
+print()
 print(
     "{} bytes used, {} bytes free in flash firmware space out of {} bytes ({}kB).".format(
         used_flash, free_flash, firmware_region, firmware_region / 1024

tools/build_release_files.py

@@ -11,14 +11,15 @@
 import shutil
 import build_board_info as build_info
 import time
+import json
 
 sys.path.append("../docs")
 from shared_bindings_matrix import get_settings_from_makefile
 
 for port in build_info.SUPPORTED_PORTS:
     result = subprocess.run("rm -rf ../ports/{port}/build*".format(port=port), shell=True)
 
-PARALLEL = "-j 5"
+PARALLEL = "-j 4"
 if "GITHUB_ACTION" in os.environ:
     PARALLEL = "-j 2"
 
@@ -29,6 +30,11 @@
 
 sha, version = build_info.get_version_info()
 
+build_all = os.environ.get("GITHUB_EVENT_NAME") != "pull_request"
+
+LANGUAGE_FIRST = "en_US"
+LANGUAGE_THRESHOLD = 10 * 1024
+
 languages = build_info.get_languages()
 
 all_languages = build_info.get_languages(list_all=True)
@@ -44,6 +50,9 @@
     board_info = all_boards[board]
     board_settings = get_settings_from_makefile("../ports/" + board_info["port"], board)
 
+    languages.remove(LANGUAGE_FIRST)
+    languages.insert(0, LANGUAGE_FIRST)
+
     for language in languages:
         bin_directory = "../bin/{board}/{language}".format(board=board, language=language)
         os.makedirs(bin_directory, exist_ok=True)
@@ -130,4 +139,16 @@
         # Flush so we will see something before 10 minutes has passed.
         print(flush=True)
 
+        if (not build_all) and (language is LANGUAGE_FIRST) and (exit_status is 0):
+            try:
+                with open(
+                    f"../ports/{board_info['port']}/{build_dir}/firmware.size.json", "r"
+                ) as f:
+                    firmware = json.load(f)
+                    if firmware["used_flash"] + LANGUAGE_THRESHOLD < firmware["firmware_region"]:
+                        print("Skipping languages")
+                        break
+            except FileNotFoundError:
+                pass
+
 sys.exit(exit_status)