Merge pull request #6629 from Neradoc/board-extensions-in-boards-dir

Build extensions in mpconfigport and mpconfigboard

Author: tannewt

conf.py

@@ -52,8 +52,8 @@
 #modules_support_matrix = shared_bindings_matrix.support_matrix_excluded_boards()
 modules_support_matrix = shared_bindings_matrix.support_matrix_by_board()
 modules_support_matrix_reverse = defaultdict(list)
-for board, modules in modules_support_matrix.items():
-    for module in modules[0]:
+for board, matrix_info in modules_support_matrix.items():
+    for module in matrix_info["modules"]:
         modules_support_matrix_reverse[module].append(board)
 
 modules_support_matrix_reverse = dict(

docs/shared_bindings_matrix.py

@@ -32,7 +32,7 @@
 
 SUPPORTED_PORTS = ['atmel-samd', 'broadcom', 'cxd56', 'espressif', 'litex', 'mimxrt10xx', 'nrf', 'raspberrypi', 'stm']
 
-aliases_by_board = {
+ALIASES_BY_BOARD = {
     "circuitplayground_express": [
         "circuitplayground_express_4h",
         "circuitplayground_express_digikey_pycon2019",
@@ -43,7 +43,7 @@
     "pewpew10": ["pewpew13"],
 }
 
-aliases_brand_names = {
+ALIASES_BRAND_NAMES = {
     "circuitplayground_express_4h":
         "Adafruit Circuit Playground Express 4-H",
     "circuitplayground_express_digikey_pycon2019":
@@ -58,36 +58,64 @@
         "PewPew 13",
 }
 
-additional_modules = {
+ADDITIONAL_MODULES = {
     "fontio": "CIRCUITPY_DISPLAYIO",
     "terminalio": "CIRCUITPY_DISPLAYIO",
     "adafruit_bus_device": "CIRCUITPY_BUSDEVICE",
     "adafruit_pixelbuf": "CIRCUITPY_PIXELBUF",
     "usb": "CIRCUITPY_USB_HOST",
 }
 
-frozen_excludes = ["examples", "docs", "tests", "utils", "conf.py", "setup.py"]
+FROZEN_EXCLUDES = ["examples", "docs", "tests", "utils", "conf.py", "setup.py"]
 """Files and dirs at the root of a frozen directory that should be ignored.
 This is the same list as in the preprocess_frozen_modules script."""
 
 repository_urls = {}
 """Cache of repository URLs for frozen modules."""
 
 def get_circuitpython_root_dir():
-    """ The path to the root './circuitpython' directory
+    """ The path to the root './circuitpython' directory.
     """
     file_path = pathlib.Path(__file__).resolve()
     root_dir = file_path.parent.parent
 
     return root_dir
 
 def get_shared_bindings():
-    """ Get a list of modules in shared-bindings based on folder names
+    """ Get a list of modules in shared-bindings based on folder names.
     """
     shared_bindings_dir = get_circuitpython_root_dir() / "shared-bindings"
     return [item.name for item in shared_bindings_dir.iterdir()] + ["binascii", "errno", "json", "re", "ulab"]
 
 
+def get_board_mapping():
+    """
+    Compiles the list of boards from the directories, with aliases and mapping
+    to the port.
+    """
+    boards = {}
+    for port in SUPPORTED_PORTS:
+        board_path = os.path.join("../ports", port, "boards")
+        for board_path in os.scandir(board_path):
+            if board_path.is_dir():
+                board_files = os.listdir(board_path.path)
+                board_id = board_path.name
+                aliases = ALIASES_BY_BOARD.get(board_path.name, [])
+                boards[board_id] = {
+                    "port": port,
+                    "download_count": 0,
+                    "aliases": aliases,
+                }
+                for alias in aliases:
+                    boards[alias] = {
+                        "port": port,
+                        "download_count": 0,
+                        "alias": True,
+                        "aliases": [],
+                    }
+    return boards
+
+
 def read_mpconfig():
     """ Open 'circuitpy_mpconfig.mk' and return the contents.
     """
@@ -112,8 +140,8 @@ def build_module_map():
     full_build = False
     for module in modules:
         full_name = module
-        if module in additional_modules:
-            search_identifier = additional_modules[module]
+        if module in ADDITIONAL_MODULES:
+            search_identifier = ADDITIONAL_MODULES[module]
         else:
             search_identifier = 'CIRCUITPY_'+module.lstrip("_").upper()
         re_pattern = f"{re.escape(search_identifier)}\s*\??=\s*(.+)"
@@ -204,27 +232,33 @@ def get_repository_url(directory):
     repository_urls[directory] = path
     return path
 
-def frozen_modules_from_dirs(frozen_mpy_dirs):
+def frozen_modules_from_dirs(frozen_mpy_dirs, withurl):
     """
     Go through the list of frozen directories and extract the python modules.
     Paths are of the type:
         $(TOP)/frozen/Adafruit_CircuitPython_CircuitPlayground
         $(TOP)/frozen/circuitpython-stage/meowbit
     Python modules are at the root of the path, and are python files or directories
-    containing python files. Except the ones in the frozen_excludes list.
+    containing python files. Except the ones in the FROZEN_EXCLUDES list.
     """
     frozen_modules = []
     for frozen_path in filter(lambda x: x, frozen_mpy_dirs.split(" ")):
         source_dir = get_circuitpython_root_dir() / frozen_path[7:]
         url_repository = get_repository_url(source_dir)
         for sub in source_dir.glob("*"):
-            if sub.name in frozen_excludes:
+            if sub.name in FROZEN_EXCLUDES:
                 continue
             if sub.name.endswith(".py"):
-                frozen_modules.append((sub.name[:-3], url_repository))
+                if withurl:
+                    frozen_modules.append((sub.name[:-3], url_repository))
+                else:
+                    frozen_modules.append(sub.name[:-3])
                 continue
             if next(sub.glob("**/*.py"), None): # tests if not empty
-                frozen_modules.append((sub.name, url_repository))
+                if withurl:
+                    frozen_modules.append((sub.name, url_repository))
+                else:
+                    frozen_modules.append(sub.name)
     return frozen_modules
 
 def lookup_setting(settings, key, default=''):
@@ -244,7 +278,7 @@ def all_ports_all_boards(ports=SUPPORTED_PORTS):
                 continue
             yield (port, entry)
 
-def support_matrix_by_board(use_branded_name=True):
+def support_matrix_by_board(use_branded_name=True, withurl=True):
     """ Compiles a list of the available core modules available for each
         board.
     """
@@ -272,29 +306,49 @@ def support_matrix(arg):
                 board_modules.append(base[module]['name'])
         board_modules.sort()
 
+        if "CIRCUITPY_BUILD_EXTENSIONS" in settings:
+            board_extensions = [
+                extension.strip() for extension in
+                settings["CIRCUITPY_BUILD_EXTENSIONS"].split(",")
+            ]
+        else:
+            raise OSError(f"Board extensions undefined: {board_name}.")
+
         frozen_modules = []
         if "FROZEN_MPY_DIRS" in settings:
-            frozen_modules = frozen_modules_from_dirs(settings["FROZEN_MPY_DIRS"])
+            frozen_modules = frozen_modules_from_dirs(settings["FROZEN_MPY_DIRS"], withurl)
             if frozen_modules:
                 frozen_modules.sort()
 
         # generate alias boards too
-        board_matrix = [(board_name, (board_modules, frozen_modules))]
-        if entry.name in aliases_by_board:
-            for alias in aliases_by_board[entry.name]:
+        board_matrix = [(
+            board_name, {
+                "modules": board_modules,
+                "frozen_libraries": frozen_modules,
+                "extensions": board_extensions,
+            }
+        )]
+        if entry.name in ALIASES_BY_BOARD:
+            for alias in ALIASES_BY_BOARD[entry.name]:
                 if use_branded_name:
-                    if alias in aliases_brand_names:
-                        alias = aliases_brand_names[alias]
+                    if alias in ALIASES_BRAND_NAMES:
+                        alias = ALIASES_BRAND_NAMES[alias]
                     else:
                         alias = alias.replace("_"," ").title()
-                board_matrix.append( (alias, (board_modules, frozen_modules)) )
+                board_matrix.append((
+                    alias, {
+                        "modules": board_modules,
+                        "frozen_libraries": frozen_modules,
+                        "extensions": board_extensions,
+                    },
+                ))
 
         return board_matrix # this is now a list of (board,modules)
 
     executor = ThreadPoolExecutor(max_workers=os.cpu_count())
     mapped_exec = executor.map(support_matrix, all_ports_all_boards())
     # flatmap with comprehensions
-    boards = dict(sorted([board for matrix in mapped_exec for board in matrix]))
+    boards = dict(sorted([board for matrix in mapped_exec for board in matrix], key=lambda x: x[0]))
 
     return boards
 

ports/atmel-samd/boards/arduino_mkr1300/mpconfigboard.mk

@@ -6,6 +6,8 @@ USB_MANUFACTURER = "Arduino"
 CHIP_VARIANT = SAMD21G18A
 CHIP_FAMILY = samd21
 
+CIRCUITPY_BUILD_EXTENSIONS = bin,uf2
+
 INTERNAL_FLASH_FILESYSTEM = 1
 LONGINT_IMPL = NONE
 CIRCUITPY_FULL_BUILD = 0

ports/atmel-samd/boards/arduino_mkrzero/mpconfigboard.mk

@@ -6,6 +6,8 @@ USB_MANUFACTURER = "Arduino"
 CHIP_VARIANT = SAMD21G18A
 CHIP_FAMILY = samd21
 
+CIRCUITPY_BUILD_EXTENSIONS = bin,uf2
+
 INTERNAL_FLASH_FILESYSTEM = 1
 LONGINT_IMPL = NONE
 CIRCUITPY_FULL_BUILD = 0

ports/atmel-samd/boards/arduino_nano_33_iot/mpconfigboard.mk

@@ -6,6 +6,8 @@ USB_MANUFACTURER = "Arduino"
 CHIP_VARIANT = SAMD21G18A
 CHIP_FAMILY = samd21
 
+CIRCUITPY_BUILD_EXTENSIONS = bin,uf2
+
 INTERNAL_FLASH_FILESYSTEM = 1
 LONGINT_IMPL = NONE
 CIRCUITPY_FULL_BUILD = 0

ports/atmel-samd/boards/arduino_zero/mpconfigboard.mk

@@ -6,6 +6,8 @@ USB_MANUFACTURER = "Arduino"
 CHIP_VARIANT = SAMD21G18A
 CHIP_FAMILY = samd21
 
+CIRCUITPY_BUILD_EXTENSIONS = bin,uf2
+
 INTERNAL_FLASH_FILESYSTEM = 1
 LONGINT_IMPL = NONE
 CIRCUITPY_FULL_BUILD = 0

ports/atmel-samd/boards/feather_m0_adalogger/mpconfigboard.mk

@@ -6,6 +6,8 @@ USB_MANUFACTURER = "Adafruit Industries LLC"
 CHIP_VARIANT = SAMD21G18A
 CHIP_FAMILY = samd21
 
+CIRCUITPY_BUILD_EXTENSIONS = bin,uf2
+
 INTERNAL_FLASH_FILESYSTEM = 1
 LONGINT_IMPL = NONE
 CIRCUITPY_FULL_BUILD = 0

ports/atmel-samd/boards/feather_m0_basic/mpconfigboard.mk

@@ -6,6 +6,8 @@ USB_MANUFACTURER = "Adafruit Industries LLC"
 CHIP_VARIANT = SAMD21G18A
 CHIP_FAMILY = samd21
 
+CIRCUITPY_BUILD_EXTENSIONS = bin,uf2
+
 INTERNAL_FLASH_FILESYSTEM = 1
 LONGINT_IMPL = NONE
 CIRCUITPY_FULL_BUILD = 0

ports/atmel-samd/boards/feather_m0_rfm69/mpconfigboard.mk

@@ -6,6 +6,8 @@ USB_MANUFACTURER = "Adafruit Industries LLC"
 CHIP_VARIANT = SAMD21G18A
 CHIP_FAMILY = samd21
 
+CIRCUITPY_BUILD_EXTENSIONS = bin,uf2
+
 INTERNAL_FLASH_FILESYSTEM = 1
 LONGINT_IMPL = NONE
 CIRCUITPY_FULL_BUILD = 0

ports/atmel-samd/boards/feather_m0_rfm9x/mpconfigboard.mk

@@ -6,6 +6,8 @@ USB_MANUFACTURER = "Adafruit Industries LLC"
 CHIP_VARIANT = SAMD21G18A
 CHIP_FAMILY = samd21
 
+CIRCUITPY_BUILD_EXTENSIONS = bin,uf2
+
 INTERNAL_FLASH_FILESYSTEM = 1
 LONGINT_IMPL = NONE
 CIRCUITPY_FULL_BUILD = 0

ports/atmel-samd/boards/uchip/mpconfigboard.mk

@@ -6,6 +6,8 @@ USB_MANUFACTURER = "Itaca Innovation"
 CHIP_VARIANT = SAMD21E18A
 CHIP_FAMILY = samd21
 
+CIRCUITPY_BUILD_EXTENSIONS = bin,uf2
+
 INTERNAL_FLASH_FILESYSTEM = 1
 LONGINT_IMPL = NONE
 CIRCUITPY_FULL_BUILD = 0

ports/atmel-samd/mpconfigport.mk

@@ -133,3 +133,5 @@ CIRCUITPY_RGBMATRIX ?= $(CIRCUITPY_FRAMEBUFFERIO)
 
 endif # same51
 ######################################################################
+
+CIRCUITPY_BUILD_EXTENSIONS ?= uf2

ports/broadcom/boards/raspberrypi_zero/mpconfigboard.mk

@@ -4,3 +4,5 @@ USB_PRODUCT = "Zero"
 USB_MANUFACTURER = "Raspberry Pi"
 
 CHIP_VARIANT = "bcm2835"
+
+CIRCUITPY_BUILD_EXTENSIONS = disk.img.zip,kernel.img

ports/broadcom/boards/raspberrypi_zero_w/mpconfigboard.mk

@@ -4,3 +4,5 @@ USB_PRODUCT = "Zero W"
 USB_MANUFACTURER = "Raspberry Pi"
 
 CHIP_VARIANT = "bcm2835"
+
+CIRCUITPY_BUILD_EXTENSIONS = disk.img.zip,kernel.img

ports/broadcom/mpconfigport.mk

@@ -24,3 +24,5 @@ INTERNAL_FLASH_FILESYSTEM = 1
 
 USB_NUM_ENDPOINT_PAIRS = 8
 USB_HIGHSPEED = 1
+
+CIRCUITPY_BUILD_EXTENSIONS ?= disk.img.zip,kernel8.img

ports/cxd56/mpconfigport.mk

@@ -25,3 +25,5 @@ CIRCUITPY_TOUCHIO = 0
 CIRCUITPY_USB_HID = 0
 CIRCUITPY_USB_MIDI = 0
 INTERNAL_LIBM = 1
+
+CIRCUITPY_BUILD_EXTENSIONS ?= spk

ports/espressif/mpconfigport.mk

@@ -40,6 +40,7 @@ CIRCUITPY_PARALLELDISPLAY = 0
 # Protomatter needs to support ESP32.
 CIRCUITPY_RGBMATRIX = 0
 CIRCUITPY_USB = 0
+CIRCUITPY_BUILD_EXTENSIONS ?= bin
 
 else ifeq ($(IDF_TARGET),esp32c3)
 CIRCUITPY_AESIO = 0
@@ -57,17 +58,20 @@ CIRCUITPY_ROTARYIO = 0
 CIRCUITPY_TOUCHIO ?= 1
 CIRCUITPY_TOUCHIO_USE_NATIVE = 0
 CIRCUITPY_USB = 0
+CIRCUITPY_BUILD_EXTENSIONS ?= bin
 
 else ifeq ($(IDF_TARGET),esp32s3)
 CIRCUITPY_BLEIO = 1
 CIRCUITPY_BLEIO_HCI = 0
 CIRCUITPY_IMAGECAPTURE = 0
 CIRCUITPY_PARALLELDISPLAY = 0
+CIRCUITPY_BUILD_EXTENSIONS ?= bin,uf2
 
 else ifeq ($(IDF_TARGET),esp32s2)
 # No BLE on S2
 CIRCUITPY_BLEIO = 0
 CIRCUITPY_BLEIO_HCI = 0
+CIRCUITPY_BUILD_EXTENSIONS ?= bin,uf2
 endif
 
 # From ESP32-S2/S3 Technical Reference Manual:

ports/litex/mpconfigport.mk

@@ -29,3 +29,5 @@ CIRCUITPY_SDCARDIO = 0
 # Enable USB support
 CIRCUITPY_USB_HID = 1
 CIRCUITPY_USB_MIDI = 1
+
+CIRCUITPY_BUILD_EXTENSIONS ?= dfu

ports/mimxrt10xx/mpconfigport.mk

@@ -21,3 +21,5 @@ CIRCUITPY_PULSEIO = 0
 CIRCUITPY_ROTARYIO = 0
 CIRCUITPY_USB_MIDI = 1
 LONGINT_IMPL = MPZ
+
+CIRCUITPY_BUILD_EXTENSIONS ?= hex,uf2

ports/nrf/boards/electronut_labs_blip/mpconfigboard.mk

@@ -5,6 +5,8 @@ USB_MANUFACTURER = "Electronut Labs"
 
 MCU_CHIP = nrf52840
 
+CIRCUITPY_BUILD_EXTENSIONS = hex
+
 INTERNAL_FLASH_FILESYSTEM = 1
 CIRCUITPY_AUDIOIO = 0
 CIRCUITPY_DISPLAYIO = 1

ports/nrf/boards/makerdiary_nrf52840_mdk/mpconfigboard.mk

@@ -5,5 +5,7 @@ USB_MANUFACTURER = "makerdiary"
 
 MCU_CHIP = nrf52840
 
+CIRCUITPY_BUILD_EXTENSIONS = hex
+
 QSPI_FLASH_FILESYSTEM = 1
 EXTERNAL_FLASH_DEVICES = "MX25R6435F"

ports/nrf/boards/makerdiary_nrf52840_mdk_usb_dongle/mpconfigboard.mk

@@ -5,4 +5,6 @@ USB_MANUFACTURER = "makerdiary"
 
 MCU_CHIP = nrf52840
 
+CIRCUITPY_BUILD_EXTENSIONS = hex,uf2
+
 INTERNAL_FLASH_FILESYSTEM = 1

ports/nrf/boards/microbit_v2/mpconfigboard.mk

@@ -3,6 +3,8 @@ CIRCUITPY_CREATION_ID = 0x80D8
 
 MCU_CHIP = nrf52833
 
+CIRCUITPY_BUILD_EXTENSIONS = combined.hex
+
 INTERNAL_FLASH_FILESYSTEM = 1
 
 # USB pins aren't used.