Enable boot-loader builds

To enable a boot-loader style application, override
"targets.bootloader_exec" or "targets.restrict_size" on a particular
target. These parameters are a bin or hex file, and an integer, in bytes,
respectively. If either override is present, then an application region
is created after the boot-loader region, when "targets.bootloader_exec"
is present, and before post-application, when "targets.restric_size" is
present. The size of the boot-loader region is read from the file
provided in the configuration.

Author: theotherjimmy

tools/build_api.py

@@ -19,12 +19,13 @@
 import tempfile
 from types import ListType
 from shutil import rmtree
-from os.path import join, exists, dirname, basename, abspath, normpath
-from os import linesep, remove
+from os.path import join, exists, dirname, basename, abspath, normpath, splitext
+from os import linesep, remove, makedirs
 from time import time
+from intelhex import IntelHex
 
 from tools.utils import mkdir, run_cmd, run_cmd_ext, NotSupportedException,\
-    ToolException, InvalidReleaseTargetException
+    ToolException, InvalidReleaseTargetException, intelhex_offset
 from tools.paths import MBED_CMSIS_PATH, MBED_TARGETS_PATH, MBED_LIBRARIES,\
     MBED_HEADER, MBED_DRIVERS, MBED_PLATFORM, MBED_HAL, MBED_CONFIG_FILE,\
     MBED_LIBRARIES_DRIVERS, MBED_LIBRARIES_PLATFORM, MBED_LIBRARIES_HAL,\
@@ -274,6 +275,29 @@ def get_mbed_official_release(version):
 
     return mbed_official_release
 
+def add_regions_to_profile(profile, config, toolchain_class):
+    """Add regions to the build profile, if there are any.
+
+    Positional Arguments:
+    profile - the profile to update
+    config - the configuration object that owns the region
+    toolchain_class - the class of the toolchain being used
+    """
+    regions = list(config.regions)
+    for region in regions:
+        for define in [(region.name.upper() + "_ADDR", region.start),
+                       (region.name.upper() + "_SIZE", region.size)]:
+            profile["common"].append("-D%s=0x%x" %  define)
+    active_region = [r for r in regions if r.active][0]
+    for define in [("MBED_APP_START", active_region.start),
+                   ("MBED_APP_SIZE", active_region.size)]:
+        profile["ld"].append(toolchain_class.make_ld_define(*define))
+
+    print("Using regions in this build:")
+    for region in regions:
+        print("  Region %s size 0x%x, offset 0x%x"
+              % (region.name, region.size, region.start))
+
 
 def prepare_toolchain(src_paths, target, toolchain_name,
                       macros=None, clean=False, jobs=1,
@@ -307,14 +331,16 @@ def prepare_toolchain(src_paths, target, toolchain_name,
     # If the configuration object was not yet created, create it now
     config = config or Config(target, src_paths, app_config=app_config)
     target = config.target
-
-    # Toolchain instance
     try:
-        toolchain = TOOLCHAIN_CLASSES[toolchain_name](
-            target, notify, macros, silent,
-            extra_verbose=extra_verbose, build_profile=build_profile)
+        cur_tc = TOOLCHAIN_CLASSES[toolchain_name]
     except KeyError:
         raise KeyError("Toolchain %s not supported" % toolchain_name)
+    if config.has_regions:
+        add_regions_to_profile(build_profile, config, cur_tc)
+
+    # Toolchain instance
+    toolchain = cur_tc(target, notify, macros, silent,
+                       extra_verbose=extra_verbose, build_profile=build_profile)
 
     toolchain.config = config
     toolchain.jobs = jobs
@@ -323,6 +349,41 @@ def prepare_toolchain(src_paths, target, toolchain_name,
 
     return toolchain
 
+def merge_region_list(region_list, destination, padding=b'\xFF'):
+    """Merege the region_list into a single image
+
+    Positional Arguments:
+    region_list - list of regions, which should contain filenames
+    destination - file name to write all regions to
+    padding - bytes to fill gapps with
+    """
+    merged = IntelHex()
+
+    print("Merging Regions:")
+
+    for region in region_list:
+        if region.active and not region.filename:
+            raise ToolException("Active region has no contents: No file found.")
+        if region.filename:
+            print("  Filling region %s with %s" % (region.name, region.filename))
+            part = intelhex_offset(region.filename, offset=region.start)
+            part_size = (part.maxaddr() - part.minaddr()) + 1
+            if part_size > region.size:
+                raise ToolException("Contents of region %s does not fit"
+                                    % region.name)
+            merged.merge(part)
+            pad_size = region.size - part_size
+            if pad_size > 0 and region != region_list[-1]:
+                print("  Padding region %s with 0x%x bytes" % (region.name, pad_size))
+                merged.puts(merged.maxaddr() + 1, padding * pad_size)
+
+    if not exists(dirname(destination)):
+        makedirs(dirname(destination))
+    print("Space used after regions merged: 0x%x" %
+          (merged.maxaddr() - merged.minaddr() + 1))
+    with open(destination, "wb+") as output:
+        merged.tofile(output, format='bin')
+
 def scan_resources(src_paths, toolchain, dependencies_paths=None,
                    inc_dirs=None, base_path=None):
     """ Scan resources using initialized toolcain
@@ -453,7 +514,15 @@ def build_project(src_paths, build_path, target, toolchain_name,
         resources.objects.extend(objects)
 
         # Link Program
-        res, _ = toolchain.link_program(resources, build_path, name)
+        if toolchain.config.has_regions:
+            res, _ = toolchain.link_program(resources, build_path, name + "_application")
+            region_list = list(toolchain.config.regions)
+            region_list = [r._replace(filename=res) if r.active else r
+                           for r in region_list]
+            res = join(build_path, name) + ".bin"
+            merge_region_list(region_list, res)
+        else:
+            res, _ = toolchain.link_program(resources, build_path, name)
 
         memap_instance = getattr(toolchain, 'memap_instance', None)
         memap_table = ''

tools/config.py

@@ -18,8 +18,12 @@
 from copy import deepcopy
 import os
 import sys
+from collections import namedtuple
+from os.path import splitext
+from intelhex import IntelHex
 # Implementation of mbed configuration mechanism
-from tools.utils import json_file_to_dict
+from tools.utils import json_file_to_dict, intelhex_offset
+from tools.arm_pack_manager import Cache
 from tools.targets import CUMULATIVE_ATTRIBUTES, TARGET_MAP, \
     generate_py_target, get_resolution_order
 
@@ -328,6 +332,8 @@ def _process_macros(mlist, macros, unit_name, unit_kind):
         macros[macro.macro_name] = macro
 
 
+Region = namedtuple("Region", "name start size active filename")
+
 class Config(object):
     """'Config' implements the mbed configuration mechanism"""
 
@@ -346,6 +352,8 @@ class Config(object):
                             "macros", "__config_path"])
     }
 
+    __unused_overrides = set(["target.bootloader_img", "target.restrict_size"])
+
     # Allowed features in configurations
     __allowed_features = [
         "UVISOR", "BLE", "CLIENT", "IPV4", "LWIP", "COMMON_PAL", "STORAGE", "NANOSTACK",
@@ -455,6 +463,52 @@ def add_config_files(self, flist):
                        self.lib_config_data[cfg["name"]]["__config_path"]))
             self.lib_config_data[cfg["name"]] = cfg
 
+    @property
+    def has_regions(self):
+        """Does this config have regions defined?"""
+        if 'target_overrides' in self.app_config_data:
+            target_overrides = self.app_config_data['target_overrides'].get(
+                self.target.name, {})
+            return ('target.bootloader_img' in target_overrides or
+                    'target.restrict_size' in target_overrides)
+        else:
+            return False
+
+    @property
+    def regions(self):
+        """Generate a list of regions from the config"""
+        cmsis_part = Cache(False, False).index[self.target.device_name]
+        start = 0
+        target_overrides = self.app_config_data['target_overrides'].get(
+            self.target.name, {})
+        try:
+            rom_size = int(cmsis_part['memory']['IROM1']['size'], 0)
+            rom_start = int(cmsis_part['memory']['IROM1']['start'], 0)
+        except KeyError:
+            raise ConfigException("Not enough information in CMSIS packs to "
+                                  "build a bootloader project")
+        if 'target.bootloader_img' in target_overrides:
+            filename = target_overrides['target.bootloader_img']
+            part = intelhex_offset(filename, offset=rom_start)
+            if part.minaddr() != rom_start:
+                raise ConfigException("bootloader executable does not "
+                                      "start at 0x%x" % rom_start)
+            part_size = (part.maxaddr() - part.minaddr()) + 1
+            yield Region("bootloader", rom_start + start, part_size, False,
+                         filename)
+            start += part_size
+        if 'target.restrict_size' in target_overrides:
+            new_size = int(target_overrides['target.restrict_size'], 0)
+            yield Region("application", rom_start + start, new_size, True, None)
+            start += new_size
+            yield Region("post_application", rom_start +start, rom_size - start,
+                         False, None)
+        else:
+            yield Region("application", rom_start + start, rom_size - start,
+                         True, None)
+        if start > rom_size:
+            raise ConfigException("Not enough memory on device to fit all "
+                                  "application regions")
 
     def _process_config_and_overrides(self, data, params, unit_name, unit_kind):
         """Process "config_parameters" and "target_config_overrides" into a
@@ -508,6 +562,8 @@ def _process_config_and_overrides(self, data, params, unit_name, unit_kind):
                     if full_name in params:
                         params[full_name].set_value(val, unit_name, unit_kind,
                                                     label)
+                    elif name in self.__unused_overrides:
+                        pass
                     else:
                         self.config_errors.append(
                             ConfigException(
@@ -560,6 +616,8 @@ def get_target_config_data(self):
                 rel_names = [tgt for tgt, _ in
                              get_resolution_order(self.target.json_data, tname,
                                                   [])]
+                if full_name in self.__unused_overrides:
+                    continue
                 if (full_name not in params) or \
                    (params[full_name].defined_by[7:] not in rel_names):
                     raise ConfigException(

tools/test/build_api/build_api_test.py

@@ -68,6 +68,7 @@ def test_always_complete_build(self, *_):
             toolchain.config_file = "junk"
             toolchain.compile_sources(res, self.build_path)
 
+            print notify.mock_calls
             assert any('percent' in msg[0] and msg[0]['percent'] == 100.0
                        for _, msg, _ in notify.mock_calls if msg)
 
@@ -81,10 +82,13 @@ def test_prepare_toolchain_app_config(self, mock_config_init):
         :return:
         """
         app_config = "app_config"
-        mock_config_init.return_value = namedtuple("Config", "target")(
-            namedtuple("Target",
-                       "init_hooks name features core")(lambda _, __ : None,
-                                                        "Junk", [], "Cortex-M3"))
+        mock_target = namedtuple("Target",
+                                 "init_hooks name features core")(lambda _, __ : None,
+                                                                  "Junk", [], "Cortex-M3")
+        mock_config_init.return_value = namedtuple("Config",
+                                                   "target has_regions")(
+                                                       mock_target,
+                                                       False)
 
         prepare_toolchain(self.src_paths, self.target, self.toolchain_name,
                           app_config=app_config)
@@ -100,10 +104,13 @@ def test_prepare_toolchain_no_app_config(self, mock_config_init):
         :param mock_config_init: mock of Config __init__
         :return:
         """
-        mock_config_init.return_value = namedtuple("Config", "target")(
-            namedtuple("Target",
-                       "init_hooks name features core")(lambda _, __ : None,
-                                                        "Junk", [], "Cortex-M3"))
+        mock_target = namedtuple("Target",
+                                 "init_hooks name features core")(lambda _, __ : None,
+                                                                  "Junk", [], "Cortex-M3")
+        mock_config_init.return_value = namedtuple("Config",
+                                                   "target has_regions")(
+                                                       mock_target,
+                                                       False)
 
         prepare_toolchain(self.src_paths, self.target, self.toolchain_name)
 
@@ -127,6 +134,7 @@ def test_build_project_app_config(self, mock_prepare_toolchain, mock_exists, _,
         app_config = "app_config"
         mock_exists.return_value = False
         mock_prepare_toolchain().link_program.return_value = 1, 2
+        mock_prepare_toolchain().config = namedtuple("Config", "has_regions")(None)
 
         build_project(self.src_paths, self.build_path, self.target,
                       self.toolchain_name, app_config=app_config)
@@ -154,6 +162,7 @@ def test_build_project_no_app_config(self, mock_prepare_toolchain, mock_exists,
         mock_exists.return_value = False
         # Needed for the unpacking of the returned value
         mock_prepare_toolchain().link_program.return_value = 1, 2
+        mock_prepare_toolchain().config = namedtuple("Config", "has_regions")(None)
 
         build_project(self.src_paths, self.build_path, self.target,
                       self.toolchain_name)

tools/utils.py

@@ -28,6 +28,7 @@
 import json
 from collections import OrderedDict
 import logging
+from intelhex import IntelHex
 
 def remove_if_in(lst, thing):
     if thing in lst:
@@ -514,3 +515,16 @@ def print_large_string(large_string):
         else:
             end_index = ((string_part + 1) * string_limit) - 1
             print large_string[start_index:end_index],
+
+def intelhex_offset(filename, offset):
+    """Load a hex or bin file at a particular offset"""
+    _, inteltype = splitext(filename)
+    ih = IntelHex()
+    if inteltype == ".bin":
+        ih.loadbin(filename, offset=offset)
+    elif inteltype == ".hex":
+        ih.loadhex(filename)
+    else:
+        raise ToolException("File %s does not have a known binary file type"
+                            % filename)
+    return ih