Merge pull request #4103 from andrewleech/extra_targets

Add extra_targets.json support to build tools

Author: theotherjimmy

tools/build.py

@@ -32,6 +32,7 @@
 from tools.targets import TARGET_NAMES, TARGET_MAP
 from tools.options import get_default_options_parser
 from tools.options import extract_profile
+from tools.options import extract_mcus
 from tools.build_api import build_library, build_mbed_libs, build_lib
 from tools.build_api import mcu_toolchain_matrix
 from tools.build_api import print_build_results
@@ -134,7 +135,7 @@
 
 
     # Get target list
-    targets = options.mcu if options.mcu else TARGET_NAMES
+    targets = extract_mcus(parser, options) if options.mcu else TARGET_NAMES
 
     # Get toolchains list
     toolchains = options.tool if options.tool else TOOLCHAINS

tools/get_config.py

@@ -26,6 +26,7 @@
 
 from tools.utils import args_error
 from tools.options import get_default_options_parser
+from tools.options import extract_mcus
 from tools.build_api import get_config
 from config import Config
 from utils import argparse_filestring_type
@@ -49,7 +50,7 @@
     # Target
     if options.mcu is None :
         args_error(parser, "argument -m/--mcu is required")
-    target = options.mcu[0]
+    target = extract_mcus(parser, options)[0]
 
     # Toolchain
     if options.tool is None:

tools/make.py

@@ -42,6 +42,7 @@
 from tools.targets import TARGET_MAP
 from tools.options import get_default_options_parser
 from tools.options import extract_profile
+from tools.options import extract_mcus
 from tools.build_api import build_project
 from tools.build_api import mcu_toolchain_matrix
 from tools.build_api import mcu_toolchain_list
@@ -200,7 +201,7 @@
     # Target
     if options.mcu is None :
         args_error(parser, "argument -m/--mcu is required")
-    mcu = options.mcu[0]
+    mcu = extract_mcus(parser, options)[0]
 
     # Toolchain
     if options.tool is None:

tools/options.py

@@ -17,9 +17,9 @@
 from json import load
 from os.path import join, dirname
 from os import listdir
-from argparse import ArgumentParser
+from argparse import ArgumentParser, ArgumentTypeError
 from tools.toolchains import TOOLCHAINS
-from tools.targets import TARGET_NAMES
+from tools.targets import TARGET_NAMES, Target, update_target_data
 from tools.utils import argparse_force_uppercase_type, \
     argparse_lowercase_hyphen_type, argparse_many, \
     argparse_filestring_type, args_error, argparse_profile_filestring_type,\
@@ -47,10 +47,7 @@ def get_default_options_parser(add_clean=True, add_options=True,
     parser.add_argument("-m", "--mcu",
                         help=("build for the given MCU (%s)" %
                               ', '.join(targetnames)),
-                        metavar="MCU",
-                        type=argparse_many(
-                            argparse_force_uppercase_type(
-                                targetnames, "MCU")))
+                        metavar="MCU")
 
     parser.add_argument("-t", "--tool",
                         help=("build using the given TOOLCHAIN (%s)" %
@@ -130,3 +127,19 @@ def mcu_is_enabled(parser, mcu):
              "See https://developer.mbed.org/platforms/Renesas-GR-PEACH/#important-notice "
              "for more information") % (mcu, mcu))
     return True
+    
+def extract_mcus(parser, options):
+    try:
+        if options.source_dir:
+            for source_dir in options.source_dir:
+                Target.add_extra_targets(source_dir)
+            update_target_data()
+    except KeyError:
+        pass
+    targetnames = TARGET_NAMES
+    targetnames.sort()
+    try:
+        return argparse_many(argparse_force_uppercase_type(targetnames, "MCU"))(options.mcu)
+    except ArgumentTypeError as exc:
+        args_error(parser, "argument -m/--mcu: {}".format(str(exc)))
+

tools/project.py

@@ -20,7 +20,7 @@
 from tools.utils import argparse_force_lowercase_type
 from tools.utils import argparse_force_uppercase_type
 from tools.utils import print_large_string
-from tools.options import extract_profile, list_profiles
+from tools.options import extract_profile, list_profiles, extract_mcus
 
 def setup_project(ide, target, program=None, source_dir=None, build=None, export_path=None):
     """Generate a name, if not provided, and find dependencies
@@ -247,7 +247,8 @@ def main():
     profile = extract_profile(parser, options, toolchain_name, fallback="debug")
     if options.clean:
         rmtree(BUILD_DIR)
-    export(options.mcu, options.ide, build=options.build,
+    mcu = extract_mcus(parser, options)[0]
+    export(mcu, options.ide, build=options.build,
            src=options.source_dir, macros=options.macros,
            project_id=options.program, zip_proj=zip_proj,
            build_profile=profile, app_config=options.app_config)

tools/targets/__init__.py

@@ -22,7 +22,7 @@
 import inspect
 import sys
 from copy import copy
-from collections import namedtuple
+from collections import namedtuple, Mapping
 from tools.targets.LPC import patch
 from tools.paths import TOOLS_BOOTLOADERS
 from tools.utils import json_file_to_dict
@@ -125,18 +125,38 @@ class Target(namedtuple("Target", "name json_data resolution_order resolution_or
     # Current/new location of the 'targets.json' file
     __targets_json_location = None
 
+    # Extra custom targets files
+    __extra_target_json_files = []
+
     @staticmethod
     @cached
     def get_json_target_data():
         """Load the description of JSON target data"""
-        return json_file_to_dict(Target.__targets_json_location or
-                                 Target.__targets_json_location_default)
+        targets = json_file_to_dict(Target.__targets_json_location or
+                                    Target.__targets_json_location_default)
+
+        for extra_target in Target.__extra_target_json_files:
+            for k, v in json_file_to_dict(extra_target).iteritems():
+                if k in targets:
+                    print 'WARNING: Custom target "%s" cannot replace existing target.' % k
+                else:
+                    targets[k] = v
+
+        return targets
+
+    @staticmethod
+    def add_extra_targets(source_dir):
+        extra_targets_file = os.path.join(source_dir, "custom_targets.json")
+        if os.path.exists(extra_targets_file):
+            Target.__extra_target_json_files.append(extra_targets_file)
+            CACHES.clear()
 
     @staticmethod
     def set_targets_json_location(location=None):
         """Set the location of the targets.json file"""
         Target.__targets_json_location = (location or
                                           Target.__targets_json_location_default)
+        Target.__extra_target_json_files = []
         # Invalidate caches, since the location of the JSON file changed
         CACHES.clear()
 
@@ -507,14 +527,20 @@ def binary_hook(t_self, resources, elf, binf):
 ################################################################################
 
 # Instantiate all public targets
-TARGETS = [Target.get_target(name) for name, value
-           in Target.get_json_target_data().items()
-           if value.get("public", True)]
+def update_target_data():
+    TARGETS[:] = [Target.get_target(tgt) for tgt, obj
+                  in Target.get_json_target_data().items()
+                  if obj.get("public", True)]
+    # Map each target name to its unique instance
+    TARGET_MAP.clear()
+    TARGET_MAP.update(dict([(tgt.name, tgt) for tgt in TARGETS]))
+    TARGET_NAMES[:] = TARGET_MAP.keys()
 
-# Map each target name to its unique instance
-TARGET_MAP = dict([(t.name, t) for t in TARGETS])
+TARGETS = []
+TARGET_MAP = dict()
+TARGET_NAMES = []
 
-TARGET_NAMES = TARGET_MAP.keys()
+update_target_data()
 
 # Some targets with different name have the same exporters
 EXPORT_MAP = {}
@@ -537,9 +563,5 @@ def set_targets_json_location(location=None):
     # re-initialization does not create new variables, it keeps the old ones
     # instead. This ensures compatibility with code that does
     # "from tools.targets import TARGET_NAMES"
-    TARGETS[:] = [Target.get_target(tgt) for tgt, obj
-                  in Target.get_json_target_data().items()
-                  if obj.get("public", True)]
-    TARGET_MAP.clear()
-    TARGET_MAP.update(dict([(tgt.name, tgt) for tgt in TARGETS]))
-    TARGET_NAMES[:] = TARGET_MAP.keys()
+    update_target_data()
+

tools/test.py

@@ -28,7 +28,7 @@
 
 from tools.config import ConfigException
 from tools.test_api import test_path_to_name, find_tests, print_tests, build_tests, test_spec_from_test_builds
-from tools.options import get_default_options_parser, extract_profile
+from tools.options import get_default_options_parser, extract_profile, extract_mcus
 from tools.build_api import build_project, build_library
 from tools.build_api import print_build_memory_usage
 from tools.build_api import merge_build_data
@@ -114,7 +114,7 @@
         # Target
         if options.mcu is None :
             args_error(parser, "argument -m/--mcu is required")
-        mcu = options.mcu[0]
+        mcu = extract_mcus(parser, options)[0]
 
         # Toolchain
         if options.tool is None:

tools/test/targets/target_test.py

@@ -15,18 +15,23 @@
  See the License for the specific language governing permissions and
  limitations under the License.
 """
-
+import os
 import sys
+import shutil
+import tempfile
 from os.path import join, abspath, dirname
+from contextlib import contextmanager
 import unittest
 
 # Be sure that the tools directory is in the search path
+
 ROOT = abspath(join(dirname(__file__), "..", "..", ".."))
 sys.path.insert(0, ROOT)
 
-from tools.targets import TARGETS
+from tools.targets import TARGETS, TARGET_MAP, Target, update_target_data
 from tools.arm_pack_manager import Cache
 
+
 class TestTargets(unittest.TestCase):
 
     def test_device_name(self):
@@ -39,5 +44,98 @@ def test_device_name(self):
                             "Target %s contains invalid device_name %s" %
                             (target.name, target.device_name))
 
+    @contextmanager
+    def temp_target_file(self, extra_target, json_filename='custom_targets.json'):
+        """Create an extra targets temp file in a context manager"""
+        tempdir = tempfile.mkdtemp()
+        try:
+            targetfile = os.path.join(tempdir, json_filename)
+            with open(targetfile, 'w') as f:
+                f.write(extra_target)
+            yield tempdir
+        finally:
+            # Reset extra targets
+            Target.set_targets_json_location()
+            # Delete temp files
+            shutil.rmtree(tempdir)
+
+    def test_add_extra_targets(self):
+        """Search for extra targets json in a source folder"""
+        test_target_json = """
+        { 
+            "Test_Target": {
+                "inherits": ["Target"]
+            }
+        }
+        """
+        with self.temp_target_file(test_target_json) as source_dir:
+            Target.add_extra_targets(source_dir=source_dir)
+            update_target_data()
+
+            assert 'Test_Target' in TARGET_MAP
+            assert TARGET_MAP['Test_Target'].core is None, \
+                   "attributes should be inherited from Target"
+
+    def test_modify_existing_target(self):
+        """Set default targets file, then override base Target definition"""
+        initial_target_json = """
+        {
+            "Target": {
+                "core": null,
+                "default_toolchain": "ARM",
+                "supported_toolchains": null,
+                "extra_labels": [],
+                "is_disk_virtual": false,
+                "macros": [],
+                "device_has": [],
+                "features": [],
+                "detect_code": [],
+                "public": false,
+                "default_lib": "std",
+                "bootloader_supported": false
+            },
+            "Test_Target": {
+                "inherits": ["Target"],
+                "core": "Cortex-M4",
+                "supported_toolchains": ["ARM"]
+            }
+        }"""
+
+        test_target_json = """
+        { 
+            "Target": {
+                "core": "Cortex-M0",
+                "default_toolchain": "GCC_ARM",
+                "supported_toolchains": null,
+                "extra_labels": [],
+                "is_disk_virtual": false,
+                "macros": [],
+                "device_has": [],
+                "features": [],
+                "detect_code": [],
+                "public": false,
+                "default_lib": "std",
+                "bootloader_supported": true
+            }
+        }
+        """
+
+        with self.temp_target_file(initial_target_json, json_filename="targets.json") as targets_dir:
+            Target.set_targets_json_location(os.path.join(targets_dir, "targets.json"))
+            update_target_data()
+            assert TARGET_MAP["Test_Target"].core == "Cortex-M4"
+            assert TARGET_MAP["Test_Target"].default_toolchain == 'ARM'
+            assert TARGET_MAP["Test_Target"].bootloader_supported == False
+
+            with self.temp_target_file(test_target_json) as source_dir:
+                Target.add_extra_targets(source_dir=source_dir)
+                update_target_data()
+
+                assert TARGET_MAP["Test_Target"].core == "Cortex-M4"
+                # The existing target should not be modified by custom targets
+                assert TARGET_MAP["Test_Target"].default_toolchain != 'GCC_ARM'
+                assert TARGET_MAP["Test_Target"].bootloader_supported != True
+
+
 if __name__ == '__main__':
     unittest.main()