Merge pull request #9561 from theotherjimmy/test-resources

Tools changes for bare metal

Author: Cruz Monrreal

features/frameworks/greentea-client/mbed_lib.json

@@ -0,0 +1,3 @@
+{
+    "name": "greentea-client"
+}

features/frameworks/mbed-client-cli/mbed_lib.json

@@ -0,0 +1,3 @@
+{
+    "name": "mbed-client-cli"
+}

features/frameworks/mbed-client-randlib/mbed_lib.json

@@ -0,0 +1,3 @@
+{
+    "name": "mbed-client-randlib"
+}

features/frameworks/mbed-coap/mbed_lib.json

@@ -0,0 +1,3 @@
+{
+    "name": "mbed-coap"
+}

features/frameworks/nanostack-libservice/mbed_lib.json

@@ -0,0 +1,3 @@
+{
+    "name": "nanostack-libservice"
+}

features/frameworks/unity/mbed_lib.json

@@ -0,0 +1,3 @@
+{
+    "name": "unity"
+}

features/mbedtls/mbed_lib.json

@@ -0,0 +1,3 @@
+{
+    "name": "mbedtls"
+}

features/nanostack/coap-service/mbed_lib.json

@@ -0,0 +1,3 @@
+{
+    "name": "coap-service"
+}

features/nanostack/mbed-mesh-api/mbed_lib.json

@@ -1,5 +1,6 @@
 {
     "name": "mbed-mesh-api",
+    "requires": ["nanostack"],
     "config": {
         "heap-size": {
             "help": "Nanostack's heap size [bytes: 0-65534]",

features/nanostack/nanostack-interface/mbed_lib.json

@@ -0,0 +1,4 @@
+{
+    "name": "nanostack-interface",
+    "requires": ["nanostack"]
+}

features/nanostack/sal-stack-nanostack-eventloop/mbed_lib.json

@@ -1,5 +1,6 @@
 {
     "name": "nanostack-eventloop",
+    "requires": ["nanostack-hal"],
     "config": {
         "use_platform_tick_timer": {
             "help": "Use platform provided low resolution tick timer for eventloop",

features/nanostack/sal-stack-nanostack/mbed_lib.json

@@ -1,5 +1,6 @@
 {
     "name": "nanostack",
+    "requires": ["nanostack-eventloop", "coap-service"],
     "config": {
         "configuration": {
             "help": "Build time configuration. Refer to Handbook for valid values. Default: full stack",

features/nfc/mbed_lib.json

@@ -0,0 +1,3 @@
+{
+    "name": "nfc"
+}

features/storage/system_storage/mbed_lib.json

@@ -0,0 +1,3 @@
+{
+    "name": "system-storage"
+}

tools/build_api.py

@@ -156,7 +156,7 @@ def get_config(src_paths, target, toolchain_name=None, app_config=None):
 
     cfg, macros = config.get_config_data()
     features = config.get_features()
-    return cfg, macros, features
+    return cfg, macros, features, res
 
 def is_official_target(target_name, version):
     """ Returns True, None if a target is part of the official release for the

tools/config/__init__.py

@@ -545,7 +545,6 @@ def add_config_files(self, flist):
             # Check that we didn't already process this file
             if full_path in self.processed_configs:
                 continue
-            self.processed_configs[full_path] = True
             # Read the library configuration and add a "__full_config_path"
             # attribute to it
             try:
@@ -570,6 +569,12 @@ def add_config_files(self, flist):
                 raise ConfigException("; ".join(
                     self.format_validation_error(x, config_file)
                     for x in errors))
+            if "requires" in self.app_config_data:
+                if cfg["name"] not in self.app_config_data["requires"]:
+                    continue
+                self.app_config_data["requires"].extend(cfg.get("requires", []))
+
+            self.processed_configs[full_path] = True
 
             cfg["__config_path"] = full_path
 
@@ -1279,6 +1284,7 @@ def load_resources(self, resources):
         """
         # Update configuration files until added features creates no changes
         prev_features = set()
+        prev_requires = set()
         while True:
             # Add/update the configuration with any .json files found while
             # scanning
@@ -1288,14 +1294,37 @@ def load_resources(self, resources):
 
             # Add features while we find new ones
             features = set(self.get_features())
-            if features == prev_features:
+            requires = set(self.app_config_data.get("requires", []))
+            if features == prev_features and requires == prev_requires:
                 break
 
             resources.add_features(features)
 
             prev_features = features
+            prev_requires = requires
         self.validate_config()
-
+        missing_requirements = {}
+        for name, lib in self.lib_config_data.items():
+            for req in lib.get("requires", []):
+                if req not in self.lib_config_data:
+                    missing_requirements.setdefault(name, [])
+                    missing_requirements[name].append(req)
+        if missing_requirements:
+            message = "; ".join(
+                "library '{}' requires {} which is not present".format(
+                    name, ", ".join("'{}'".format(i) for i in missing)
+                )
+                for name, missing in missing_requirements.items()
+            )
+            raise ConfigException(message)
+        all_json_paths = [
+            cfg["__config_path"] for cfg in self.lib_config_data.values()
+        ]
+        included_json_files = [
+            ref for ref in resources.get_file_refs(FileType.JSON)
+            if abspath(ref.path) in all_json_paths
+        ]
+        resources.filter_by_libraries(included_json_files)
         if  (hasattr(self.target, "release_versions") and
              "5" not in self.target.release_versions and
              "rtos" in self.lib_config_data):

tools/config/definitions.json

@@ -6,6 +6,13 @@
       "type": "string"
     }
   },
+  "requires_definition": {
+    "description": "Required libraries",
+    "type": "array",
+    "items": {
+      "$ref": "#/name_definition"
+    }
+  },
   "macro_definition": {
     "description": "A list of extra macros that will be defined when compiling a project that includes this library.",
     "type": "array",

tools/config/schema_app.json

@@ -16,6 +16,9 @@
     "macros": {
       "$ref": "definitions.json#/macro_definition"
     },
+    "requires": {
+      "$ref": "definitions.json#/requires_definition"
+    },
     "artifact_name": {
       "type": "string"
     }

tools/config/schema_lib.json

@@ -13,6 +13,9 @@
     "target_overrides": {
       "$ref": "definitions.json#/target_overrides_definition"
     },
+    "requires": {
+      "$ref": "definitions.json#/requires_definition"
+    },
     "macros": {
       "$ref": "definitions.json#/macro_definition"
     }

tools/get_config.py

@@ -56,7 +56,7 @@
     options.prefix = options.prefix or [""]
 
     try:
-        params, macros, features = get_config(
+        params, macros, features, _ = get_config(
             options.source_dir,
             target,
             options.tool[0] if options.tool else None,

tools/resources/__init__.py

@@ -80,9 +80,17 @@
     'ARMC6': 'ARMC6',
 }
 
+MBED_LIB_FILENAME = 'mbed_lib.json'
+MBED_APP_FILENAME = 'mbed_app.json'
+CONFIG_FILES = set([
+    MBED_LIB_FILENAME,
+    MBED_APP_FILENAME
+])
+
 
 FileRef = namedtuple("FileRef", "name path")
 
+
 class FileType(object):
     C_SRC = "c"
     CPP_SRC = "c++"
@@ -126,6 +134,9 @@ def __init__(self, notify, collect_ignores=False):
         # publicly accessible things
         self.ignored_dirs = []
 
+        # library requirements
+        self._libs_filtered = None
+
         # Pre-mbed 2.0 ignore dirs
         self._legacy_ignore_dirs = (LEGACY_IGNORE_DIRS)
 
@@ -260,9 +271,47 @@ def add_file_ref(self, file_type, file_name, file_path):
                 file_name = file_name.replace(sep, self._sep)
             self._file_refs[file_type].add(FileRef(file_name, file_path))
 
+    def _include_file(self, ref):
+        """Determine if a given file ref should be included in the build
+
+        Files may be part of a library if a parent directory contains an
+        mbed_lib.json. If a file is part of a library, include or exclude
+        it based on the library it's part of.
+        If a file is not part of a library, it's included.
+        """
+        _, path = ref
+        cur_dir = dirname(path)
+        included_lib_paths = [dirname(e.path) for e in self._libs_filtered]
+        excluded_lib_paths = [dirname(e.path) for e in self._excluded_libs]
+        while dirname(cur_dir) != cur_dir:
+            if cur_dir in included_lib_paths:
+                return True
+            elif cur_dir in excluded_lib_paths:
+                return False
+            cur_dir = dirname(cur_dir)
+        return True
+
     def get_file_refs(self, file_type):
         """Return a list of FileRef for every file of the given type"""
-        return list(self._file_refs[file_type])
+        if self._libs_filtered is None:
+            return list(self._file_refs[file_type])
+        else:
+            return [
+                ref for ref in self._file_refs[file_type]
+                if self._include_file(ref)
+            ]
+
+    def filter_by_libraries(self, libraries_included):
+        """
+        Call after completely done scanning to filter resources based on
+        libraries
+        """
+        self._libs_filtered = set(libraries_included)
+        all_library_refs = set(
+            ref for ref in self._file_refs[FileType.JSON]
+            if ref.name.endswith(MBED_LIB_FILENAME)
+        )
+        self._excluded_libs = all_library_refs - self._libs_filtered
 
     def _get_from_refs(self, file_type, key):
         return sorted([key(f) for f in self.get_file_refs(file_type)])

tools/test/config/config_test.py

@@ -26,6 +26,14 @@
 from tools.build_api import get_config
 from tools.targets import set_targets_json_location, Target, TARGET_NAMES
 from tools.config import ConfigException, Config, ConfigParameter, ConfigMacro
+from tools.resources import Resources
+
+NOT_CONFIG = [
+    "expected_macros",
+    "expected_features",
+    "included_source",
+    "excluded_source",
+]
 
 def compare_config(cfg, expected):
     """Compare the output of config against a dictionary of known good results
@@ -40,7 +48,7 @@ def compare_config(cfg, expected):
     except KeyError:
         return "Unexpected key '%s' in configuration data" % k
     for k in expected:
-        if k not in ["expected_macros", "expected_features"] + list(cfg.keys()):
+        if k not in NOT_CONFIG + list(cfg.keys()):
             return "Expected key '%s' was not found in configuration data" % k
     return ""
 
@@ -73,7 +81,7 @@ def test_config(name):
     set_targets_json_location(targets_json if isfile(targets_json) else None)
     for target, expected in test_data.items():
         try:
-            cfg, macros, features = get_config(test_dir, target, "GCC_ARM")
+            cfg, macros, features, resources = get_config(test_dir, target, "GCC_ARM")
             res = compare_config(cfg, expected)
             assert not(res), res
             expected_macros = expected.get("expected_macros", None)
@@ -84,6 +92,24 @@ def test_config(name):
                 assert sorted(expected_macros) == sorted(macros)
             if expected_features is not None:
                 assert sorted(expected_features) == sorted(features)
+
+            included_source = [
+                join(test_dir, src) for src in
+                expected.get("included_source", [])
+            ]
+            excluded_source = [
+                join(test_dir, src) for src in
+                expected.get("excluded_source", [])
+            ]
+            for typ in Resources.ALL_FILE_TYPES:
+                for _, path in resources.get_file_refs(typ):
+                    if included_source and path in included_source:
+                        included_source.remove(path)
+                    if excluded_source:
+                        assert(path not in excluded_source)
+            assert(not included_source)
+            if included_source:
+                assert(False)
         except ConfigException as e:
             err_msg = str(e)
             if "exception_msg" not in expected:

tools/test/config/feature_recursive_add/FEATURE_BOOTLOADER/lib1/lib1.c

@@ -1,5 +1,6 @@
 {
     "test_target": {
-        "expected_features": ["BOOTLOADER", "STORAGE"]
+        "expected_features": ["BOOTLOADER", "STORAGE"],
+        "included_source": ["FEATURE_BOOTLOADER/lib1/lib1.c", "FEATURE_STORAGE/lib2/lib2.c"]
     }
 }

tools/test/config/feature_recursive_add/FEATURE_STORAGE/lib2/lib2.c

@@ -0,0 +1,7 @@
+{
+    "name": "lib1",
+    "requires": ["lib2"],
+    "config": {
+        "test": "BAD"
+    }
+}

tools/test/config/feature_recursive_add/test_data.json

@@ -0,0 +1,9 @@
+{
+    "name": "lib2",
+    "requires": ["lib3"],
+    "config": {
+        "test": {
+            "value": "BAD"
+        }
+    }
+}

tools/test/config/lib_requires_missing_lib/lib1/mbed_lib.json

@@ -0,0 +1,9 @@
+{
+    "requires" : ["lib1"],
+    "target_overrides": {
+        "test_target": {
+            "lib2.test": "GOOD",
+            "lib1.test": "GOOD"
+        }
+    }
+}

tools/test/config/lib_requires_missing_lib/lib2/mbed_lib.json

@@ -0,0 +1,9 @@
+{
+    "test_target": {
+        "supported_toolchains": ["GCC_ARM"],
+        "core": "Cortex-M0",
+        "extra_labels": [],
+        "features": [],
+        "default_lib": "std"
+    }
+}

tools/test/config/lib_requires_missing_lib/mbed_app.json

@@ -0,0 +1,5 @@
+{
+    "test_target": {
+    	"exception_msg": "'lib2' requires 'lib3' which is not present"
+    }
+}

tools/test/config/lib_requires_missing_lib/targets.json

@@ -0,0 +1,8 @@
+{
+    "name": "lib2",
+    "config": {
+        "test": {
+            "value": "BAD"
+        }
+    }
+}

tools/test/config/lib_requires_missing_lib/test_data.json

@@ -0,0 +1,6 @@
+{
+    "name": "lib1",
+    "config": {
+        "test": "BAD"
+    }
+}