Update project generator script to allow working with any source

Update test exporters script
Update travis build script
Added config system CLI script
Fixed building of legacy libraries and layouts

Author: screamerbg

tools/build.py

@@ -192,7 +192,7 @@
     if options.usb_host:
         libraries.append("usb_host")
     if options.dsp:
-        libraries.extend(["cmsis_dsp", "dsp"])
+        libraries.extend(["dsp"])
     if options.fat:
         libraries.extend(["fat"])
     if options.ublox:

tools/build_api.py

@@ -394,33 +394,145 @@ def build_library(src_paths, build_path, target, toolchain_name,
         # Let Exception propagate
         raise e
 
-def build_lib(lib_id, target, toolchain, options=None, verbose=False, clean=False, macros=None, notify=None, jobs=1, silent=False, report=None, properties=None, extra_verbose=False):
-    """ Wrapper for build_library function.
+######################
+### Legacy methods ###
+######################
+
+def build_lib(lib_id, target, toolchain_name, options=None, verbose=False, clean=False, macros=None, notify=None, jobs=1, silent=False, report=None, properties=None, extra_verbose=False):
+    """ Legacy method for building mbed libraries
         Function builds library in proper directory using all dependencies and macros defined by user.
     """
     lib = Library(lib_id)
-    if lib.is_supported(target, toolchain):
-        # We need to combine macros from parameter list with macros from library definition
-        MACROS = lib.macros if lib.macros else []
-        if macros:
-            MACROS.extend(macros)
-
-        return build_library(lib.source_dir, lib.build_dir, target, toolchain, lib.dependencies, options,
-                      verbose=verbose,
-                      silent=silent,
-                      clean=clean,
-                      macros=MACROS,
-                      notify=notify,
-                      inc_dirs=lib.inc_dirs,
-                      inc_dirs_ext=lib.inc_dirs_ext,
-                      jobs=jobs,
-                      report=report,
-                      properties=properties,
-                      extra_verbose=extra_verbose)
-    else:
+    if not lib.is_supported(target, toolchain_name):
         print 'Library "%s" is not yet supported on target %s with toolchain %s' % (lib_id, target.name, toolchain)
         return False
+    
+    # We need to combine macros from parameter list with macros from library definition
+    MACROS = lib.macros if lib.macros else []
+    if macros:
+        macros.extend(MACROS)
+    else:
+        macros = MACROS
+
+    src_paths = lib.source_dir
+    build_path = lib.build_dir
+    dependencies_paths = lib.dependencies
+    inc_dirs = lib.inc_dirs
+    inc_dirs_ext = lib.inc_dirs_ext
+    
+    """ src_path: the path of the source directory
+    build_path: the path of the build directory
+    target: ['LPC1768', 'LPC11U24', 'LPC2368']
+    toolchain: ['ARM', 'uARM', 'GCC_ARM', 'GCC_CR']
+    library_paths: List of paths to additional libraries
+    clean: Rebuild everything if True
+    notify: Notify function for logs
+    verbose: Write the actual tools command lines if True
+    inc_dirs: additional include directories which should be included in build
+    inc_dirs_ext: additional include directories which should be copied to library directory
+    """
+    if type(src_paths) != ListType:
+        src_paths = [src_paths]
+
+    # The first path will give the name to the library
+    name = basename(src_paths[0])
+
+    if report != None:
+        start = time()
+        id_name = name.upper()
+        description = name
+        vendor_label = target.extra_labels[0]
+        cur_result = None
+        prep_report(report, target.name, toolchain_name, id_name)
+        cur_result = create_result(target.name, toolchain_name, id_name, description)
 
+        if properties != None:
+            prep_properties(properties, target.name, toolchain_name, vendor_label)
+
+    for src_path in src_paths:
+        if not exists(src_path):
+            error_msg = "The library source folder does not exist: %s", src_path
+
+            if report != None:
+                cur_result["output"] = error_msg
+                cur_result["result"] = "FAIL"
+                add_result_to_report(report, cur_result)
+
+            raise Exception(error_msg)
+
+    try:
+        # Toolchain instance
+        toolchain = TOOLCHAIN_CLASSES[toolchain_name](target, options, macros=macros, notify=notify, silent=silent, extra_verbose=extra_verbose)
+        toolchain.VERBOSE = verbose
+        toolchain.jobs = jobs
+        toolchain.build_all = clean
+
+        toolchain.info("Building library %s (%s, %s)" % (name.upper(), target.name, toolchain_name))
+
+        # Scan Resources
+        resources = []
+        for src_path in src_paths:
+            resources.append(toolchain.scan_resources(src_path))
+
+        # Add extra include directories / files which are required by library
+        # This files usually are not in the same directory as source files so
+        # previous scan will not include them
+        if inc_dirs_ext is not None:
+            for inc_ext in inc_dirs_ext:
+                resources.append(toolchain.scan_resources(inc_ext))
+
+        # Dependencies Include Paths
+        dependencies_include_dir = []
+        if dependencies_paths is not None:
+            for path in dependencies_paths:
+                lib_resources = toolchain.scan_resources(path)
+                dependencies_include_dir.extend(lib_resources.inc_dirs)
+
+        if inc_dirs:
+            dependencies_include_dir.extend(inc_dirs)
+
+        # Create the desired build directory structure
+        bin_path = join(build_path, toolchain.obj_path)
+        mkdir(bin_path)
+        tmp_path = join(build_path, '.temp', toolchain.obj_path)
+        mkdir(tmp_path)
+
+        # Copy Headers
+        for resource in resources:
+            toolchain.copy_files(resource.headers, build_path, rel_path=resource.base_path)
+        dependencies_include_dir.extend(toolchain.scan_resources(build_path).inc_dirs)
+
+        # Compile Sources
+        objects = []
+        for resource in resources:
+            objects.extend(toolchain.compile_sources(resource, tmp_path, dependencies_include_dir))
+
+        needed_update = toolchain.build_library(objects, bin_path, name)
+
+        if report != None and needed_update:
+            end = time()
+            cur_result["elapsed_time"] = end - start
+            cur_result["output"] = toolchain.get_output()
+            cur_result["result"] = "OK"
+
+            add_result_to_report(report, cur_result)
+
+    except Exception, e:
+        if report != None:
+            end = time()
+            cur_result["result"] = "FAIL"
+            cur_result["elapsed_time"] = end - start
+
+            toolchain_output = toolchain.get_output()
+            if toolchain_output:
+                cur_result["output"] += toolchain_output
+
+            cur_result["output"] += str(e)
+
+            add_result_to_report(report, cur_result)
+
+        # Let Exception propagate
+        raise e
 
 # We do have unique legacy conventions about how we build and package the mbed library
 def build_mbed_libs(target, toolchain_name, options=None, verbose=False, clean=False, macros=None, notify=None, jobs=1, silent=False, report=None, properties=None, extra_verbose=False):
@@ -543,6 +655,7 @@ def build_mbed_libs(target, toolchain_name, options=None, verbose=False, clean=F
         # Let Exception propagate
         raise e
 
+
 def get_unique_supported_toolchains():
     """ Get list of all unique toolchains supported by targets """
     unique_supported_toolchains = []

tools/get_config.py

@@ -0,0 +1,94 @@
+#! /usr/bin/env python2
+"""
+mbed SDK
+Copyright (c) 2011-2013 ARM Limited
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+import sys
+from os.path import isdir, abspath, dirname, join
+from os import _exit
+
+# Be sure that the tools directory is in the search path
+ROOT = abspath(join(dirname(__file__), ".."))
+sys.path.insert(0, ROOT)
+
+from tools.utils import args_error
+from tools.options import get_default_options_parser
+from tools.build_api import get_config
+from config import Config
+try:
+    import tools.private_settings as ps
+except:
+    ps = object()
+
+if __name__ == '__main__':
+    # Parse Options
+    parser = get_default_options_parser(add_clean=False, add_options=False)
+    parser.add_option("--source", dest="source_dir",
+                      default=None, help="The source (input) directory", action="append")
+    parser.add_option("--prefix", dest="prefix", action="append",
+                      default=None, help="Restrict listing to parameters that have this prefix")
+    parser.add_option("-v", "--verbose", action="store_true", dest="verbose",
+                      default=False, help="Verbose diagnostic output")
+
+    (options, args) = parser.parse_args()
+
+    for path in options.source_dir :
+        if not isdir(path) :
+            args_error(parser, "[ERROR] you passed \"{}\" to --source, which does not exist".
+                       format(path))
+    # Target
+    if options.mcu is None :
+        args_error(parser, "[ERROR] You should specify an MCU")
+    target = options.mcu
+
+    # Toolchain
+    if options.tool is None:
+        args_error(parser, "[ERROR] You should specify a TOOLCHAIN")
+    toolchain = options.tool
+
+    options.prefix = options.prefix or [""]
+
+    try:
+        params, macros = get_config(options.source_dir, target, toolchain)
+        if not params and not macros:
+            print "No configuration data available."
+            _exit(0)
+        if params:
+            print "Configuration parameters"
+            print "------------------------"
+            for p in params:
+                for s in options.prefix:
+                    if p.startswith(s):
+                        print(str(params[p]) if not options.verbose else params[p].get_verbose_description())
+                        break
+            print ""
+
+        print "Macros"
+        print "------"
+        if macros:
+            print 'Defined with "macros":', macros
+        print "Generated from configuration parameters:", Config.parameters_to_macros(params)
+
+    except KeyboardInterrupt, e:
+        print "\n[CTRL+c] exit"
+    except Exception,e:
+        if options.verbose:
+            import traceback
+            traceback.print_exc(file=sys.stdout)
+        else:
+            print "[ERROR] %s" % str(e)
+
+        sys.exit(1)

tools/libraries.py

@@ -59,17 +59,11 @@
     },
 
     # DSP libraries
-    {
-        "id": "cmsis_dsp",
-        "source_dir": DSP_CMSIS,
-        "build_dir": DSP_LIBRARIES,
-        "dependencies": [MBED_LIBRARIES],
-    },
     {
         "id": "dsp",
-        "source_dir": DSP_ABSTRACTION,
+        "source_dir": [DSP_ABSTRACTION, DSP_CMSIS],
         "build_dir": DSP_LIBRARIES,
-        "dependencies": [MBED_LIBRARIES, DSP_CMSIS],
+        "dependencies": [MBED_LIBRARIES]
     },
 
     # File system libraries