Ignore build directory from scan resources

This is a bug fix for the following bug (Github issue #437):

If two builds were run specifying a non-default build folder, the second
build would fail to link with duplicate symbols and may not fit on the
device. The root of this problem is that these non-default build folders
are not ignored by scan-resources, and therefore included in the build.

We fix this bug by ignoring the build directory passed into the tools.

Author: theotherjimmy

tools/build_api.py

@@ -121,7 +121,7 @@ def get_config(src_paths, target, toolchain_name):
         src_paths = [src_paths]
 
     # Pass all params to the unified prepare_resources()
-    toolchain = prepare_toolchain(src_paths, target, toolchain_name)
+    toolchain = prepare_toolchain(src_paths, None, target, toolchain_name)
 
     # Scan src_path for config files
     resources = toolchain.scan_resources(src_paths[0])
@@ -299,7 +299,7 @@ def add_regions_to_profile(profile, config, toolchain_class):
               % (region.name, region.size, region.start))
 
 
-def prepare_toolchain(src_paths, target, toolchain_name,
+def prepare_toolchain(src_paths, build_dir, target, toolchain_name,
                       macros=None, clean=False, jobs=1,
                       notify=None, silent=False, verbose=False,
                       extra_verbose=False, config=None,
@@ -339,7 +339,7 @@ def prepare_toolchain(src_paths, target, toolchain_name,
         add_regions_to_profile(build_profile, config, cur_tc)
 
     # Toolchain instance
-    toolchain = cur_tc(target, notify, macros, silent,
+    toolchain = cur_tc(target, notify, macros, silent, build_dir=build_dir,
                        extra_verbose=extra_verbose, build_profile=build_profile)
 
     toolchain.config = config
@@ -475,8 +475,8 @@ def build_project(src_paths, build_path, target, toolchain_name,
 
     # Pass all params to the unified prepare_toolchain()
     toolchain = prepare_toolchain(
-        src_paths, target, toolchain_name, macros=macros, clean=clean,
-        jobs=jobs, notify=notify, silent=silent, verbose=verbose,
+        src_paths, build_path, target, toolchain_name, macros=macros,
+        clean=clean, jobs=jobs, notify=notify, silent=silent, verbose=verbose,
         extra_verbose=extra_verbose, config=config, app_config=app_config,
         build_profile=build_profile)
 
@@ -509,8 +509,7 @@ def build_project(src_paths, build_path, target, toolchain_name,
             resources.linker_script = linker_script
 
         # Compile Sources
-        objects = toolchain.compile_sources(resources, build_path,
-                                            resources.inc_dirs)
+        objects = toolchain.compile_sources(resources, resources.inc_dirs)
         resources.objects.extend(objects)
 
         # Link Program
@@ -629,9 +628,9 @@ def build_library(src_paths, build_path, target, toolchain_name,
 
     # Pass all params to the unified prepare_toolchain()
     toolchain = prepare_toolchain(
-        src_paths, target, toolchain_name, macros=macros, clean=clean,
-        jobs=jobs, notify=notify, silent=silent, verbose=verbose,
-        extra_verbose=extra_verbose, app_config=app_config,
+        src_paths, build_path, target, toolchain_name, macros=macros,
+        clean=clean, jobs=jobs, notify=notify, silent=silent,
+        verbose=verbose, extra_verbose=extra_verbose, app_config=app_config,
         build_profile=build_profile)
 
     # The first path will give the name to the library
@@ -687,8 +686,7 @@ def build_library(src_paths, build_path, target, toolchain_name,
                                  resources=resources)
 
         # Compile Sources
-        objects = toolchain.compile_sources(resources, abspath(tmp_path),
-                                            resources.inc_dirs)
+        objects = toolchain.compile_sources(resources, resources.inc_dirs)
         resources.objects.extend(objects)
 
         if archive:
@@ -815,6 +813,7 @@ def build_lib(lib_id, target, toolchain_name, verbose=False,
         toolchain.VERBOSE = verbose
         toolchain.jobs = jobs
         toolchain.build_all = clean
+        toolchain.build_dir = build_path
 
         toolchain.info("Building library %s (%s, %s)" %
                        (name.upper(), target.name, toolchain_name))
@@ -869,8 +868,7 @@ def build_lib(lib_id, target, toolchain_name, verbose=False,
         # Compile Sources
         objects = []
         for resource in resources:
-            objects.extend(toolchain.compile_sources(resource, tmp_path,
-                                                     dependencies_include_dir))
+            objects.extend(toolchain.compile_sources(resource, dependencies_include_dir))
 
         needed_update = toolchain.build_library(objects, bin_path, name)
 
@@ -962,6 +960,11 @@ def build_mbed_libs(target, toolchain_name, verbose=False,
         toolchain.jobs = jobs
         toolchain.build_all = clean
 
+        tmp_path = join(MBED_LIBRARIES, '.temp', toolchain.obj_path)
+        mkdir(tmp_path)
+
+        toolchain.build_dir = tmp_path
+
         # Take into account the library configuration (MBED_CONFIG_FILE)
         config = Config(target)
         toolchain.config = config
@@ -973,8 +976,6 @@ def build_mbed_libs(target, toolchain_name, verbose=False,
         build_toolchain = join(build_target, "TOOLCHAIN_" + toolchain.name)
         mkdir(build_toolchain)
 
-        tmp_path = join(MBED_LIBRARIES, '.temp', toolchain.obj_path)
-        mkdir(tmp_path)
 
         # CMSIS
         toolchain.info("Building library %s (%s, %s)" %
@@ -1015,7 +1016,7 @@ def build_mbed_libs(target, toolchain_name, verbose=False,
         toolchain.copy_files(hal_implementation.linker_script, build_toolchain)
         toolchain.copy_files(hal_implementation.bin_files, build_toolchain)
         incdirs = toolchain.scan_resources(build_target).inc_dirs
-        objects = toolchain.compile_sources(hal_implementation, tmp_path,
+        objects = toolchain.compile_sources(hal_implementation,
                                             library_incdirs + incdirs)
         toolchain.copy_files(objects, build_toolchain)
 
@@ -1024,7 +1025,7 @@ def build_mbed_libs(target, toolchain_name, verbose=False,
         for dir in [MBED_DRIVERS, MBED_PLATFORM, MBED_HAL]:
             mbed_resources += toolchain.scan_resources(dir)
 
-        objects = toolchain.compile_sources(mbed_resources, tmp_path,
+        objects = toolchain.compile_sources(mbed_resources,
                                             library_incdirs + incdirs)
 
         # A number of compiled files need to be copied as objects as opposed to

tools/project_api.py

@@ -186,10 +186,10 @@ def export_project(src_paths, export_path, target, ide, libraries_paths=None,
     _, toolchain_name = get_exporter_toolchain(ide)
 
     # Pass all params to the unified prepare_resources()
-    toolchain = prepare_toolchain(paths, target, toolchain_name, macros=macros,
-                                  jobs=jobs, notify=notify, silent=silent,
-                                  verbose=verbose, extra_verbose=extra_verbose,
-                                  config=config, build_profile=build_profile)
+    toolchain = prepare_toolchain(
+        paths, export_path, target, toolchain_name, macros=macros, jobs=jobs,
+        notify=notify, silent=silent, verbose=verbose,
+        extra_verbose=extra_verbose, config=config, build_profile=build_profile)
     # The first path will give the name to the library
     if name is None:
         name = basename(normpath(abspath(src_paths[0])))

tools/test/build_api/build_api_test.py

@@ -58,17 +58,16 @@ def tearDown(self):
     @patch('tools.utils.run_cmd', return_value=("", "", 0))
     def test_always_complete_build(self, *_):
         with MagicMock() as notify:
-            toolchain = prepare_toolchain(self.src_paths, self.target,
+            toolchain = prepare_toolchain(self.src_paths, self.build_path, self.target,
                                           self.toolchain_name, notify=notify)
 
             res = scan_resources(self.src_paths, toolchain)
 
             toolchain.RESPONSE_FILES=False
             toolchain.config_processed = True
             toolchain.config_file = "junk"
-            toolchain.compile_sources(res, self.build_path)
+            toolchain.compile_sources(res)
 
-            print notify.mock_calls
             assert any('percent' in msg[0] and msg[0]['percent'] == 100.0
                        for _, msg, _ in notify.mock_calls if msg)
 
@@ -90,7 +89,7 @@ def test_prepare_toolchain_app_config(self, mock_config_init):
                                                        mock_target,
                                                        False)
 
-        prepare_toolchain(self.src_paths, self.target, self.toolchain_name,
+        prepare_toolchain(self.src_paths, None, self.target, self.toolchain_name,
                           app_config=app_config)
 
         mock_config_init.assert_called_once_with(self.target, self.src_paths,
@@ -112,7 +111,7 @@ def test_prepare_toolchain_no_app_config(self, mock_config_init):
                                                        mock_target,
                                                        False)
 
-        prepare_toolchain(self.src_paths, self.target, self.toolchain_name)
+        prepare_toolchain(self.src_paths, None, self.target, self.toolchain_name)
 
         mock_config_init.assert_called_once_with(self.target, self.src_paths,
                                                  app_config=None)

tools/toolchains/__init__.py

@@ -256,7 +256,8 @@ class mbedToolchain:
 
     profile_template = {'common':[], 'c':[], 'cxx':[], 'asm':[], 'ld':[]}
 
-    def __init__(self, target, notify=None, macros=None, silent=False, extra_verbose=False, build_profile=None):
+    def __init__(self, target, notify=None, macros=None, silent=False,
+                 extra_verbose=False, build_profile=None, build_dir=None):
         self.target = target
         self.name = self.__class__.__name__
 
@@ -295,7 +296,7 @@ def __init__(self, target, notify=None, macros=None, silent=False, extra_verbose
         self.build_all = False
 
         # Build output dir
-        self.build_dir = None
+        self.build_dir = build_dir
         self.timestamp = time()
 
         # Output build naming based on target+toolchain combo (mbed 2.0 builds)
@@ -580,7 +581,8 @@ def _add_dir(self, path, resources, base_path, exclude_paths=None):
                     self.add_ignore_patterns(root, base_path, lines)
 
             # Skip the whole folder if ignored, e.g. .mbedignore containing '*'
-            if self.is_ignored(join(relpath(root, base_path),"")):
+            if (self.is_ignored(join(relpath(root, base_path),"")) or
+                self.build_dir == join(relpath(root, base_path))):
                 dirs[:] = []
                 continue
 
@@ -773,7 +775,7 @@ def get_arch_file(self, objects):
 
     # THIS METHOD IS BEING CALLED BY THE MBED ONLINE BUILD SYSTEM
     # ANY CHANGE OF PARAMETERS OR RETURN VALUES WILL BREAK COMPATIBILITY
-    def compile_sources(self, resources, build_path, inc_dirs=None):
+    def compile_sources(self, resources, inc_dirs=None):
         # Web IDE progress bar for project build
         files_to_compile = resources.s_sources + resources.c_sources + resources.cpp_sources
         self.to_be_compiled = len(files_to_compile)
@@ -790,8 +792,6 @@ def compile_sources(self, resources, build_path, inc_dirs=None):
         inc_paths = sorted(set(inc_paths))
         # Unique id of all include paths
         self.inc_md5 = md5(' '.join(inc_paths)).hexdigest()
-        # Where to store response files
-        self.build_dir = build_path
 
         objects = []
         queue = []
@@ -804,7 +804,8 @@ def compile_sources(self, resources, build_path, inc_dirs=None):
         # Sort compile queue for consistency
         files_to_compile.sort()
         for source in files_to_compile:
-            object = self.relative_object_path(build_path, resources.file_basepath[source], source)
+            object = self.relative_object_path(
+                self.build_dir, resources.file_basepath[source], source)
 
             # Queue mode (multiprocessing)
             commands = self.compile_command(source, object, inc_paths)

tools/toolchains/arm.py

@@ -40,8 +40,10 @@ def check_executable():
         return mbedToolchain.generic_check_executable("ARM", 'armcc', 2, 'bin')
 
     def __init__(self, target, notify=None, macros=None,
-                 silent=False, extra_verbose=False, build_profile=None):
+                 silent=False, extra_verbose=False, build_profile=None,
+                 build_dir=None):
         mbedToolchain.__init__(self, target, notify, macros, silent,
+                               build_dir=build_dir,
                                extra_verbose=extra_verbose,
                                build_profile=build_profile)
 

tools/toolchains/gcc.py

@@ -29,10 +29,11 @@ class GCC(mbedToolchain):
     INDEX_PATTERN  = re.compile('(?P<col>\s*)\^')
 
     def __init__(self, target,  notify=None, macros=None,
-                 silent=False, extra_verbose=False, build_profile=None):
+                 silent=False, extra_verbose=False, build_profile=None,
+                 build_dir=None):
         mbedToolchain.__init__(self, target, notify, macros, silent,
                                extra_verbose=extra_verbose,
-                               build_profile=build_profile)
+                               build_profile=build_profile, build_dir=build_dir)
 
         tool_path=TOOLCHAIN_PATHS['GCC_ARM']
         # Add flags for current size setting

tools/toolchains/iar.py

@@ -37,8 +37,10 @@ def check_executable():
         return mbedToolchain.generic_check_executable("IAR", 'iccarm', 2, "bin")
 
     def __init__(self, target, notify=None, macros=None,
-                 silent=False, extra_verbose=False, build_profile=None):
+                 silent=False, extra_verbose=False, build_profile=None,
+                 build_dir=None):
         mbedToolchain.__init__(self, target, notify, macros, silent,
+                               build_dir=build_dir,
                                extra_verbose=extra_verbose,
                                build_profile=build_profile)
         if target.core == "Cortex-M7F" or target.core == "Cortex-M7FD":