Calculate md5 of all include paths in compile_sources() and remove calculation from <toolchaon>get_compile_options(), thus significantly reduce repetitive md5 calculations

Unify handling of the include response file in mbedToolchain::get_inc_file()
Sanitize obsolete no-longer needed methods

Author: screamerbg

tools/toolchains/__init__.py

@@ -30,6 +30,7 @@
 from tools.utils import run_cmd, mkdir, rel_path, ToolException, NotSupportedException, split_path
 from tools.settings import BUILD_OPTIONS, MBED_ORG_USER
 import tools.hooks as hooks
+from hashlib import md5
 
 
 #Disables multiprocessing if set to higher number than the host machine CPUs
@@ -478,18 +479,35 @@ def relative_object_path(self, build_path, base_dir, source):
         mkdir(obj_dir)
         return join(obj_dir, name + '.o')
 
+    def get_inc_file(self, includes):
+        include_file = join(self.temp_dir, "includes_%s.txt" % self.inc_md5)
+        if not exists(include_file):
+            with open(include_file, "wb") as f:
+                cmd_list = []
+                for c in includes:
+                    if c:
+                        cmd_list.append(('-I%s' % c).replace("\\", "/"))                    
+                string = " ".join(cmd_list)
+                f.write(string)
+        return include_file
+
     def compile_sources(self, resources, build_path, inc_dirs=None):
         # Web IDE progress bar for project build
         files_to_compile = resources.s_sources + resources.c_sources + resources.cpp_sources
         self.to_be_compiled = len(files_to_compile)
         self.compiled = 0
-        self.temp_dir = build_path
 
         inc_paths = resources.inc_dirs
         if inc_dirs is not None:
             inc_paths.extend(inc_dirs)
-        # De-duplicate include paths and sort for consistency
+        # De-duplicate include paths
+        inc_paths = set(inc_paths)
+        # Sort include paths for consistency
         inc_paths = sorted(set(inc_paths))
+        # Unique id of all include paths
+        self.inc_md5 = md5(' '.join(inc_paths)).hexdigest()
+        # Where to store response files
+        self.temp_dir = build_path
 
         objects = []
         queue = []

tools/toolchains/arm.py

@@ -16,7 +16,6 @@
 """
 import re
 from os.path import join, dirname, splitext, basename, exists
-from hashlib import md5
 
 from tools.toolchains import mbedToolchain
 from tools.settings import ARM_BIN, ARM_INC, ARM_LIB, MY_ARM_CLIB, ARM_CPPLIB, GOANNA_PATH
@@ -79,11 +78,6 @@ def __init__(self, target, options=None, notify=None, macros=None, silent=False,
         self.ar = join(ARM_BIN, "armar")
         self.elf2bin = join(ARM_BIN, "fromelf")
 
-    def remove_option(self, option):
-        for tool in [self.asm, self.cc, self.cppc]:
-            if option in tool:
-                tool.remove(option)
-
     def parse_dependencies(self, dep_path):
         dependencies = []
         for line in open(dep_path).readlines():
@@ -113,25 +107,13 @@ def parse_output(self, output):
                     match.group('message')
                 )
 
-    def get_dep_opt(self, dep_path):
+    def get_dep_option(self, object):
+        base, _ = splitext(object)
+        dep_path = base + '.d'
         return ["--depend", dep_path]
 
-    def get_compile_options(self, defines, includes):
-        cmd = []
-        
-        sum = md5(' '.join(includes)).hexdigest()
-        options_file = join(self.temp_dir, "options_%s.txt" % sum)
-        if not exists(options_file):
-            with open(options_file, "wb") as f:
-                cmd_list = ['-D%s' % d for d in defines]
-                for c in includes:
-                    if c:
-                        cmd_list.append(('-I%s' % c) if not c.startswith('-') else c)                    
-                string = " ".join(cmd_list).replace("\\", "/")
-                f.write(string)
-        cmd.extend(['--via', options_file])
-        
-        return cmd
+    def get_compile_options(self, defines, includes):        
+        return ['-D%s' % d for d in defines] + ['--via', self.get_inc_file(includes)]
 
     @hook_tool
     def assemble(self, source, object, includes):
@@ -158,9 +140,7 @@ def compile(self, cc, source, object, includes):
         # Build compile command
         cmd = cc + self.get_compile_options(self.get_symbols(), includes)
 
-        base, _ = splitext(object)
-        dep_path = base + '.d'
-        cmd.extend(self.get_dep_opt(dep_path))
+        cmd.extend(self.get_dep_option(object))
 
         cmd.extend(["-o", object, source])
 

tools/toolchains/gcc.py

@@ -16,7 +16,6 @@
 """
 import re
 from os.path import join, basename, splitext, dirname, exists
-from hashlib import md5
 
 from tools.toolchains import mbedToolchain
 from tools.settings import GCC_ARM_PATH, GCC_CR_PATH
@@ -69,7 +68,7 @@ def __init__(self, target, options=None, notify=None, macros=None, silent=False,
             "-Wno-unused-parameter", "-Wno-missing-field-initializers",
             "-fmessage-length=0", "-fno-exceptions", "-fno-builtin",
             "-ffunction-sections", "-fdata-sections",
-            "-MMD", "-fno-delete-null-pointer-checks", "-fomit-frame-pointer"
+            "-fno-delete-null-pointer-checks", "-fomit-frame-pointer"
             ] + self.cpu
 
         if "save-asm" in self.options:
@@ -162,22 +161,13 @@ def parse_output(self, output):
                     message + match.group('message')
                 )
 
+    def get_dep_option(self, object):
+        base, _ = splitext(object)
+        dep_path = base + '.d'
+        return ["-MD", "-MF", dep_path]
+
     def get_compile_options(self, defines, includes):
-        cmd = []
-        
-        sum = md5(' '.join(includes)).hexdigest()
-        options_file = join(self.temp_dir, "options_%s.txt" % sum)
-        if not exists(options_file):
-            with open(options_file, "wb") as f:
-                cmd_list = ['-D%s' % d for d in defines]
-                for c in includes:
-                    if c:
-                        cmd_list.append(('-I%s' % c) if not c.startswith('-') else c)                    
-                string = " ".join(cmd_list).replace("\\", "/")
-                f.write(string)
-        cmd.extend(['@%s' % options_file])
-        
-        return cmd
+        return ['-D%s' % d for d in defines] + ['@%s' % self.get_inc_file(includes)]
 
     @hook_tool
     def assemble(self, source, object, includes):
@@ -193,8 +183,12 @@ def assemble(self, source, object, includes):
     @hook_tool
     def compile(self, cc, source, object, includes):
         # Build compile command
-        cmd = cc + self.get_compile_options(self.get_symbols(), includes) + ["-o", object, source]
+        cmd = cc + self.get_compile_options(self.get_symbols(), includes)
 
+        cmd.extend(self.get_dep_option(object))
+
+        cmd.extend(["-o", object, source])
+        
         # Call cmdline hook
         cmd = self.hook.get_cmdline_compiler(cmd)
 

tools/toolchains/iar.py

@@ -17,7 +17,6 @@
 import re
 from os import remove
 from os.path import join, exists, dirname, splitext, exists
-from hashlib import md5
 
 from tools.toolchains import mbedToolchain
 from tools.settings import IAR_PATH
@@ -73,6 +72,10 @@ def __init__(self, target, options=None, notify=None, macros=None, silent=False,
         self.ar = join(IAR_BIN, "iarchive")
         self.elf2bin = join(IAR_BIN, "ielftool")
 
+    def parse_dependencies(self, dep_path):
+        return [path.strip() for path in open(dep_path).readlines()
+                if (path and not path.isspace())]
+
     def parse_output(self, output):
         for line in output.splitlines():
             match = IAR.DIAGNOSTIC_PATTERN.match(line)
@@ -94,32 +97,17 @@ def parse_output(self, output):
                     match.group('message')
                 )
 
-    def get_dep_opt(self, dep_path):
+    def get_dep_option(self, object):
+        base, _ = splitext(object)
+        dep_path = base + '.d'
         return ["--dependencies", dep_path]
 
-    def cc_extra(self, base):
+    def cc_extra(self, object):
+        base, _ = splitext(object)
         return ["-l", base + '.s']
 
-    def parse_dependencies(self, dep_path):
-        return [path.strip() for path in open(dep_path).readlines()
-                if (path and not path.isspace())]
-
     def get_compile_options(self, defines, includes):
-        cmd = []
-        
-        sum = md5(' '.join(includes)).hexdigest()
-        options_file = join(self.temp_dir, "options_%s.txt" % sum)
-        if not exists(options_file):
-            with open(options_file, "wb") as f:
-                cmd_list = ['-D%s' % d for d in defines]
-                for c in includes:
-                    if c:
-                        cmd_list.append(('-I%s' % c) if not c.startswith('-') else c)                    
-                string = " ".join(cmd_list).replace("\\", "/")
-                f.write(string)
-        cmd.extend(['-f', options_file])
-        
-        return cmd
+        return ['-D%s' % d for d in defines] + ['-f', self.get_inc_file(includes)]
 
     @hook_tool
     def assemble(self, source, object, includes):
@@ -137,11 +125,9 @@ def compile(self, cc, source, object, includes):
         # Build compile command
         cmd = cc +  self.get_compile_options(self.get_symbols(), includes)
 
-        base, _ = splitext(object)
-        dep_path = base + '.d'
-        cmd.extend(self.get_dep_opt(dep_path))
+        cmd.extend(self.get_dep_option(object))
 
-        cmd.extend(self.cc_extra(base))
+        cmd.extend(self.cc_extra(object))
 
         cmd.extend(["-o", object, source])