Merge pull request #4783 from theotherjimmy/refactor-exporter-ignores

Collect ignored directories in scan and use in exporters

Author: theotherjimmy

tools/build_api.py

@@ -400,7 +400,7 @@ def merge_region_list(region_list, destination, padding=b'\xFF'):
         merged.tofile(output, format='bin')
 
 def scan_resources(src_paths, toolchain, dependencies_paths=None,
-                   inc_dirs=None, base_path=None):
+                   inc_dirs=None, base_path=None, collect_ignores=False):
     """ Scan resources using initialized toolcain
 
     Positional arguments
@@ -412,9 +412,11 @@ def scan_resources(src_paths, toolchain, dependencies_paths=None,
     """
 
     # Scan src_path
-    resources = toolchain.scan_resources(src_paths[0], base_path=base_path)
+    resources = toolchain.scan_resources(src_paths[0], base_path=base_path,
+                                         collect_ignores=collect_ignores)
     for path in src_paths[1:]:
-        resources.add(toolchain.scan_resources(path, base_path=base_path))
+        resources.add(toolchain.scan_resources(path, base_path=base_path,
+                                               collect_ignores=collect_ignores))
 
     # Scan dependency paths for include dirs
     if dependencies_paths is not None:

tools/export/__init__.py

@@ -312,7 +312,7 @@ def export_project(src_paths, export_path, target, ide, libraries_paths=None,
         name = basename(normpath(abspath(src_paths[0])))
 
     # Call unified scan_resources
-    resource_dict = {loc: scan_resources(path, toolchain, inc_dirs=inc_dirs)
+    resource_dict = {loc: scan_resources(path, toolchain, inc_dirs=inc_dirs, collect_ignores=True)
                      for loc, path in src_paths.iteritems()}
     resources = Resources()
     toolchain.build_dir = export_path

tools/export/gnuarmeclipse/__init__.py

@@ -419,105 +419,10 @@ def compute_exclusions(self):
         """
         source_folders = [self.filter_dot(s) for s in set(dirname(
             src) for src in self.resources.c_sources + self.resources.cpp_sources + self.resources.s_sources)]
-        if '.' in source_folders:
-            source_folders.remove('.')
-
-        # print 'source folders'
-        # print source_folders
-
-        # Source folders were converted before and are guaranteed to
-        # use the POSIX separator.
-        top_folders = [f for f in set(s.split('/')[0]
-                                      for s in source_folders)]
-        # print 'top folders'
-        # print top_folders
-
-        self.source_tree = {}
-        for top_folder in top_folders:
-            for root, dirs, files in os.walk(top_folder, topdown=True):
-                # print root, dirs, files
-
-                # Paths returned by os.walk() must be split with os.dep
-                # to accomodate Windows weirdness.
-                parts = root.split(os.sep)
-
-                # Ignore paths that include parts starting with dot.
-                skip = False
-                for part in parts:
-                    if part.startswith('.'):
-                        skip = True
-                        break
-                if skip:
-                    continue
-
-                # Further process only leaf paths, (that do not have
-                # sub-folders).
-                if len(dirs) == 0:
-                    # The path is reconstructed using POSIX separators.
-                    self.add_source_folder_to_tree('/'.join(parts))
-
-        for folder in source_folders:
-            self.add_source_folder_to_tree(folder, True)
-
-        # print
-        # print self.source_tree
-        # self.dump_paths(self.source_tree)
-        # self.dump_tree(self.source_tree)
-
-        # print 'excludings'
-        self.excluded_folders = ['BUILD']
-        self.recurse_excludings(self.source_tree)
 
+        self.excluded_folders = set(self.resources.ignored_dirs) - set(self.resources.inc_dirs)
         print 'Source folders: {0}, with {1} exclusions'.format(len(source_folders), len(self.excluded_folders))
 
-    def add_source_folder_to_tree(self, path, is_used=False):
-        """
-        Decompose a path in an array of folder names and create the tree.
-        On the second pass the nodes should be already there; mark them
-        as used.
-        """
-        # print path, is_used
-
-        # All paths arriving here are guaranteed to use the POSIX
-        # separators, os.walk() paths were also explicitly converted.
-        parts = path.split('/')
-        # print parts
-        node = self.source_tree
-        prev = None
-        for part in parts:
-            if part not in node.keys():
-                new_node = {}
-                new_node['name'] = part
-                new_node['children'] = {}
-                if prev != None:
-                    new_node['parent'] = prev
-                node[part] = new_node
-            node[part]['is_used'] = is_used
-            prev = node[part]
-            node = node[part]['children']
-
-    def recurse_excludings(self, nodes):
-        """
-        Recurse the tree and collect all unused folders; descend
-        the hierarchy only for used nodes.
-        """
-        for k in nodes.keys():
-            node = nodes[k]
-            if node['is_used'] == False:
-                parts = []
-                cnode = node
-                while True:
-                    parts.insert(0, cnode['name'])
-                    if 'parent' not in cnode:
-                        break
-                    cnode = cnode['parent']
-
-                # Compose a POSIX path.
-                path = '/'.join(parts)
-                # print path
-                self.excluded_folders.append(path)
-            else:
-                self.recurse_excludings(node['children'])
 
     # -------------------------------------------------------------------------
 

tools/toolchains/__init__.py

@@ -116,8 +116,9 @@ def closure(v=v):
         self.eager = {}
 
 class Resources:
-    def __init__(self, base_path=None):
+    def __init__(self, base_path=None, collect_ignores=False):
         self.base_path = base_path
+        self.collect_ignores = collect_ignores
 
         self.file_basepath = {}
 
@@ -148,6 +149,7 @@ def __init__(self, base_path=None):
 
         # Features
         self.features = LazyDict()
+        self.ignored_dirs = []
 
     def __add__(self, resources):
         if resources is None:
@@ -161,6 +163,10 @@ def __radd__(self, resources):
         else:
             return self.add(resources)
 
+    def ignore_dir(self, directory):
+        if self.collect_ignores:
+            self.ignored_dirs.append(directory)
+
     def add(self, resources):
         for f,p in resources.file_basepath.items():
             self.file_basepath[f] = p
@@ -190,6 +196,7 @@ def add(self, resources):
         self.json_files += resources.json_files
 
         self.features.update(resources.features)
+        self.ignored_dirs += resources.ignored_dirs
 
         return self
 
@@ -612,10 +619,11 @@ def add_ignore_patterns(self, root, base_path, patterns):
     # The parameter *base_path* is used to set the base_path attribute of the Resources
     # object and the parameter *exclude_paths* is used by the directory traversal to
     # exclude certain paths from the traversal.
-    def scan_resources(self, path, exclude_paths=None, base_path=None):
+    def scan_resources(self, path, exclude_paths=None, base_path=None,
+                       collect_ignores=False):
         self.progress("scan", path)
 
-        resources = Resources(path)
+        resources = Resources(path, collect_ignores=collect_ignores)
         if not base_path:
             if isfile(path):
                 base_path = dirname(path)
@@ -656,8 +664,10 @@ def _add_dir(self, path, resources, base_path, exclude_paths=None):
                     self.add_ignore_patterns(root, base_path, lines)
 
             # Skip the whole folder if ignored, e.g. .mbedignore containing '*'
-            if (self.is_ignored(join(relpath(root, base_path),"")) or
-                self.build_dir == join(relpath(root, base_path))):
+            root_path =join(relpath(root, base_path))
+            if  (self.is_ignored(join(root_path,"")) or
+                 self.build_dir == root_path):
+                resources.ignore_dir(root_path)
                 dirs[:] = []
                 continue
 
@@ -676,18 +686,21 @@ def _add_dir(self, path, resources, base_path, exclude_paths=None):
                     self.is_ignored(join(relpath(root, base_path), d,"")) or
                     # Ignore TESTS dir
                     (d == 'TESTS')):
+                        resources.ignore_dir(dir_path)
                         dirs.remove(d)
                 elif d.startswith('FEATURE_'):
                     # Recursively scan features but ignore them in the current scan.
                     # These are dynamically added by the config system if the conditions are matched
                     def closure (dir_path=dir_path, base_path=base_path):
                         return self.scan_resources(dir_path, base_path=base_path)
                     resources.features.add_lazy(d[8:], closure)
+                    resources.ignore_dir(dir_path)
                     dirs.remove(d)
                 elif exclude_paths:
                     for exclude_path in exclude_paths:
                         rel_path = relpath(dir_path, exclude_path)
                         if not (rel_path.startswith('..')):
+                            resources.ignore_dir(dir_path)
                             dirs.remove(d)
                             break