Adding .mbedignore logic to find_tests function

Addresses issues #247 and #197

Author: bridadan

tools/build_api.py

@@ -19,12 +19,13 @@
 import tempfile
 import colorama
 
-
+from copy import copy
 from types import ListType
 from shutil import rmtree
 from os.path import join, exists, basename, abspath, normpath
-from os import getcwd
+from os import getcwd, walk
 from time import time
+import fnmatch
 
 from tools.utils import mkdir, run_cmd, run_cmd_ext, NotSupportedException, ToolException
 from tools.paths import MBED_TARGETS_PATH, MBED_LIBRARIES, MBED_API, MBED_HAL, MBED_COMMON
@@ -827,3 +828,63 @@ def write_build_report(build_report, template_filename, filename):
 
     with open(filename, 'w+') as f:
         f.write(template.render(failing_builds=build_report_failing, passing_builds=build_report_passing))
+
+
+def scan_for_source_paths(path, exclude_paths=None):
+    ignorepatterns = []
+    paths = []
+    
+    def is_ignored(file_path):
+        for pattern in ignorepatterns:
+            if fnmatch.fnmatch(file_path, pattern):
+                return True
+        return False
+    
+    
+    """ os.walk(top[, topdown=True[, onerror=None[, followlinks=False]]])
+    When topdown is True, the caller can modify the dirnames list in-place
+    (perhaps using del or slice assignment), and walk() will only recurse into
+    the subdirectories whose names remain in dirnames; this can be used to prune
+    the search, impose a specific order of visiting, or even to inform walk()
+    about directories the caller creates or renames before it resumes walk()
+    again. Modifying dirnames when topdown is False is ineffective, because in
+    bottom-up mode the directories in dirnames are generated before dirpath
+    itself is generated.
+    """
+    for root, dirs, files in walk(path, followlinks=True):
+        # Remove ignored directories
+        # Check if folder contains .mbedignore
+        if ".mbedignore" in files :
+            with open (join(root,".mbedignore"), "r") as f:
+                lines=f.readlines()
+                lines = [l.strip() for l in lines] # Strip whitespaces
+                lines = [l for l in lines if l != ""] # Strip empty lines
+                lines = [l for l in lines if not re.match("^#",l)] # Strip comment lines
+                # Append root path to glob patterns
+                # and append patterns to ignorepatterns
+                ignorepatterns.extend([join(root,line.strip()) for line in lines])
+
+        for d in copy(dirs):
+            dir_path = join(root, d)
+
+            # Always ignore hidden directories
+            if d.startswith('.'):
+                dirs.remove(d)
+
+            # Remove dirs that already match the ignorepatterns
+            # to avoid travelling into them and to prevent them
+            # on appearing in include path.
+            if is_ignored(join(dir_path,"")):
+                dirs.remove(d)
+
+            if exclude_paths:
+                for exclude_path in exclude_paths:
+                    rel_path = relpath(dir_path, exclude_path)
+                    if not (rel_path.startswith('..')):
+                        dirs.remove(d)
+                        break
+
+        # Add root to include paths
+        paths.append(root)
+
+    return paths

tools/test_api.py

@@ -55,6 +55,7 @@
 from tools.build_api import prep_properties
 from tools.build_api import create_result
 from tools.build_api import add_result_to_report
+from tools.build_api import scan_for_source_paths
 from tools.libraries import LIBRARIES, LIBRARY_MAP
 from tools.toolchains import TOOLCHAIN_BIN_PATH
 from tools.test_exporters import ReportExporter, ResultExporterType
@@ -1965,6 +1966,12 @@ def test_path_to_name(path):
 def find_tests(base_dir):
     """Given any directory, walk through the subdirectories and find all tests"""
 
+    def is_subdir(path, directory):
+        path = os.path.realpath(path)
+        directory = os.path.realpath(directory)
+        relative = os.path.relpath(path, directory)
+        return not (relative.startswith(os.pardir + os.sep) and relative.startswith(os.pardir))
+    
     def find_tests_in_tests_directory(directory):
         """Given a 'TESTS' directory, return a dictionary of test names and test paths.
         The formate of the dictionary is {"test-name": "./path/to/test"}"""
@@ -1975,10 +1982,11 @@ def find_tests_in_tests_directory(directory):
             if d != "host_tests":
                 # Loop on test case directories
                 for td in os.listdir(os.path.join(directory, d)):
-                    # Add test case to the results if it is a directory
-                    test_case_path = os.path.join(directory, d, td)
-                    if os.path.isdir(test_case_path):
-                        tests[test_path_to_name(test_case_path)] = test_case_path
+                    # Add test case to the results if it is a directory and not "host_tests"
+                    if td != "host_tests":
+                        test_case_path = os.path.join(directory, d, td)
+                        if os.path.isdir(test_case_path):
+                            tests[test_path_to_name(test_case_path)] = test_case_path
 
         return tests
 
@@ -1994,20 +2002,27 @@ def find_tests_in_tests_directory(directory):
         # Not pointing at a "TESTS" directory, so go find one!
         tests = {}
 
-        for root, dirs, files in os.walk(base_dir):
-            # Don't search build directories
-            if '.build' in dirs:
-                dirs.remove('.build')
+        dirs = scan_for_source_paths(base_dir)
+        
+        test_and_sub_dirs = [x for x in dirs if tests_path in x]
+        test_dirs = []
+        for potential_test_dir in test_and_sub_dirs:
+            good_to_add = True
+            if test_dirs:
+                for test_dir in test_dirs:
+                    if is_subdir(potential_test_dir, test_dir):
+                        good_to_add = False
+                        break
 
-            # If a "TESTS" directory is found, find the tests inside of it
-            if tests_path in dirs:
-                # Remove it from the directory walk
-                dirs.remove(tests_path)
-                
-                # Get the tests inside of the "TESTS" directory
-                new_tests = find_tests_in_tests_directory(os.path.join(root, tests_path))
-                if new_tests:
-                    tests.update(new_tests)
+            if good_to_add:
+                test_dirs.append(potential_test_dir)
+        
+        # Only look at valid paths
+        for path in test_dirs:
+            # Get the tests inside of the "TESTS" directory
+            new_tests = find_tests_in_tests_directory(path)
+            if new_tests:
+                tests.update(new_tests)
 
         return tests