Progen build tests and project_api

sarahmarshy · sarahmarshy · commit c6aa902441ba · 2016-07-14T11:56:24.000-05:00
diff --git a/tools/build_test.py b/tools/build_test.py
@@ -0,0 +1,95 @@
+import sys
+from os.path import join, abspath, dirname, exists, basename
+ROOT = abspath(join(dirname(__file__), ".."))
+sys.path.insert(0, ROOT)
+
+import argparse
+
+from export import EXPORTERS
+from targets import TARGET_NAMES, TARGET_MAP
+from project_api import setup_project, perform_export, print_results, get_test_from_name
+from project_generator_definitions.definitions import ProGenDef
+from utils import args_error
+
+class BuildTest():
+    def __init__(self, desired_ides, tests, targets):
+        self.target_ides = {}
+        for target in targets:
+            self.target_ides[target] =[]
+            for ide in desired_ides:
+                if target in EXPORTERS[ide].TARGETS:
+                    self.target_ides[target].append(ide)
+            if len(self.target_ides[target]) == 0:
+                del self.target_ides[target]
+
+        successes, failures = self._generate_and_build(tests)
+        print_results(successes, failures)
+
+    @staticmethod
+    def get_pgen_targets(ides):
+        targs = []
+        for ide in ides:
+            for target in TARGET_NAMES:
+                if target not in targs and hasattr(TARGET_MAP[target],'progen') \
+                        and ProGenDef(ide).is_supported(TARGET_MAP[target].progen['target']):
+                    targs.append(target)
+        return targs
+
+    def _generate_and_build(self, tests):
+        successes = []
+        failures = []
+        for mcu, ides in self.target_ides.items():
+            for test in tests:
+                test = get_test_from_name(test)
+                for ide in ides:
+                    project_dir, project_name, project_temp = setup_project(mcu, ide, test)
+                    tmp_path, report = perform_export(project_dir, project_name, ide, mcu, project_temp,
+                                                      progen_build = True)
+                    if report['success']:
+                        successes.append("build for %s::%s\t%s" % (mcu, ide, project_name))
+                    else:
+                        failures.append("%s::%s\t%s for %s" % (mcu, ide, report['errormsg'], project_name))
+        return successes, failures
+
+
+if __name__ == '__main__':
+    accepted_ides = ["iar", "uvision", "uvision5"]
+    accepted_targets = sorted(BuildTest.get_pgen_targets(accepted_ides))
+    default_tests = ["MBED_BLINKY"]
+    parser = argparse.ArgumentParser(description = "Test progen builders. Leave any flag off to run with all possible options.")
+    parser.add_argument("-i", "--IDEs",
+                      nargs = '+',
+                      dest="ides",
+                      help="tools you wish to perfrom build tests. (%s)" % ', '.join(accepted_ides),
+                      default = accepted_ides)
+
+    parser.add_argument("-t", "--tests",
+                    nargs='+',
+                    dest="tests",
+                    help="names of desired test programs",
+                    default = default_tests)
+
+    parser.add_argument("-m", "--mcus",
+                      nargs='+',
+                      dest ="targets",
+                      help="generate project for the given MCUs (%s)" % '\n '.join(accepted_targets),
+                      default = TARGET_NAMES)
+
+    options = parser.parse_args()
+
+    tests = options.tests
+    ides = options.ides
+    targets = options.targets
+
+    if any(get_test_from_name(test) is None for test in tests):
+        args_error(parser, "[ERROR] test name not recognized")
+
+    if any(target not in accepted_targets for target in targets):
+        args_error(parser, "[ERROR] mcu must be one of the following:\n %s" % '\n '.join(accepted_targets))
+
+    if any(ide not in accepted_ides for ide in ides):
+        args_error(parser, "[ERROR] ide must be in %s" % ', '.join(accepted_ides))
+
+    b = BuildTest(ides, tests, targets)
+
+
diff --git a/tools/export/__init__.py b/tools/export/__init__.py
@@ -21,7 +21,7 @@
 
 from tools.utils import mkdir
 from tools.export import uvision4, uvision5, codered, gccarm, ds5_5, iar, emblocks, coide, kds, zip, simplicityv3, atmelstudio, sw4stm32, e2studio
-from tools.export.exporters import zip_working_directory_and_clean_up, OldLibrariesException
+from tools.export.exporters import zip_working_directory_and_clean_up, OldLibrariesException, FailedBuildException
 from tools.targets import TARGET_NAMES, EXPORT_MAP, TARGET_MAP
 
 from project_generator_definitions.definitions import ProGenDef
@@ -58,7 +58,8 @@ def online_build_url_resolver(url):
 
 
 def export(project_path, project_name, ide, target, destination='/tmp/',
-           tempdir=None, clean=True, extra_symbols=None, make_zip=True, sources_relative=False, build_url_resolver=online_build_url_resolver):
+           tempdir=None, pgen_build = False, clean=True, extra_symbols=None, make_zip=True, sources_relative=False,
+           build_url_resolver=online_build_url_resolver, **kwargs):
     # Convention: we are using capitals for toolchain and target names
     if target is not None:
         target = target.upper()
@@ -67,6 +68,10 @@ def export(project_path, project_name, ide, target, destination='/tmp/',
         tempdir = tempfile.mkdtemp()
 
     use_progen = False
+    build = False
+    if 'progen_build' in kwargs and kwargs.get('progen_build') == True:
+        build = True
+
     supported = True
     report = {'success': False, 'errormsg':''}
     
@@ -103,10 +108,19 @@ def export(project_path, project_name, ide, target, destination='/tmp/',
                 try:
                     exporter = Exporter(target, tempdir, project_name, build_url_resolver, extra_symbols=extra_symbols, sources_relative=sources_relative)
                     exporter.scan_and_copy_resources(project_path, tempdir, sources_relative)
-                    exporter.generate()
-                    report['success'] = True
+                    if build:
+                        #try to build with pgen ide builders
+                        try:
+                            exporter.generate(progen_build=True)
+                            report['success'] = True
+                        except FailedBuildException, f:
+                            report['errormsg'] = "Build Failed"
+                    else:
+                        exporter.generate()
+                        report['success'] = True
                 except OldLibrariesException, e:
                     report['errormsg'] = ERROR_MESSAGE_NOT_EXPORT_LIBS
+
             else:
                 report['errormsg'] = ERROR_MESSAGE_UNSUPPORTED_TOOLCHAIN % (target, ide)
 
diff --git a/tools/export/exporters.py b/tools/export/exporters.py
@@ -21,6 +21,8 @@
 
 class OldLibrariesException(Exception): pass
 
+class FailedBuildException(Exception) : pass
+
 class Exporter(object):
     TEMPLATE_DIR = dirname(__file__)
     DOT_IN_RELATIVE_PATH = False
@@ -107,14 +109,18 @@ def progen_get_project_data(self):
         }
         return project_data
 
-    def progen_gen_file(self, tool_name, project_data):
-        """ Generate project using ProGen Project API """
+    def progen_gen_file(self, tool_name, project_data, pgen_build = False):
+        """" Generate project using ProGen Project API """
         settings = ProjectSettings()
         project = Project(self.program_name, [project_data], settings)
         # TODO: Fix this, the inc_dirs are not valid (our scripts copy files), therefore progen
         # thinks it is not dict but a file, and adds them to workspace.
         project.project['common']['include_paths'] = self.resources.inc_dirs
         project.generate(tool_name, copied=not self.sources_relative)
+        if pgen_build:
+            result = project.build(tool_name)
+            if result == -1:
+                raise FailedBuildException("Build Failed")
 
     def __scan_all(self, path):
         resources = []
diff --git a/tools/export/iar.py b/tools/export/iar.py
@@ -46,7 +46,7 @@ class IAREmbeddedWorkbench(Exporter):
             # target is not supported yet
             continue
 
-    def generate(self):
+    def generate(self, **kwargs):
         """ Generates the project files """
         project_data = self.progen_get_project_data()
         tool_specific = {}
@@ -75,7 +75,10 @@ def generate(self):
         # VLA is enabled via template IccAllowVLA
         project_data['tool_specific']['iar']['misc']['c_flags'].remove("--vla")
         project_data['common']['build_dir'] = os.path.join(project_data['common']['build_dir'], 'iar_arm')
-        self.progen_gen_file('iar_arm', project_data)
+        if 'progen_build' in kwargs:
+            self.progen_gen_file('iar_arm', project_data, True)
+        else:
+            self.progen_gen_file('iar_arm', project_data)
 
 # Currently not used, we should reuse folder_name to create virtual folders
 class IarFolder():
diff --git a/tools/export/uvision4.py b/tools/export/uvision4.py
@@ -50,7 +50,7 @@ class Uvision4(Exporter):
     def get_toolchain(self):
         return TARGET_MAP[self.target].default_toolchain
 
-    def generate(self):
+    def generate(self, **kwargs):
         """ Generates the project files """
         project_data = self.progen_get_project_data()
         tool_specific = {}
@@ -96,4 +96,7 @@ def generate(self):
             i += 1
         project_data['common']['macros'].append('__ASSERT_MSG')
         project_data['common']['build_dir'] = join(project_data['common']['build_dir'], 'uvision4')
-        self.progen_gen_file('uvision', project_data)
+        if 'progen_build' in kwargs and kwargs.get('progen_build') == True:
+            self.progen_gen_file('uvision', project_data, True)
+        else:
+            self.progen_gen_file('uvision', project_data)
diff --git a/tools/export/uvision5.py b/tools/export/uvision5.py
@@ -50,7 +50,7 @@ class Uvision5(Exporter):
     def get_toolchain(self):
         return TARGET_MAP[self.target].default_toolchain
 
-    def generate(self):
+    def generate(self, **kwargs):
         """ Generates the project files """
         project_data = self.progen_get_project_data()
         tool_specific = {}
@@ -95,4 +95,7 @@ def generate(self):
                 project_data['common']['macros'].pop(i)
             i += 1
         project_data['common']['macros'].append('__ASSERT_MSG')
-        self.progen_gen_file('uvision5', project_data)
+        if 'progen_build' in kwargs and kwargs.get('progen_build') == True:
+            self.progen_gen_file('uvision5', project_data, True)
+        else:
+            self.progen_gen_file('uvision5', project_data)
diff --git a/tools/project.py b/tools/project.py
@@ -5,19 +5,16 @@
 
 from shutil import move, rmtree
 from argparse import ArgumentParser
-from os import path
 
-from tools.paths import EXPORT_DIR, EXPORT_WORKSPACE, EXPORT_TMP
-from tools.paths import MBED_BASE, MBED_LIBRARIES
-from tools.export import export, setup_user_prj, EXPORTERS, mcu_ide_matrix
-from tools.utils import args_error, mkdir
-from tools.tests import TESTS, Test, TEST_MAP
+from tools.paths import EXPORT_DIR
+from tools.export import EXPORTERS, mcu_ide_matrix
+from tools.utils import args_error
+from tools.tests import TESTS, TEST_MAP
 from tools.tests import test_known, test_name_known
 from tools.targets import TARGET_NAMES
-from tools.libraries import LIBRARIES
-from utils import argparse_lowercase_type, argparse_uppercase_type, argparse_filestring_type, argparse_many
+from utils import argparse_filestring_type, argparse_many
 from utils import argparse_force_lowercase_type, argparse_force_uppercase_type
-
+from project_api import setup_project, perform_export, print_results, get_test_from_name
 
 
 if __name__ == '__main__':
@@ -128,77 +125,68 @@
         if exists(EXPORT_DIR):
             rmtree(EXPORT_DIR)
 
+
+    # Target
+    if options.mcu is None :
+        args_error(parser, "[ERROR] You should specify an MCU")
+    mcus = options.mcu
+
+    # IDE
+    if options.ide is None:
+        args_error(parser, "[ERROR] You should specify an IDE")
+    ide = options.ide
+
+    p, n, src= options.program, options.program_name, options.source_dir
+
+    if src is None:
+        if p is not None and n is not None:
+            args_error(parser, "[ERROR] specify either '-n' or '-p', not both")
+        if n:
+            p = get_test_from_name(n)
+            if p is None:
+                args_error(parser, "[ERROR] Program with name '%s' not found" % n)
+
+        if p is None or (p < 0) or (p > (len(TESTS) - 1)):
+            message = "[ERROR] You have to specify one of the following tests:\n"
+            message += '\n'.join(map(str, sorted(TEST_MAP.values())))
+            args_error(parser, message)
+
+
     # Export results
     successes = []
     failures = []
-    zip = True
-    clean = True
+    zip = src is None
+    clean = src is None
 
     # source_dir = use relative paths, otherwise sources are copied
     sources_relative = True if options.source_dir else False
+    for mcu in mcus.split(','):
+
+        lib_symbols = []
+        if options.macros:
+            lib_symbols += options.macros
+        project_dir, project_name, project_temp = setup_project(mcu,
+                      ide,
+                      p,
+                      src,
+                      options.macros,
+                      options.build)
+
+        tmp_path, report = perform_export(project_dir, project_name, ide, mcu,
+                                          project_temp, clean, zip, lib_symbols,
+                                          sources_relative)
+
 
-    for mcu in options.mcu:
-        # Program Number or name
-        p, src, ide = options.program, options.source_dir, options.ide
-
-        if src:
-            # --source is used to generate IDE files to toolchain directly in the source tree and doesn't generate zip file
-            project_dir = options.source_dir
-            project_name = TESTS[p] if p else "Unnamed_project"
-            project_temp = path.join(options.source_dir[0], 'projectfiles', '%s_%s' % (ide, mcu))
-            mkdir(project_temp)
-            lib_symbols = []
-            if options.macros:
-                lib_symbols += options.macros
-            zip = False   # don't create zip
-            clean = False # don't cleanup because we use the actual source tree to generate IDE files
-        else:
-            test = Test(p)
-
-            # Some libraries have extra macros (called by exporter symbols) to we need to pass
-            # them to maintain compilation macros integrity between compiled library and
-            # header files we might use with it
-            lib_symbols = []
-            if options.macros:
-                lib_symbols += options.macros
-            for lib in LIBRARIES:
-                if lib['build_dir'] in test.dependencies:
-                    lib_macros = lib.get('macros', None)
-                    if lib_macros is not None:
-                        lib_symbols.extend(lib_macros)
-
-            if not options.build:
-                # Substitute the library builds with the sources
-                # TODO: Substitute also the other library build paths
-                if MBED_LIBRARIES in test.dependencies:
-                    test.dependencies.remove(MBED_LIBRARIES)
-                    test.dependencies.append(MBED_BASE)
-
-            # Build the project with the same directory structure of the mbed online IDE
-            project_name = test.id
-            project_dir = [join(EXPORT_WORKSPACE, project_name)]
-            project_temp = EXPORT_TMP
-            setup_user_prj(project_dir[0], test.source_dir, test.dependencies)
-
-        # Export to selected toolchain
-        tmp_path, report = export(project_dir, project_name, ide, mcu, project_dir[0], project_temp, clean=clean, make_zip=zip, extra_symbols=lib_symbols, sources_relative=sources_relative)
         if report['success']:
             if not zip:
                 zip_path = join(project_temp, project_name)
             else:
                 zip_path = join(EXPORT_DIR, "%s_%s_%s.zip" % (project_name, ide, mcu))
                 move(tmp_path, zip_path)
+
             successes.append("%s::%s\t%s"% (mcu, ide, zip_path))
         else:
             failures.append("%s::%s\t%s"% (mcu, ide, report['errormsg']))
 
     # Prints export results
-    print
-    if len(successes) > 0:
-        print "Successful exports:"
-        for success in successes:
-            print "  * %s"% success
-    if len(failures) > 0:
-        print "Failed exports:"
-        for failure in failures:
-            print "  * %s"% failure
+    print_results(successes, failures)
diff --git a/tools/project_api.py b/tools/project_api.py
