Progen build tests

requirements.txt

@@ -3,8 +3,8 @@ PySerial>=2.7
 PrettyTable>=0.7.2
 Jinja2>=2.7.3
 IntelHex>=1.3
-project-generator>=0.9.3,<0.10.0
 project_generator_definitions>=0.2.26,<0.3.0
+project-generator>=0.9.7,<0.10.0
 junit-xml
 pyYAML
 requests

tools/export/__init__.py

@@ -21,7 +21,7 @@
 
 from tools.utils import mkdir
 from tools.export import uvision4, uvision5, codered, gccarm, ds5_5, iar, emblocks, coide, kds, zip, simplicityv3, atmelstudio, sw4stm32, e2studio
-from tools.export.exporters import zip_working_directory_and_clean_up, OldLibrariesException
+from tools.export.exporters import zip_working_directory_and_clean_up, OldLibrariesException, FailedBuildException
 from tools.targets import TARGET_NAMES, EXPORT_MAP, TARGET_MAP
 
 from project_generator_definitions.definitions import ProGenDef
@@ -58,7 +58,8 @@ def online_build_url_resolver(url):
 
 
 def export(project_path, project_name, ide, target, destination='/tmp/',
-           tempdir=None, clean=True, extra_symbols=None, make_zip=True, sources_relative=False, build_url_resolver=online_build_url_resolver):
+           tempdir=None, pgen_build = False, clean=True, extra_symbols=None, make_zip=True, sources_relative=False,
+           build_url_resolver=online_build_url_resolver, progen_build=False):
     # Convention: we are using capitals for toolchain and target names
     if target is not None:
         target = target.upper()
@@ -67,8 +68,11 @@ def export(project_path, project_name, ide, target, destination='/tmp/',
         tempdir = tempfile.mkdtemp()
 
     use_progen = False
+
     supported = True
-    report = {'success': False, 'errormsg':''}
+    report = {'success': False, 'errormsg':'', 'skip': False}
+
+
 
     if ide is None or ide == "zip":
         # Simple ZIP exporter
@@ -83,6 +87,7 @@ def export(project_path, project_name, ide, target, destination='/tmp/',
     else:
         if ide not in EXPORTERS:
             report['errormsg'] = ERROR_MESSAGE_UNSUPPORTED_TOOLCHAIN % (target, ide)
+            report['skip'] = True
         else:
             Exporter = EXPORTERS[ide]
             target = EXPORT_MAP.get(target, target)
@@ -91,24 +96,35 @@ def export(project_path, project_name, ide, target, destination='/tmp/',
                     use_progen = True
             except AttributeError:
                 pass
+
+            if target not in Exporter.TARGETS or Exporter.TOOLCHAIN not in TARGET_MAP[target].supported_toolchains:
+                supported = False
+
             if use_progen:
                 if not ProGenDef(ide).is_supported(TARGET_MAP[target].progen['target']):
                     supported = False
-            else:
-                if target not in Exporter.TARGETS:
-                    supported = False
 
             if supported:
                 # target checked, export
                 try:
                     exporter = Exporter(target, tempdir, project_name, build_url_resolver, extra_symbols=extra_symbols, sources_relative=sources_relative)
                     exporter.scan_and_copy_resources(project_path, tempdir, sources_relative)
-                    exporter.generate()
-                    report['success'] = True
+                    if progen_build:
+                        #try to build with pgen ide builders
+                        try:
+                            exporter.generate(progen_build=True)
+                            report['success'] = True
+                        except FailedBuildException, f:
+                            report['errormsg'] = "Build Failed"
+                    else:
+                        exporter.generate()
+                        report['success'] = True
                 except OldLibrariesException, e:
                     report['errormsg'] = ERROR_MESSAGE_NOT_EXPORT_LIBS
+
             else:
                 report['errormsg'] = ERROR_MESSAGE_UNSUPPORTED_TOOLCHAIN % (target, ide)
+                report['skip'] = True
 
     zip_path = None
     if report['success']:

tools/export/exporters.py

@@ -21,6 +21,8 @@
 
 class OldLibrariesException(Exception): pass
 
+class FailedBuildException(Exception) : pass
+
 class Exporter(object):
     TEMPLATE_DIR = dirname(__file__)
     DOT_IN_RELATIVE_PATH = False
@@ -107,14 +109,19 @@ def progen_get_project_data(self):
         }
         return project_data
 
-    def progen_gen_file(self, tool_name, project_data):
-        """ Generate project using ProGen Project API """
+    def progen_gen_file(self, tool_name, project_data, progen_build=False):
+        """" Generate project using ProGen Project API """
         settings = ProjectSettings()
         project = Project(self.program_name, [project_data], settings)
         # TODO: Fix this, the inc_dirs are not valid (our scripts copy files), therefore progen
         # thinks it is not dict but a file, and adds them to workspace.
         project.project['common']['include_paths'] = self.resources.inc_dirs
         project.generate(tool_name, copied=not self.sources_relative)
+        if progen_build:
+            print("Project exported, building...")
+            result = project.build(tool_name)
+            if result == -1:
+                raise FailedBuildException("Build Failed")
 
     def __scan_all(self, path):
         resources = []

tools/export/iar.py

@@ -46,7 +46,7 @@ class IAREmbeddedWorkbench(Exporter):
             # target is not supported yet
             continue
 
-    def generate(self):
+    def generate(self, progen_build=False):
         """ Generates the project files """
         project_data = self.progen_get_project_data()
         tool_specific = {}
@@ -75,7 +75,10 @@ def generate(self):
         # VLA is enabled via template IccAllowVLA
         project_data['tool_specific']['iar']['misc']['c_flags'].remove("--vla")
         project_data['common']['build_dir'] = os.path.join(project_data['common']['build_dir'], 'iar_arm')
-        self.progen_gen_file('iar_arm', project_data)
+        if progen_build:
+            self.progen_gen_file('iar_arm', project_data, True)
+        else:
+            self.progen_gen_file('iar_arm', project_data)
 
 # Currently not used, we should reuse folder_name to create virtual folders
 class IarFolder():

tools/export/uvision4.py

@@ -50,7 +50,7 @@ class Uvision4(Exporter):
     def get_toolchain(self):
         return TARGET_MAP[self.target].default_toolchain
 
-    def generate(self):
+    def generate(self, progen_build=False):
         """ Generates the project files """
         project_data = self.progen_get_project_data()
         tool_specific = {}
@@ -96,4 +96,7 @@ def generate(self):
             i += 1
         project_data['common']['macros'].append('__ASSERT_MSG')
         project_data['common']['build_dir'] = join(project_data['common']['build_dir'], 'uvision4')
-        self.progen_gen_file('uvision', project_data)
+        if progen_build:
+            self.progen_gen_file('uvision', project_data, True)
+        else:
+            self.progen_gen_file('uvision', project_data)

tools/export/uvision5.py

@@ -50,7 +50,7 @@ class Uvision5(Exporter):
     def get_toolchain(self):
         return TARGET_MAP[self.target].default_toolchain
 
-    def generate(self):
+    def generate(self, progen_build=False):
         """ Generates the project files """
         project_data = self.progen_get_project_data()
         tool_specific = {}
@@ -94,5 +94,8 @@ def generate(self):
             if macro.startswith('MBED_USERNAME'):
                 project_data['common']['macros'].pop(i)
             i += 1
-        project_data['common']['macros'].append('__ASSERT_MSG')
-        self.progen_gen_file('uvision5', project_data)
+
+        if progen_build:
+            self.progen_gen_file('uvision5', project_data, True)
+        else:
+            self.progen_gen_file('uvision5', project_data)

tools/project.py

@@ -5,19 +5,16 @@
 
 from shutil import move, rmtree
 from argparse import ArgumentParser
-from os import path
 
-from tools.paths import EXPORT_DIR, EXPORT_WORKSPACE, EXPORT_TMP
-from tools.paths import MBED_BASE, MBED_LIBRARIES
-from tools.export import export, setup_user_prj, EXPORTERS, mcu_ide_matrix
-from tools.utils import args_error, mkdir
-from tools.tests import TESTS, Test, TEST_MAP
+from tools.paths import EXPORT_DIR
+from tools.export import EXPORTERS, mcu_ide_matrix
+from tools.utils import args_error
+from tools.tests import TESTS, TEST_MAP
 from tools.tests import test_known, test_name_known
 from tools.targets import TARGET_NAMES
-from tools.libraries import LIBRARIES
-from utils import argparse_lowercase_type, argparse_uppercase_type, argparse_filestring_type, argparse_many
+from utils import argparse_filestring_type, argparse_many
 from utils import argparse_force_lowercase_type, argparse_force_uppercase_type
-
+from project_api import setup_project, perform_export, print_results, get_test_from_name
 
 
 if __name__ == '__main__':
@@ -128,77 +125,68 @@
         if exists(EXPORT_DIR):
             rmtree(EXPORT_DIR)
 
+
+    # Target
+    if options.mcu is None :
+        args_error(parser, "[ERROR] You should specify an MCU")
+    mcus = options.mcu
+
+    # IDE
+    if options.ide is None:
+        args_error(parser, "[ERROR] You should specify an IDE")
+    ide = options.ide
+
+    # Program Number or name
+    p, n, src= options.program, options.program_name, options.source_dir
+
+    if src is None:
+        if p is not None and n is not None:
+            args_error(parser, "[ERROR] specify either '-n' or '-p', not both")
+        if n:
+            p = get_test_from_name(n)
+            if p is None:
+                args_error(parser, "[ERROR] Program with name '%s' not found" % n)
+
+        if p is None or (p < 0) or (p > (len(TESTS) - 1)):
+            message = "[ERROR] You have to specify one of the following tests:\n"
+            message += '\n'.join(map(str, sorted(TEST_MAP.values())))
+            args_error(parser, message)
+
+
     # Export results
     successes = []
     failures = []
-    zip = True
-    clean = True
+
+    # source is used to generate IDE files to toolchain directly in the source tree and doesn't generate zip file
+    zip = src is None
+    clean = src is None
 
     # source_dir = use relative paths, otherwise sources are copied
     sources_relative = True if options.source_dir else False
+    for mcu in mcus.split(','):
+
+        lib_symbols = []
+        if options.macros:
+            lib_symbols += options.macros
+        project_dir, project_name, project_temp = setup_project(mcu, ide, p, src,
+                                                                options.macros,
+                                                                options.build)
+
+        tmp_path, report = perform_export(project_dir, project_name, ide, mcu,
+                                          project_temp, clean, zip, lib_symbols,
+                                          sources_relative)
+
 
-    for mcu in options.mcu:
-        # Program Number or name
-        p, src, ide = options.program, options.source_dir, options.ide
-
-        if src:
-            # --source is used to generate IDE files to toolchain directly in the source tree and doesn't generate zip file
-            project_dir = options.source_dir
-            project_name = TESTS[p] if p else "Unnamed_project"
-            project_temp = path.join(options.source_dir[0], 'projectfiles', '%s_%s' % (ide, mcu))
-            mkdir(project_temp)
-            lib_symbols = []
-            if options.macros:
-                lib_symbols += options.macros
-            zip = False   # don't create zip
-            clean = False # don't cleanup because we use the actual source tree to generate IDE files
-        else:
-            test = Test(p)
-
-            # Some libraries have extra macros (called by exporter symbols) to we need to pass
-            # them to maintain compilation macros integrity between compiled library and
-            # header files we might use with it
-            lib_symbols = []
-            if options.macros:
-                lib_symbols += options.macros
-            for lib in LIBRARIES:
-                if lib['build_dir'] in test.dependencies:
-                    lib_macros = lib.get('macros', None)
-                    if lib_macros is not None:
-                        lib_symbols.extend(lib_macros)
-
-            if not options.build:
-                # Substitute the library builds with the sources
-                # TODO: Substitute also the other library build paths
-                if MBED_LIBRARIES in test.dependencies:
-                    test.dependencies.remove(MBED_LIBRARIES)
-                    test.dependencies.append(MBED_BASE)
-
-            # Build the project with the same directory structure of the mbed online IDE
-            project_name = test.id
-            project_dir = [join(EXPORT_WORKSPACE, project_name)]
-            project_temp = EXPORT_TMP
-            setup_user_prj(project_dir[0], test.source_dir, test.dependencies)
-
-        # Export to selected toolchain
-        tmp_path, report = export(project_dir, project_name, ide, mcu, project_dir[0], project_temp, clean=clean, make_zip=zip, extra_symbols=lib_symbols, sources_relative=sources_relative)
         if report['success']:
             if not zip:
                 zip_path = join(project_temp, project_name)
             else:
                 zip_path = join(EXPORT_DIR, "%s_%s_%s.zip" % (project_name, ide, mcu))
                 move(tmp_path, zip_path)
+
             successes.append("%s::%s\t%s"% (mcu, ide, zip_path))
         else:
             failures.append("%s::%s\t%s"% (mcu, ide, report['errormsg']))
 
     # Prints export results
-    print
-    if len(successes) > 0:
-        print "Successful exports:"
-        for success in successes:
-            print "  * %s"% success
-    if len(failures) > 0:
-        print "Failed exports:"
-        for failure in failures:
-            print "  * %s"% failure
+    print_results(successes, failures)