Merge pull request #3172 from sarahmarshy/export-test-rev

[Exporters] New export-build tests

Author: sg-

tools/export/__init__.py

@@ -18,7 +18,6 @@
 from tools.export import codered, ds5_5, iar, makefile
 from tools.export import emblocks, coide, kds, simplicityv3, atmelstudio
 from tools.export import sw4stm32, e2studio, zip, cmsis, uvision, cdt
-from tools.export.exporters import OldLibrariesException, FailedBuildException
 from tools.targets import TARGET_NAMES
 
 EXPORTERS = {

tools/export/cmsis/__init__.py

@@ -30,38 +30,41 @@ class DeviceCMSIS():
     """CMSIS Device class
 
     Encapsulates target information retrieved by arm-pack-manager"""
-    def __init__(self, target):
-        cache = Cache(True, False)
 
-        t = TARGET_MAP[target]
-        self.core = t.core
-        try:
-            cpu_name = t.device_name
-            target_info = cache.index[cpu_name]
-        # Target does not have device name or pdsc file
-        except:
-            try:
-                # Try to find the core as a generic CMSIS target
-                cpu_name = self.cpu_cmsis()
-                target_info = cache.index[cpu_name]
-            except:
-                raise TargetNotSupportedException("Target not in CMSIS packs")
-
-        self.target_info = target_info
+    CACHE = Cache(True, False)
+    def __init__(self, target):
+        target_info = self.check_supported(target)
+        if not target_info:
+            raise TargetNotSupportedException("Target not supported in CMSIS pack")
 
         self.url = target_info['pdsc_file']
         self.pack_url, self.pack_id = ntpath.split(self.url)
-        self.dname = cpu_name
+        self.dname = target_info["_cpu_name"]
+        self.core = target_info["_core"]
         self.dfpu = target_info['processor']['fpu']
         self.debug, self.dvendor = self.vendor_debug(target_info['vendor'])
         self.dendian = target_info['processor'].get('endianness','Little-endian')
         self.debug_svd = target_info.get('debug', '')
         self.compile_header = target_info['compile']['header']
+        self.target_info = target_info
 
-    def check_version(self, filename):
-        with open(filename) as data_file:
-            data = json.load(data_file)
-            return data.get("version", "0") == "0.1.0"
+    @staticmethod
+    def check_supported(target):
+        t = TARGET_MAP[target]
+        try:
+            cpu_name = t.device_name
+            target_info = DeviceCMSIS.CACHE.index[cpu_name]
+        # Target does not have device name or pdsc file
+        except:
+            try:
+                # Try to find the core as a generic CMSIS target
+                cpu_name = DeviceCMSIS.cpu_cmsis(t.core)
+                target_info = DeviceCMSIS.index[cpu_name]
+            except:
+                return False
+        target_info["_cpu_name"] = cpu_name
+        target_info["_core"] = t.core
+        return target_info
 
     def vendor_debug(self, vendor):
         reg = "([\w\s]+):?\d*?"
@@ -74,9 +77,9 @@ def vendor_debug(self, vendor):
         }
         return debug_map.get(vendor_match, "CMSIS-DAP"), vendor_match
 
-    def cpu_cmsis(self):
+    @staticmethod
+    def cpu_cmsis(cpu):
         #Cortex-M4F => ARMCM4_FP, Cortex-M0+ => ARMCM0P
-        cpu = self.core
         cpu = cpu.replace("Cortex-","ARMC")
         cpu = cpu.replace("+","P")
         cpu = cpu.replace("F","_FP")

tools/export/exporters.py

@@ -11,16 +11,6 @@
 from tools.targets import TARGET_MAP
 
 
-class OldLibrariesException(Exception):
-    """Exception that indicates an export can not complete due to an out of date
-    library version.
-    """
-    pass
-
-class FailedBuildException(Exception):
-    """Exception that indicates that a build failed"""
-    pass
-
 class TargetNotSupportedException(Exception):
     """Indicates that an IDE does not support a particular MCU"""
     pass
@@ -119,13 +109,6 @@ def get_source_paths(self):
             source_files.extend(getattr(self.resources, key))
         return list(set([os.path.dirname(src) for src in source_files]))
 
-    def check_supported(self):
-        """Indicated if this combination of IDE and MCU is supported"""
-        if self.target not in self.TARGETS or \
-           self.TOOLCHAIN not in TARGET_MAP[self.target].supported_toolchains:
-            raise TargetNotSupportedException()
-        return True
-
     def gen_file(self, template_file, data, target_file):
         """Generates a project file from a template using jinja"""
         jinja_loader = FileSystemLoader(
@@ -153,9 +136,31 @@ def make_key(self, src):
     def group_project_files(self, sources):
         """Group the source files by their encompassing directory
         Positional Arguments:
-        sources - array of sourc locations
+        sources - array of source locations
 
         Returns a dictionary of {group name: list of source locations}
         """
         data = sorted(sources, key=self.make_key)
         return {k: list(g) for k,g in groupby(data, self.make_key)}
+
+    @staticmethod
+    def build(project_name, log_name='build_log.txt', cleanup=True):
+        """Invoke exporters build command within a subprocess.
+        This method is assumed to be executed at the same level as exporter
+        project files and project source code.
+        See uvision/__init__.py, iar/__init__.py, and makefile/__init__.py for
+        example implemenation.
+
+        Positional Arguments:
+        project_name - the name of the project to build; often required by
+        exporter's build command.
+
+        Keyword Args:
+        log_name - name of the build log to create. Written and printed out,
+        deleted if cleanup = True
+        cleanup - a boolean dictating whether exported project files and
+        build log are removed after build
+
+        Returns -1 on failure and 0 on success
+        """
+        raise NotImplemented("Implement in derived Exporter class.")

tools/export/iar/__init__.py

@@ -2,12 +2,12 @@
 from os.path import sep, join, exists
 from collections import namedtuple
 from subprocess import Popen, PIPE
-from distutils.spawn import find_executable
+import shutil
 import re
 import sys
 
 from tools.targets import TARGET_MAP
-from tools.export.exporters import Exporter, FailedBuildException
+from tools.export.exporters import Exporter
 import json
 from tools.export.cmsis import DeviceCMSIS
 from multiprocessing import cpu_count
@@ -29,7 +29,8 @@ class IAR(Exporter):
     #iar_definitions.json
     TARGETS = [target for target, obj in TARGET_MAP.iteritems()
                if hasattr(obj, 'device_name') and
-               obj.device_name in IAR_DEFS.keys()]
+               obj.device_name in IAR_DEFS.keys() and "IAR" in obj.supported_toolchains
+               and DeviceCMSIS.check_supported(target)]
 
     SPECIAL_TEMPLATES = {
         'rz_a1h'  : 'iar/iar_rz_a1h.ewp.tmpl',
@@ -120,22 +121,13 @@ def generate(self):
         self.gen_file('iar/ewd.tmpl', ctx, self.project_name + ".ewd")
         self.gen_file(self.get_ewp_template(), ctx, self.project_name + ".ewp")
 
-    def build(self):
+    @staticmethod
+    def build(project_name, cleanup=True):
         """ Build IAR project """
         # > IarBuild [project_path] -build [project_name]
-        proj_file = join(self.export_dir, self.project_name + ".ewp")
 
-        if find_executable("IarBuild"):
-            iar_exe = "IarBuild.exe"
-        else:
-            iar_exe = join('C:', sep,
-                          'Program Files (x86)', 'IAR Systems',
-                          'Embedded Workbench 7.5', 'common', 'bin',
-                          'IarBuild.exe')
-            if not exists(iar_exe):
-                raise Exception("IarBuild.exe not found. Add to path.")
-
-        cmd = [iar_exe, proj_file, '-build', self.project_name]
+        proj_file = project_name + ".ewp"
+        cmd = ["IarBuild.exe", proj_file, '-build', project_name]
 
         # IAR does not support a '0' option to automatically use all
         # available CPUs, so we use Python's multiprocessing library
@@ -156,7 +148,14 @@ def build(self):
             m = re.match(error_re, line)
             if m is not None:
                 num_errors = int(m.group(1))
+
+        if cleanup:
+            os.remove(project_name + ".ewp")
+            os.remove(project_name + ".ewd")
+            os.remove(project_name + ".eww")
+            shutil.rmtree('.build')
+
         if num_errors !=0:
             # Seems like something went wrong.
-            raise FailedBuildException("Project: %s build failed with %s erros" % (
-            proj_file, num_errors))
+            return -1
+        return 0

tools/export/makefile/__init__.py

@@ -16,7 +16,10 @@
 """
 from os.path import splitext, basename, relpath, join, abspath, dirname,\
     exists
-from os import curdir, getcwd
+from os import remove
+import sys
+from subprocess import check_output, CalledProcessError, Popen, PIPE
+import shutil
 from jinja2.exceptions import TemplateNotFound
 from tools.export.exporters import Exporter
 from tools.utils import NotSupportedException
@@ -102,6 +105,38 @@ def generate(self):
         else:
             raise NotSupportedException("This make tool is in development")
 
+    @staticmethod
+    def build(project_name, log_name="build_log.txt", cleanup=True):
+        """ Build Make project """
+        # > Make -j
+        cmd = ["make", "-j"]
+        p = Popen(cmd, stdout=PIPE, stderr=PIPE)
+        ret = p.communicate()
+        out, err = ret[0], ret[1]
+        ret_code = p.returncode
+        with open(log_name, 'w+') as f:
+            f.write("=" * 10 + "OUT" + "=" * 10 + "\n")
+            f.write(out)
+            f.write("=" * 10 + "ERR" + "=" * 10 + "\n")
+            f.write(err)
+            if ret_code == 0:
+                f.write("SUCCESS")
+            else:
+                f.write("FAILURE")
+        with open(log_name, 'r') as f:
+            print "\n".join(f.readlines())
+        sys.stdout.flush()
+
+        if cleanup:
+            remove("Makefile")
+            remove(log_name)
+            if exists('.build'):
+                shutil.rmtree('.build')
+        if ret_code != 0:
+            # Seems like something went wrong.
+            return -1
+        return 0
+
 
 class GccArm(Makefile):
     """GCC ARM specific makefile target"""

tools/export/uvision/__init__.py

@@ -3,13 +3,13 @@
 import ntpath
 import copy
 from collections import namedtuple
-from distutils.spawn import find_executable
+import shutil
 import subprocess
 import re
 
 from tools.arm_pack_manager import Cache
 from tools.targets import TARGET_MAP
-from tools.export.exporters import Exporter, FailedBuildException
+from tools.export.exporters import Exporter
 from tools.export.cmsis import DeviceCMSIS
 
 cache_d = False
@@ -117,10 +117,15 @@ class Uvision(Exporter):
     project file (.uvprojx).
     The needed information can be viewed in uvision.tmpl
     """
-    NAME = 'cmsis'
+    NAME = 'uvision5'
     TOOLCHAIN = 'ARM'
-    TARGETS = [target for target, obj in TARGET_MAP.iteritems()
-               if "ARM" in obj.supported_toolchains]
+    TARGETS = []
+    for target, obj in TARGET_MAP.iteritems():
+        if not ("ARM" in obj.supported_toolchains and hasattr(obj, "device_name")):
+            continue
+        if not DeviceCMSIS.check_supported(target):
+            continue
+        TARGETS.append(target)
     #File associations within .uvprojx file
     file_types = {'.cpp': 8, '.c': 1, '.s': 2,
                   '.obj': 3, '.o': 3, '.lib': 4,
@@ -200,35 +205,24 @@ def generate(self):
         self.gen_file('uvision/uvision.tmpl', ctx, self.project_name+".uvprojx")
         self.gen_file('uvision/uvision_debug.tmpl', ctx, self.project_name + ".uvoptx")
 
-    def build(self):
-        ERRORLEVEL = {
-            0: 'success (0 warnings, 0 errors)',
-            1: 'warnings',
-            2: 'errors',
-            3: 'fatal errors',
-            11: 'cant write to project file',
-            12: 'device error',
-            13: 'error writing',
-            15: 'error reading xml file',
-        }
+    @staticmethod
+    def build(project_name, log_name='build_log.txt', cleanup=True):
+        """ Build Uvision project """
+        # > UV4.exe -r -j0 -o [log_name] [project_name].uvprojx
         success = 0
         warn  = 1
-        if find_executable("UV4"):
-            uv_exe = "UV4.exe"
-        else:
-            uv_exe = join('C:', sep,
-            'Keil_v5', 'UV4', 'UV4.exe')
-            if not exists(uv_exe):
-                raise Exception("UV4.exe not found. Add to path.")
-        cmd = [uv_exe, '-r', '-j0', '-o', join(self.export_dir,'build_log.txt'), join(self.export_dir,self.project_name+".uvprojx")]
+        cmd = ["UV4.exe", '-r', '-j0', '-o', log_name, project_name+".uvprojx"]
         ret_code = subprocess.call(cmd)
-        with open(join(self.export_dir, 'build_log.txt'), 'r') as build_log:
+        with open(log_name, 'r') as build_log:
             print build_log.read()
+        if cleanup:
+            os.remove(log_name)
+            os.remove(project_name+".uvprojx")
+            os.remove(project_name+".uvoptx")
+            shutil.rmtree(".build")
+
 
         if ret_code != success and ret_code != warn:
             # Seems like something went wrong.
-            raise FailedBuildException("Project: %s build failed with the status: %s" % (
-                self.project_name, ERRORLEVEL.get(ret_code, "Unknown")))
-        else:
-            return "Project: %s build succeeded with the status: %s" % (
-            self.project_name, ERRORLEVEL.get(ret_code, "Unknown"))
+            return -1
+        return 0

tools/project.py

@@ -233,7 +233,9 @@ def main():
     if (options.program is None) and (not options.source_dir):
         args_error(parser, "one of -p, -n, or --source is required")
         # Export to selected toolchain
-    _, toolchain_name = get_exporter_toolchain(options.ide)
+    exporter, toolchain_name = get_exporter_toolchain(options.ide)
+    if options.mcu not in exporter.TARGETS:
+        args_error(parser, "%s not supported by %s"%(options.mcu,options.ide))
     profile = extract_profile(parser, options, toolchain_name)
     export(options.mcu, options.ide, build=options.build,
            src=options.source_dir, macros=options.macros,

tools/project_api.py

@@ -86,7 +86,6 @@ def generate_project_files(resources, export_path, target, name, toolchain, ide,
     exporter_cls, _ = get_exporter_toolchain(ide)
     exporter = exporter_cls(target, export_path, name, toolchain,
                             extra_symbols=macros, resources=resources)
-    exporter.check_supported()
     exporter.generate()
     files = exporter.generated_files
     return files, exporter