Merge pull request #5848 from theotherjimmy/compile-py3

Python2+3: mbed compile, mbed test --compile,  python unit tests

Author: cmonr

.travis.yml

@@ -10,7 +10,7 @@ env:
       --data @- << DATA\n{
       "state": "$0",
       "description": "$1",
-      "context": "travis-ci/$NAME",
+      "context": "travis-ci/$NAME/$(python --version)",
       "target_url": "https://travis-ci.org/$TRAVIS_REPO_SLUG/jobs/$TRAVIS_JOB_ID"
       }\nDATA'
 
@@ -73,6 +73,10 @@ matrix:
 
     - env:
         - NAME=tools
+      python:
+        - '2.7'
+        - '3.5'
+        - '3.6'
       install:
         # Install dependencies
         - sudo apt-get install gcc-arm-embedded
@@ -175,7 +179,7 @@ matrix:
         - mkdir BUILD
       script:
         # Run local mbed 2 testing
-        - python2 -u tools/build_travis.py --vendor "${NAME#mbed2-}"
+        - python -u tools/build_travis.py --vendor "${NAME#mbed2-}"
     - <<: *mbed-2
       env: NAME=mbed2-STM
     - <<: *mbed-2
@@ -190,3 +194,8 @@ matrix:
       env: NAME=mbed2-NUVOTON
     - <<: *mbed-2
       env: NAME=mbed2-RENESAS
+      # Change python version here only because 3x the other jobs does not add any more coverage
+      python:
+        - '2.7'
+        - '3.5'
+        - '3.6'

tools/arm_pack_manager/__init__.py

@@ -1,10 +1,16 @@
-from urllib2 import urlopen, URLError
+try:
+    from urllib2 import urlopen, URLError
+except ImportError:
+    from urllib.request import urlopen, URLError
 from bs4 import BeautifulSoup
 from os.path import join, dirname, basename
 from os import makedirs
 from errno import EEXIST
 from threading import Thread
-from Queue import Queue
+try:
+    from Queue import Queue
+except ImportError:
+    from queue import Queue
 from re import compile, sub
 from sys import stderr, stdout
 from itertools import takewhile

tools/arm_pack_manager/pack_manager.py

@@ -1,11 +1,12 @@
+from __future__ import print_function, division, absolute_import
 import argparse
 from os.path import basename
 from tools.arm_pack_manager import Cache
 from os.path import basename, join, dirname, exists
 from os import makedirs
 from itertools import takewhile
 from fuzzywuzzy import process
-from tools.arm_pack_manager import Cache
+from .arm_pack_manager import Cache
 
 parser = argparse.ArgumentParser(description='A Handy little utility for keeping your cache of pack files up to date.')
 subparsers = parser.add_subparsers(title="Commands")
@@ -69,7 +70,7 @@ def fuzzy_find(matches, urls) :
         for key, value in process.extract(match, urls, limit=None) :
             choices.setdefault(key, 0)
             choices[key] += value
-    choices = sorted([(v, k) for k, v in choices.iteritems()], reverse=True)
+    choices = sorted([(v, k) for k, v in choices.items()], reverse=True)
     if not choices : return []
     elif len(choices) == 1 : return [choices[0][1]]
     elif choices[0][0] > choices[1][0] : choices = choices[:1]
@@ -133,12 +134,12 @@ def command_find_part (cache, matches, long=False, intersection=True,
         aliases = sum([fuzzy_find([m], cache.aliases.keys()) for m in matches], [])
     if print_parts:
         for part in choices :
-            print part
+            print(part)
             if long :
                 pp.pprint(cache.index[part])
     if print_aliases:
         for alias in aliases :
-            print alias
+            print(alias)
             if long :
                 pp.pprint(cache.index[cache.aliases[alias]])
 
@@ -155,7 +156,7 @@ def command_dump_parts (cache, out, parts, intersection=False) :
     else :
         for part in parts :
             index.update(dict(cache.find_device(part)))
-    for n, p in index.iteritems() :
+    for n, p in index.items() :
         try :
             if not exists(join(out, dirname(p['algorithm']['file']))) :
                 makedirs(join(out, dirname(p['algorithm']['file'])))

tools/build.py

@@ -17,6 +17,8 @@
 
 LIBRARIES BUILD
 """
+from __future__ import print_function, division, absolute_import
+
 import sys
 from time import time
 from os.path import join, abspath, dirname
@@ -130,7 +132,7 @@
 
     # Only prints matrix of supported toolchains
     if options.supported_toolchains:
-        print mcu_toolchain_matrix(platform_filter=options.general_filter_regex)
+        print(mcu_toolchain_matrix(platform_filter=options.general_filter_regex))
         exit(0)
 
 
@@ -184,7 +186,7 @@
             tt_id = "%s::%s" % (toolchain, target)
             if toolchain not in TARGET_MAP[target].supported_toolchains:
                 # Log this later
-                print "%s skipped: toolchain not supported" % tt_id
+                print("%s skipped: toolchain not supported" % tt_id)
                 skipped.append(tt_id)
             else:
                 try:
@@ -224,26 +226,24 @@
                         successes.append(tt_id)
                     else:
                         skipped.append(tt_id)
-                except Exception, e:
+                except Exception as e:
                     if options.verbose:
                         import traceback
                         traceback.print_exc(file=sys.stdout)
                         sys.exit(1)
                     failures.append(tt_id)
-                    print e
+                    print(e)
 
 
     # Write summary of the builds
-    print
-    print "Completed in: (%.2f)s" % (time() - start)
-    print
+    print("\nCompleted in: (%.2f)s\n" % (time() - start))
 
     for report, report_name in [(successes, "Build successes:"),
                                 (skipped, "Build skipped:"),
                                 (failures, "Build failures:"),
                                ]:
         if report:
-            print print_build_results(report, report_name),
+            print(print_build_results(report, report_name))
 
     if failures:
         sys.exit(1)

tools/build_api.py

@@ -14,33 +14,35 @@
 See the License for the specific language governing permissions and
 limitations under the License.
 """
+from __future__ import print_function, division, absolute_import
 
 import re
 import tempfile
 import datetime
 import uuid
-from types import ListType
 from shutil import rmtree
 from os.path import join, exists, dirname, basename, abspath, normpath, splitext
 from os.path import relpath
 from os import linesep, remove, makedirs
 from time import time
 from intelhex import IntelHex
 from json import load, dump
-from tools.arm_pack_manager import Cache
-
-from tools.utils import mkdir, run_cmd, run_cmd_ext, NotSupportedException,\
-    ToolException, InvalidReleaseTargetException, intelhex_offset
-from tools.paths import MBED_CMSIS_PATH, MBED_TARGETS_PATH, MBED_LIBRARIES,\
-    MBED_HEADER, MBED_DRIVERS, MBED_PLATFORM, MBED_HAL, MBED_CONFIG_FILE,\
-    MBED_LIBRARIES_DRIVERS, MBED_LIBRARIES_PLATFORM, MBED_LIBRARIES_HAL,\
-    BUILD_DIR
-from tools.targets import TARGET_NAMES, TARGET_MAP
-from tools.libraries import Library
-from tools.toolchains import TOOLCHAIN_CLASSES
 from jinja2 import FileSystemLoader
 from jinja2.environment import Environment
-from tools.config import Config
+
+from .arm_pack_manager import Cache
+from .utils import (mkdir, run_cmd, run_cmd_ext, NotSupportedException,
+                    ToolException, InvalidReleaseTargetException,
+                    intelhex_offset)
+from .paths import (MBED_CMSIS_PATH, MBED_TARGETS_PATH, MBED_LIBRARIES,
+                    MBED_HEADER, MBED_DRIVERS, MBED_PLATFORM, MBED_HAL,
+                    MBED_CONFIG_FILE, MBED_LIBRARIES_DRIVERS,
+                    MBED_LIBRARIES_PLATFORM, MBED_LIBRARIES_HAL,
+                    BUILD_DIR)
+from .targets import TARGET_NAMES, TARGET_MAP
+from .libraries import Library
+from .toolchains import TOOLCHAIN_CLASSES
+from .config import Config
 
 RELEASE_VERSIONS = ['2', '5']
 
@@ -124,7 +126,7 @@ def get_config(src_paths, target, toolchain_name):
     toolchain_name - the string that identifies the build tools
     """
     # Convert src_paths to a list if needed
-    if type(src_paths) != ListType:
+    if not isinstance(src_paths, list):
         src_paths = [src_paths]
 
     # Pass all params to the unified prepare_resources()
@@ -404,7 +406,7 @@ def scan_resources(src_paths, toolchain, dependencies_paths=None,
 
     # Add additional include directories if passed
     if inc_dirs:
-        if type(inc_dirs) == ListType:
+        if isinstance(inc_dirs, list):
             resources.inc_dirs.extend(inc_dirs)
         else:
             resources.inc_dirs.append(inc_dirs)
@@ -462,7 +464,7 @@ def build_project(src_paths, build_path, target, toolchain_name,
     """
 
     # Convert src_path to a list if needed
-    if type(src_paths) != ListType:
+    if not isinstance(src_paths, list):
         src_paths = [src_paths]
     # Extend src_paths wiht libraries_paths
     if libraries_paths is not None:
@@ -533,9 +535,9 @@ def build_project(src_paths, build_path, target, toolchain_name,
                     memap_bars = memap_instance.generate_output('bars',
                             real_stats_depth, None,
                             getattr(toolchain.target, 'device_name', None))
-                    print memap_bars
+                    print(memap_bars)
                 else:
-                    print memap_table
+                    print(memap_table)
 
             # Write output to file in JSON format
             map_out = join(build_path, name + "_map.json")
@@ -619,7 +621,7 @@ def build_library(src_paths, build_path, target, toolchain_name,
     """
 
     # Convert src_path to a list if needed
-    if type(src_paths) != ListType:
+    if not isinstance(src_paths, list):
         src_paths = [src_paths]
 
     # Build path
@@ -788,7 +790,7 @@ def build_lib(lib_id, target, toolchain_name, verbose=False,
     inc_dirs = lib.inc_dirs
     inc_dirs_ext = lib.inc_dirs_ext
 
-    if type(src_paths) != ListType:
+    if not isinstance(src_paths, list):
         src_paths = [src_paths]
 
     # The first path will give the name to the library

tools/build_everything.py

@@ -154,7 +154,7 @@
         test_builds = {}
         total_build_success = True
 
-        for target_name, target_toolchains in build_config.iteritems():
+        for target_name, target_toolchains in build_config.items():
             target = TARGET_MAP[target_name]
 
             for target_toolchain in target_toolchains:

tools/build_travis.py

@@ -18,6 +18,7 @@
 See the License for the specific language governing permissions and
 limitations under the License.
 """
+from __future__ import print_function, division, absolute_import
 
 import os
 import sys
@@ -382,11 +383,12 @@ def run_builds(dry_run, vendor):
                 toolchain_list = build["toolchains"]
                 if type(toolchain_list) != type([]): toolchain_list = [toolchain_list]
                 for toolchain in toolchain_list:
-                    cmdline = "python tools/build.py -m %s -t %s -c --silent "% (build["target"], toolchain)
+                    cmdline = ("%s tools/build.py -m %s -t %s -c --silent "%
+                               (sys.executable, build["target"], toolchain))
                     libs = build.get("libs", [])
                     if libs:
                         cmdline = cmdline + " ".join(["--" + l for l in libs])
-                    print "Executing: " + cmdline
+                    print("Executing: %s" % cmdline)
                     if not dry_run:
                         if os.system(cmdline) != 0:
                             sys.exit(1)
@@ -408,19 +410,15 @@ def run_test_linking(dry_run, vendor):
                     for test_lib in tests:
                         test_names = tests[test_lib]
                         test_lib_switch = "--" + test_lib if test_lib else ""
-                        cmdline = "python tools/make.py -m %s -t %s -c --silent %s -n %s " % (link["target"], toolchain, test_lib_switch, ",".join(test_names))
-                        print "Executing: " + cmdline
+                        cmdline = ("%s tools/make.py -m %s -t %s -c --silent %s "
+                                   "-n %s" % (sys.executable, link["target"],
+                                               toolchain, test_lib_switch,
+                                               ",".join(test_names)))
+                        print("Executing: %s" % cmdline)
                         if not dry_run:
                             if os.system(cmdline) != 0:
                                 sys.exit(1)
 
-def run_test_testsuite(dry_run, vendor):
-    cmdline = "python tools/singletest.py --version"
-    print "Executing: " + cmdline
-    if not dry_run:
-        if os.system(cmdline) != 0:
-            sys.exit(1)
-
 if __name__ == "__main__":
     parser = ArgumentParser()
 
@@ -434,4 +432,3 @@ def run_test_testsuite(dry_run, vendor):
 
     run_builds("-s" in sys.argv, options.vendor)
     run_test_linking("-s" in sys.argv, options.vendor)
-    run_test_testsuite("-s" in sys.argv, options.vendor)