Initial Python 3 compatibility fixes

benchmark/scripts/Benchmark_Driver

@@ -26,6 +26,7 @@ class `BenchmarkDoctor` analyzes performance tests, implements `check` COMMAND.
 """
 
 import argparse
+import functools
 import glob
 import logging
 import math
@@ -37,6 +38,9 @@ import time
 
 from compare_perf_tests import LogParser
 
+if not hasattr(sys.modules['__builtin__'], 'reduce'):
+    reduce = functools.reduce
+
 DRIVER_DIR = os.path.dirname(os.path.realpath(__file__))
 
 
@@ -178,14 +182,10 @@ class BenchmarkDriver(object):
 
         Returns the aggregated result of independent benchmark invocations.
         """
-        def merge_results(a, b):
-            a.merge(b)
-            return a
-
-        return reduce(merge_results,
-                      [self.run(test, measure_memory=True,
+        result_sets = [self.run(test, measure_memory=True,
                                 num_iters=1, quantile=20)
-                       for _ in range(self.args.independent_samples)])
+                       for _ in range(self.args.independent_samples)]
+        return reduce(lambda m, res: (m.merge(res), m)[1], result_sets)
 
     def log_results(self, output, log_file=None):
         """Log output to `log_file`.
@@ -303,8 +303,12 @@ class MarkdownReportHandler(logging.StreamHandler):
         msg = self.format(record)
         stream = self.stream
         try:
-            if (isinstance(msg, unicode) and
-                    getattr(stream, 'encoding', None)):
+            is_text = isinstance(msg, unicode)
+        except NameError:
+            is_text = isinstance(msg, bytes)
+
+        try:
+            if is_text and getattr(stream, 'encoding', None):
                 stream.write(msg.encode(stream.encoding))
             else:
                 stream.write(msg)
@@ -414,6 +418,7 @@ class BenchmarkDoctor(object):
         name = measurements['name']
         setup, ratio = BenchmarkDoctor._setup_overhead(measurements)
         setup = 0 if ratio < 0.05 else setup
+
         runtime = min(
             [(result.samples.min - correction) for i_series in
              [BenchmarkDoctor._select(measurements, num_iters=i)

benchmark/scripts/compare_perf_tests.py

@@ -30,12 +30,15 @@ class `ReportFormatter` creates the test comparison report in specified format.
 from __future__ import print_function
 
 import argparse
+import functools
 import re
 import sys
 from bisect import bisect, bisect_left, bisect_right
 from collections import namedtuple
 from math import ceil, sqrt
 
+if not hasattr(sys.modules['__builtin__'], 'reduce'):
+    reduce = functools.reduce
 
 class Sample(namedtuple('Sample', 'i num_iters runtime')):
     u"""Single benchmark measurement.
@@ -185,13 +188,14 @@ def sd(self):
                 sqrt(self.S_runtime / (self.count - 1)))
 
     @staticmethod
-    def running_mean_variance((k, M_, S_), x):
+    def running_mean_variance(kMS, x):
         """Compute running variance, B. P. Welford's method.
 
         See Knuth TAOCP vol 2, 3rd edition, page 232, or
         https://www.johndcook.com/blog/standard_deviation/
         M is mean, Standard Deviation is defined as sqrt(S/k-1)
         """
+        k, M_, S_ = kMS
         k = float(k + 1)
         M = M_ + (x - M_) / k
         S = S_ + (x - M_) * (x - M)

benchmark/scripts/test_Benchmark_Driver.py

@@ -18,7 +18,11 @@
 import time
 import unittest
 
-from StringIO import StringIO
+try:
+    from StringIO import StringIO # py2
+except ModuleNotFoundError:
+    from io import StringIO # py3
+
 from imp import load_source
 
 from compare_perf_tests import PerformanceTestResult

benchmark/scripts/test_utils.py

@@ -25,7 +25,11 @@
 import logging
 import sys
 
-from StringIO import StringIO
+try:
+    from StringIO import StringIO # py2
+except ModuleNotFoundError:
+    from io import StringIO # py3
+
 from contextlib import contextmanager
 
 

stdlib/public/core/IntegerTypes.swift.gyb

@@ -14,10 +14,10 @@
 # Utility code for later in this template
 #
 
+from __future__ import division
 from SwiftIntTypes import all_integer_types, int_max_bits, should_define_truncating_bit_pattern_init
 from SwiftFloatingPointTypes import getFtoIBounds
 
-from string import maketrans, capitalize
 from itertools import chain
 
 # Number of bits in the Builtin.Word type
@@ -1651,7 +1651,7 @@ extension ${Self} : Hashable {
     return Hasher._hash(
       seed: seed,
       bytes: UInt64(truncatingIfNeeded: ${U}${Self}(_value)),
-      count: ${bits / 8})
+      count: ${bits // 8})
     % end
   }
 }

stdlib/public/core/SIMDVectorTypes.swift.gyb

@@ -1,4 +1,10 @@
 %{
+from __future__ import division
+try:
+	xrange
+except NameError:
+	xrange = range
+
 from SwiftIntTypes import all_integer_types
 word_bits = int(CMAKE_SIZEOF_VOID_P) * 8
 storagescalarCounts = [2,4,8,16,32,64]
@@ -46,9 +52,9 @@ public struct SIMD${n}<Scalar> : SIMD where Scalar: SIMDScalar {
 
   /// Creates a new vector from the given elements.
   @_transparent
-  public init(${', '.join(['_ v' + str(i) + ': Scalar' for i in range(n)])}) {
+  public init(${', '.join(['_ v' + str(i) + ': Scalar' for i in list(range(n))])}) {
     self.init()
-% for i in range(n):
+% for i in list(range(n)):
     self[${i}] = v${i}
 % end
   }
@@ -65,7 +71,7 @@ public struct SIMD${n}<Scalar> : SIMD where Scalar: SIMDScalar {
     self.init(${', '.join('xyzw'[:n])})
   }
 
-%  for i in range(n):
+%  for i in list(range(n)):
   /// The ${ordinalPositions[i]} element of the vector.
   @_transparent
   public var ${'xyzw'[i]}: Scalar {
@@ -78,17 +84,17 @@ public struct SIMD${n}<Scalar> : SIMD where Scalar: SIMDScalar {
 % if n >= 4:
   /// Creates a new vector from two half-length vectors.
   @_transparent
-  public init(lowHalf: SIMD${n/2}<Scalar>, highHalf: SIMD${n/2}<Scalar>) {
+  public init(lowHalf: SIMD${n//2}<Scalar>, highHalf: SIMD${n//2}<Scalar>) {
     self.init()
     self.lowHalf = lowHalf
     self.highHalf = highHalf
   }
 
-%  for (half,indx) in [('low','i'), ('high',str(n/2)+'+i'), ('even','2*i'), ('odd','2*i+1')]:
+%  for (half,indx) in [('low','i'), ('high',str(n//2)+'+i'), ('even','2*i'), ('odd','2*i+1')]:
   /// A half-length vector made up of the ${half} elements of the vector.
-  public var ${half}Half: SIMD${n/2}<Scalar> {
+  public var ${half}Half: SIMD${n//2}<Scalar> {
     @inlinable get {
-      var result = SIMD${n/2}<Scalar>()
+      var result = SIMD${n//2}<Scalar>()
       for i in result.indices { result[i] = self[${indx}] }
       return result
     }
@@ -184,7 +190,7 @@ extension ${Self} : SIMDScalar {
   public typealias SIMDMaskScalar = ${Mask}
 
 % for n in storagescalarCounts:
-%  bytes = n * self_type.bits / 8
+%  bytes = n * self_type.bits // 8
   /// Storage for a vector of ${spelledNumbers[n]} integers.
   @_fixed_layout
   @_alignment(${bytes if bytes <= 16 else 16})
@@ -229,7 +235,7 @@ extension ${Self} : SIMDScalar {
   public typealias SIMDMaskScalar = Int${bits}
 
 % for n in storagescalarCounts:
-%  bytes = n * bits / 8
+%  bytes = n * bits // 8
   /// Storage for a vector of ${spelledNumbers[n]} floating-point values.
   @_fixed_layout
   @_alignment(${bytes if bytes <= 16 else 16})

stdlib/public/core/Tuple.swift.gyb

@@ -111,7 +111,7 @@ public func >=(lhs: (), rhs: ()) -> Bool {
 %   equatableTypeParams = ", ".join(["{} : Equatable".format(c) for c in typeParams])
 
 %   originalTuple = "(\"a\", {})".format(", ".join(map(str, range(1, arity))))
-%   greaterTuple = "(\"a\", {})".format(", ".join(map(str, range(1, arity - 1) + [arity])))
+%   greaterTuple = "(\"a\", {})".format(", ".join(map(str, list(range(1, arity - 1)) + [arity])))
 
 /// Returns a Boolean value indicating whether the corresponding components of
 /// two tuples are equal.

test/Driver/response-file-merge-modules.swift

@@ -1,4 +1,4 @@
-// RUN: %{python} -c 'for i in range(500001): print "-DTEST_" + str(i)' > %t.resp
+// RUN: %{python} -c 'for i in range(500001): print("-DTEST_{}".format(i))' > %t.resp
 // RUN: %swiftc_driver -driver-print-jobs -module-name merge -emit-module %S/Inputs/main.swift %S/Inputs/lib.swift @%t.resp 2>&1 > %t.jobs.txt
 // RUN: %FileCheck %s < %t.jobs.txt -check-prefix=MERGE
 

test/Driver/response-file.swift

@@ -2,7 +2,7 @@
 // RUN: %target-build-swift -typecheck @%t.0.resp %s 2>&1 | %FileCheck %s -check-prefix=SHORT
 // SHORT: warning: result of call to 'abs' is unused
 
-// RUN: %{python} -c 'for a in ["A", "B", "C", "D"]: print "-DTEST1" + a' > %t.1.resp
+// RUN: %{python} -c 'for a in ["A", "B", "C", "D"]: print("-DTEST1{}".format(a))' > %t.1.resp
 // RUN: %target-build-swift -typecheck @%t.1.resp %s 2>&1 | %FileCheck %s -check-prefix=MULTIPLE
 // MULTIPLE: warning: result of call to 'abs' is unused
 
@@ -24,7 +24,7 @@
 // RUN: %target-build-swift -typecheck @%t.4B.resp 2>&1 | %FileCheck %s -check-prefix=RECURSIVE
 // RECURSIVE: warning: result of call to 'abs' is unused
 
-// RUN: %{python} -c 'for i in range(500001): print "-DTEST5_" + str(i)' > %t.5.resp
+// RUN: %{python} -c 'for i in range(500001): print("-DTEST5_{}".format(i))' > %t.5.resp
 // RUN: %target-build-swift -typecheck @%t.5.resp %s 2>&1 | %FileCheck %s -check-prefix=LONG
 // LONG: warning: result of call to 'abs' is unused
 // RUN: %empty-directory(%t/tmp)

test/lit.cfg

@@ -1120,7 +1120,7 @@ source_compiler_rt_libs(compiler_rt_dir)
 
 def check_runtime_libs(features_to_check):
     for lib in config.compiler_rt_libs:
-        for (libname, feature) in features_to_check.iteritems():
+        for (libname, feature) in features_to_check.items():
             if lib.startswith("libclang_rt." + libname + "_"):
                 config.available_features.add(feature)
 

test/swift_test.py

@@ -30,7 +30,7 @@ def __init__(self, coverage_mode=None, execute_external=True):
             self.coverage_mode = coverage_mode
         self.skipped_tests = set()
 
-    def before_test(self, test, litConfig):
+    def before_test(self, test, litconfig):
         if self.coverage_mode:
             # FIXME: The compiler crashers run so fast they fill up the
             # merger's queue (and therefore the build bot's disk)
@@ -53,12 +53,12 @@ def before_test(self, test, litConfig):
                     os.path.join(test.config.swift_test_results_dir,
                                  "swift-%4m.profraw")
 
-    def after_test(self, test, litConfig, result):
+    def after_test(self, test, litconfig, result):
         if test.getSourcePath() in self.skipped_tests:
             self.skipped_tests.remove(test.getSourcePath())
         return result
 
-    def execute(self, test, litConfig):
-        self.before_test(test, litConfig)
-        result = super(SwiftTest, self).execute(test, litConfig)
-        return self.after_test(test, litConfig, result)
+    def execute(self, test, litconfig):
+        self.before_test(test, litconfig)
+        result = super(SwiftTest, self).execute(test, litconfig)
+        return self.after_test(test, litconfig, result)

unittests/Reflection/RemoteMirrorInterop/test.py

@@ -48,15 +48,15 @@
 for i in range(len(swiftcs) + 1):
     for localMirrorlibs in itertools.combinations(mirrorlibs, i):
         for i, arg in enumerate(absoluteArgs):
-            print 'Testing', arg, 'with mirror libs:'
+            print('Testing {} with mirror libs:'.format(arg))
             for l in localMirrorlibs:
                 print '\t', l
             callArgs = ['/tmp/test']
             dylibPath = os.path.join('/tmp', 'libtest' + str(i) + '.dylib')
             callArgs.append(dylibPath)
             callArgs += list(localMirrorlibs)
-            print ' '.join(callArgs)
+            print(' '.join(callArgs))
             subprocess.call(callArgs)
-            print 'DONE'
-            print ''
+            print('DONE')
+            print('')
         print localMirrorlibs

utils/PathSanitizingFileCheck

@@ -51,7 +51,7 @@ constants.""")
 
     p = subprocess.Popen(
         [args.file_check_path] + unknown_args, stdin=subprocess.PIPE)
-    stdout, stderr = p.communicate(stdin)
+    stdout, stderr = p.communicate(stdin.encode('utf-8'))
     if stdout is not None:
         print(stdout)
     if stderr is not None:

utils/bug_reducer/bug_reducer/swift_tools.py

@@ -3,7 +3,10 @@
 import os
 import subprocess
 
-import subprocess_utils
+try:
+	from . import subprocess_utils
+except ValueError:
+	import subprocess_utils
 
 DRY_RUN = False
 SQUELCH_STDERR = True

utils/bug_reducer/tests/test_funcbugreducer.py

@@ -45,7 +45,8 @@ def setUp(self):
         self.module_cache = os.path.join(self.tmp_dir, 'module_cache')
         self.sdk = subprocess.check_output(['xcrun', '--sdk', 'macosx',
                                             '--toolchain', 'Default',
-                                            '--show-sdk-path']).strip("\n")
+                                            '--show-sdk-path']
+                                          ).decode('utf-8').strip("\n")
         self.tools = swift_tools.SwiftTools(self.build_dir)
         self.passes = ['--pass=-bug-reducer-tester']
 
@@ -111,7 +112,7 @@ def test_basic(self):
             '--extra-silopt-arg=-bug-reducer-tester-failure-kind=opt-crasher'
         ]
         args.extend(self.passes)
-        output = self.run_check_output(args).split("\n")
+        output = self.run_check_output(args).decode('utf-8').split("\n")
         self.assertTrue("*** Successfully Reduced file!" in output)
         self.assertTrue("*** Final Functions: " +
                         "$s9testbasic6foo413yyF" in output)

utils/bug_reducer/tests/test_optbugreducer.py

@@ -45,7 +45,8 @@ def setUp(self):
         self.module_cache = os.path.join(self.tmp_dir, 'module_cache')
         self.sdk = subprocess.check_output(['xcrun', '--sdk', 'macosx',
                                             '--toolchain', 'Default',
-                                            '--show-sdk-path']).strip("\n")
+                                            '--show-sdk-path']
+                                          ).decode('utf-8').strip("\n")
         self.tools = swift_tools.SwiftTools(self.build_dir)
         json_data = json.loads(subprocess.check_output(
             [self.tools.sil_passpipeline_dumper, '-Performance']))
@@ -54,7 +55,7 @@ def setUp(self):
             for z in y:
                 self.passes.append('--pass=-' + z[1])
         random.seed(0xf487c07f)
-        random.shuffle(self.passes)
+        random.shuffle(self.passes, random=random.random)
         self.passes.insert(random.randint(0, len(self.passes)),
                            '--pass=-bug-reducer-tester')
 
@@ -101,7 +102,7 @@ def test_basic(self):
             '--extra-arg=-bug-reducer-tester-failure-kind=opt-crasher'
         ]
         args.extend(self.passes)
-        output = subprocess.check_output(args).split("\n")
+        output = subprocess.check_output(args).decode('utf-8').split("\n")
         self.assertTrue('*** Found miscompiling passes!' in output)
         self.assertTrue('*** Final Passes: --bug-reducer-tester' in output)
         re_end = 'testoptbugreducer_testbasic_initial'
@@ -130,7 +131,7 @@ def test_suffix_in_need_of_prefix(self):
             '--extra-arg=-bug-reducer-tester-failure-kind=opt-crasher'
         ]
         args.extend(self.passes)
-        output = subprocess.check_output(args).split("\n")
+        output = subprocess.check_output(args).decode('utf-8').split("\n")
         self.assertTrue('*** Found miscompiling passes!' in output)
         self.assertTrue('*** Final Passes: --bug-reducer-tester' in output)
         re_end = 'testoptbugreducer_testsuffixinneedofprefix_initial'
@@ -160,7 +161,7 @@ def test_reduce_function(self):
             '--reduce-sil'
         ]
         args.extend(self.passes)
-        output = subprocess.check_output(args).split("\n")
+        output = subprocess.check_output(args).decode('utf-8').split("\n")
         self.assertTrue('*** Found miscompiling passes!' in output)
         self.assertTrue(
             '*** Final Functions: $s18testreducefunction6foo413yyF' in output)