tests: make jobstats and process-stats-dir python3 compatible

This makes the two utilities Python 2 and Python 3 compatible.  This
repairs the NameLookup tests with Python 3.

Author: compnerd

utils/jobstats/jobstats.py

@@ -90,7 +90,7 @@ def merged_with(self, other, merge_by="sum"):
                                     else max(a, b)),
                "max": lambda a, b: max(a, b)}
         op = ops[merge_by]
-        for k, v in self.stats.items() + other.stats.items():
+        for k, v in list(self.stats.items()) + list(other.stats.items()):
             if k in merged_stats:
                 merged_stats[k] = op(v, merged_stats[k])
             else:
@@ -317,8 +317,7 @@ def load_stats_dir(path, select_module=[], select_stat=[],
             jobargs = [mg["input"], mg["triple"], mg["out"], mg["opt"]]
 
             if platform.system() == 'Windows':
-                p = unicode(u"\\\\?\\%s" % os.path.abspath(os.path.join(root,
-                                                                        f)))
+                p = str(u"\\\\?\\%s" % os.path.abspath(os.path.join(root, f)))
             else:
                 p = os.path.join(root, f)
 

utils/process-stats-dir.py

@@ -17,6 +17,7 @@
 
 import argparse
 import csv
+import io
 import itertools
 import json
 import os
@@ -25,7 +26,14 @@
 import sys
 import time
 import urllib
-import urllib2
+if sys.version_info[0] < 3:
+    import urllib2
+    Request = urllib2.Request
+    URLOpen = urllib2.urlopen
+else:
+    import urllib.request, urllib.parse, urllib.error
+    Request = urllib.request.Request
+    URLOpen = urllib.request.urlopen
 from collections import namedtuple
 from operator import attrgetter
 
@@ -49,7 +57,8 @@ def stat_name_minus_module(name):
 def vars_of_args(args):
     vargs = vars(args)
     if args.select_stats_from_csv_baseline is not None:
-        b = read_stats_dict_from_csv(args.select_stats_from_csv_baseline)
+        with io.open(args.select_stats_from_csv_baseline, 'r', encoding='utf-8') as f:
+            b = read_stats_dict_from_csv(f)
         # Sniff baseline stat-names to figure out if they're module-qualified
         # even when the user isn't asking us to _output_ module-grouped data.
         all_triples = all(len(k.split('.')) == 3 for k in b.keys())
@@ -105,18 +114,18 @@ def write_lnt_values(args):
             json.dump(j, args.output, indent=4)
         else:
             url = args.lnt_submit
-            print "\nsubmitting to LNT server: " + url
+            print("\nsubmitting to LNT server: " + url)
             json_report = {'input_data': json.dumps(j), 'commit': '1'}
             data = urllib.urlencode(json_report)
-            response_str = urllib2.urlopen(urllib2.Request(url, data))
+            response_str = URLOpen(Request(url, data))
             response = json.loads(response_str.read())
-            print "### response:"
-            print response
+            print("### response:")
+            print(response)
             if 'success' in response:
-                print "server response:\tSuccess"
+                print("server response:\tSuccess")
             else:
-                print "server response:\tError"
-                print "error:\t", response['error']
+                print("server response:\tError")
+                print("error:\t", response['error'])
                 sys.exit(1)
 
 
@@ -187,8 +196,8 @@ def update_epoch_value(d, name, epoch, value):
             epoch = existing_epoch
         else:
             (_, delta_pct) = diff_and_pct(existing_value, value)
-            print ("note: changing value %d -> %d (%.2f%%) for %s" %
-                   (existing_value, value, delta_pct, name))
+            print("note: changing value %d -> %d (%.2f%%) for %s" %
+                  (existing_value, value, delta_pct, name))
             changed = 1
     d[name] = (epoch, value)
     return (epoch, value, changed)
@@ -233,22 +242,28 @@ def set_csv_baseline(args):
     existing = None
     vargs = vars_of_args(args)
     if os.path.exists(args.set_csv_baseline):
-        with open(args.set_csv_baseline, "r") as f:
+        with io.open(args.set_csv_baseline, "r", encoding='utf-8', newline='\n') as f:
             ss = vargs['select_stat']
             existing = read_stats_dict_from_csv(f, select_stat=ss)
-            print ("updating %d baseline entries in %s" %
-                   (len(existing), args.set_csv_baseline))
+            print("updating %d baseline entries in %s" %
+                  (len(existing), args.set_csv_baseline))
     else:
-        print "making new baseline " + args.set_csv_baseline
+        print("making new baseline " + args.set_csv_baseline)
     fieldnames = ["epoch", "name", "value"]
-    with open(args.set_csv_baseline, "wb") as f:
+
+    def _open(path):
+        if sys.version_info[0] < 3:
+            return open(path, 'wb')
+        return io.open(path, "w", encoding='utf-8', newline='\n')
+
+    with _open(args.set_csv_baseline) as f:
         out = csv.DictWriter(f, fieldnames, dialect='excel-tab',
                              quoting=csv.QUOTE_NONNUMERIC)
         m = merge_all_jobstats((s for d in args.remainder
                                 for s in load_stats_dir(d, **vargs)),
                                **vargs)
         if m is None:
-            print "no stats found"
+            print("no stats found")
             return 1
         changed = 0
         newepoch = int(time.time())
@@ -265,7 +280,7 @@ def set_csv_baseline(args):
                               name=name,
                               value=int(value)))
         if existing is not None:
-            print "changed %d entries in baseline" % changed
+            print("changed %d entries in baseline" % changed)
     return 0
 
 
@@ -402,8 +417,8 @@ def keyfunc(e):
 
 def compare_to_csv_baseline(args):
     vargs = vars_of_args(args)
-    old_stats = read_stats_dict_from_csv(args.compare_to_csv_baseline,
-                                         select_stat=vargs['select_stat'])
+    with io.open(args.compare_to_csv_baseline, 'r', encoding='utf-8') as f:
+        old_stats = read_stats_dict_from_csv(f, select_stat=vargs['select_stat'])
     m = merge_all_jobstats((s for d in args.remainder
                             for s in load_stats_dir(d, **vargs)),
                            **vargs)
@@ -585,7 +600,7 @@ def main():
                         action="append",
                         help="Select specific statistics")
     parser.add_argument("--select-stats-from-csv-baseline",
-                        type=argparse.FileType('rb', 0), default=None,
+                        type=str, default=None,
                         help="Select statistics present in a CSV baseline")
     parser.add_argument("--exclude-timers",
                         default=False,
@@ -635,8 +650,7 @@ def main():
                        help="summarize the 'incrementality' of a build")
     modes.add_argument("--set-csv-baseline", type=str, default=None,
                        help="Merge stats from a stats-dir into a CSV baseline")
-    modes.add_argument("--compare-to-csv-baseline",
-                       type=argparse.FileType('rb', 0), default=None,
+    modes.add_argument("--compare-to-csv-baseline", type=str, default=None,
                        metavar="BASELINE.csv",
                        help="Compare stats dir to named CSV baseline")
     modes.add_argument("--compare-stats-dirs",