Skip to content

tests: make jobstats and process-stats-dir python3 compatible #32604

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Jun 30, 2020
Merged

tests: make jobstats and process-stats-dir python3 compatible #32604

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
cb0a064
tests: make `jobstats` and `process-stats-dir` python3 compatible
compnerd Jun 24, 2020
cfebd57
test: adjust style for python linter (NFC)
compnerd Jun 30, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 2 additions & 3 deletions utils/jobstats/jobstats.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -90,7 +90,7 @@ def merged_with(self, other, merge_by="sum"):
else max(a, b)),
"max": lambda a, b: max(a, b)}
op = ops[merge_by]
for k, v in self.stats.items() + other.stats.items():
for k, v in list(self.stats.items()) + list(other.stats.items()):
if k in merged_stats:
merged_stats[k] = op(v, merged_stats[k])
else:
Expand Down Expand Up @@ -317,8 +317,7 @@ def load_stats_dir(path, select_module=[], select_stat=[],
jobargs = [mg["input"], mg["triple"], mg["out"], mg["opt"]]

if platform.system() == 'Windows':
p = unicode(u"\\\\?\\%s" % os.path.abspath(os.path.join(root,
f)))
p = str(u"\\\\?\\%s" % os.path.abspath(os.path.join(root, f)))
else:
p = os.path.join(root, f)

Expand Down
62 changes: 39 additions & 23 deletions utils/process-stats-dir.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@

import argparse
import csv
import io
import itertools
import json
import os
Expand All @@ -25,13 +26,22 @@
import sys
import time
import urllib
import urllib2
from collections import namedtuple
from operator import attrgetter

from jobstats import (list_stats_dir_profiles,
load_stats_dir, merge_all_jobstats)

if sys.version_info[0] < 3:
import urllib2
Request = urllib2.Request
URLOpen = urllib2.urlopen
else:
import urllib.request
import urllib.parse
import urllib.error
Request = urllib.request.Request
URLOpen = urllib.request.urlopen

MODULE_PAT = re.compile(r'^(\w+)\.')

Expand All @@ -49,7 +59,8 @@ def stat_name_minus_module(name):
def vars_of_args(args):
vargs = vars(args)
if args.select_stats_from_csv_baseline is not None:
b = read_stats_dict_from_csv(args.select_stats_from_csv_baseline)
with io.open(args.select_stats_from_csv_baseline, 'r', encoding='utf-8') as f:
b = read_stats_dict_from_csv(f)
# Sniff baseline stat-names to figure out if they're module-qualified
# even when the user isn't asking us to _output_ module-grouped data.
all_triples = all(len(k.split('.')) == 3 for k in b.keys())
Expand Down Expand Up @@ -105,18 +116,18 @@ def write_lnt_values(args):
json.dump(j, args.output, indent=4)
else:
url = args.lnt_submit
print "\nsubmitting to LNT server: " + url
print("\nsubmitting to LNT server: " + url)
json_report = {'input_data': json.dumps(j), 'commit': '1'}
data = urllib.urlencode(json_report)
response_str = urllib2.urlopen(urllib2.Request(url, data))
response_str = URLOpen(Request(url, data))
response = json.loads(response_str.read())
print "### response:"
print response
print("### response:")
print(response)
if 'success' in response:
print "server response:\tSuccess"
print("server response:\tSuccess")
else:
print "server response:\tError"
print "error:\t", response['error']
print("server response:\tError")
print("error:\t", response['error'])
sys.exit(1)


Expand Down Expand Up @@ -187,8 +198,8 @@ def update_epoch_value(d, name, epoch, value):
epoch = existing_epoch
else:
(_, delta_pct) = diff_and_pct(existing_value, value)
print ("note: changing value %d -> %d (%.2f%%) for %s" %
(existing_value, value, delta_pct, name))
print("note: changing value %d -> %d (%.2f%%) for %s" %
(existing_value, value, delta_pct, name))
changed = 1
d[name] = (epoch, value)
return (epoch, value, changed)
Expand Down Expand Up @@ -233,22 +244,28 @@ def set_csv_baseline(args):
existing = None
vargs = vars_of_args(args)
if os.path.exists(args.set_csv_baseline):
with open(args.set_csv_baseline, "r") as f:
with io.open(args.set_csv_baseline, "r", encoding='utf-8', newline='\n') as f:
ss = vargs['select_stat']
existing = read_stats_dict_from_csv(f, select_stat=ss)
print ("updating %d baseline entries in %s" %
(len(existing), args.set_csv_baseline))
print("updating %d baseline entries in %s" %
(len(existing), args.set_csv_baseline))
else:
print "making new baseline " + args.set_csv_baseline
print("making new baseline " + args.set_csv_baseline)
fieldnames = ["epoch", "name", "value"]
with open(args.set_csv_baseline, "wb") as f:

def _open(path):
if sys.version_info[0] < 3:
return open(path, 'wb')
return io.open(path, "w", encoding='utf-8', newline='\n')

with _open(args.set_csv_baseline) as f:
out = csv.DictWriter(f, fieldnames, dialect='excel-tab',
quoting=csv.QUOTE_NONNUMERIC)
m = merge_all_jobstats((s for d in args.remainder
for s in load_stats_dir(d, **vargs)),
**vargs)
if m is None:
print "no stats found"
print("no stats found")
return 1
changed = 0
newepoch = int(time.time())
Expand All @@ -265,7 +282,7 @@ def set_csv_baseline(args):
name=name,
value=int(value)))
if existing is not None:
print "changed %d entries in baseline" % changed
print("changed %d entries in baseline" % changed)
return 0


Expand Down Expand Up @@ -402,8 +419,8 @@ def keyfunc(e):

def compare_to_csv_baseline(args):
vargs = vars_of_args(args)
old_stats = read_stats_dict_from_csv(args.compare_to_csv_baseline,
select_stat=vargs['select_stat'])
with io.open(args.compare_to_csv_baseline, 'r', encoding='utf-8') as f:
old_stats = read_stats_dict_from_csv(f, select_stat=vargs['select_stat'])
m = merge_all_jobstats((s for d in args.remainder
for s in load_stats_dir(d, **vargs)),
**vargs)
Expand Down Expand Up @@ -585,7 +602,7 @@ def main():
action="append",
help="Select specific statistics")
parser.add_argument("--select-stats-from-csv-baseline",
type=argparse.FileType('rb', 0), default=None,
type=str, default=None,
help="Select statistics present in a CSV baseline")
parser.add_argument("--exclude-timers",
default=False,
Expand Down Expand Up @@ -635,8 +652,7 @@ def main():
help="summarize the 'incrementality' of a build")
modes.add_argument("--set-csv-baseline", type=str, default=None,
help="Merge stats from a stats-dir into a CSV baseline")
modes.add_argument("--compare-to-csv-baseline",
type=argparse.FileType('rb', 0), default=None,
modes.add_argument("--compare-to-csv-baseline", type=str, default=None,
metavar="BASELINE.csv",
help="Compare stats dir to named CSV baseline")
modes.add_argument("--compare-stats-dirs",
Expand Down