---

yaml
---
r: 345151
b: refs/heads/master
c: 957076f
h: refs/heads/master
i:
  345149: 5648666
  345147: 425269e
  345143: 3c2d063
  345135: 5b204e3
  345119: d0a53df
  345087: 88749b1

Author: swift-ci

[refs]

@@ -1,5 +1,5 @@
 ---
-refs/heads/master: 76376696451bb1cfc6fb9170000192f2867f7657
+refs/heads/master: 957076fa0e559e17422afa9fdd17003638112831
 refs/heads/master-next: 203b3026584ecad859eb328b2e12490099409cd5
 refs/tags/osx-passed: b6b74147ef8a386f532cf9357a1bde006e552c54
 refs/tags/swift-2.2-SNAPSHOT-2015-12-01-a: 6bb18e013c2284f2b45f5f84f2df2887dc0f7dea

trunk/utils/jobstats/__init__.py

@@ -20,4 +20,4 @@
 __versioninfo__ = (0, 1, 0)
 __version__ = '.'.join(str(v) for v in __versioninfo__)
 
-from .jobstats import JobStats, load_stats_dir, merge_all_jobstats # noqa
+from .jobstats import JobStats, JobProfs, load_stats_dir, merge_all_jobstats, list_stats_dir_profiles # noqa

trunk/utils/jobstats/jobstats.py

@@ -23,20 +23,14 @@
 import re
 
 
-class JobStats(object):
-    """Object holding the stats of a single job run during a compilation,
-    corresponding to a single JSON file produced by a single job process
-    passed -stats-output-dir."""
+class JobData(object):
 
-    def __init__(self, jobkind, jobid, module, start_usec, dur_usec,
-                 jobargs, stats):
+    def __init__(self, jobkind, jobid, module, jobargs):
         self.jobkind = jobkind
         self.jobid = jobid
         self.module = module
-        self.start_usec = start_usec
-        self.dur_usec = dur_usec
         self.jobargs = jobargs
-        self.stats = stats
+        (self.input, self.triple, self.out, self.opt) = jobargs[0:4]
 
     def is_driver_job(self):
         """Return true iff self measures a driver job"""
@@ -46,6 +40,29 @@ def is_frontend_job(self):
         """Return true iff self measures a frontend job"""
         return self.jobkind == 'frontend'
 
+
+class JobProfs(JobData):
+    """Object denoting the profile of a single job run during a compilation,
+    corresponding to a single directory of profiles produced by a single
+    job process passed -stats-output-dir."""
+
+    def __init__(self, jobkind, jobid, module, jobargs, profiles):
+        self.profiles = profiles
+        super(JobProfs, self).__init__(jobkind, jobid, module, jobargs)
+
+
+class JobStats(JobData):
+    """Object holding the stats of a single job run during a compilation,
+    corresponding to a single JSON file produced by a single job process
+    passed -stats-output-dir."""
+
+    def __init__(self, jobkind, jobid, module, start_usec, dur_usec,
+                 jobargs, stats):
+        self.start_usec = start_usec
+        self.dur_usec = dur_usec
+        self.stats = stats
+        super(JobStats, self).__init__(jobkind, jobid, module, jobargs)
+
     def driver_jobs_ran(self):
         """Return the count of a driver job's ran sub-jobs"""
         assert(self.is_driver_job())
@@ -190,6 +207,11 @@ def to_lnt_test_obj(self, args):
               r"-(?P<pid>\d+)(-.*)?.json$")
 FILEPAT = re.compile(FILEPATSTR)
 
+PROFILEPATSTR = (r"^profile-(?P<start>\d+)-swift-(?P<kind>\w+)-" +
+                 AUXPATSTR +
+                 r"-(?P<pid>\d+)(-.*)?.dir$")
+PROFILEPAT = re.compile(PROFILEPATSTR)
+
 
 def match_auxpat(s):
     m = AUXPAT.match(s)
@@ -215,6 +237,64 @@ def match_filepat(s):
         return None
 
 
+def match_profilepat(s):
+    m = PROFILEPAT.match(s)
+    if m is not None:
+        return m.groupdict()
+    else:
+        return None
+
+
+def find_profiles_in(profiledir, select_stat=[]):
+    sre = re.compile('.*' if len(select_stat) == 0 else
+                     '|'.join(select_stat))
+    profiles = None
+    for profile in os.listdir(profiledir):
+        if profile.endswith(".svg"):
+            continue
+        if sre.search(profile) is None:
+            continue
+        fullpath = os.path.join(profiledir, profile)
+        s = os.stat(fullpath)
+        if s.st_size != 0:
+            if profiles is None:
+                profiles = dict()
+            try:
+                (counter, profiletype) = os.path.splitext(profile)
+                # drop leading period from extension
+                profiletype = profiletype[1:]
+                if profiletype not in profiles:
+                    profiles[profiletype] = dict()
+                profiles[profiletype][counter] = fullpath
+            except:
+                pass
+    return profiles
+
+
+def list_stats_dir_profiles(path, select_module=[], select_stat=[], **kwargs):
+    """Finds all stats-profiles in path, returning list of JobProfs objects"""
+    jobprofs = []
+    for root, dirs, files in os.walk(path):
+        for d in dirs:
+            mg = match_profilepat(d)
+            if not mg:
+                continue
+            # NB: "pid" in fpat is a random number, not unix pid.
+            jobkind = mg['kind']
+            jobid = int(mg['pid'])
+            module = mg["module"]
+            if len(select_module) != 0 and module not in select_module:
+                continue
+            jobargs = [mg["input"], mg["triple"], mg["out"], mg["opt"]]
+
+            e = JobProfs(jobkind=jobkind, jobid=jobid,
+                         module=module, jobargs=jobargs,
+                         profiles=find_profiles_in(os.path.join(root, d),
+                                                   select_stat))
+            jobprofs.append(e)
+    return jobprofs
+
+
 def load_stats_dir(path, select_module=[], select_stat=[],
                    exclude_timers=False, merge_timers=False, **kwargs):
     """Loads all stats-files found in path into a list of JobStats objects"""

trunk/utils/process-stats-dir.py

@@ -28,7 +28,9 @@
 import urllib2
 from collections import namedtuple
 from operator import attrgetter
-from jobstats import load_stats_dir, merge_all_jobstats
+
+from jobstats import (list_stats_dir_profiles,
+                      load_stats_dir, merge_all_jobstats)
 
 
 MODULE_PAT = re.compile('^(\w+)\.')
@@ -492,6 +494,56 @@ def evaluate_delta(args):
         return 1
 
 
+def render_profiles(args):
+    flamegraph_pl = args.flamegraph_script
+    if flamegraph_pl is None:
+        import distutils.spawn
+        flamegraph_pl = distutils.spawn.find_executable("flamegraph.pl")
+    if flamegraph_pl is None:
+        print("Need flamegraph.pl in $PATH, or pass --flamegraph-script")
+
+    vargs = vars_of_args(args)
+    for statsdir in args.remainder:
+        jobprofs = list_stats_dir_profiles(statsdir, **vargs)
+        index_path = os.path.join(statsdir, "profile-index.html")
+        all_profile_types = set([k for keys in [j.profiles.keys()
+                                                for j in jobprofs
+                                                if j.profiles is not None]
+                                 for k in keys])
+        with open(index_path, "wb") as index:
+            for ptype in all_profile_types:
+                index.write("<h2>Profile type: " + ptype + "</h2>\n")
+                index.write("<ul>\n")
+                for j in jobprofs:
+                    if j.is_frontend_job():
+                        index.write("    <li>" +
+                                    ("Module %s :: %s" %
+                                     (j.module, " ".join(j.jobargs))) + "\n")
+                        index.write("    <ul>\n")
+                        profiles = sorted(j.profiles.get(ptype, {}).items())
+                        for counter, path in profiles:
+                            title = ("Module: %s, File: %s, "
+                                     "Counter: %s, Profile: %s" %
+                                     (j.module, j.input, counter, ptype))
+                            subtitle = j.triple + ", -" + j.opt
+                            svg = os.path.abspath(path + ".svg")
+                            with open(path) as p, open(svg, "wb") as g:
+                                import subprocess
+                                print("Building flamegraph " + svg)
+                                subprocess.check_call([flamegraph_pl,
+                                                       "--title", title,
+                                                       "--subtitle", subtitle],
+                                                      stdin=p, stdout=g)
+                            link = ("<tt><a href=\"file://%s\">%s</a></tt>" %
+                                    (svg, counter))
+                            index.write("        <li>" + link + "\n")
+                        index.write("    </ul>\n")
+                        index.write("    </li>\n")
+        if args.browse_profiles:
+            import webbrowser
+            webbrowser.open_new_tab("file://" + os.path.abspath(index_path))
+
+
 def main():
     parser = argparse.ArgumentParser()
     parser.add_argument("--verbose", action="store_true",
@@ -596,6 +648,12 @@ def main():
                        help="evaluate an expression of stat-names")
     modes.add_argument("--evaluate-delta", type=str, default=None,
                        help="evaluate an expression of stat-deltas")
+    modes.add_argument("--render-profiles", action="store_true",
+                       help="render any profiles to SVG flamegraphs")
+    parser.add_argument("--flamegraph-script", type=str, default=None,
+                        help="path to flamegraph.pl")
+    parser.add_argument("--browse-profiles", action="store_true",
+                        help="open web browser tabs with rendered profiles")
     parser.add_argument('remainder', nargs=argparse.REMAINDER,
                         help="stats-dirs to process")
 
@@ -622,6 +680,8 @@ def main():
         return evaluate(args)
     elif args.evaluate_delta:
         return evaluate_delta(args)
+    elif args.render_profiles:
+        return render_profiles(args)
     return None