Merge pull request #7590 from theotherjimmy/differential-memap

Tools: Differential Memap

Author: Cruz Monrreal

tools/memap.py

@@ -5,9 +5,9 @@
 
 from abc import abstractmethod, ABCMeta
 from sys import stdout, exit, argv
-from os import sep
+from os import sep, rename, remove
 from os.path import (basename, dirname, join, relpath, abspath, commonprefix,
-                     splitext)
+                     splitext, exists)
 import re
 import csv
 import json
@@ -20,6 +20,7 @@
 
 from .utils import (argparse_filestring_type, argparse_lowercase_hyphen_type,
                     argparse_uppercase_type)
+from .settings import COMPARE_FIXED
 
 
 class _Parser(object):
@@ -455,6 +456,7 @@ class MemapParser(object):
     """
 
     print_sections = ('.text', '.data', '.bss')
+    delta_sections = ('.text-delta', '.data-delta', '.bss-delta')
 
 
     # sections to print info (generic for all toolchains)
@@ -466,6 +468,7 @@ def __init__(self):
         # list of all modules and their sections
         # full list - doesn't change with depth
         self.modules = dict()
+        self.old_modules = None
         # short version with specific depth
         self.short_modules = dict()
 
@@ -510,8 +513,17 @@ def reduce_depth(self, depth):
                 new_name = join(*split_name[:depth])
                 self.short_modules.setdefault(new_name, defaultdict(int))
                 for section_idx, value in v.items():
-                    self.short_modules[new_name].setdefault(section_idx, 0)
                     self.short_modules[new_name][section_idx] += self.modules[module_name][section_idx]
+                    self.short_modules[new_name][section_idx + '-delta'] += self.modules[module_name][section_idx]
+            if self.old_modules:
+                for module_name, v in self.old_modules.items():
+                    split_name = module_name.split(sep)
+                    if split_name[0] == '':
+                        split_name = split_name[1:]
+                    new_name = join(*split_name[:depth])
+                    self.short_modules.setdefault(new_name, defaultdict(int))
+                    for section_idx, value in v.items():
+                        self.short_modules[new_name][section_idx + '-delta'] -= self.old_modules[module_name][section_idx]
 
     export_formats = ["json", "csv-ci", "html", "table"]
 
@@ -557,7 +569,7 @@ def _move_up_tree(tree, next_module):
             if child["name"] == next_module:
                 return child
         else:
-            new_module = {"name": next_module, "value": 0}
+            new_module = {"name": next_module, "value": 0, "delta": 0}
             tree["children"].append(new_module)
             return new_module
 
@@ -567,9 +579,9 @@ def generate_html(self, file_desc):
         Positional arguments:
         file_desc - the file to write out the final report to
         """
-        tree_text = {"name": ".text", "value": 0}
-        tree_bss = {"name": ".bss", "value": 0}
-        tree_data = {"name": ".data", "value": 0}
+        tree_text = {"name": ".text", "value": 0, "delta": 0}
+        tree_bss = {"name": ".bss", "value": 0, "delta": 0}
+        tree_data = {"name": ".data", "value": 0, "delta": 0}
         for name, dct in self.modules.items():
             cur_text = tree_text
             cur_bss = tree_bss
@@ -578,14 +590,17 @@ def generate_html(self, file_desc):
             while True:
                 try:
                     cur_text["value"] += dct['.text']
+                    cur_text["delta"] += dct['.text']
                 except KeyError:
                     pass
                 try:
                     cur_bss["value"] += dct['.bss']
+                    cur_bss["delta"] += dct['.bss']
                 except KeyError:
                     pass
                 try:
                     cur_data["value"] += dct['.data']
+                    cur_data["delta"] += dct['.data']
                 except KeyError:
                     pass
                 if not modules:
@@ -594,15 +609,44 @@ def generate_html(self, file_desc):
                 cur_text = self._move_up_tree(cur_text, next_module)
                 cur_data = self._move_up_tree(cur_data, next_module)
                 cur_bss = self._move_up_tree(cur_bss, next_module)
+        if self.old_modules:
+            for name, dct in self.old_modules.items():
+                cur_text = tree_text
+                cur_bss = tree_bss
+                cur_data = tree_data
+                modules = name.split(sep)
+                while True:
+                    try:
+                        cur_text["delta"] -= dct['.text']
+                    except KeyError:
+                        pass
+                    try:
+                        cur_bss["delta"] -= dct['.bss']
+                    except KeyError:
+                        pass
+                    try:
+                        cur_data["delta"] -= dct['.data']
+                    except KeyError:
+                        pass
+                    if not modules:
+                        break
+                    next_module = modules.pop(0)
+                    if not any(cld['name'] == next_module for cld in cur_text['children']):
+                        break
+                    cur_text = self._move_up_tree(cur_text, next_module)
+                    cur_data = self._move_up_tree(cur_data, next_module)
+                    cur_bss = self._move_up_tree(cur_bss, next_module)
 
         tree_rom = {
             "name": "ROM",
             "value": tree_text["value"] + tree_data["value"],
+            "delta": tree_text["delta"] + tree_data["delta"],
             "children": [tree_text, tree_data]
         }
         tree_ram = {
             "name": "RAM",
             "value": tree_bss["value"] + tree_data["value"],
+            "delta": tree_bss["delta"] + tree_data["delta"],
             "children": [tree_bss, tree_data]
         }
 
@@ -634,6 +678,14 @@ def generate_json(self, file_desc):
         file_desc.write('\n')
         return None
 
+    RAM_FORMAT_STR = (
+        "Total Static RAM memory (data + bss): {}({:+}) bytes\n"
+    )
+
+    ROM_FORMAT_STR = (
+        "Total Flash memory (text + data): {}({:+}) bytes\n"
+    )
+
     def generate_csv(self, file_desc):
         """Generate a CSV file from a memoy map
 
@@ -646,7 +698,7 @@ def generate_csv(self, file_desc):
         module_section = []
         sizes = []
         for i in sorted(self.short_modules):
-            for k in self.print_sections:
+            for k in self.print_sections + self.delta_sections:
                 module_section.append((i + k))
                 sizes += [self.short_modules[i][k]]
 
@@ -681,23 +733,29 @@ def generate_table(self, file_desc):
             row = [i]
 
             for k in self.print_sections:
-                row.append(self.short_modules[i][k])
+                row.append("{}({:+})".format(self.short_modules[i][k],
+                                             self.short_modules[i][k + "-delta"]))
 
             table.add_row(row)
 
         subtotal_row = ['Subtotals']
         for k in self.print_sections:
-            subtotal_row.append(self.subtotal[k])
+            subtotal_row.append("{}({:+})".format(
+                self.subtotal[k], self.subtotal[k + '-delta']))
 
         table.add_row(subtotal_row)
 
         output = table.get_string()
         output += '\n'
 
-        output += "Total Static RAM memory (data + bss): %s bytes\n" % \
-                        str(self.mem_summary['static_ram'])
-        output += "Total Flash memory (text + data): %s bytes\n" % \
-                        str(self.mem_summary['total_flash'])
+        output += self.RAM_FORMAT_STR.format(
+            self.mem_summary['static_ram'],
+            self.mem_summary['static_ram_delta']
+        )
+        output += self.ROM_FORMAT_STR.format(
+            self.mem_summary['total_flash'],
+            self.mem_summary['total_flash_delta']
+        )
 
         return output
 
@@ -706,16 +764,24 @@ def generate_table(self, file_desc):
     def compute_report(self):
         """ Generates summary of memory usage for main areas
         """
-        for k in self.sections:
-            self.subtotal[k] = 0
+        self.subtotal = defaultdict(int)
 
         for mod in self.modules.values():
             for k in self.sections:
                 self.subtotal[k] += mod[k]
+                self.subtotal[k + '-delta'] += mod[k]
+        if self.old_modules:
+            for mod in self.old_modules.values():
+                for k in self.sections:
+                    self.subtotal[k + '-delta'] -= mod[k]
 
         self.mem_summary = {
-            'static_ram': (self.subtotal['.data'] + self.subtotal['.bss']),
+            'static_ram': self.subtotal['.data'] + self.subtotal['.bss'],
+            'static_ram_delta':
+            self.subtotal['.data-delta'] + self.subtotal['.bss-delta'],
             'total_flash': (self.subtotal['.text'] + self.subtotal['.data']),
+            'total_flash_delta':
+            self.subtotal['.text-delta'] + self.subtotal['.data-delta'],
         }
 
         self.mem_report = []
@@ -724,7 +790,8 @@ def compute_report(self):
                 self.mem_report.append({
                     "module": name,
                     "size":{
-                        k: sizes.get(k, 0) for k in self.print_sections
+                        k: sizes.get(k, 0) for k in (self.print_sections + 
+                                                     self.delta_sections)
                     }
                 })
 
@@ -741,16 +808,26 @@ def parse(self, mapfile, toolchain):
         """
         self.tc_name = toolchain.title()
         if toolchain in ("ARM", "ARM_STD", "ARM_MICRO", "ARMC6"):
-            parser = _ArmccParser()
+            parser = _ArmccParser
         elif toolchain == "GCC_ARM" or toolchain == "GCC_CR":
-            parser = _GccParser()
+            parser = _GccParser
         elif toolchain == "IAR":
-            parser = _IarParser()
+            parser = _IarParser
         else:
             return False
         try:
             with open(mapfile, 'r') as file_input:
-                self.modules = parser.parse_mapfile(file_input)
+                self.modules = parser().parse_mapfile(file_input)
+            try:
+                with open("%s.old" % mapfile, 'r') as old_input:
+                    self.old_modules = parser().parse_mapfile(old_input)
+            except IOError:
+                self.old_modules = None
+            if not COMPARE_FIXED:
+                old_mapfile = "%s.old" % mapfile
+                if exists(old_mapfile):
+                    remove(old_mapfile)
+                rename(mapfile, old_mapfile)
             return True
 
         except IOError as error:

tools/memap_flamegraph.html

@@ -80,16 +80,35 @@ <h3 class="text-muted">{{name}} Memory Details</h3>
       .direction("s")
       .offset([8, 0])
       .attr('class', 'd3-flame-graph-tip')
-      .html(function(d) { return "module: " + d.data.name + ", bytes: " + d.data.value; });
+      .html(function(d) { return "module: " + d.data.name + ", bytes: " + d.data.value + ", delta: " + d.data.delta; });
+    var colorizer = function (d) {
+        if (d.data.delta > 0) {
+            ratio = (d.data.value - d.data.delta) / d.data.value;
+            green = ("0" + (Number(ratio * 0xFF | 0).toString(16))).slice(-2).toUpperCase();
+            blue = ("0" + (Number(ratio * 0xEE | 0).toString(16))).slice(-2).toUpperCase();
+            console.log(d.data.name, green, blue);
+            return "#EE" + green + blue
+        } else if (d.data.delta < 0) {
+            ratio = (d.data.value + d.data.delta) / d.data.value;
+            green = ("0" + (Number(ratio * 0xFF | 0).toString(16))).slice(-2).toUpperCase();
+            red = ("0" + (Number(ratio * 0xFF | 0).toString(16))).slice(-2).toUpperCase();
+            console.log(d.data.name, red, green);
+            return "#" + red + green + "EE";
+        } else {
+            return "#FFFFEE";
+        }
+    }
     var flameGraph_rom = d3.flameGraph()
       .transitionDuration(250)
       .transitionEase(d3.easeCubic)
       .sort(true)
+      .color(colorizer)
       .tooltip(tip);
     var flameGraph_ram = d3.flameGraph()
       .transitionDuration(250)
       .transitionEase(d3.easeCubic)
       .sort(true)
+      .color(colorizer)
       .tooltip(tip);
     var rom_elem = d3.select("#chart-rom");
     flameGraph_rom.width(rom_elem.node().getBoundingClientRect().width);

tools/settings.py

@@ -58,6 +58,9 @@
 # Print compiler warnings and errors as link format
 PRINT_COMPILER_OUTPUT_AS_LINK = False
 
+# Compare against a fixed build of the project for space consumption
+COMPARE_FIXED = False
+
 # Print warnings/errors in color
 COLOR = False
 
@@ -91,7 +94,7 @@
             print("WARNING: MBED_%s set as environment variable but doesn't"
                   " exist" % _n)
 
-_ENV_VARS = ['PRINT_COMPILER_OUTPUT_AS_LINK', 'COLOR']
+_ENV_VARS = ['PRINT_COMPILER_OUTPUT_AS_LINK', 'COLOR', 'COMPARE_FIXED']
 for _n in _ENV_VARS:
     value = getenv('MBED_%s' % _n)
     if value: