Merge pull request #2047 from PrzemekWirkus/devel_mmap_proj

[Tools] Add summary for test building

Author: 0xc0170

tools/build_api.py

@@ -176,7 +176,7 @@ def build_project(src_path, build_path, target, toolchain_name,
 
     if report != None:
         start = time()
-        
+
         # If project_id is specified, use that over the default name
         id_name = project_id.upper() if project_id else name.upper()
         description = project_description if project_description else name
@@ -232,6 +232,7 @@ def build_project(src_path, build_path, target, toolchain_name,
             cur_result["elapsed_time"] = end - start
             cur_result["output"] = toolchain.get_output()
             cur_result["result"] = "OK"
+            cur_result["memory_usage"] = toolchain.map_outputs
 
             add_result_to_report(report, cur_result)
 
@@ -294,7 +295,7 @@ def build_library(src_paths, build_path, target, toolchain_name,
 
     if report != None:
         start = time()
-        
+
         # If project_id is specified, use that over the default name
         id_name = project_id.upper() if project_id else name.upper()
         description = name
@@ -377,7 +378,7 @@ def build_library(src_paths, build_path, target, toolchain_name,
         toolchain.copy_files(resources.libraries, build_path, resources=resources)
         if resources.linker_script:
             toolchain.copy_files(resources.linker_script, build_path, resources=resources)
-            
+
         if resource.hex_files:
             toolchain.copy_files(resources.hex_files, build_path, resources=resources)
 
@@ -399,12 +400,12 @@ def build_library(src_paths, build_path, target, toolchain_name,
     except Exception, e:
         if report != None:
             end = time()
-            
+
             if isinstance(e, ToolException):
                 cur_result["result"] = "FAIL"
             elif isinstance(e, NotSupportedException):
                 cur_result["result"] = "NOT_SUPPORTED"
-            
+
             cur_result["elapsed_time"] = end - start
 
             toolchain_output = toolchain.get_output()
@@ -428,7 +429,7 @@ def build_lib(lib_id, target, toolchain_name, options=None, verbose=False, clean
     if not lib.is_supported(target, toolchain_name):
         print 'Library "%s" is not yet supported on target %s with toolchain %s' % (lib_id, target.name, toolchain)
         return False
-    
+
     # We need to combine macros from parameter list with macros from library definition
     MACROS = lib.macros if lib.macros else []
     if macros:
@@ -441,7 +442,7 @@ def build_lib(lib_id, target, toolchain_name, options=None, verbose=False, clean
     dependencies_paths = lib.dependencies
     inc_dirs = lib.inc_dirs
     inc_dirs_ext = lib.inc_dirs_ext
-    
+
     """ src_path: the path of the source directory
     build_path: the path of the build directory
     target: ['LPC1768', 'LPC11U24', 'LPC2368']
@@ -522,7 +523,7 @@ def build_lib(lib_id, target, toolchain_name, options=None, verbose=False, clean
         # Copy Headers
         for resource in resources:
             toolchain.copy_files(resource.headers, build_path, resources=resource)
-            
+
         dependencies_include_dir.extend(toolchain.scan_resources(build_path).inc_dirs)
 
         # Compile Sources
@@ -716,7 +717,7 @@ def mcu_toolchain_matrix(verbose_html=False, platform_filter=None):
                 perm_counter += 1
             else:
                 text = "-"
-            
+
             row.append(text)
         pt.add_row(row)
 
@@ -942,6 +943,49 @@ def print_build_results(result_list, build_name):
         result += "\n"
     return result
 
+def print_build_memory_usage_results(report):
+    """ Generate result table with memory usage values for build results
+        Agregates (puts together) reports obtained from self.get_memory_summary()
+        @param report Report generated during build procedure. See
+    """
+    from prettytable import PrettyTable
+    columns_text = ['name', 'target', 'toolchain']
+    columns_int = ['static_ram', 'stack', 'heap', 'total_ram', 'total_flash']
+    table = PrettyTable(columns_text + columns_int)
+
+    for col in columns_text:
+        table.align[col] = 'l'
+
+    for col in columns_int:
+        table.align[col] = 'r'
+
+    for target in report:
+        for toolchain in report[target]:
+            for name in report[target][toolchain]:
+                for dlist in report[target][toolchain][name]:
+                    for dlistelem in dlist:
+                        # Get 'memory_usage' record and build table with statistics
+                        record = dlist[dlistelem]
+                        if 'memory_usage' in record and record['memory_usage']:
+                            # Note that summary should be in the last record of
+                            # 'memory_usage' section. This is why we are grabbing
+                            # last "[-1]" record.
+                            row = [
+                                record['description'],
+                                record['target_name'],
+                                record['toolchain_name'],
+                                record['memory_usage'][-1]['summary']['static_ram'],
+                                record['memory_usage'][-1]['summary']['stack'],
+                                record['memory_usage'][-1]['summary']['heap'],
+                                record['memory_usage'][-1]['summary']['total_ram'],
+                                record['memory_usage'][-1]['summary']['total_flash'],
+                            ]
+                            table.add_row(row)
+
+    result = "Memory map breakdown for built projects (values in Bytes):\n"
+    result += table.get_string(sortby='name')
+    return result
+
 def write_build_report(build_report, template_filename, filename):
     build_report_failing = []
     build_report_passing = []
@@ -963,14 +1007,14 @@ def write_build_report(build_report, template_filename, filename):
 def scan_for_source_paths(path, exclude_paths=None):
     ignorepatterns = []
     paths = []
-    
+
     def is_ignored(file_path):
         for pattern in ignorepatterns:
             if fnmatch.fnmatch(file_path, pattern):
                 return True
         return False
-    
-    
+
+
     """ os.walk(top[, topdown=True[, onerror=None[, followlinks=False]]])
     When topdown is True, the caller can modify the dirnames list in-place
     (perhaps using del or slice assignment), and walk() will only recurse into

tools/memap.py

@@ -26,9 +26,9 @@ def __init__(self):
 
         self.misc_flash_sections = ('.interrupts', '.flash_config')
 
-        self.other_sections = ('.interrupts_ram', '.init', '.ARM.extab', \
-                               '.ARM.exidx', '.ARM.attributes', '.eh_frame', \
-                               '.init_array', '.fini_array', '.jcr', '.stab', \
+        self.other_sections = ('.interrupts_ram', '.init', '.ARM.extab',
+                               '.ARM.exidx', '.ARM.attributes', '.eh_frame',
+                               '.init_array', '.fini_array', '.jcr', '.stab',
                                '.stabstr', '.ARM.exidx', '.ARM')
 
         # sections to print info (generic for all toolchains)
@@ -43,6 +43,9 @@ def __init__(self):
         # list of all object files and mappting to module names
         self.object_to_module = dict()
 
+        # Memory usage summary structure
+        self.mem_summary = dict()
+
     def module_add(self, module_name, size, section):
         """
         Adds a module / section to the list
@@ -67,7 +70,7 @@ def check_new_section_gcc(self, line):
                 return i  # should name of the section (assuming it's a known one)
 
         if line.startswith('.'):
-            return 'unknown'     # all others are clasified are unknown
+            return 'unknown'     # all others are classified are unknown
         else:
             return False         # everything else, means no change in section
 
@@ -363,11 +366,12 @@ def generate_output(self, export_format, file_output=None):
 
         # Create table
         columns = ['Module']
-        for i in list(self.print_sections):
-            columns.append(i)
+        columns.extend(self.print_sections)
 
         table = PrettyTable(columns)
         table.align["Module"] = "l"
+        for col in self.print_sections:
+            table.align[col] = 'r'
 
         for i in list(self.print_sections):
             table.align[i] = 'r'
@@ -388,8 +392,12 @@ def generate_output(self, export_format, file_output=None):
             for k in self.print_sections:
                 row.append(self.modules[i][k])
 
-            json_obj.append({"module":i, "size":{\
-                k:self.modules[i][k] for k in self.print_sections}})
+            json_obj.append({
+                "module":i,
+                "size":{
+                    k:self.modules[i][k] for k in self.print_sections
+                }
+            })
 
             table.add_row(row)
 
@@ -399,16 +407,19 @@ def generate_output(self, export_format, file_output=None):
 
         table.add_row(subtotal_row)
 
+        summary = {
+            'summary':{
+                'static_ram':(subtotal['.data']+subtotal['.bss']),
+                'heap':(subtotal['.heap']),
+                'stack':(subtotal['.stack']),
+                'total_ram':(subtotal['.data']+subtotal['.bss']+subtotal['.heap']+subtotal['.stack']),
+                'total_flash':(subtotal['.text']+subtotal['.data']+misc_flash_mem),
+            }
+        }
+
         if export_format == 'json':
-            json_obj.append({\
-                  'summary':{\
-                  'total_static_ram':(subtotal['.data']+subtotal['.bss']),\
-                  'allocated_heap':(subtotal['.heap']),\
-                  'allocated_stack':(subtotal['.stack']),\
-                  'total_ram':(subtotal['.data']+subtotal['.bss']+subtotal['.heap']+subtotal['.stack']),\
-                  'total_flash':(subtotal['.text']+subtotal['.data']+misc_flash_mem),}})
-
-            file_desc.write(json.dumps(json_obj, indent=4))
+            json_to_file = json_obj + [summary]
+            file_desc.write(json.dumps(json_to_file, indent=4))
             file_desc.write('\n')
 
         elif export_format == 'csv-ci': # CSV format for the CI system
@@ -467,33 +478,38 @@ def generate_output(self, export_format, file_output=None):
         if file_desc is not sys.stdout:
             file_desc.close()
 
+        self.mem_summary = json_obj + [summary]
+
         return True
 
+    def get_memory_summary(self):
+        """! Object is available only after self.generate_output('json') is called
+        @return Return memory summary object
+        """
+        return self.mem_summary
+
     def parse(self, mapfile, toolchain):
         """
         Parse and decode map file depending on the toolchain
         """
 
+        result = True
         try:
-            file_input = open(mapfile, 'rt')
+            with open(mapfile, 'rt') as file_input:
+                if toolchain == "ARM" or toolchain == "ARM_STD" or toolchain == "ARM_MICRO":
+                    self.search_objects(os.path.abspath(mapfile), "ARM")
+                    self.parse_map_file_armcc(file_input)
+                elif toolchain == "GCC_ARM":
+                    self.parse_map_file_gcc(file_input)
+                elif toolchain == "IAR":
+                    self.search_objects(os.path.abspath(mapfile), toolchain)
+                    self.parse_map_file_iar(file_input)
+                else:
+                    result = False
         except IOError as error:
             print "I/O error({0}): {1}".format(error.errno, error.strerror)
-            return False
-
-        if toolchain == "ARM" or toolchain == "ARM_STD" or toolchain == "ARM_MICRO":
-            self.search_objects(os.path.abspath(mapfile), "ARM")
-            self.parse_map_file_armcc(file_input)
-        elif toolchain == "GCC_ARM":
-            self.parse_map_file_gcc(file_input)
-        elif toolchain == "IAR":
-            self.search_objects(os.path.abspath(mapfile), toolchain)
-            self.parse_map_file_iar(file_input)
-        else:
-            return False
-
-        file_input.close()
-
-        return True
+            result = False
+        return result
 
 def main():