SPM: Overhaul the psa build system to integrate with mbed-cli (#6867)

SPM: PSA build system integration with tools

Author: orenc17

tools/build_api.py

@@ -47,7 +47,7 @@
 from .libraries import Library
 from .toolchains import TOOLCHAIN_CLASSES
 from .config import Config
-from .spm import generate_spm_data
+from .spm import generate_spm_data, generate_partitions_sources
 
 RELEASE_VERSIONS = ['2', '5']
 
@@ -509,7 +509,8 @@ def build_project(src_paths, build_path, target, toolchain_name,
                   notify=None, name=None, macros=None, inc_dirs=None, jobs=1,
                   report=None, properties=None, project_id=None,
                   project_description=None, config=None,
-                  app_config=None, build_profile=None, stats_depth=None, ignore=None):
+                  app_config=None, build_profile=None, stats_depth=None,
+                  ignore=None, psa_manifests=None):
     """ Build a project. A project may be a test or a user program.
 
     Positional arguments:
@@ -580,11 +581,20 @@ def build_project(src_paths, build_path, target, toolchain_name,
         # Call unified scan_resources
         resources = scan_resources(src_paths, toolchain, inc_dirs=inc_dirs)
 
+        if psa_manifests is None:
+            psa_manifests = resources.psa_manifests
+        else:
+            psa_manifests.extend(resources.psa_manifests)
+
+        for f in generate_partitions_sources(psa_manifests):
+            resources.add(toolchain.scan_resources(
+                f, base_path=resources.base_path))
+
         # Skip SPM sources for Mbed OS 2 builds
         # Directories scanned would not include the root of Mbed OS for legacy builds
         if 'rtos' in toolchain.config.lib_config_data:
             # Generate SPM additional code from manifests
-            psa_files_dir = generate_spm_data(resources.psa_manifests, build_path)
+            psa_files_dir = generate_spm_data(psa_manifests, build_path)
             resources.add(toolchain.scan_resources(psa_files_dir))
 
         # Change linker script if specified
@@ -658,7 +668,7 @@ def build_library(src_paths, build_path, target, toolchain_name,
                   archive=True, notify=None, macros=None, inc_dirs=None, jobs=1,
                   report=None, properties=None, project_id=None,
                   remove_config_header_file=False, app_config=None,
-                  build_profile=None, ignore=None):
+                  build_profile=None, ignore=None, psa_manifests=None):
     """ Build a library
 
     Positional arguments:
@@ -745,6 +755,15 @@ def build_library(src_paths, build_path, target, toolchain_name,
                                    dependencies_paths=dependencies_paths,
                                    inc_dirs=inc_dirs)
 
+        if psa_manifests is None:
+            psa_manifests = resources.psa_manifests
+        else:
+            psa_manifests.extend(resources.psa_manifests)
+
+        for f in generate_partitions_sources(psa_manifests):
+            resources.add(toolchain.scan_resources(
+                f, base_path=resources.base_path))
+
         # Copy headers, objects and static libraries - all files needed for
         # static lib, PSA manifests
         toolchain.copy_files(resources.headers, build_path, resources=resources)
@@ -761,10 +780,6 @@ def build_library(src_paths, build_path, target, toolchain_name,
             toolchain.copy_files(resources.hex_files, build_path,
                                  resources=resources)
 
-        # if resources.psa_manifests:
-        #     toolchain.copy_files(resources.psa_manifests, build_path,
-        #                          resources=resources)
-
         # Compile Sources
         objects = toolchain.compile_sources(resources, resources.inc_dirs)
         resources.objects.extend(objects)

tools/make.py

@@ -54,7 +54,6 @@
 from utils import argparse_many
 from utils import argparse_dir_not_parent
 from tools.toolchains import mbedToolchain, TOOLCHAIN_CLASSES, TOOLCHAIN_PATHS
-from tools.spm import generate_partitions_sources, scan_for_manifests
 
 if __name__ == '__main__':
     # Parse Options
@@ -242,8 +241,6 @@
                            "Currently set search path: %s"
                            %(toolchain, search_path))
 
-    generate_partitions_sources(scan_for_manifests(options.source_dir))
-
     # Test
     build_data_blob = {} if options.build_data else None
     for test_no in p:

tools/spm/__init__.py

@@ -1,8 +1,7 @@
-from .generate_partition_code import generate_partitions_sources, \
-    generate_spm_data, scan_for_manifests
+from .generate_partition_code import \
+    generate_partitions_sources, generate_spm_data
 
 __all__ = [
     'generate_partitions_sources',
     'generate_spm_data',
-    'scan_for_manifests'
 ]

tools/spm/generate_partition_code.py

@@ -627,9 +627,6 @@ def generate_partitions_sources(manifest_files, extra_filters=None):
         manifest = Manifest.from_json(manifest_file)
         manifests.append(manifest)
 
-    # Validate the correctness of the manifest collection.
-    validate_partition_manifests(manifests)
-
     generated_folders = []
     for manifest in manifests:
         manifest_output_folder = manifest.autogen_folder
@@ -645,8 +642,7 @@ def generate_partitions_sources(manifest_files, extra_filters=None):
                 manifest_output_folder,
                 extra_filters=extra_filters
             )
-
-        generated_folders.append(manifest_output_folder)
+            generated_folders.append(manifest_output_folder)
 
     return generated_folders
 
@@ -682,6 +678,9 @@ def generate_spm_data(manifest_files, output_dir, extra_filters=None):
                 manifest_obj.autogen_folder).values()
         )
 
+    # Validate the correctness of the manifest collection.
+    validate_partition_manifests(manifests)
+
     if is_up_to_date(
             manifest_files,
             MANIFEST_TEMPLATES + COMMON_TEMPLATES,
@@ -699,18 +698,3 @@ def generate_spm_data(manifest_files, output_dir, extra_filters=None):
         autogen_folder,
         extra_filters=extra_filters
     )
-
-
-def scan_for_manifests(src_dirs):
-    """
-    Scan a list of directories for PSA manifests.
-    
-    :param src_dirs: List of directories
-    :return: List of PSA manifests
-    """
-    manifests = set()
-    for src_dir in src_dirs:
-        for root, dirnames, filenames in os.walk(src_dir, followlinks=True):
-            for filename in fnmatch.filter(filenames, MANIFEST_FILE_PATTERN):
-                manifests.add(os.path.join(root, filename))
-    return list(manifests)

tools/test.py

@@ -43,7 +43,6 @@
 from tools.utils import argparse_dir_not_parent
 from tools.toolchains import mbedToolchain, TOOLCHAIN_PATHS, TOOLCHAIN_CLASSES
 from tools.settings import CLI_COLOR_MAP
-from tools.spm import generate_partitions_sources, scan_for_manifests
 
 if __name__ == '__main__':
     try:
@@ -196,10 +195,9 @@
             if not base_source_paths:
                 base_source_paths = ['.']
 
-            generate_partitions_sources(scan_for_manifests(base_source_paths))
-
             build_report = {}
             build_properties = {}
+            psa_manifests = []
 
             library_build_success = False
             profile = extract_profile(parser, options, toolchain)
@@ -214,7 +212,8 @@
                               notify=notify, archive=False,
                               app_config=config,
                               build_profile=profile,
-                              ignore=options.ignore)
+                              ignore=options.ignore,
+                              psa_manifests=psa_manifests)
 
                 library_build_success = True
             except ToolException as e:
@@ -243,7 +242,8 @@
                         app_config=config,
                         build_profile=profile,
                         stats_depth=options.stats_depth,
-                        ignore=options.ignore)
+                        ignore=options.ignore,
+                        psa_manifests=psa_manifests)
 
                 # If a path to a test spec is provided, write it to a file
                 if options.test_spec:

tools/test_api.py

@@ -2211,7 +2211,8 @@ def build_tests(tests, base_source_paths, build_path, target, toolchain_name,
                 clean=False, notify=None, jobs=1, macros=None,
                 silent=False, report=None, properties=None,
                 continue_on_build_fail=False, app_config=None,
-                build_profile=None, stats_depth=None, ignore=None):
+                build_profile=None, stats_depth=None, ignore=None,
+                psa_manifests=None):
     """Given the data structure from 'find_tests' and the typical build parameters,
     build all the tests
 
@@ -2264,7 +2265,8 @@ def build_tests(tests, base_source_paths, build_path, target, toolchain_name,
             'build_profile': build_profile,
             'toolchain_paths': TOOLCHAIN_PATHS,
             'stats_depth': stats_depth,
-            'notify': MockNotifier()
+            'notify': MockNotifier(),
+            'psa_manifests': psa_manifests
         }
 
         results.append(p.apply_async(build_test_worker, args, kwargs))