config: Extract memory definitions from sources

Memory regions may be defined in config sources with MBED_ROM_START and
MBED_ROM_SIZE or MBED_RAM_START and MBED_RAM_SIZE. These were not
extracted by the config system, and so were ignored when generating a
config in mbed-tools.

These attributes are extracted from config files, adding defined memory
regions to the Config. Both SIZE and START are required together to
add a region, and repeated definition of a memory region is ignored.

Author: wernerlewis

news/222.bugfix

@@ -0,0 +1 @@
+Add support for MBED_ROM_START, MBED_ROM_SIZE, MBED_RAM_START and MBED_RAM_SIZE in config system.

src/mbed_tools/build/_internal/config/config.py

@@ -8,7 +8,7 @@
 from collections import UserDict
 from typing import Any, Iterable, Hashable, Callable, List
 
-from mbed_tools.build._internal.config.source import Override, ConfigSetting
+from mbed_tools.build._internal.config.source import Memory, Override, ConfigSetting
 
 logger = logging.getLogger(__name__)
 
@@ -18,13 +18,15 @@ class Config(UserDict):
 
     This object understands how to populate the different 'config sections' which all have different rules for how the
     settings are collected.
-    Applies overrides, appends macros and updates config settings.
+    Applies overrides, appends macros, updates memories, and updates config settings.
     """
 
     def __setitem__(self, key: Hashable, item: Any) -> None:
         """Set an item based on its key."""
         if key == CONFIG_SECTION:
             self._update_config_section(item)
+        elif key == MEMORIES_SECTION:
+            self._update_memories_section(item)
         elif key == OVERRIDES_SECTION:
             self._handle_overrides(item)
         elif key == MACROS_SECTION:
@@ -67,6 +69,20 @@ def _update_config_section(self, config_settings: List[ConfigSetting]) -> None:
 
         self.data[CONFIG_SECTION] = self.data.get(CONFIG_SECTION, []) + config_settings
 
+    def _update_memories_section(self, memories: List[Memory]) -> None:
+        defined_memories = self.data.get(MEMORIES_SECTION, [])
+        for memory in memories:
+            logger.debug(f"Adding memory settings `{memory.name}: start={memory.start} size={memory.size}`")
+            prev_defined = next((mem for mem in defined_memories if mem.name == memory.name), None)
+            if prev_defined is None:
+                defined_memories.append(memory)
+            else:
+                logger.warning(
+                    f"You are attempting to redefine `{memory.name}` from {prev_defined.namespace}.\n"
+                    f"The values from `{memory.namespace}` will be ignored"
+                )
+        self.data[MEMORIES_SECTION] = defined_memories
+
     def _find_first_config_setting(self, predicate: Callable) -> Any:
         """Find first config setting based on `predicate`.
 
@@ -89,6 +105,7 @@ def _find_first_config_setting(self, predicate: Callable) -> Any:
 
 CONFIG_SECTION = "config"
 MACROS_SECTION = "macros"
+MEMORIES_SECTION = "memories"
 OVERRIDES_SECTION = "overrides"
 
 

src/mbed_tools/build/_internal/config/source.py

@@ -28,8 +28,8 @@ def prepare(
 ) -> dict:
     """Prepare a config source for entry into the Config object.
 
-    Extracts config and override settings from the source. Flattens these nested dictionaries out into lists of
-    objects which are namespaced in the way the Mbed config system expects.
+    Extracts memory, config and override settings from the source. Flattens these nested dictionaries out into
+    lists of objects which are namespaced in the way the Mbed config system expects.
 
     Args:
         input_data: The raw config JSON object parsed from the config file.
@@ -46,6 +46,11 @@ def prepare(
     for key in data:
         data[key] = _sanitise_value(data[key])
 
+    memories = _extract_memories(namespace, data)
+
+    if memories:
+        data["memories"] = memories
+
     if "config" in data:
         data["config"] = _extract_config_settings(namespace, data["config"])
 
@@ -78,6 +83,31 @@ def __post_init__(self) -> None:
         self.value = _sanitise_value(self.value)
 
 
+@dataclass
+class Memory:
+    """Representation of a defined RAM/ROM region."""
+
+    name: str
+    namespace: str
+    start: str
+    size: str
+
+    def __post_init__(self) -> None:
+        """Convert start and size to hex format strings."""
+        try:
+            self.start = hex(int(self.start, 0))
+        except ValueError:
+            raise ValueError(
+                f"Value of MBED_{self.name}_START in {self.namespace}, {self.start} is invalid: must be an integer"
+            )
+        try:
+            self.size = hex(int(self.size, 0))
+        except ValueError:
+            raise ValueError(
+                f"Value of MBED_{self.name}_SIZE in {self.namespace}, {self.size} is invalid: must be an integer"
+            )
+
+
 @dataclass
 class Override:
     """Representation of a config override.
@@ -128,6 +158,27 @@ def _extract_config_settings(namespace: str, config_data: dict) -> List[ConfigSe
     return settings
 
 
+def _extract_memories(namespace: str, data: dict) -> List[Memory]:
+    memories = []
+    for mem in ["rom", "ram"]:
+        start_attr = f"mbed_{mem}_start"
+        size_attr = f"mbed_{mem}_size"
+        start = data.get(start_attr)
+        size = data.get(size_attr)
+
+        if size is not None and start is not None:
+            logger.debug(f"Extracting MBED_{mem.upper()} definitions in {namespace}: _START={start}, _SIZE={size}.")
+
+            memory = Memory(mem.upper(), namespace, start, size)
+            memories.append(memory)
+        elif start is not None or size is not None:
+            raise ValueError(
+                f"{size_attr.upper()} and {start_attr.upper()} must be defined together. Only "
+                f"{'START' if start is not None else 'SIZE'} is defined in the lib {namespace}."
+            )
+    return memories
+
+
 def _extract_target_overrides(
     namespace: str, override_data: dict, allowed_target_labels: Iterable[str]
 ) -> List[Override]:

tests/build/_internal/config/test_config.py

@@ -2,10 +2,11 @@
 # Copyright (c) 2020-2021 Arm Limited and Contributors. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
 #
+import logging
 import pytest
 
 from mbed_tools.build._internal.config.config import Config
-from mbed_tools.build._internal.config.source import prepare, ConfigSetting, Override
+from mbed_tools.build._internal.config.source import prepare, ConfigSetting, Memory, Override
 
 
 class TestConfig:
@@ -24,6 +25,17 @@ def test_raises_when_trying_to_add_duplicate_config_setting(self):
         with pytest.raises(ValueError, match="lib.param already defined"):
             conf.update(prepare({"config": {"param": {"value": 0}}}, source_name="lib"))
 
+    def test_logs_ignore_mbed_ram_repeated(self, caplog):
+        caplog.set_level(logging.DEBUG)
+        input_dict = {"mbed_ram_size": "0x80000", "mbed_ram_start": "0x24000000"}
+        input_dict2 = {"mbed_ram_size": "0x78000", "mbed_ram_start": "0x24200000"}
+
+        conf = Config(prepare(input_dict, source_name="lib1"))
+        conf.update(prepare(input_dict2, source_name="lib2"))
+
+        assert "values from `lib2` will be ignored" in caplog.text
+        assert conf["memories"] == [Memory("RAM", "lib1", "0x24000000", "0x80000")]
+
     def test_target_overrides_handled(self):
         conf = Config(
             {

tests/build/_internal/config/test_source.py

@@ -2,8 +2,10 @@
 # Copyright (c) 2020-2021 Arm Limited and Contributors. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
 #
+import pytest
+
 from mbed_tools.build._internal.config import source
-from mbed_tools.build._internal.config.source import Override
+from mbed_tools.build._internal.config.source import Memory, Override
 
 
 class TestPrepareSource:
@@ -118,3 +120,48 @@ def test_converts_config_setting_value_lists_to_sets(self):
         assert conf["config"][0].value == {"ETHERNET", "WIFI"}
         assert conf["sectors"] == {0, 2048}
         assert conf["header_info"] == {0, 2048, "bobbins", "magic"}
+
+    def test_memory_attr_extracted(self):
+        lib = {
+            "mbed_ram_size": "0x80000",
+            "mbed_ram_start": "0x24000000",
+            "mbed_rom_size": "0x200000",
+            "mbed_rom_start": "0x08000000",
+        }
+
+        conf = source.prepare(lib, "lib")
+
+        assert Memory("RAM", "lib", "0x24000000", "0x80000") in conf["memories"]
+        assert Memory("ROM", "lib", "0x8000000", "0x200000") in conf["memories"]
+
+    def test_memory_attr_converted_as_hex(self):
+        input_dict = {"mbed_ram_size": "1024", "mbed_ram_start": "0x24000000"}
+
+        conf = source.prepare(input_dict, source_name="lib")
+
+        memory, *_ = conf["memories"]
+        assert memory.size == "0x400"
+
+    def test_raises_memory_size_not_integer(self):
+        input_dict = {"mbed_ram_size": "NOT INT", "mbed_ram_start": "0x24000000"}
+
+        with pytest.raises(ValueError, match="_SIZE in lib, NOT INT is invalid: must be an integer"):
+            source.prepare(input_dict, "lib")
+
+    def test_raises_memory_start_not_integer(self):
+        input_dict = {"mbed_ram_size": "0x80000", "mbed_ram_start": "NOT INT"}
+
+        with pytest.raises(ValueError, match="_START in lib, NOT INT is invalid: must be an integer"):
+            source.prepare(input_dict, "lib")
+
+    def test_raises_memory_size_defined_not_start(self):
+        input_dict = {"mbed_ram_size": "0x80000"}
+
+        with pytest.raises(ValueError, match="Only SIZE is defined"):
+            source.prepare(input_dict)
+
+    def test_raises_memory_start_defined_not_size(self):
+        input_dict = {"mbed_ram_start": "0x24000000"}
+
+        with pytest.raises(ValueError, match="Only START is defined"):
+            source.prepare(input_dict)