Allow configuration to be loaded from multiple sources

This means that a user of this toolset can add their own configuration
as a separate file, which overrides the supplied defaults as
necessary.  One use is to provide the configuration for an internal
lab, or internal kernels.

This adds an `--extra-config` parameter in addition to the existing
`--yaml-config` default configuration. It's an extra parameter because
we want to keep the implicit default configuration in these cases,
rather than have to specify a path to it ourselves (which might drift,
for example, and in any case is less usable).

Signed-off-by: Ed Smith <[email protected]>

Author: eds-collabora

kci_bisect

@@ -62,6 +62,6 @@ class cmd_get_mbox(Command):
 
 if __name__ == '__main__':
     opts = parse_opts("kci_bisect", globals())
-    configs = kernelci.config.load(opts.yaml_config)
+    configs = kernelci.config.load_all(opts.get_yaml_configs())
     status = opts.command(configs, opts)
     sys.exit(0 if status is True else 1)

kci_build

@@ -476,6 +476,6 @@ class cmd_push_tarball(Command):
 
 if __name__ == '__main__':
     opts = parse_opts("kci_build", globals())
-    configs = kernelci.config.load(opts.yaml_config)
+    configs = kernelci.config.load_all(opts.get_yaml_configs())
     status = opts.command(configs, opts)
     sys.exit(0 if status is True else 1)

kci_data

@@ -115,6 +115,6 @@ class cmd_create_user(Command):
 
 if __name__ == '__main__':
     opts = parse_opts("kci_data", globals())
-    configs = kernelci.config.load(opts.yaml_config)
+    configs = kernelci.config.load_all(opts.get_yaml_configs())
     status = opts.command(configs, opts)
     sys.exit(0 if status is True else 1)

kci_rootfs

@@ -145,6 +145,6 @@ class cmd_upload(Command):
 
 if __name__ == '__main__':
     opts = parse_opts("kci_rootfs", globals())
-    configs = kernelci.config.load(opts.yaml_config)
+    configs = kernelci.config.load_all(opts.get_yaml_configs())
     status = opts.command(configs, opts)
     sys.exit(0 if status is True else 1)

kci_test

@@ -390,6 +390,6 @@ class cmd_submit(Command):
 
 if __name__ == '__main__':
     opts = parse_opts("kci_test", globals())
-    configs = kernelci.config.load(opts.yaml_config)
+    configs = kernelci.config.load_all(opts.get_yaml_configs())
     status = opts.command(configs, opts)
     sys.exit(0 if status is True else 1)

kernelci/cli.py

@@ -523,6 +523,10 @@ def get_missing_args(self):
                 missing_args.append(arg_name)
         return missing_args
 
+    def get_yaml_configs(self):
+        """Get a list of all the YAML configuration files."""
+        return [self._cli_args.yaml_config] + self._cli_args.extra_config
+
 
 def make_parser(title, default_config_path):
     """Helper to make a parser object from argparse.
@@ -531,10 +535,21 @@ def make_parser(title, default_config_path):
     *default_config_path* is the default YAML config directory to use
     """
     parser = argparse.ArgumentParser(title)
-    parser.add_argument("--yaml-config", default=default_config_path,
-                        help="Path to the directory with YAML config files")
-    parser.add_argument("--settings",
-                        help="Path to the settings file")
+    parser.add_argument(
+        "--yaml-config",
+        default=default_config_path,
+        help="Path to the Kernel CI directory with YAML config files",
+    )
+    parser.add_argument(
+        "--extra-config",
+        action='append',
+        default=list(),
+        help="Path to additional YAML site config files",
+    )
+    parser.add_argument(
+        "--settings",
+        help="Path to the settings file",
+    )
     return parser
 
 

kernelci/config/__init__.py

@@ -64,7 +64,9 @@ def load_yaml(config_path, validate_entries=None):
     will be merged together under the same top-level dictionary key.
 
     *config_path* is the path to the YAML config directory, or alternative a
-                  single YAML file
+                  single YAML file.
+
+    *validate_entries* is currently unused.
     """
     config = dict()
     for yaml_path, data in _iterate_yaml_files(config_path):
@@ -79,6 +81,64 @@ def load_yaml(config_path, validate_entries=None):
     return config
 
 
+def _merge_trees(old, update):
+    """Merge two values loaded from YAML
+
+    This combines two values recursively that have been loaded from
+    YAML.  The data from *update* will be overlaid onto *old*
+    according to the rules:
+
+    - If *old* and *update* are dictionaries, their keys will be
+      unified, with the values for any keys present in both
+      dictionaries being merged recursively.
+
+    - If *old* and *update* are lists, the result is the concatenation
+      of the two lists.
+
+    - Otherwise, *update* replaces *old*.
+
+    Neither *old* nor *update* is modified; any modifications required
+    lead to a new value being returned.
+
+    """
+    if isinstance(old, dict) and isinstance(update, dict):
+        res = dict()
+        for k in (set(old) | set(update)):
+            if (k in old) and (k in update):
+                res[k] = _merge_trees(old[k], update[k])
+            elif k in old:
+                res[k] = old[k]
+            else:
+                res[k] = update[k]
+        return res
+    elif isinstance(old, list) and isinstance(update, list):
+        return old + update
+    else:
+        return update
+
+
+def load_all_yaml(config_paths, validate_entries=None):
+    """Load the YAML configuration
+
+    Load all the YAML files in all the specific configuration
+    directories and aggregate them together. Later directories take
+    precedence over earlier ones. This enables combining sources of
+    configuration data from multiple places.
+
+    *config_paths* is an ordered list of YAML configuration
+                   directories or YAML files, with later entries
+                   having higher priority.
+
+    *validate_entries* is passed to `load_yaml()`
+
+    """
+    config = dict()
+    for path in config_paths:
+        data = load_yaml(path, validate_entries)
+        config = _merge_trees(config, data)
+    return config
+
+
 def from_data(data):
     """Create configuration objects from the YAML data
 
@@ -104,9 +164,27 @@ def load(config_path):
     Load all the YAML files found in the configuration directory then create
     a dictionary containing the configuration objects and return it.
 
-    *config_path* is the path to the YAML config directory
+    *config_path* is the path to the YAML config directory or a
+    unified file
+
     """
     if config_path is None:
         return {}
     data = load_yaml(config_path)
     return from_data(data)
+
+
+def load_all(config_paths):
+    """Load the configuration from YAML files
+
+    Load all the YAML files found in all the configuration
+    directories, then create a dictionary containing the configuration
+    objects and return it. Note that the config paths are in priority
+    order, with later entries overriding earlier ones.
+
+    *config_paths* is a list of YAML config directories (or unified
+     files)
+
+    """
+    data = load_all_yaml(config_paths)
+    return from_data(data)