gh-418 Implement data input plug-in engine and integration with SCP service

docs/api/rest/config.md

@@ -54,7 +54,11 @@ curl --location --request GET 'http://localhost:5000/config/ae'
     "grouping": "0020,000D",
     "timeout": 5,
     "ignoredSopClasses": ["1.2.840.10008.5.1.4.1.1.1.1"],
-    "allowedSopClasses": ["1.2.840.10008.5.1.4.1.1.1.2"]
+    "allowedSopClasses": ["1.2.840.10008.5.1.4.1.1.1.2"],
+    "pluginAssemblies": [
+        "Monai.Deploy.InformaticsGateway.Test.Plugins.TestInputDataPluginAddWorkflow, Monai.Deploy.InformaticsGateway.Test.Plugins",
+        "Monai.Deploy.InformaticsGateway.Test.Plugins.TestInputDataPluginModifyDicomFile, Monai.Deploy.InformaticsGateway.Test.Plugins"
+    ]
   },
   {
     "name": "liver-seg",
@@ -151,6 +155,10 @@ curl --location --request POST 'http://localhost:5000/config/ae/' \
             "timeout": 5,
             "workflows": [
                 "3f6a08a1-0dea-44e9-ab82-1ff1adf43a8e"
+            ],
+            "pluginAssemblies": [
+                "Monai.Deploy.InformaticsGateway.Test.Plugins.TestInputDataPluginAddWorkflow, Monai.Deploy.InformaticsGateway.Test.Plugins",
+                "Monai.Deploy.InformaticsGateway.Test.Plugins.TestInputDataPluginModifyDicomFile, Monai.Deploy.InformaticsGateway.Test.Plugins"
             ]
         }
     }'
@@ -162,7 +170,11 @@ curl --location --request POST 'http://localhost:5000/config/ae/' \
 {
   "name": "breast-tumor",
   "aeTitle": "BREASTV1",
-  "workflows": ["3f6a08a1-0dea-44e9-ab82-1ff1adf43a8e"]
+  "workflows": ["3f6a08a1-0dea-44e9-ab82-1ff1adf43a8e"],
+  "pluginAssemblies": [
+      "Monai.Deploy.InformaticsGateway.Test.Plugins.TestInputDataPluginAddWorkflow, Monai.Deploy.InformaticsGateway.Test.Plugins",
+      "Monai.Deploy.InformaticsGateway.Test.Plugins.TestInputDataPluginModifyDicomFile, Monai.Deploy.InformaticsGateway.Test.Plugins"
+  ]
 }
 ```
 
@@ -210,6 +222,10 @@ curl --location --request PUT 'http://localhost:5000/config/ae/' \
             "timeout": 3,
             "workflows": [
                 "3f6a08a1-0dea-44e9-ab82-1ff1adf43a8e"
+            ],
+            "pluginAssemblies": [
+                "Monai.Deploy.InformaticsGateway.Test.Plugins.TestInputDataPluginAddWorkflow, Monai.Deploy.InformaticsGateway.Test.Plugins",
+                "Monai.Deploy.InformaticsGateway.Test.Plugins.TestInputDataPluginModifyDicomFile, Monai.Deploy.InformaticsGateway.Test.Plugins"
             ]
         }
     }'
@@ -222,6 +238,10 @@ curl --location --request PUT 'http://localhost:5000/config/ae/' \
   "name": "breast-tumor",
   "aeTitle": "BREASTV1",
   "workflows": ["3f6a08a1-0dea-44e9-ab82-1ff1adf43a8e"],
+  "pluginAssemblies": [
+      "Monai.Deploy.InformaticsGateway.Test.Plugins.TestInputDataPluginAddWorkflow, Monai.Deploy.InformaticsGateway.Test.Plugins",
+      "Monai.Deploy.InformaticsGateway.Test.Plugins.TestInputDataPluginModifyDicomFile, Monai.Deploy.InformaticsGateway.Test.Plugins"
+  ],
   "timeout": 3
 }
 ```

docs/setup/setup.md

@@ -311,6 +311,19 @@ mig-cli aet add -a BrainAET -grouping 0020,000E, -t 30
 The command creates a new listening AE Title with AE Title `BrainAET`. The listening AE Title
 will group instances by the Series Instance UID (0020,000E) with a timeout value of 30 seconds.
 
+
+### Optional: Input Data Plug-ins
+
+Each listening AE Title may be configured with zero or more plug-ins to maniulate incoming DICOM files before saving to the storage
+service and dispatching a workflow request. To include input data plug-ins, first create your plug-ins by implementing the
+[IInputDataPlugin](xref:Monai.Deploy.InformaticsGateway.Api.IInputDataPlugin) interface and then use `-p` argument with the fully
+qualified type name with the `mig-cli aet add` command. For example, the following command adds `MyNamespace.AnonymizePlugin`
+and `MyNamespace.FixSeriesData` plug-ins from the `MyNamespace.Plugins` assembly file.
+
+```bash
+mig-cli aet add -a BrainAET -grouping 0020,000E, -t 30 -p "MyNamespace.AnonymizePlugin, MyNamespace.Plugins" "MyNamespace.FixSeriesData, MyNamespace.Plugins"
+```
+
 > [!Note]
 > `-grouping` is optional, with a default value of 0020,000D.
 > `-t` is optional, with a default value of 5 seconds.

src/Api/IInputDataPlugin.cs

@@ -0,0 +1,32 @@
+/*
+ * Copyright 2023 MONAI Consortium
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+using System.Threading.Tasks;
+using FellowOakDicom;
+using Monai.Deploy.InformaticsGateway.Api.Storage;
+
+namespace Monai.Deploy.InformaticsGateway.Api
+{
+    /// <summary>
+    /// <c>IInputDataPlugin</c> enables lightweight data processing over incoming data received from supported data ingestion
+    /// services.
+    /// Refer to <see cref="IInputDataPluginEngine" /> for additional details.
+    /// </summary>
+    public interface IInputDataPlugin
+    {
+        Task<(DicomFile dicomFile, FileStorageMetadata fileMetadata)> Execute(DicomFile dicomFile, FileStorageMetadata fileMetadata);
+    }
+}

src/Api/IInputDataPluginEngine.cs

@@ -0,0 +1,40 @@
+/*
+ * Copyright 2023 MONAI Consortium
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+using System.Collections.Generic;
+using System.Threading.Tasks;
+using FellowOakDicom;
+using Monai.Deploy.InformaticsGateway.Api.Storage;
+
+namespace Monai.Deploy.InformaticsGateway.Api
+{
+    /// <summary>
+    /// <c>IInputDataPluginEngine</c> processes incoming data receivied from various supported services through
+    /// a list of plug-ins based on <see cref="IInputDataPlugin"/>.
+    /// Rules:
+    /// <list type="bullet">
+    /// <item>A list of plug-ins can be included with each export request, and each plug-in is executed in the order stored, processing one file at a time, enabling piping of the data before each file is exported.</item>
+    /// <item>Plugins MUST be lightweight and not hinder the export process</item>
+    /// <item>Plugins SHALL not accumulate files in memory or storage for bulk processing</item>
+    /// </list>
+    /// </summary>
+    public interface IInputDataPluginEngine
+    {
+        void Configure(IReadOnlyList<string> pluginAssemblies);
+
+        Task<(DicomFile dicomFile, FileStorageMetadata fileMetadata)> ExecutePlugins(DicomFile dicomFile, FileStorageMetadata fileMetadata);
+    }
+}

src/Api/MonaiApplicationEntity.cs

@@ -72,6 +72,11 @@ public class MonaiApplicationEntity : MongoDBEntityBase
         /// </summary>
         public List<string> Workflows { get; set; } = default!;
 
+        /// <summary>
+        /// Optional list of data input plug-in type names to be executed by the <see cref="IInputDataPluginEngine"/>.
+        /// </summary>
+        public List<string> PluginAssemblies { get; set; } = default!;
+
         /// <summary>
         /// Optional field to specify SOP Class UIDs to ignore.
         /// <see cref="IgnoredSopClasses"/> and <see cref="AllowedSopClasses"/> are mutually exclusive.
@@ -128,6 +133,8 @@ public void SetDefaultValues()
             IgnoredSopClasses ??= new List<string>();
 
             AllowedSopClasses ??= new List<string>();
+
+            PluginAssemblies ??= new List<string>();
         }
 
         public override string ToString()

src/CLI/Commands/AetCommand.cs

@@ -97,6 +97,12 @@ private void SetupAddAetCommand()
                 IsRequired = false,
             };
             addCommand.AddOption(allowedSopsOption);
+            var plugins = new Option<List<string>>(new string[] { "-p", "--plugins" }, description: "A space separated list of fully qualified type names of the plug-ins (surround each plug-in with double quotes)")
+            {
+                AllowMultipleArgumentsPerToken = true,
+                IsRequired = false,
+            };
+            addCommand.AddOption(plugins);
 
             addCommand.Handler = CommandHandler.Create<MonaiApplicationEntity, IHost, bool, CancellationToken>(AddAeTitlehandlerAsync);
         }
@@ -130,6 +136,12 @@ private void SetupEditAetCommand()
                 IsRequired = false,
             };
             addCommand.AddOption(allowedSopsOption);
+            var plugins = new Option<List<string>>(new string[] { "-p", "--plugins" }, description: "A space separated list of fully qualified type names of the plug-ins (surround each plug-in with double quotes)")
+            {
+                AllowMultipleArgumentsPerToken = true,
+                IsRequired = false,
+            };
+            addCommand.AddOption(plugins);
 
             addCommand.Handler = CommandHandler.Create<MonaiApplicationEntity, IHost, bool, CancellationToken>(EditAeTitleHandlerAsync);
         }
@@ -274,8 +286,7 @@ private async Task<int> AddAeTitlehandlerAsync(MonaiApplicationEntity entity, IH
                 }
                 if (result.AllowedSopClasses.Any())
                 {
-                    logger.MonaiAeAllowedSops(string.Join(',', result.AllowedSopClasses));
-                    logger.AcceptedSopClassesWarning();
+                    logger.MonaiAePlugins(string.Join(',', result.AllowedSopClasses));
                 }
             }
             catch (ConfigurationException ex)
@@ -330,6 +341,10 @@ private async Task<int> EditAeTitleHandlerAsync(MonaiApplicationEntity entity, I
                     logger.MonaiAeAllowedSops(string.Join(',', result.AllowedSopClasses));
                     logger.AcceptedSopClassesWarning();
                 }
+                if (result.AllowedSopClasses.Any())
+                {
+                    logger.MonaiAePlugins(string.Join(',', result.AllowedSopClasses));
+                }
             }
             catch (ConfigurationException ex)
             {

src/CLI/Logging/Log.cs

@@ -187,6 +187,9 @@ public static partial class Log
         [LoggerMessage(EventId = 30061, Level = LogLevel.Critical, Message = "Error updating SCP Application Entity {aeTitle}: {message}")]
         public static partial void ErrorUpdatingMonaiApplicationEntity(this ILogger logger, string aeTitle, string message);
 
+        [LoggerMessage(EventId = 30062, Level = LogLevel.Information, Message = "\tPlug-ins: {plugins}")]
+        public static partial void MonaiAePlugins(this ILogger logger, string plugins);
+
         // Docker Runner
         [LoggerMessage(EventId = 31000, Level = LogLevel.Debug, Message = "Checking for existing {applicationName} ({version}) containers...")]
         public static partial void CheckingExistingAppContainer(this ILogger logger, string applicationName, string version);

src/CLI/Test/AetCommandTest.cs

@@ -100,7 +100,7 @@ public async Task AetAdd_Command()
             {
                 Name = result.CommandResult.Children[0].Tokens[0].Value,
                 AeTitle = result.CommandResult.Children[1].Tokens[0].Value,
-                Workflows = result.CommandResult.Children[2].Tokens.Select(p => p.Value).ToList()
+                Workflows = result.CommandResult.Children[2].Tokens.Select(p => p.Value).ToList(),
             };
             Assert.Equal("MyName", entity.Name);
             Assert.Equal("MyAET", entity.AeTitle);
@@ -123,6 +123,44 @@ public async Task AetAdd_Command()
                     It.IsAny<CancellationToken>()), Times.Once());
         }
 
+        [Fact(DisplayName = "aet add comand with plug-ins")]
+        public async Task AetAdd_Command_WithPlugins()
+        {
+            var command = "aet add -n MyName -a MyAET --workflows App MyCoolApp TheApp --plugins \"PluginTypeA\" \"PluginTypeB\"";
+            var result = _paser.Parse(command);
+            Assert.Equal(ExitCodes.Success, result.Errors.Count);
+
+            var entity = new MonaiApplicationEntity()
+            {
+                Name = result.CommandResult.Children[0].Tokens[0].Value,
+                AeTitle = result.CommandResult.Children[1].Tokens[0].Value,
+                Workflows = result.CommandResult.Children[2].Tokens.Select(p => p.Value).ToList(),
+                PluginAssemblies = result.CommandResult.Children[3].Tokens.Select(p => p.Value).ToList(),
+            };
+            Assert.Equal("MyName", entity.Name);
+            Assert.Equal("MyAET", entity.AeTitle);
+            Assert.Collection(entity.Workflows,
+                item => item.Equals("App"),
+                item => item.Equals("MyCoolApp"),
+                item => item.Equals("TheApp"));
+            Assert.Collection(entity.PluginAssemblies,
+                item => item.Equals("PluginTypeA"),
+                item => item.Equals("PluginTypeB"));
+
+            _informaticsGatewayClient.Setup(p => p.MonaiScpAeTitle.Create(It.IsAny<MonaiApplicationEntity>(), It.IsAny<CancellationToken>()))
+                .ReturnsAsync(entity);
+
+            int exitCode = await _paser.InvokeAsync(command);
+
+            Assert.Equal(ExitCodes.Success, exitCode);
+
+            _informaticsGatewayClient.Verify(p => p.ConfigureServiceUris(It.IsAny<Uri>()), Times.Once());
+            _informaticsGatewayClient.Verify(
+                p => p.MonaiScpAeTitle.Create(
+                    It.Is<MonaiApplicationEntity>(o => o.AeTitle == entity.AeTitle && o.Name == entity.Name && Enumerable.SequenceEqual(o.Workflows, entity.Workflows)),
+                    It.IsAny<CancellationToken>()), Times.Once());
+        }
+
         [Fact(DisplayName = "aet add comand with allowed & ignored SOP classes")]
         public async Task AetAdd_Command_AllowedIgnoredSopClasses()
         {
@@ -335,7 +373,7 @@ public async Task AetList_Command_Empty()
         [Fact(DisplayName = "aet update command")]
         public async Task AetUpdate_Command()
         {
-            var command = "aet update -n MyName --workflows App MyCoolApp TheApp -i A B C -s D E F";
+            var command = "aet update -n MyName --workflows App MyCoolApp TheApp -i A B C -s D E F -p PlugInAssemblyA PlugInAssemblyB";
             var result = _paser.Parse(command);
             Assert.Equal(ExitCodes.Success, result.Errors.Count);
 
@@ -346,6 +384,7 @@ public async Task AetUpdate_Command()
                 Workflows = result.CommandResult.Children[1].Tokens.Select(p => p.Value).ToList(),
                 IgnoredSopClasses = result.CommandResult.Children[2].Tokens.Select(p => p.Value).ToList(),
                 AllowedSopClasses = result.CommandResult.Children[3].Tokens.Select(p => p.Value).ToList(),
+                PluginAssemblies = result.CommandResult.Children[4].Tokens.Select(p => p.Value).ToList(),
             };
 
             Assert.Equal("MyName", entity.Name);
@@ -362,6 +401,9 @@ public async Task AetUpdate_Command()
                 item => item.Equals("A"),
                 item => item.Equals("B"),
                 item => item.Equals("C"));
+            Assert.Collection(entity.PluginAssemblies,
+                item => item.Equals("PlugInAssemblyA"),
+                item => item.Equals("PlugInAssemblyB"));
 
             _informaticsGatewayClient.Setup(p => p.MonaiScpAeTitle.Update(It.IsAny<MonaiApplicationEntity>(), It.IsAny<CancellationToken>()))
                 .ReturnsAsync(entity);

src/Database/EntityFramework/Configuration/MonaiApplicationEntityConfiguration.cs

@@ -1,5 +1,5 @@
 /*
- * Copyright 2021-2022 MONAI Consortium
+ * Copyright 2021-2023 MONAI Consortium
  * Copyright 2021 NVIDIA Corporation
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
@@ -25,6 +25,7 @@
 namespace Monai.Deploy.InformaticsGateway.Database.EntityFramework.Configuration
 {
 #pragma warning disable CS8604, CS8603
+
     internal class MonaiApplicationEntityConfiguration : IEntityTypeConfiguration<MonaiApplicationEntity>
     {
         public void Configure(EntityTypeBuilder<MonaiApplicationEntity> builder)
@@ -51,6 +52,11 @@ public void Configure(EntityTypeBuilder<MonaiApplicationEntity> builder)
                         v => JsonSerializer.Serialize(v, jsonSerializerSettings),
                         v => JsonSerializer.Deserialize<List<string>>(v, jsonSerializerSettings))
                 .Metadata.SetValueComparer(valueComparer);
+            builder.Property(j => j.PluginAssemblies)
+                .HasConversion(
+                        v => JsonSerializer.Serialize(v, jsonSerializerSettings),
+                        v => JsonSerializer.Deserialize<List<string>>(v, jsonSerializerSettings))
+                .Metadata.SetValueComparer(valueComparer);
             builder.Property(j => j.IgnoredSopClasses)
                 .HasConversion(
                         v => JsonSerializer.Serialize(v, jsonSerializerSettings),
@@ -67,5 +73,6 @@ public void Configure(EntityTypeBuilder<MonaiApplicationEntity> builder)
             builder.Ignore(p => p.Id);
         }
     }
+
 #pragma warning restore CS8604, CS8603
 }