Skip to content

Commit 90b31a6

Browse files
mocsharpmocsharp
authored
gh-418 Implement data input plug-in engine and integration with SCP service (#426)
* gh-418 Implement data input plug-in engine and integration with SCP service - Implement IInputDataPluginEngine and IInputDataPlugin - Add unit tests - Update SCP feature in integration test * gh-418 Update CLI to support --plugins and update API doc * gh-418 Rename BackgroundProcessingAsync Signed-off-by: Victor Chang <[email protected]>
1 parent b44839a commit 90b31a6

34 files changed

+1255
-195
lines changed

‎docs/api/rest/config.md

Lines changed: 22 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -54,7 +54,11 @@ curl --location --request GET 'http://localhost:5000/config/ae'
5454
"grouping": "0020,000D",
5555
"timeout": 5,
5656
"ignoredSopClasses": ["1.2.840.10008.5.1.4.1.1.1.1"],
57-
"allowedSopClasses": ["1.2.840.10008.5.1.4.1.1.1.2"]
57+
"allowedSopClasses": ["1.2.840.10008.5.1.4.1.1.1.2"],
58+
"pluginAssemblies": [
59+
"Monai.Deploy.InformaticsGateway.Test.Plugins.TestInputDataPluginAddWorkflow, Monai.Deploy.InformaticsGateway.Test.Plugins",
60+
"Monai.Deploy.InformaticsGateway.Test.Plugins.TestInputDataPluginModifyDicomFile, Monai.Deploy.InformaticsGateway.Test.Plugins"
61+
]
5862
},
5963
{
6064
"name": "liver-seg",
@@ -151,6 +155,10 @@ curl --location --request POST 'http://localhost:5000/config/ae/' \
151155
"timeout": 5,
152156
"workflows": [
153157
"3f6a08a1-0dea-44e9-ab82-1ff1adf43a8e"
158+
],
159+
"pluginAssemblies": [
160+
"Monai.Deploy.InformaticsGateway.Test.Plugins.TestInputDataPluginAddWorkflow, Monai.Deploy.InformaticsGateway.Test.Plugins",
161+
"Monai.Deploy.InformaticsGateway.Test.Plugins.TestInputDataPluginModifyDicomFile, Monai.Deploy.InformaticsGateway.Test.Plugins"
154162
]
155163
}
156164
}'
@@ -162,7 +170,11 @@ curl --location --request POST 'http://localhost:5000/config/ae/' \
162170
{
163171
"name": "breast-tumor",
164172
"aeTitle": "BREASTV1",
165-
"workflows": ["3f6a08a1-0dea-44e9-ab82-1ff1adf43a8e"]
173+
"workflows": ["3f6a08a1-0dea-44e9-ab82-1ff1adf43a8e"],
174+
"pluginAssemblies": [
175+
"Monai.Deploy.InformaticsGateway.Test.Plugins.TestInputDataPluginAddWorkflow, Monai.Deploy.InformaticsGateway.Test.Plugins",
176+
"Monai.Deploy.InformaticsGateway.Test.Plugins.TestInputDataPluginModifyDicomFile, Monai.Deploy.InformaticsGateway.Test.Plugins"
177+
]
166178
}
167179
```
168180

@@ -210,6 +222,10 @@ curl --location --request PUT 'http://localhost:5000/config/ae/' \
210222
"timeout": 3,
211223
"workflows": [
212224
"3f6a08a1-0dea-44e9-ab82-1ff1adf43a8e"
225+
],
226+
"pluginAssemblies": [
227+
"Monai.Deploy.InformaticsGateway.Test.Plugins.TestInputDataPluginAddWorkflow, Monai.Deploy.InformaticsGateway.Test.Plugins",
228+
"Monai.Deploy.InformaticsGateway.Test.Plugins.TestInputDataPluginModifyDicomFile, Monai.Deploy.InformaticsGateway.Test.Plugins"
213229
]
214230
}
215231
}'
@@ -222,6 +238,10 @@ curl --location --request PUT 'http://localhost:5000/config/ae/' \
222238
"name": "breast-tumor",
223239
"aeTitle": "BREASTV1",
224240
"workflows": ["3f6a08a1-0dea-44e9-ab82-1ff1adf43a8e"],
241+
"pluginAssemblies": [
242+
"Monai.Deploy.InformaticsGateway.Test.Plugins.TestInputDataPluginAddWorkflow, Monai.Deploy.InformaticsGateway.Test.Plugins",
243+
"Monai.Deploy.InformaticsGateway.Test.Plugins.TestInputDataPluginModifyDicomFile, Monai.Deploy.InformaticsGateway.Test.Plugins"
244+
],
225245
"timeout": 3
226246
}
227247
```

‎docs/setup/setup.md

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -311,6 +311,19 @@ mig-cli aet add -a BrainAET -grouping 0020,000E, -t 30
311311
The command creates a new listening AE Title with AE Title `BrainAET`. The listening AE Title
312312
will group instances by the Series Instance UID (0020,000E) with a timeout value of 30 seconds.
313313

314+
315+
### Optional: Input Data Plug-ins
316+
317+
Each listening AE Title may be configured with zero or more plug-ins to maniulate incoming DICOM files before saving to the storage
318+
service and dispatching a workflow request. To include input data plug-ins, first create your plug-ins by implementing the
319+
[IInputDataPlugin](xref:Monai.Deploy.InformaticsGateway.Api.IInputDataPlugin) interface and then use `-p` argument with the fully
320+
qualified type name with the `mig-cli aet add` command. For example, the following command adds `MyNamespace.AnonymizePlugin`
321+
and `MyNamespace.FixSeriesData` plug-ins from the `MyNamespace.Plugins` assembly file.
322+
323+
```bash
324+
mig-cli aet add -a BrainAET -grouping 0020,000E, -t 30 -p "MyNamespace.AnonymizePlugin, MyNamespace.Plugins" "MyNamespace.FixSeriesData, MyNamespace.Plugins"
325+
```
326+
314327
> [!Note]
315328
> `-grouping` is optional, with a default value of 0020,000D.
316329
> `-t` is optional, with a default value of 5 seconds.

‎src/Api/IInputDataPlugin.cs

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
/*
2+
* Copyright 2023 MONAI Consortium
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
using System.Threading.Tasks;
18+
using FellowOakDicom;
19+
using Monai.Deploy.InformaticsGateway.Api.Storage;
20+
21+
namespace Monai.Deploy.InformaticsGateway.Api
22+
{
23+
/// <summary>
24+
/// <c>IInputDataPlugin</c> enables lightweight data processing over incoming data received from supported data ingestion
25+
/// services.
26+
/// Refer to <see cref="IInputDataPluginEngine" /> for additional details.
27+
/// </summary>
28+
public interface IInputDataPlugin
29+
{
30+
Task<(DicomFile dicomFile, FileStorageMetadata fileMetadata)> Execute(DicomFile dicomFile, FileStorageMetadata fileMetadata);
31+
}
32+
}

‎src/Api/IInputDataPluginEngine.cs

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
/*
2+
* Copyright 2023 MONAI Consortium
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
using System.Collections.Generic;
18+
using System.Threading.Tasks;
19+
using FellowOakDicom;
20+
using Monai.Deploy.InformaticsGateway.Api.Storage;
21+
22+
namespace Monai.Deploy.InformaticsGateway.Api
23+
{
24+
/// <summary>
25+
/// <c>IInputDataPluginEngine</c> processes incoming data receivied from various supported services through
26+
/// a list of plug-ins based on <see cref="IInputDataPlugin"/>.
27+
/// Rules:
28+
/// <list type="bullet">
29+
/// <item>A list of plug-ins can be included with each export request, and each plug-in is executed in the order stored, processing one file at a time, enabling piping of the data before each file is exported.</item>
30+
/// <item>Plugins MUST be lightweight and not hinder the export process</item>
31+
/// <item>Plugins SHALL not accumulate files in memory or storage for bulk processing</item>
32+
/// </list>
33+
/// </summary>
34+
public interface IInputDataPluginEngine
35+
{
36+
void Configure(IReadOnlyList<string> pluginAssemblies);
37+
38+
Task<(DicomFile dicomFile, FileStorageMetadata fileMetadata)> ExecutePlugins(DicomFile dicomFile, FileStorageMetadata fileMetadata);
39+
}
40+
}

‎src/Api/MonaiApplicationEntity.cs

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,11 @@ public class MonaiApplicationEntity : MongoDBEntityBase
7272
/// </summary>
7373
public List<string> Workflows { get; set; } = default!;
7474

75+
/// <summary>
76+
/// Optional list of data input plug-in type names to be executed by the <see cref="IInputDataPluginEngine"/>.
77+
/// </summary>
78+
public List<string> PluginAssemblies { get; set; } = default!;
79+
7580
/// <summary>
7681
/// Optional field to specify SOP Class UIDs to ignore.
7782
/// <see cref="IgnoredSopClasses"/> and <see cref="AllowedSopClasses"/> are mutually exclusive.
@@ -128,6 +133,8 @@ public void SetDefaultValues()
128133
IgnoredSopClasses ??= new List<string>();
129134

130135
AllowedSopClasses ??= new List<string>();
136+
137+
PluginAssemblies ??= new List<string>();
131138
}
132139

133140
public override string ToString()

‎src/CLI/Commands/AetCommand.cs

Lines changed: 17 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -97,6 +97,12 @@ private void SetupAddAetCommand()
9797
IsRequired = false,
9898
};
9999
addCommand.AddOption(allowedSopsOption);
100+
var plugins = new Option<List<string>>(new string[] { "-p", "--plugins" }, description: "A space separated list of fully qualified type names of the plug-ins (surround each plug-in with double quotes)")
101+
{
102+
AllowMultipleArgumentsPerToken = true,
103+
IsRequired = false,
104+
};
105+
addCommand.AddOption(plugins);
100106

101107
addCommand.Handler = CommandHandler.Create<MonaiApplicationEntity, IHost, bool, CancellationToken>(AddAeTitlehandlerAsync);
102108
}
@@ -130,6 +136,12 @@ private void SetupEditAetCommand()
130136
IsRequired = false,
131137
};
132138
addCommand.AddOption(allowedSopsOption);
139+
var plugins = new Option<List<string>>(new string[] { "-p", "--plugins" }, description: "A space separated list of fully qualified type names of the plug-ins (surround each plug-in with double quotes)")
140+
{
141+
AllowMultipleArgumentsPerToken = true,
142+
IsRequired = false,
143+
};
144+
addCommand.AddOption(plugins);
133145

134146
addCommand.Handler = CommandHandler.Create<MonaiApplicationEntity, IHost, bool, CancellationToken>(EditAeTitleHandlerAsync);
135147
}
@@ -274,8 +286,7 @@ private async Task<int> AddAeTitlehandlerAsync(MonaiApplicationEntity entity, IH
274286
}
275287
if (result.AllowedSopClasses.Any())
276288
{
277-
logger.MonaiAeAllowedSops(string.Join(',', result.AllowedSopClasses));
278-
logger.AcceptedSopClassesWarning();
289+
logger.MonaiAePlugins(string.Join(',', result.AllowedSopClasses));
279290
}
280291
}
281292
catch (ConfigurationException ex)
@@ -330,6 +341,10 @@ private async Task<int> EditAeTitleHandlerAsync(MonaiApplicationEntity entity, I
330341
logger.MonaiAeAllowedSops(string.Join(',', result.AllowedSopClasses));
331342
logger.AcceptedSopClassesWarning();
332343
}
344+
if (result.AllowedSopClasses.Any())
345+
{
346+
logger.MonaiAePlugins(string.Join(',', result.AllowedSopClasses));
347+
}
333348
}
334349
catch (ConfigurationException ex)
335350
{

‎src/CLI/Logging/Log.cs

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -187,6 +187,9 @@ public static partial class Log
187187
[LoggerMessage(EventId = 30061, Level = LogLevel.Critical, Message = "Error updating SCP Application Entity {aeTitle}: {message}")]
188188
public static partial void ErrorUpdatingMonaiApplicationEntity(this ILogger logger, string aeTitle, string message);
189189

190+
[LoggerMessage(EventId = 30062, Level = LogLevel.Information, Message = "\tPlug-ins: {plugins}")]
191+
public static partial void MonaiAePlugins(this ILogger logger, string plugins);
192+
190193
// Docker Runner
191194
[LoggerMessage(EventId = 31000, Level = LogLevel.Debug, Message = "Checking for existing {applicationName} ({version}) containers...")]
192195
public static partial void CheckingExistingAppContainer(this ILogger logger, string applicationName, string version);

‎src/CLI/Test/AetCommandTest.cs

Lines changed: 44 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -100,7 +100,7 @@ public async Task AetAdd_Command()
100100
{
101101
Name = result.CommandResult.Children[0].Tokens[0].Value,
102102
AeTitle = result.CommandResult.Children[1].Tokens[0].Value,
103-
Workflows = result.CommandResult.Children[2].Tokens.Select(p => p.Value).ToList()
103+
Workflows = result.CommandResult.Children[2].Tokens.Select(p => p.Value).ToList(),
104104
};
105105
Assert.Equal("MyName", entity.Name);
106106
Assert.Equal("MyAET", entity.AeTitle);
@@ -123,6 +123,44 @@ public async Task AetAdd_Command()
123123
It.IsAny<CancellationToken>()), Times.Once());
124124
}
125125

126+
[Fact(DisplayName = "aet add comand with plug-ins")]
127+
public async Task AetAdd_Command_WithPlugins()
128+
{
129+
var command = "aet add -n MyName -a MyAET --workflows App MyCoolApp TheApp --plugins \"PluginTypeA\" \"PluginTypeB\"";
130+
var result = _paser.Parse(command);
131+
Assert.Equal(ExitCodes.Success, result.Errors.Count);
132+
133+
var entity = new MonaiApplicationEntity()
134+
{
135+
Name = result.CommandResult.Children[0].Tokens[0].Value,
136+
AeTitle = result.CommandResult.Children[1].Tokens[0].Value,
137+
Workflows = result.CommandResult.Children[2].Tokens.Select(p => p.Value).ToList(),
138+
PluginAssemblies = result.CommandResult.Children[3].Tokens.Select(p => p.Value).ToList(),
139+
};
140+
Assert.Equal("MyName", entity.Name);
141+
Assert.Equal("MyAET", entity.AeTitle);
142+
Assert.Collection(entity.Workflows,
143+
item => item.Equals("App"),
144+
item => item.Equals("MyCoolApp"),
145+
item => item.Equals("TheApp"));
146+
Assert.Collection(entity.PluginAssemblies,
147+
item => item.Equals("PluginTypeA"),
148+
item => item.Equals("PluginTypeB"));
149+
150+
_informaticsGatewayClient.Setup(p => p.MonaiScpAeTitle.Create(It.IsAny<MonaiApplicationEntity>(), It.IsAny<CancellationToken>()))
151+
.ReturnsAsync(entity);
152+
153+
int exitCode = await _paser.InvokeAsync(command);
154+
155+
Assert.Equal(ExitCodes.Success, exitCode);
156+
157+
_informaticsGatewayClient.Verify(p => p.ConfigureServiceUris(It.IsAny<Uri>()), Times.Once());
158+
_informaticsGatewayClient.Verify(
159+
p => p.MonaiScpAeTitle.Create(
160+
It.Is<MonaiApplicationEntity>(o => o.AeTitle == entity.AeTitle && o.Name == entity.Name && Enumerable.SequenceEqual(o.Workflows, entity.Workflows)),
161+
It.IsAny<CancellationToken>()), Times.Once());
162+
}
163+
126164
[Fact(DisplayName = "aet add comand with allowed & ignored SOP classes")]
127165
public async Task AetAdd_Command_AllowedIgnoredSopClasses()
128166
{
@@ -335,7 +373,7 @@ public async Task AetList_Command_Empty()
335373
[Fact(DisplayName = "aet update command")]
336374
public async Task AetUpdate_Command()
337375
{
338-
var command = "aet update -n MyName --workflows App MyCoolApp TheApp -i A B C -s D E F";
376+
var command = "aet update -n MyName --workflows App MyCoolApp TheApp -i A B C -s D E F -p PlugInAssemblyA PlugInAssemblyB";
339377
var result = _paser.Parse(command);
340378
Assert.Equal(ExitCodes.Success, result.Errors.Count);
341379

@@ -346,6 +384,7 @@ public async Task AetUpdate_Command()
346384
Workflows = result.CommandResult.Children[1].Tokens.Select(p => p.Value).ToList(),
347385
IgnoredSopClasses = result.CommandResult.Children[2].Tokens.Select(p => p.Value).ToList(),
348386
AllowedSopClasses = result.CommandResult.Children[3].Tokens.Select(p => p.Value).ToList(),
387+
PluginAssemblies = result.CommandResult.Children[4].Tokens.Select(p => p.Value).ToList(),
349388
};
350389

351390
Assert.Equal("MyName", entity.Name);
@@ -362,6 +401,9 @@ public async Task AetUpdate_Command()
362401
item => item.Equals("A"),
363402
item => item.Equals("B"),
364403
item => item.Equals("C"));
404+
Assert.Collection(entity.PluginAssemblies,
405+
item => item.Equals("PlugInAssemblyA"),
406+
item => item.Equals("PlugInAssemblyB"));
365407

366408
_informaticsGatewayClient.Setup(p => p.MonaiScpAeTitle.Update(It.IsAny<MonaiApplicationEntity>(), It.IsAny<CancellationToken>()))
367409
.ReturnsAsync(entity);

‎src/Database/EntityFramework/Configuration/MonaiApplicationEntityConfiguration.cs

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
/*
2-
* Copyright 2021-2022 MONAI Consortium
2+
* Copyright 2021-2023 MONAI Consortium
33
* Copyright 2021 NVIDIA Corporation
44
*
55
* Licensed under the Apache License, Version 2.0 (the "License");
@@ -25,6 +25,7 @@
2525
namespace Monai.Deploy.InformaticsGateway.Database.EntityFramework.Configuration
2626
{
2727
#pragma warning disable CS8604, CS8603
28+
2829
internal class MonaiApplicationEntityConfiguration : IEntityTypeConfiguration<MonaiApplicationEntity>
2930
{
3031
public void Configure(EntityTypeBuilder<MonaiApplicationEntity> builder)
@@ -51,6 +52,11 @@ public void Configure(EntityTypeBuilder<MonaiApplicationEntity> builder)
5152
v => JsonSerializer.Serialize(v, jsonSerializerSettings),
5253
v => JsonSerializer.Deserialize<List<string>>(v, jsonSerializerSettings))
5354
.Metadata.SetValueComparer(valueComparer);
55+
builder.Property(j => j.PluginAssemblies)
56+
.HasConversion(
57+
v => JsonSerializer.Serialize(v, jsonSerializerSettings),
58+
v => JsonSerializer.Deserialize<List<string>>(v, jsonSerializerSettings))
59+
.Metadata.SetValueComparer(valueComparer);
5460
builder.Property(j => j.IgnoredSopClasses)
5561
.HasConversion(
5662
v => JsonSerializer.Serialize(v, jsonSerializerSettings),
@@ -67,5 +73,6 @@ public void Configure(EntityTypeBuilder<MonaiApplicationEntity> builder)
6773
builder.Ignore(p => p.Id);
6874
}
6975
}
76+
7077
#pragma warning restore CS8604, CS8603
7178
}

0 commit comments

Comments
 (0)