Refactor storage layer (#75)

* Refactor messaging broker layer

Signed-off-by: Victor Chang <[email protected]>

* Refactor storage layer

Signed-off-by: Victor Chang <[email protected]>

* Fix default storage service assembly name

Signed-off-by: Victor Chang <[email protected]>

* Fix integration test configs

Signed-off-by: Victor Chang <[email protected]>

Author: mocsharp

README.md

@@ -43,7 +43,7 @@ Please refer to the latest [user guide](./docs/setup/setup.md) for installation
 During development, change any settings inside the `appsettings.Development.json` file.
 First, export the following environment variable before executing `dotnet run`:
 
-#### Linux 
+#### Bash 
 
 ```bash
 export DOTNET_ENVIRONMENT=Development

src/Api/Test/Monai.Deploy.InformaticsGateway.Api.Test.csproj

@@ -21,7 +21,7 @@ SPDX-License-Identifier: Apache License 2.0
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
     </PackageReference>
     <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.1.0" />
-    <PackageReference Include="System.IO.Abstractions.TestingHelpers" Version="13.2.47" />
+    <PackageReference Include="System.IO.Abstractions.TestingHelpers" Version="16.1.25" />
     <PackageReference Include="xRetry" Version="1.8.0" />
     <PackageReference Include="xunit" Version="2.4.1" />
     <PackageReference Include="xunit.runner.visualstudio" Version="2.4.3">

src/CLI/Monai.Deploy.InformaticsGateway.CLI.csproj

@@ -51,6 +51,6 @@ SPDX-License-Identifier: Apache License 2.0
     <PackageReference Include="System.CommandLine" Version="2.0.0-beta1.21308.1" />
     <PackageReference Include="System.CommandLine.Hosting" Version="0.3.0-alpha.21216.1" />
     <PackageReference Include="System.CommandLine.Rendering" Version="0.3.0-alpha.21216.1" />
-    <PackageReference Include="System.IO.Abstractions" Version="16.1.23" />
+    <PackageReference Include="System.IO.Abstractions" Version="16.1.25" />
   </ItemGroup>
 </Project>

src/CLI/Test/Monai.Deploy.InformaticsGateway.CLI.Test.csproj

@@ -23,7 +23,7 @@
     <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.1.0" />
     <PackageReference Include="Moq" Version="4.17.2" />
     <PackageReference Include="System.CommandLine.Hosting" Version="0.3.0-alpha.21216.1" />
-    <PackageReference Include="System.IO.Abstractions.TestingHelpers" Version="16.1.23" />
+    <PackageReference Include="System.IO.Abstractions.TestingHelpers" Version="16.1.25" />
     <PackageReference Include="xRetry" Version="1.8.0" />
     <PackageReference Include="xunit" Version="2.4.1" />
     <PackageReference Include="xunit.runner.visualstudio" Version="2.4.3">

src/Common/Monai.Deploy.InformaticsGateway.Common.csproj

@@ -20,7 +20,7 @@
     <PackageReference Include="GitVersion.MsBuild" Version="5.9.0">
       <PrivateAssets>All</PrivateAssets>
     </PackageReference>
-    <PackageReference Include="System.IO.Abstractions" Version="16.1.23" />
+    <PackageReference Include="System.IO.Abstractions" Version="16.1.25" />
     <PackageReference Include="System.Threading.Tasks.Dataflow" Version="6.0.0" />
   </ItemGroup>
 

src/Common/Test/Monai.Deploy.InformaticsGateway.Common.Test.csproj

@@ -19,7 +19,7 @@ SPDX-License-Identifier: Apache License 2.0
     </PackageReference>
     <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.1.0" />
     <PackageReference Include="Moq" Version="4.17.2" />
-    <PackageReference Include="System.IO.Abstractions.TestingHelpers" Version="16.1.23" />
+    <PackageReference Include="System.IO.Abstractions.TestingHelpers" Version="16.1.25" />
     <PackageReference Include="xunit" Version="2.4.1" />
     <PackageReference Include="xunit.runner.visualstudio" Version="2.4.3">
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>

src/Configuration/ConfigurationException.cs

@@ -13,6 +13,10 @@ namespace Monai.Deploy.InformaticsGateway.Configuration
     [Serializable]
     public class ConfigurationException : Exception
     {
+        public ConfigurationException()
+        {
+        }
+
         public ConfigurationException(string message) : base(message)
         {
         }

src/Configuration/Monai.Deploy.InformaticsGateway.Configuration.csproj

@@ -20,7 +20,7 @@ SPDX-License-Identifier: Apache License 2.0
     <PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="6.0.1" />
     <PackageReference Include="Microsoft.Extensions.Options" Version="6.0.0" />
     <PackageReference Include="Newtonsoft.Json" Version="13.0.1" />
-    <PackageReference Include="System.IO.Abstractions" Version="16.1.23" />
+    <PackageReference Include="System.IO.Abstractions" Version="16.1.25" />
   </ItemGroup>
 
   <ItemGroup>
@@ -37,5 +37,6 @@ SPDX-License-Identifier: Apache License 2.0
     <ProjectReference Include="..\Api\Monai.Deploy.InformaticsGateway.Api.csproj" />
     <ProjectReference Include="..\Common\Monai.Deploy.InformaticsGateway.Common.csproj" />
     <ProjectReference Include="..\MessageBroker\RabbitMq\Monai.Deploy.MessageBroker.csproj" />
+    <ProjectReference Include="..\Storage\MinIo\Monai.Deploy.Storage.csproj" />
   </ItemGroup>
 </Project>

src/Configuration/ServiceCredentials.cs

@@ -3,27 +3,27 @@
 // SPDX-License-Identifier: Apache License 2.0
 
 using System.IO.Abstractions;
+using Microsoft.Extensions.Configuration;
+using Monai.Deploy.Storage.Common;
 using Newtonsoft.Json;
 
 namespace Monai.Deploy.InformaticsGateway.Configuration
 {
-    public class StorageConfiguration
+    public class StorageConfiguration : StorageServiceConfiguration
     {
         private readonly IFileSystem _fileSystem;
 
         public StorageConfiguration() : this(new FileSystem())
-        {
-        }
+        { }
 
-        public StorageConfiguration(IFileSystem fileSystem)
-            => _fileSystem = fileSystem ?? throw new System.ArgumentNullException(nameof(fileSystem));
+        public StorageConfiguration(IFileSystem fileSystem) => _fileSystem = fileSystem ?? throw new System.ArgumentNullException(nameof(fileSystem));
 
         /// <summary>
         /// Gets or sets temporary storage path.
         /// This is used to store all instances received to a temporary folder.
         /// </summary>
         /// <value></value>
-        [JsonProperty(PropertyName = "temporary")]
+        [ConfigurationKeyName("temporary")]
         public string Temporary { get; set; } = "./payloads";
 
         /// <summary>
@@ -33,7 +33,7 @@ public StorageConfiguration(IFileSystem fileSystem)
         /// is above the watermark.
         /// </summary>
         /// <value></value>
-        [JsonProperty(PropertyName = "watermarkPercent")]
+        [ConfigurationKeyName("watermarkPercent")]
         public uint Watermark { get; set; } = 75;
 
         /// <summary>
@@ -43,36 +43,15 @@ public StorageConfiguration(IFileSystem fileSystem)
         /// is less than the value.
         /// </summary>
         /// <value></value>
-        [JsonProperty(PropertyName = "reserveSpaceGB")]
+        [ConfigurationKeyName("reserveSpaceGB")]
         public uint ReserveSpaceGB { get; set; } = 5;
 
-        /// <summary>
-        /// Gets or sets the a fully qualified type name of the storage service.
-        /// The spcified type must implement <typeparam name="Monai.Deploy.InformaticsGateway.Api.Storage.IStorageService">IStorageService</typeparam> interface.
-        /// The default storage service configured is MinIO.
-        /// </summary>
-
-        [JsonProperty(PropertyName = "storageService")]
-        public string StorageService { get; set; } = "Monai.Deploy.InformaticsGateway.Storage.MinIoStorageService, Monai.Deploy.InformaticsGateway.Storage.MinIo";
-
-        /// <summary>
-        /// Gets or sets credentials used to access the storage service.
-        /// </summary>
-        [JsonProperty(PropertyName = "storageServiceCredentials")]
-        public ServiceCredentials StorageServiceCredentials { get; set; }
-
         /// <summary>
         /// Gets or sets retry options relate to saving files to temporary storage, processing payloads and uploading payloads to the storage service.
         /// </summary>
-        [JsonProperty(PropertyName = "reties")]
+        [ConfigurationKeyName("reties")]
         public RetryConfiguration Retries { get; set; } = new RetryConfiguration();
 
-        /// <summary>
-        /// Gets or set whether to use secured connection to the storage service.  Default is true.
-        /// </summary>
-        [JsonProperty(PropertyName = "securedConnection")]
-        public bool SecuredConnection { get; set; } = true;
-
         /// <summary>
         /// Gets or set number of threads used for payload upload. Default is 1;
         /// </summary>
@@ -90,7 +69,7 @@ public string TemporaryDataDirFullPath
         /// <summary>
         /// Gets or sets the name of the bucket where payloads are uploaded to.
         /// </summary>
-        [JsonProperty(PropertyName = "storageServiceBucketName")]
-        public string StorageServiceBucketName { get; set; }
+        [ConfigurationKeyName("bucketName")]
+        public string StorageServiceBucketName { get; set; } = string.Empty;
     }
 }

src/Configuration/StorageConfiguration.cs

@@ -24,7 +24,7 @@ SPDX-License-Identifier: Apache License 2.0
     </PackageReference>
     <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.1.0" />
     <PackageReference Include="Moq" Version="4.17.2" />
-    <PackageReference Include="System.IO.Abstractions.TestingHelpers" Version="16.1.23" />
+    <PackageReference Include="System.IO.Abstractions.TestingHelpers" Version="16.1.25" />
     <PackageReference Include="xunit" Version="2.4.1" />
     <PackageReference Include="xunit.runner.visualstudio" Version="2.4.3">
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>

src/Configuration/Test/Monai.Deploy.InformaticsGateway.Configuration.Test.csproj

@@ -55,7 +55,7 @@
         <ProjectReference Include="..\Database\Monai.Deploy.InformaticsGateway.Database.csproj" />
         <ProjectReference Include="..\DicomWebClient\Monai.Deploy.InformaticsGateway.DicomWeb.Client.csproj" />
         <ProjectReference Include="..\MessageBroker\RabbitMq\Monai.Deploy.MessageBroker.csproj" />
-        <ProjectReference Include="..\Storage\MinIo\Monai.Deploy.InformaticsGateway.Storage.MinIo.csproj" />
+        <ProjectReference Include="..\Storage\MinIo\Monai.Deploy.Storage.csproj" />
     </ItemGroup>
 
     <ItemGroup>

src/InformaticsGateway/Monai.Deploy.InformaticsGateway.csproj

@@ -13,7 +13,6 @@
 using Microsoft.Extensions.Hosting;
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Options;
-using Monai.Deploy.InformaticsGateway.Api.Storage;
 using Monai.Deploy.InformaticsGateway.Common;
 using Monai.Deploy.InformaticsGateway.Configuration;
 using Monai.Deploy.InformaticsGateway.Database;
@@ -23,9 +22,12 @@
 using Monai.Deploy.InformaticsGateway.Services.Http;
 using Monai.Deploy.InformaticsGateway.Services.Scp;
 using Monai.Deploy.InformaticsGateway.Services.Storage;
-using Monai.Deploy.InformaticsGateway.Storage;
 using Monai.Deploy.MessageBroker;
+using Monai.Deploy.MessageBroker.Common;
 using Monai.Deploy.MessageBroker.RabbitMq;
+using Monai.Deploy.Storage;
+using Monai.Deploy.Storage.Common;
+using Monai.Deploy.Storage.MinIo;
 
 namespace Monai.Deploy.InformaticsGateway
 {
@@ -76,6 +78,17 @@ internal static IHostBuilder CreateHostBuilder(string[] args) =>
                         .PostConfigure(options =>
                         {
                         });
+                    services.AddOptions<MessageBrokerServiceConfiguration>()
+                        .Bind(hostContext.Configuration.GetSection("InformaticsGateway:messaging"))
+                        .PostConfigure(options =>
+                        {
+                        });
+                    services.AddOptions<StorageServiceConfiguration>()
+                        .Bind(hostContext.Configuration.GetSection("InformaticsGateway:storage"))
+                        .PostConfigure(options =>
+                        {
+                        });
+
                     services.TryAddEnumerable(ServiceDescriptor.Singleton<IValidateOptions<InformaticsGatewayConfiguration>, ConfigurationValidator>());
 
                     services.AddDbContext<InformaticsGatewayContext>(
@@ -99,7 +112,7 @@ internal static IHostBuilder CreateHostBuilder(string[] args) =>
                         var options = implementationFactory.GetService<IOptions<InformaticsGatewayConfiguration>>();
                         var serviceProvider = implementationFactory.GetService<IServiceProvider>();
                         var logger = implementationFactory.GetService<ILogger<Program>>();
-                        return serviceProvider.LocateService<IStorageService>(logger, options.Value.Storage.StorageService);
+                        return serviceProvider.LocateService<IStorageService>(logger, options.Value.Storage.ServiceAssemblyName);
                     });
 
                     services.AddSingleton<RabbitMqMessagePublisherService>();

src/InformaticsGateway/Program.cs

@@ -25,6 +25,7 @@
 using Monai.Deploy.InformaticsGateway.Services.Storage;
 using Monai.Deploy.MessageBroker;
 using Monai.Deploy.MessageBroker.Messages;
+using Monai.Deploy.Storage;
 
 namespace Monai.Deploy.InformaticsGateway.Services.Connectors
 {

src/InformaticsGateway/Services/Connectors/PayloadNotificationService.cs

@@ -16,7 +16,6 @@
 using Microsoft.Extensions.Options;
 using Monai.Deploy.InformaticsGateway.Api;
 using Monai.Deploy.InformaticsGateway.Api.Rest;
-using Monai.Deploy.InformaticsGateway.Api.Storage;
 using Monai.Deploy.InformaticsGateway.Common;
 using Monai.Deploy.InformaticsGateway.Configuration;
 using Monai.Deploy.InformaticsGateway.Logging;
@@ -25,6 +24,7 @@
 using Monai.Deploy.MessageBroker;
 using Monai.Deploy.MessageBroker.Common;
 using Monai.Deploy.MessageBroker.Messages;
+using Monai.Deploy.Storage;
 using Polly;
 
 namespace Monai.Deploy.InformaticsGateway.Services.Export

src/InformaticsGateway/Services/Export/ExportServiceBase.cs

@@ -25,7 +25,7 @@
     <PackageReference Include="Microsoft.EntityFrameworkCore.InMemory" Version="6.0.3" />
     <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.1.0" />
     <PackageReference Include="Moq" Version="4.17.2" />
-    <PackageReference Include="System.IO.Abstractions.TestingHelpers" Version="16.1.23" />
+    <PackageReference Include="System.IO.Abstractions.TestingHelpers" Version="16.1.25" />
     <PackageReference Include="xRetry" Version="1.8.0" />
     <PackageReference Include="xunit" Version="2.4.1" />
     <PackageReference Include="xunit.runner.visualstudio" Version="2.4.3">

src/InformaticsGateway/Test/Monai.Deploy.InformaticsGateway.Test.csproj

@@ -19,6 +19,7 @@
 using Monai.Deploy.InformaticsGateway.SharedTest;
 using Monai.Deploy.MessageBroker;
 using Monai.Deploy.MessageBroker.Messages;
+using Monai.Deploy.Storage;
 using Moq;
 using xRetry;
 using Xunit;

src/InformaticsGateway/Test/Services/Connectors/PayloadNotificationServiceTest.cs

@@ -14,7 +14,6 @@
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Options;
 using Monai.Deploy.InformaticsGateway.Api.Rest;
-using Monai.Deploy.InformaticsGateway.Api.Storage;
 using Monai.Deploy.InformaticsGateway.Common;
 using Monai.Deploy.InformaticsGateway.Configuration;
 using Monai.Deploy.InformaticsGateway.DicomWeb.Client;
@@ -25,6 +24,7 @@
 using Monai.Deploy.MessageBroker;
 using Monai.Deploy.MessageBroker.Common;
 using Monai.Deploy.MessageBroker.Messages;
+using Monai.Deploy.Storage;
 using Moq;
 using Moq.Protected;
 using xRetry;

src/InformaticsGateway/Test/Services/Export/DicomWebExportServiceTest.cs

@@ -9,14 +9,14 @@
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Options;
-using Monai.Deploy.InformaticsGateway.Api.Storage;
 using Monai.Deploy.InformaticsGateway.Configuration;
 using Monai.Deploy.InformaticsGateway.Services.Export;
 using Monai.Deploy.InformaticsGateway.Services.Storage;
 using Monai.Deploy.InformaticsGateway.SharedTest;
 using Monai.Deploy.MessageBroker;
 using Monai.Deploy.MessageBroker.Common;
 using Monai.Deploy.MessageBroker.Messages;
+using Monai.Deploy.Storage;
 using Moq;
 using xRetry;
 using Xunit;

src/InformaticsGateway/Test/Services/Export/ExportServiceBaseTest.cs

@@ -12,7 +12,6 @@
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Options;
 using Monai.Deploy.InformaticsGateway.Api;
-using Monai.Deploy.InformaticsGateway.Api.Storage;
 using Monai.Deploy.InformaticsGateway.Common;
 using Monai.Deploy.InformaticsGateway.Configuration;
 using Monai.Deploy.InformaticsGateway.Repositories;
@@ -22,6 +21,7 @@
 using Monai.Deploy.MessageBroker;
 using Monai.Deploy.MessageBroker.Common;
 using Monai.Deploy.MessageBroker.Messages;
+using Monai.Deploy.Storage;
 using Moq;
 using xRetry;
 using Xunit;

src/InformaticsGateway/Test/Services/Export/ScuExportServiceTest.cs

@@ -11,14 +11,14 @@
     },
     "messaging": {
       "publisherSettings": {
-        "endpoint": "192.168.20.156",
+        "endpoint": "localhost",
         "username": "mdig",
         "password": "helloworld",
         "virtualHost": "monaideploy",
         "exchange": "monaideploy"
       },
       "subscriberSettings": {
-        "endpoint": "192.168.20.156",
+        "endpoint": "localhost",
         "username": "mdig",
         "password": "helloworld",
         "virtualHost": "monaideploy",
@@ -28,13 +28,13 @@
     },
     "storage": {
       "temporary": "./payloads",
-      "storageServiceCredentials": {
-        "endpoint": "192.168.20.156:9000",
+      "bucketName": "igbucket",
+      "settings": {
+        "endpoint": "localhost:9000",
         "accessKey": "admin",
-        "accessToken": "password"
-      },
-      "securedConnection": false,
-      "storageServiceBucketName": "igbucket"
+        "accessToken": "password",
+        "securedConnection": false
+      }
     }
   },
   "Logging": {

src/InformaticsGateway/appsettings.Development.json

@@ -5,7 +5,7 @@
 
 namespace Monai.Deploy.MessageBroker.Common
 {
-    public abstract class MessageBrokerServiceConfiguration
+    public class MessageBrokerServiceConfiguration
     {
         /// <summary>
         /// Gets or sets the a fully qualified type name of the message publisher service.