Skip to content

Commit 09887b1

Browse files
mocsharpmocsharp
authored
Ability to switch temporary storage to use either memory or disk (#166)
* Ability to switch temporary storage to use either memory or disk Signed-off-by: Victor Chang <[email protected]> * Fix configuration name for temp data storage. Signed-off-by: Victor Chang <[email protected]> * Validate storage configurations based on temp storage location Signed-off-by: Victor Chang <[email protected]> * Log time for a payload to complete end-to-end within the service. Signed-off-by: Victor Chang <[email protected]> Signed-off-by: Victor Chang <[email protected]>
1 parent 5b05935 commit 09887b1

26 files changed

+353
-80
lines changed

‎src/Api/Storage/Payload.cs

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,8 @@ public enum PayloadState
6666

6767
public bool HasTimedOut { get => ElapsedTime().TotalSeconds >= Timeout; }
6868

69+
public TimeSpan Elapsed { get { return DateTime.UtcNow.Subtract(DateTimeCreated); } }
70+
6971
public string CallingAeTitle { get => Files.OfType<DicomFileStorageMetadata>().Select(p => p.CallingAeTitle).FirstOrDefault(); }
7072

7173
public string CalledAeTitle { get => Files.OfType<DicomFileStorageMetadata>().Select(p => p.CalledAeTitle).FirstOrDefault(); }

‎src/Api/Storage/StorageObjectMetadata.cs

Lines changed: 22 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616

1717
using System;
1818
using System.IO;
19+
using System.Runtime;
1920
using System.Text.Json.Serialization;
2021
using Ardalis.GuardClauses;
2122

@@ -123,9 +124,27 @@ public void SetUploaded(string bucketName)
123124

124125
if (Data is not null && Data.CanSeek)
125126
{
126-
Data.Close();
127-
Data.Dispose();
128-
Data = null;
127+
if (Data is FileStream fileStream)
128+
{
129+
var filename = fileStream.Name;
130+
Data.Close();
131+
Data.Dispose();
132+
Data = null;
133+
System.IO.File.Delete(filename);
134+
}
135+
else // MemoryStream
136+
{
137+
Data.Close();
138+
Data.Dispose();
139+
Data = null;
140+
141+
// When IG stores all received/downloaded data in-memory using MemoryStream, LOH grows tremendously and thus impacts the performance and
142+
// memory usage. The following makes sure LOH is compacted after the data is uploaded.
143+
GCSettings.LargeObjectHeapCompactionMode = GCLargeObjectHeapCompactionMode.CompactOnce;
144+
#pragma warning disable S1215 // "GC.Collect" should not be called
145+
GC.Collect();
146+
#pragma warning restore S1215 // "GC.Collect" should not be called
147+
}
129148
}
130149
}
131150

‎src/Configuration/ConfigurationValidator.cs

Lines changed: 37 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,8 @@
1717

1818
using System;
1919
using System.Collections.Generic;
20+
using System.IO;
21+
using System.IO.Abstractions;
2022
using System.Text.RegularExpressions;
2123
using Microsoft.Extensions.Logging;
2224
using Microsoft.Extensions.Options;
@@ -29,16 +31,18 @@ namespace Monai.Deploy.InformaticsGateway.Configuration
2931
public class ConfigurationValidator : IValidateOptions<InformaticsGatewayConfiguration>
3032
{
3133
private readonly ILogger<ConfigurationValidator> _logger;
34+
private readonly IFileSystem _fileSystem;
3235
private readonly List<string> _validationErrors;
3336

3437
/// <summary>
3538
/// Initializes an instance of the <see cref="ConfigurationValidator"/> class.
3639
/// </summary>
3740
/// <param name="configuration">InformaticsGatewayConfiguration to be validated</param>
3841
/// <param name="logger">Logger to be used by ConfigurationValidator</param>
39-
public ConfigurationValidator(ILogger<ConfigurationValidator> logger)
42+
public ConfigurationValidator(ILogger<ConfigurationValidator> logger, IFileSystem fileSystem)
4043
{
4144
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
45+
_fileSystem = fileSystem ?? throw new ArgumentNullException(nameof(fileSystem));
4246
_validationErrors = new List<string>();
4347
}
4448

@@ -82,6 +86,38 @@ private bool IsStorageValid(StorageConfiguration storage)
8286
valid &= IsValueInRange("InformaticsGateway>storage>reserveSpaceGB", 1, 999, storage.ReserveSpaceGB);
8387
valid &= IsValueInRange("InformaticsGateway>storage>payloadProcessThreads", 1, 128, storage.PayloadProcessThreads);
8488
valid &= IsValueInRange("InformaticsGateway>storage>concurrentUploads", 1, 128, storage.ConcurrentUploads);
89+
valid &= IsValueInRange("InformaticsGateway>storage>memoryThreshold", 1, int.MaxValue, storage.MemoryThreshold);
90+
91+
if (storage.TemporaryDataStorage == TemporaryDataStorageLocation.Disk)
92+
{
93+
valid &= IsNotNullOrWhiteSpace("InformaticsGateway>storage>bufferRootPath", storage.BufferStorageRootPath);
94+
valid &= IsValidDirectory("InformaticsGateway>storage>bufferRootPath", storage.BufferStorageRootPath);
95+
valid &= IsValueInRange("InformaticsGateway>storage>bufferSize", 1, int.MaxValue, storage.BufferSize);
96+
}
97+
98+
return valid;
99+
}
100+
101+
private bool IsValidDirectory(string source, string directory)
102+
{
103+
var valid = true;
104+
try
105+
{
106+
if (!_fileSystem.Directory.Exists(directory))
107+
{
108+
valid = false;
109+
_validationErrors.Add($"Directory `{directory}` specified in `{source}` does not exist.");
110+
}
111+
else
112+
{
113+
using var _ = _fileSystem.File.Create(Path.Combine(directory, Path.GetRandomFileName()), 1, FileOptions.DeleteOnClose);
114+
}
115+
}
116+
catch (Exception ex)
117+
{
118+
valid = false;
119+
_validationErrors.Add($"Directory `{directory}` specified in `{source}` is not accessible: {ex.Message}.");
120+
}
85121
return valid;
86122
}
87123

‎src/Configuration/StorageConfiguration.cs

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,13 +22,28 @@ namespace Monai.Deploy.InformaticsGateway.Configuration
2222
{
2323
public class StorageConfiguration : StorageServiceConfiguration
2424
{
25+
/// <summary>
26+
/// Gets or sets whether to store temporary data in <c>Memory</c> or on <c>Disk</c>.
27+
/// Defaults to <c>Memory</c>.
28+
/// </summary>
29+
[ConfigurationKeyName("tempStorageLocation")]
30+
public TemporaryDataStorageLocation TemporaryDataStorage { get; set; } = TemporaryDataStorageLocation.Memory;
31+
2532
/// <summary>
2633
/// Gets or sets the path used for buffering incoming data.
2734
/// Defaults to <c>./temp</c>.
2835
/// </summary>
2936
[ConfigurationKeyName("bufferRootPath")]
3037
public string BufferStorageRootPath { get; set; } = "./temp";
3138

39+
/// <summary>
40+
/// Gets or sets the number of bytes buffered for reads and writes to the temporary file.
41+
/// Defaults to <c>128000</c>.
42+
/// </summary>
43+
[ConfigurationKeyName("bufferSize")]
44+
public int BufferSize { get; set; } = 128000;
45+
46+
3247
/// <summary>
3348
/// Gets or set the maximum memory buffer size in bytes with default to 30MiB.
3449
/// </summary>

‎src/Configuration/TemporaryDataStorageLocation.cs

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
/*
2+
* Copyright 2022 MONAI Consortium
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
namespace Monai.Deploy.InformaticsGateway.Configuration
18+
{
19+
public enum TemporaryDataStorageLocation
20+
{
21+
Memory,
22+
Disk
23+
}
24+
}

‎src/Configuration/Test/ConfigurationValidatorTest.cs

Lines changed: 31 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,8 @@
1515
*/
1616

1717
using System;
18+
using System.IO;
19+
using System.IO.Abstractions;
1820
using Microsoft.Extensions.Logging;
1921
using Microsoft.Extensions.Options;
2022
using Monai.Deploy.InformaticsGateway.SharedTest;
@@ -26,17 +28,19 @@ namespace Monai.Deploy.InformaticsGateway.Configuration.Test
2628
public class ConfigurationValidatorTest
2729
{
2830
private readonly Mock<ILogger<ConfigurationValidator>> _logger;
31+
private readonly Mock<IFileSystem> _fileSystem;
2932

3033
public ConfigurationValidatorTest()
3134
{
3235
_logger = new Mock<ILogger<ConfigurationValidator>>();
36+
_fileSystem = new Mock<IFileSystem>();
3337
}
3438

3539
[Fact(DisplayName = "ConfigurationValidator test with all valid settings")]
3640
public void AllValid()
3741
{
3842
var config = MockValidConfiguration();
39-
var valid = new ConfigurationValidator(_logger.Object).Validate("", config);
43+
var valid = new ConfigurationValidator(_logger.Object, _fileSystem.Object).Validate("", config);
4044
Assert.True(valid == ValidateOptionsResult.Success);
4145
}
4246

@@ -46,7 +50,7 @@ public void InvalidScpPort()
4650
var config = MockValidConfiguration();
4751
config.Dicom.Scp.Port = Int32.MaxValue;
4852

49-
var valid = new ConfigurationValidator(_logger.Object).Validate("", config);
53+
var valid = new ConfigurationValidator(_logger.Object, _fileSystem.Object).Validate("", config);
5054

5155
var validationMessage = $"Invalid port number '{Int32.MaxValue}' specified for InformaticsGateway>dicom>scp>port.";
5256
Assert.Equal(validationMessage, valid.FailureMessage);
@@ -59,7 +63,7 @@ public void InvalidScpMaxAssociations()
5963
var config = MockValidConfiguration();
6064
config.Dicom.Scp.MaximumNumberOfAssociations = 0;
6165

62-
var valid = new ConfigurationValidator(_logger.Object).Validate("", config);
66+
var valid = new ConfigurationValidator(_logger.Object, _fileSystem.Object).Validate("", config);
6367

6468
var validationMessage = $"Value of InformaticsGateway>dicom>scp>max-associations must be between {1} and {1000}.";
6569
Assert.Equal(validationMessage, valid.FailureMessage);
@@ -72,7 +76,7 @@ public void StorageWithInvalidWatermark()
7276
var config = MockValidConfiguration();
7377
config.Storage.Watermark = 1000;
7478

75-
var valid = new ConfigurationValidator(_logger.Object).Validate("", config);
79+
var valid = new ConfigurationValidator(_logger.Object, _fileSystem.Object).Validate("", config);
7680

7781
var validationMessage = "Value of InformaticsGateway>storage>watermark must be between 1 and 100.";
7882
Assert.Equal(validationMessage, valid.FailureMessage);
@@ -85,7 +89,7 @@ public void StorageWithInvalidReservedSpace()
8589
var config = MockValidConfiguration();
8690
config.Storage.ReserveSpaceGB = 9999;
8791

88-
var valid = new ConfigurationValidator(_logger.Object).Validate("", config);
92+
var valid = new ConfigurationValidator(_logger.Object, _fileSystem.Object).Validate("", config);
8993

9094
var validationMessage = "Value of InformaticsGateway>storage>reserveSpaceGB must be between 1 and 999.";
9195
Assert.Equal(validationMessage, valid.FailureMessage);
@@ -98,7 +102,7 @@ public void StorageWithInvalidTemporaryBucketName()
98102
var config = MockValidConfiguration();
99103
config.Storage.TemporaryStorageBucket = " ";
100104

101-
var valid = new ConfigurationValidator(_logger.Object).Validate("", config);
105+
var valid = new ConfigurationValidator(_logger.Object, _fileSystem.Object).Validate("", config);
102106

103107
var validationMessages = new[] { "Value for InformaticsGateway>storage>temporaryBucketName is required.", "Value for InformaticsGateway>storage>temporaryBucketName does not conform to Amazon S3 bucket naming requirements." };
104108
Assert.Equal(string.Join(Environment.NewLine, validationMessages), valid.FailureMessage);
@@ -114,7 +118,7 @@ public void StorageWithInvalidBucketName()
114118
var config = MockValidConfiguration();
115119
config.Storage.StorageServiceBucketName = "";
116120

117-
var valid = new ConfigurationValidator(_logger.Object).Validate("", config);
121+
var valid = new ConfigurationValidator(_logger.Object, _fileSystem.Object).Validate("", config);
118122

119123
var validationMessages = new[] { "Value for InformaticsGateway>storage>bucketName is required.", "Value for InformaticsGateway>storage>bucketName does not conform to Amazon S3 bucket naming requirements." };
120124
Assert.Equal(string.Join(Environment.NewLine, validationMessages), valid.FailureMessage);
@@ -124,6 +128,26 @@ public void StorageWithInvalidBucketName()
124128
}
125129
}
126130

131+
[Fact(DisplayName = "ConfigurationValidator test with inaccessible directory")]
132+
public void StorageWithInaccessbleDirectory()
133+
{
134+
_fileSystem.Setup(p => p.Directory.Exists(It.IsAny<string>())).Returns(true);
135+
_fileSystem.Setup(p => p.File.Create(It.IsAny<string>(), It.IsAny<int>(), It.IsAny<FileOptions>())).Throws(new UnauthorizedAccessException("error"));
136+
137+
var config = MockValidConfiguration();
138+
config.Storage.TemporaryDataStorage = TemporaryDataStorageLocation.Disk;
139+
config.Storage.BufferStorageRootPath = "/blabla";
140+
141+
var valid = new ConfigurationValidator(_logger.Object, _fileSystem.Object).Validate("", config);
142+
143+
var validationMessages = new[] { $"Directory `/blabla` specified in `InformaticsGateway>storage>bufferRootPath` is not accessible: error." };
144+
Assert.Equal(string.Join(Environment.NewLine, validationMessages), valid.FailureMessage);
145+
foreach (var message in validationMessages)
146+
{
147+
_logger.VerifyLogging(message, LogLevel.Error, Times.Once());
148+
}
149+
}
150+
127151
private static InformaticsGatewayConfiguration MockValidConfiguration()
128152
{
129153
var config = new InformaticsGatewayConfiguration();

‎src/Database/PayloadConfiguration.cs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,7 @@ public void Configure(EntityTypeBuilder<Payload> builder)
5757
builder.Ignore(j => j.CalledAeTitle);
5858
builder.Ignore(j => j.CallingAeTitle);
5959
builder.Ignore(j => j.HasTimedOut);
60+
builder.Ignore(j => j.Elapsed);
6061
builder.Ignore(j => j.Count);
6162
}
6263
}

‎src/InformaticsGateway/Common/FileStorageMetadataExtensions.cs

Lines changed: 63 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -14,41 +14,92 @@
1414
* limitations under the License.
1515
*/
1616

17-
using System.IO;
17+
using System.IO.Abstractions;
1818
using System.Text;
1919
using System.Threading.Tasks;
2020
using Ardalis.GuardClauses;
2121
using FellowOakDicom;
2222
using Monai.Deploy.InformaticsGateway.Api.Storage;
23+
using Monai.Deploy.InformaticsGateway.Configuration;
2324

2425
namespace Monai.Deploy.InformaticsGateway.Common
2526
{
2627
internal static class FileStorageMetadataExtensions
2728
{
28-
public static async Task SetDataStreams(this DicomFileStorageMetadata dicomFileStorageMetadata, DicomFile dicomFile, string dicomJson)
29+
public static async Task SetDataStreams(
30+
this DicomFileStorageMetadata dicomFileStorageMetadata,
31+
DicomFile dicomFile,
32+
string dicomJson,
33+
TemporaryDataStorageLocation storageLocation,
34+
IFileSystem fileSystem = null,
35+
string temporaryStoragePath = "")
2936
{
3037
Guard.Against.Null(dicomFile, nameof(dicomFile));
3138
Guard.Against.Null(dicomJson, nameof(dicomJson)); // allow empty here
3239

33-
dicomFileStorageMetadata.File.Data = new MemoryStream();
40+
switch (storageLocation)
41+
{
42+
case TemporaryDataStorageLocation.Disk:
43+
Guard.Against.Null(fileSystem, nameof(fileSystem));
44+
Guard.Against.NullOrWhiteSpace(temporaryStoragePath, nameof(temporaryStoragePath));
45+
46+
var tempFile = fileSystem.Path.Combine(temporaryStoragePath, $@"{System.DateTime.UtcNow.Ticks}.tmp");
47+
dicomFileStorageMetadata.File.Data = fileSystem.File.Create(tempFile);
48+
break;
49+
default:
50+
dicomFileStorageMetadata.File.Data = new System.IO.MemoryStream();
51+
break;
52+
}
53+
3454
await dicomFile.SaveAsync(dicomFileStorageMetadata.File.Data).ConfigureAwait(false);
35-
dicomFileStorageMetadata.File.Data.Seek(0, SeekOrigin.Begin);
55+
dicomFileStorageMetadata.File.Data.Seek(0, System.IO.SeekOrigin.Begin);
3656

37-
SetTextStream(dicomFileStorageMetadata.JsonFile, dicomJson);
57+
await SetTextStream(dicomFileStorageMetadata.JsonFile, dicomJson, storageLocation, fileSystem, temporaryStoragePath);
3858
}
3959

40-
public static void SetDataStream(this FhirFileStorageMetadata fhirFileStorageMetadata, string json)
41-
=> SetTextStream(fhirFileStorageMetadata.File, json);
60+
public static async Task SetDataStream(
61+
this FhirFileStorageMetadata fhirFileStorageMetadata,
62+
string json,
63+
TemporaryDataStorageLocation storageLocation,
64+
IFileSystem fileSystem = null,
65+
string temporaryStoragePath = "")
66+
=> await SetTextStream(fhirFileStorageMetadata.File, json, storageLocation, fileSystem, temporaryStoragePath);
4267

43-
public static void SetDataStream(this Hl7FileStorageMetadata hl7FileStorageMetadata, string message)
44-
=> SetTextStream(hl7FileStorageMetadata.File, message);
68+
public static async Task SetDataStream(
69+
this Hl7FileStorageMetadata hl7FileStorageMetadata,
70+
string message,
71+
TemporaryDataStorageLocation storageLocation,
72+
IFileSystem fileSystem = null,
73+
string temporaryStoragePath = "")
74+
=> await SetTextStream(hl7FileStorageMetadata.File, message, storageLocation, fileSystem, temporaryStoragePath);
4575

46-
private static void SetTextStream(StorageObjectMetadata storageObjectMetadata, string message)
76+
private static async Task SetTextStream(
77+
StorageObjectMetadata storageObjectMetadata,
78+
string message,
79+
TemporaryDataStorageLocation storageLocation,
80+
IFileSystem fileSystem = null,
81+
string temporaryStoragePath = "")
4782
{
4883
Guard.Against.Null(message, nameof(message)); // allow empty here
4984

50-
storageObjectMetadata.Data = new MemoryStream(Encoding.UTF8.GetBytes(message));
51-
storageObjectMetadata.Data.Seek(0, SeekOrigin.Begin);
85+
switch (storageLocation)
86+
{
87+
case TemporaryDataStorageLocation.Disk:
88+
Guard.Against.Null(fileSystem, nameof(fileSystem));
89+
Guard.Against.NullOrWhiteSpace(temporaryStoragePath, nameof(temporaryStoragePath));
90+
91+
var tempFile = fileSystem.Path.Combine(temporaryStoragePath, $@"{System.DateTime.UtcNow.Ticks}.tmp");
92+
var stream = fileSystem.File.Create(tempFile);
93+
var data = Encoding.UTF8.GetBytes(message);
94+
await stream.WriteAsync(data, 0, data.Length);
95+
storageObjectMetadata.Data = stream;
96+
break;
97+
default:
98+
storageObjectMetadata.Data = new System.IO.MemoryStream(Encoding.UTF8.GetBytes(message));
99+
break;
100+
}
101+
102+
storageObjectMetadata.Data.Seek(0, System.IO.SeekOrigin.Begin);
52103
}
53104
}
54105
}

0 commit comments

Comments
 (0)