Skip to content

Commit 126ee70

Browse files
mocsharpmocsharp
committed
gh-188 Stops accepting/retreiving data when disk space is low.
Allows users to configure watermark & reserve space similar to 0.1. Signed-off-by: Victor Chang <[email protected]>
1 parent 609f939 commit 126ee70

34 files changed

+544
-40
lines changed

‎docs/setup/schema.md

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -96,12 +96,14 @@ The `InformaticsGateway` configuration section contains the following sub-sectio
9696
}
9797
},
9898
"storage": {
99-
"bufferRootPath": "./temp",
100-
"tempStorageRootPath": "/incoming",
99+
"localTemporaryStoragePath": "/payloads",
100+
"remoteTemporaryStoragePath": "/incoming",
101101
"bucketName": "monaideploy",
102102
"storageRootPath": "/payloads",
103103
"temporaryBucketName": "monaideploy",
104104
"serviceAssemblyName": "Monai.Deploy.Storage.MinIO.MinIoStorageService, Monai.Deploy.Storage.MinIO",
105+
"watermarkPercent": 75,
106+
"reserveSpaceGB": 5,
105107
"settings": {
106108
"endpoint": "localhost:9000",
107109
"accessKey": "admin",

‎docs/setup/setup.md

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -101,9 +101,9 @@ The Informatics Gateway operates on two storage locations. In the first location
101101

102102
### Temporary Storage of Incoming Dataset
103103

104-
By default, the temporary storage location is set to `/payloads` in the `appsettings.json` file.
104+
By default, the temporary storage location is set to use `Disk` and stores any incoming files inside `/payloads`. This can be modified to user a different location, such as `Memory` or a different path.
105105

106-
To change the temporary storage location, locate the `./InformaticsGateway/storage/temporary` property in the `appsettings.json` file and modify it.
106+
To change the temporary storage path, locate the `InformaticsGateway>storage>localTemporaryStoragePath` property in the `appsettings.json` file and modify it.
107107

108108
> [!Note]
109109
> You will need to calculate the required temporary storage based on the number of studies and the size of each study.
@@ -116,7 +116,7 @@ To change the temporary storage location, locate the `./InformaticsGateway/stora
116116
> the expected number of studies and size of each study. The suggested value for `reserveSpaceGB` is 2x to 3x the
117117
> size of a single study multiplied by the number of configured AE Titles.
118118
119-
### Shared Storage
119+
### Storage Service
120120

121121
Informatics Gateway includes MinIO as the default storage service provider. To integrate with another storage service provider, please refer to the [Data Storage](https://github.com/Project-MONAI/monai-deploy-informatics-gateway/blob/main/guidelines/srs.md#data-storage) section of the SRS.
122122

@@ -138,7 +138,6 @@ Locate the storage section of the configuration in `appsettings.json`:
138138
"accessToken": "password", # Access token or password
139139
"securedConnection": false, # Indicates if connection should be secured using HTTPS
140140
"region": "local", # Region
141-
"executableLocation": "/bin/mc", # Path to minio client
142141
"serviceName": "MinIO" # Name of the service
143142
},
144143
"storageService": "Monai.Deploy.Storage.MinIO.MinIoStorageService, Monai.Deploy.Storage.MinIO", # Fully qualified type name of the storage service

‎src/Configuration/ConfigurationValidator.cs

Lines changed: 5 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -81,7 +81,7 @@ private bool IsStorageValid(StorageConfiguration storage)
8181
var valid = true;
8282
valid &= IsValidBucketName("InformaticsGateway>storage>bucketName", storage.StorageServiceBucketName);
8383
valid &= IsValidBucketName("InformaticsGateway>storage>temporaryBucketName", storage.TemporaryStorageBucket);
84-
valid &= IsNotNullOrWhiteSpace("InformaticsGateway>storage>temporary", storage.TemporaryStorageRootPath);
84+
valid &= IsNotNullOrWhiteSpace("InformaticsGateway>storage>temporary", storage.RemoteTemporaryStoragePath);
8585
valid &= IsValueInRange("InformaticsGateway>storage>watermark", 1, 100, storage.Watermark);
8686
valid &= IsValueInRange("InformaticsGateway>storage>reserveSpaceGB", 1, 999, storage.ReserveSpaceGB);
8787
valid &= IsValueInRange("InformaticsGateway>storage>payloadProcessThreads", 1, 128, storage.PayloadProcessThreads);
@@ -90,8 +90,8 @@ private bool IsStorageValid(StorageConfiguration storage)
9090

9191
if (storage.TemporaryDataStorage == TemporaryDataStorageLocation.Disk)
9292
{
93-
valid &= IsNotNullOrWhiteSpace("InformaticsGateway>storage>bufferRootPath", storage.BufferStorageRootPath);
94-
valid &= IsValidDirectory("InformaticsGateway>storage>bufferRootPath", storage.BufferStorageRootPath);
93+
valid &= IsNotNullOrWhiteSpace("InformaticsGateway>storage>localTemporaryStoragePath", storage.LocalTemporaryStoragePath);
94+
valid &= IsValidDirectory("InformaticsGateway>storage>localTemporaryStoragePath", storage.LocalTemporaryStoragePath);
9595
valid &= IsValueInRange("InformaticsGateway>storage>bufferSize", 1, int.MaxValue, storage.BufferSize);
9696
}
9797

@@ -105,13 +105,9 @@ private bool IsValidDirectory(string source, string directory)
105105
{
106106
if (!_fileSystem.Directory.Exists(directory))
107107
{
108-
valid = false;
109-
_validationErrors.Add($"Directory `{directory}` specified in `{source}` does not exist.");
110-
}
111-
else
112-
{
113-
using var _ = _fileSystem.File.Create(Path.Combine(directory, Path.GetRandomFileName()), 1, FileOptions.DeleteOnClose);
108+
_fileSystem.Directory.CreateDirectory(directory);
114109
}
110+
using var _ = _fileSystem.File.Create(Path.Combine(directory, Path.GetRandomFileName()), 1, FileOptions.DeleteOnClose);
115111
}
116112
catch (Exception ex)
117113
{

‎src/Configuration/StorageConfiguration.cs

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -33,8 +33,8 @@ public class StorageConfiguration : StorageServiceConfiguration
3333
/// Gets or sets the path used for buffering incoming data.
3434
/// Defaults to <c>./temp</c>.
3535
/// </summary>
36-
[ConfigurationKeyName("bufferRootPath")]
37-
public string BufferStorageRootPath { get; set; } = "./temp";
36+
[ConfigurationKeyName("localTemporaryStoragePath")]
37+
public string LocalTemporaryStoragePath { get; set; } = "/payloads";
3838

3939
/// <summary>
4040
/// Gets or sets the number of bytes buffered for reads and writes to the temporary file.
@@ -66,8 +66,8 @@ public class StorageConfiguration : StorageServiceConfiguration
6666
/// Gets or sets root directory path for storing incoming data in the <c>temporaryBucketName</c>.
6767
/// Defaults to <c>/incoming</c>.
6868
/// </summary>
69-
[ConfigurationKeyName("tempStorageRootPath")]
70-
public string TemporaryStorageRootPath { get; set; } = "/incoming";
69+
[ConfigurationKeyName("remoteTemporaryStoragePath")]
70+
public string RemoteTemporaryStoragePath { get; set; } = "/incoming";
7171

7272
/// <summary>
7373
/// Gets or sets the watermark for disk usage with default value of 75%,

‎src/Configuration/Test/ConfigurationValidatorTest.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -137,7 +137,7 @@ public void StorageWithInaccessbleDirectory()
137137

138138
var config = MockValidConfiguration();
139139
config.Storage.TemporaryDataStorage = TemporaryDataStorageLocation.Disk;
140-
config.Storage.BufferStorageRootPath = "/blabla";
140+
config.Storage.LocalTemporaryStoragePath = "/blabla";
141141

142142
var valid = new ConfigurationValidator(_logger.Object, _fileSystem.Object).Validate("", config);
143143

‎src/InformaticsGateway/Common/InsufficientStorageAvailableException.cs

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
// Copyright 2021 MONAI Consortium
2+
// Licensed under the Apache License, Version 2.0 (the "License");
3+
// you may not use this file except in compliance with the License.
4+
// You may obtain a copy of the License at
5+
// http://www.apache.org/licenses/LICENSE-2.0
6+
// Unless required by applicable law or agreed to in writing, software
7+
// distributed under the License is distributed on an "AS IS" BASIS,
8+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
9+
// See the License for the specific language governing permissions and
10+
// limitations under the License.
11+
12+
/*
13+
* Apache License, Version 2.0
14+
* Copyright 2021 NVIDIA Corporation
15+
*
16+
* Licensed under the Apache License, Version 2.0 (the "License");
17+
* you may not use this file except in compliance with the License.
18+
* You may obtain a copy of the License at
19+
*
20+
* http://www.apache.org/licenses/LICENSE-2.0
21+
*
22+
* Unless required by applicable law or agreed to in writing, software
23+
* distributed under the License is distributed on an "AS IS" BASIS,
24+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
25+
* See the License for the specific language governing permissions and
26+
* limitations under the License.
27+
*/
28+
29+
using System;
30+
31+
namespace Monai.Deploy.InformaticsGateway.Common
32+
{
33+
[Serializable]
34+
public class InsufficientStorageAvailableException : Exception
35+
{
36+
public InsufficientStorageAvailableException()
37+
{
38+
}
39+
40+
public InsufficientStorageAvailableException(string message) : base(message)
41+
{
42+
}
43+
44+
public InsufficientStorageAvailableException(string message, Exception innerException) : base(message, innerException)
45+
{
46+
}
47+
48+
protected InsufficientStorageAvailableException(System.Runtime.Serialization.SerializationInfo serializationInfo, System.Runtime.Serialization.StreamingContext streamingContext)
49+
{
50+
throw new NotImplementedException();
51+
}
52+
}
53+
}

‎src/InformaticsGateway/Logging/Log.100.200.ScpService.cs

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -94,5 +94,8 @@ public static partial class Log
9494

9595
[LoggerMessage(EventId = 211, Level = LogLevel.Warning, Message = "Verification service is disabled: rejecting association.")]
9696
public static partial void VerificationServiceDisabled(this ILogger logger);
97+
98+
[LoggerMessage(EventId = 212, Level = LogLevel.Error, Message = "Failed to process C-STORE request, out of storage space.")]
99+
public static partial void CStoreFailedDueToLowStorageSpace(this ILogger logger, Exception ex);
97100
}
98101
}

‎src/InformaticsGateway/Logging/Log.500.ExportService.cs

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -114,5 +114,8 @@ public static partial class Log
114114

115115
[LoggerMessage(EventId = 530, Level = LogLevel.Error, Message = "{message}")]
116116
public static partial void ExportException(this ILogger logger, string message, Exception ex);
117+
118+
[LoggerMessage(EventId = 531, Level = LogLevel.Warning, Message = "Export service paused due to insufficient storage space. Available storage space: {availableFreeSpace:D}")]
119+
public static partial void ExportServiceStoppedDueToLowStorageSpace(this ILogger logger, long availableFreeSpace);
117120
}
118121
}

‎src/InformaticsGateway/Logging/Log.600.DataRetrievalService.cs

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -91,5 +91,8 @@ public static partial class Log
9191

9292
[LoggerMessage(EventId = 622, Level = LogLevel.Warning, Message = "FHIR resource {type}/{id} contains no data.")]
9393
public static partial void FhirResourceContainsNoData(this ILogger logger, string type, string id);
94+
95+
[LoggerMessage(EventId = 623, Level = LogLevel.Warning, Message = "Data retrieval paused due to insufficient storage space. Available storage space: {availableFreeSpace:D}.")]
96+
public static partial void DataRetrievalServiceStoppedDueToLowStorageSpace(this ILogger logger, long availableFreeSpace);
9497
}
9598
}

‎src/InformaticsGateway/Program.cs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -146,6 +146,7 @@ internal static IHostBuilder CreateHostBuilder(string[] args) =>
146146
services.AddSingleton<IPayloadAssembler, PayloadAssembler>();
147147
services.AddSingleton<FellowOakDicom.Log.ILogManager, Logging.FoDicomLogManager>();
148148
services.AddSingleton<IMonaiServiceLocator, MonaiServiceLocator>();
149+
services.AddSingleton<IStorageInfoProvider, StorageInfoProvider>();
149150
services.AddSingleton<IMonaiAeChangedNotificationService, MonaiAeChangedNotificationService>();
150151
services.AddSingleton<ITcpListenerFactory, TcpListenerFactory>();
151152
services.AddSingleton<IMllpClientFactory, MllpClientFactory>();

‎src/InformaticsGateway/Services/Connectors/DataRetrievalService.cs

Lines changed: 13 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,7 @@ internal class DataRetrievalService : IHostedService, IMonaiService, IDisposable
5757
private readonly IPayloadAssembler _payloadAssembler;
5858
private readonly IDicomToolkit _dicomToolkit;
5959
private readonly IFileSystem _fileSystem;
60+
private readonly IStorageInfoProvider _storageInfoProvider;
6061
private bool _disposedValue;
6162

6263
public ServiceStatus Status { get; set; }
@@ -80,6 +81,8 @@ public DataRetrievalService(
8081
_payloadAssembler = _rootScope.ServiceProvider.GetService<IPayloadAssembler>() ?? throw new ServiceNotFoundException(nameof(IPayloadAssembler));
8182
_dicomToolkit = _rootScope.ServiceProvider.GetService<IDicomToolkit>() ?? throw new ServiceNotFoundException(nameof(IDicomToolkit));
8283
_fileSystem = _rootScope.ServiceProvider.GetService<IFileSystem>() ?? throw new ServiceNotFoundException(nameof(IFileSystem));
84+
_storageInfoProvider = _rootScope.ServiceProvider.GetService<IStorageInfoProvider>() ?? throw new ServiceNotFoundException(nameof(IStorageInfoProvider));
85+
8386
}
8487

8588
public Task StartAsync(CancellationToken cancellationToken)
@@ -110,6 +113,13 @@ private async Task BackgroundProcessing(CancellationToken cancellationToken)
110113
using var scope = _serviceScopeFactory.CreateScope();
111114
var repository = scope.ServiceProvider.GetRequiredService<IInferenceRequestRepository>();
112115

116+
if (!_storageInfoProvider.HasSpaceAvailableToRetrieve)
117+
{
118+
_logger.DataRetrievalServiceStoppedDueToLowStorageSpace(_storageInfoProvider.AvailableFreeSpace);
119+
await Task.Delay(5000, cancellationToken).ConfigureAwait(false);
120+
continue;
121+
}
122+
113123
InferenceRequest request = null;
114124
try
115125
{
@@ -333,7 +343,7 @@ private async Task<bool> RetrieveFhirResource(string transactionId, HttpClient h
333343
}
334344

335345
var fhirFile = new FhirFileStorageMetadata(transactionId, resource.Type, resource.Id, fhirFormat);
336-
await fhirFile.SetDataStream(json, _options.Value.Storage.TemporaryDataStorage, _fileSystem, _options.Value.Storage.BufferStorageRootPath);
346+
await fhirFile.SetDataStream(json, _options.Value.Storage.TemporaryDataStorage, _fileSystem, _options.Value.Storage.LocalTemporaryStoragePath);
337347
retrievedResources.Add(fhirFile.Id, fhirFile);
338348
return true;
339349
}
@@ -512,7 +522,7 @@ private async Task RetrieveInstances(string transactionId, IDicomWebClient dicom
512522
}
513523

514524
var dicomFileStorageMetadata = SaveFile(transactionId, file, uids);
515-
await dicomFileStorageMetadata.SetDataStreams(file, file.ToJson(_options.Value.Dicom.WriteDicomJson, _options.Value.Dicom.ValidateDicomOnSerialization), _options.Value.Storage.TemporaryDataStorage, _fileSystem, _options.Value.Storage.BufferStorageRootPath).ConfigureAwait(false);
525+
await dicomFileStorageMetadata.SetDataStreams(file, file.ToJson(_options.Value.Dicom.WriteDicomJson, _options.Value.Dicom.ValidateDicomOnSerialization), _options.Value.Storage.TemporaryDataStorage, _fileSystem, _options.Value.Storage.LocalTemporaryStoragePath).ConfigureAwait(false);
516526
retrievedInstance.Add(uids.Identifier, dicomFileStorageMetadata);
517527
count++;
518528
}
@@ -542,7 +552,7 @@ private async Task SaveFiles(string transactionId, IAsyncEnumerable<DicomFile> f
542552
}
543553

544554
var dicomFileStorageMetadata = SaveFile(transactionId, file, uids);
545-
await dicomFileStorageMetadata.SetDataStreams(file, file.ToJson(_options.Value.Dicom.WriteDicomJson, _options.Value.Dicom.ValidateDicomOnSerialization), _options.Value.Storage.TemporaryDataStorage, _fileSystem, _options.Value.Storage.BufferStorageRootPath).ConfigureAwait(false);
555+
await dicomFileStorageMetadata.SetDataStreams(file, file.ToJson(_options.Value.Dicom.WriteDicomJson, _options.Value.Dicom.ValidateDicomOnSerialization), _options.Value.Storage.TemporaryDataStorage, _fileSystem, _options.Value.Storage.LocalTemporaryStoragePath).ConfigureAwait(false);
546556
retrievedInstance.Add(uids.Identifier, dicomFileStorageMetadata);
547557
}
548558
}

‎src/InformaticsGateway/Services/Connectors/PayloadMoveActionHandler.cs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -152,13 +152,13 @@ private async Task MoveFile(Guid payloadId, string identity, StorageObjectMetada
152152
_logger.MovingFileToPayloadDirectory(payloadId, identity);
153153
await _storageService.CopyObjectAsync(
154154
file.TemporaryBucketName,
155-
file.GetTempStoragPath(_options.Value.Storage.TemporaryStorageRootPath),
155+
file.GetTempStoragPath(_options.Value.Storage.RemoteTemporaryStoragePath),
156156
_options.Value.Storage.StorageServiceBucketName,
157157
file.GetPayloadPath(payloadId),
158158
cancellationToken).ConfigureAwait(false);
159159

160160
_logger.DeletingFileFromTemporaryBbucket(file.TemporaryBucketName, identity, file.TemporaryPath);
161-
await _storageService.RemoveObjectAsync(file.TemporaryBucketName, file.GetTempStoragPath(_options.Value.Storage.TemporaryStorageRootPath), cancellationToken);
161+
await _storageService.RemoveObjectAsync(file.TemporaryBucketName, file.GetTempStoragPath(_options.Value.Storage.RemoteTemporaryStoragePath), cancellationToken);
162162

163163
file.SetMoved(_options.Value.Storage.StorageServiceBucketName);
164164
}

‎src/InformaticsGateway/Services/DicomWeb/DicomInstanceReaderBase.cs

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -73,11 +73,11 @@ protected async Task<Stream> ConvertStream(HttpContext httpContext, Stream sourc
7373
{
7474
lock (SyncLock)
7575
{
76-
FileSystem.Directory.CreateDirectoryIfNotExists(Configuration.Storage.BufferStorageRootPath);
76+
FileSystem.Directory.CreateDirectoryIfNotExists(Configuration.Storage.LocalTemporaryStoragePath);
7777
}
7878

79-
Logger.ConvertingStreamToFileBufferingReadStream(Configuration.Storage.MemoryThreshold, Configuration.Storage.BufferStorageRootPath);
80-
seekableStream = new FileBufferingReadStream(sourceStream, Configuration.Storage.MemoryThreshold, null, Configuration.Storage.BufferStorageRootPath);
79+
Logger.ConvertingStreamToFileBufferingReadStream(Configuration.Storage.MemoryThreshold, Configuration.Storage.LocalTemporaryStoragePath);
80+
seekableStream = new FileBufferingReadStream(sourceStream, Configuration.Storage.MemoryThreshold, null, Configuration.Storage.LocalTemporaryStoragePath);
8181
httpContext.Response.RegisterForDisposeAsync(seekableStream);
8282
await seekableStream.DrainAsync(cancellationToken).ConfigureAwait(false);
8383
}

‎src/InformaticsGateway/Services/DicomWeb/IStreamsWriter.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -169,7 +169,7 @@ private async Task SaveInstance(Stream stream, string studyInstanceUid, string w
169169
dicomInfo.SetWorkflows(workflowName);
170170
}
171171

172-
await dicomInfo.SetDataStreams(dicomFile, dicomFile.ToJson(_configuration.Value.Dicom.WriteDicomJson, _configuration.Value.Dicom.ValidateDicomOnSerialization), _configuration.Value.Storage.TemporaryDataStorage, _fileSystem, _configuration.Value.Storage.BufferStorageRootPath).ConfigureAwait(false);
172+
await dicomInfo.SetDataStreams(dicomFile, dicomFile.ToJson(_configuration.Value.Dicom.WriteDicomJson, _configuration.Value.Dicom.ValidateDicomOnSerialization), _configuration.Value.Storage.TemporaryDataStorage, _fileSystem, _configuration.Value.Storage.LocalTemporaryStoragePath).ConfigureAwait(false);
173173
_uploadQueue.Queue(dicomInfo);
174174

175175
// for DICOMweb, use correlation ID as the grouping key

0 commit comments

Comments
 (0)