Skip to content

Commit 42e4b44

Browse files
mocsharpmocsharp
committed
gh-27 add options to configure database retries (#111)
Signed-off-by: Victor Chang <[email protected]>
1 parent e38c702 commit 42e4b44

File tree

10 files changed

+98
-57
lines changed

10 files changed

+98
-57
lines changed

‎src/Configuration/DatabaseConfiguration.cs

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
/*
2+
* Copyright 2021-2022 MONAI Consortium
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
using Microsoft.Extensions.Configuration;
18+
19+
namespace Monai.Deploy.InformaticsGateway.Configuration
20+
{
21+
public class DatabaseConfiguration
22+
{
23+
/// <summary>
24+
/// Gets or sets retry options relate to reading/writing to the database.
25+
/// </summary>
26+
[ConfigurationKeyName("retries")]
27+
public RetryConfiguration Retries { get; set; } = new RetryConfiguration();
28+
}
29+
}

‎src/Configuration/InformaticsGatewayConfiguration.cs

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,12 @@ public class InformaticsGatewayConfiguration
6868
[ConfigurationKeyName("messaging")]
6969
public MessageBrokerConfiguration Messaging { get; set; }
7070

71+
/// <summary>
72+
/// Represents the <c>database</c> section of the configuration file.
73+
/// </summary>
74+
[ConfigurationKeyName("database")]
75+
public DatabaseConfiguration Database { get; set; }
76+
7177
public InformaticsGatewayConfiguration()
7278
{
7379
Dicom = new DicomConfiguration();
@@ -76,6 +82,7 @@ public InformaticsGatewayConfiguration()
7682
Fhir = new FhirConfiguration();
7783
Export = new DataExportConfiguration();
7884
Messaging = new MessageBrokerConfiguration();
85+
Database = new DatabaseConfiguration();
7986
}
8087
}
8188
}

‎src/InformaticsGateway/Repositories/InferenceRequestRepository.cs

Lines changed: 9 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -20,28 +20,31 @@
2020
using System.Threading.Tasks;
2121
using Ardalis.GuardClauses;
2222
using Microsoft.Extensions.Logging;
23+
using Microsoft.Extensions.Options;
2324
using Monai.Deploy.InformaticsGateway.Api;
2425
using Monai.Deploy.InformaticsGateway.Api.Rest;
26+
using Monai.Deploy.InformaticsGateway.Configuration;
2527
using Monai.Deploy.InformaticsGateway.Logging;
2628
using Polly;
2729

2830
namespace Monai.Deploy.InformaticsGateway.Repositories
2931
{
3032
public class InferenceRequestRepository : IInferenceRequestRepository
3133
{
32-
private const int MaxRetryLimit = 3;
33-
3434
private readonly ILogger<InferenceRequestRepository> _logger;
3535
private readonly IInformaticsGatewayRepository<InferenceRequest> _inferenceRequestRepository;
36+
private readonly IOptions<InformaticsGatewayConfiguration> _options;
3637

3738
public ServiceStatus Status { get; set; } = ServiceStatus.Unknown;
3839

3940
public InferenceRequestRepository(
4041
ILogger<InferenceRequestRepository> logger,
41-
IInformaticsGatewayRepository<InferenceRequest> inferenceRequestRepository)
42+
IInformaticsGatewayRepository<InferenceRequest> inferenceRequestRepository,
43+
IOptions<InformaticsGatewayConfiguration> options)
4244
{
4345
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
4446
_inferenceRequestRepository = inferenceRequestRepository ?? throw new ArgumentNullException(nameof(inferenceRequestRepository));
47+
_options = options ?? throw new ArgumentNullException(nameof(options));
4548
}
4649

4750
public async Task Add(InferenceRequest inferenceRequest)
@@ -52,8 +55,7 @@ public async Task Add(InferenceRequest inferenceRequest)
5255
await Policy
5356
.Handle<Exception>()
5457
.WaitAndRetryAsync(
55-
3,
56-
retryAttempt => TimeSpan.FromSeconds(Math.Pow(2, retryAttempt)),
58+
_options.Value.Database.Retries.RetryDelays,
5759
(exception, timeSpan, retryCount, context) =>
5860
{
5961
_logger.ErrorSavingInferenceRequest(timeSpan, retryCount, exception);
@@ -81,7 +83,7 @@ public async Task Update(InferenceRequest inferenceRequest, InferenceRequestStat
8183
}
8284
else
8385
{
84-
if (++inferenceRequest.TryCount > MaxRetryLimit)
86+
if (++inferenceRequest.TryCount > _options.Value.Database.Retries.DelaysMilliseconds.Length)
8587
{
8688
_logger.InferenceRequestUpdateExceededMaximumRetries();
8789
inferenceRequest.State = InferenceRequestState.Completed;
@@ -158,8 +160,7 @@ private async Task Save(InferenceRequest inferenceRequest)
158160
await Policy
159161
.Handle<Exception>()
160162
.WaitAndRetryAsync(
161-
3,
162-
retryAttempt => TimeSpan.FromSeconds(Math.Pow(2, retryAttempt)),
163+
_options.Value.Database.Retries.RetryDelays,
163164
(exception, timeSpan, retryCount, context) =>
164165
{
165166
_logger.InferenceRequestUpdateError(timeSpan, retryCount, exception);

‎src/InformaticsGateway/Services/Connectors/DataRetrievalService.cs

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -28,10 +28,12 @@
2828
using Microsoft.Extensions.DependencyInjection;
2929
using Microsoft.Extensions.Hosting;
3030
using Microsoft.Extensions.Logging;
31+
using Microsoft.Extensions.Options;
3132
using Monai.Deploy.InformaticsGateway.Api;
3233
using Monai.Deploy.InformaticsGateway.Api.Rest;
3334
using Monai.Deploy.InformaticsGateway.Api.Storage;
3435
using Monai.Deploy.InformaticsGateway.Common;
36+
using Monai.Deploy.InformaticsGateway.Configuration;
3537
using Monai.Deploy.InformaticsGateway.DicomWeb.Client;
3638
using Monai.Deploy.InformaticsGateway.DicomWeb.Client.API;
3739
using Monai.Deploy.InformaticsGateway.Logging;
@@ -45,6 +47,7 @@ namespace Monai.Deploy.InformaticsGateway.Services.Connectors
4547
internal class DataRetrievalService : IHostedService, IMonaiService, IDisposable
4648
{
4749
private readonly IServiceScopeFactory _serviceScopeFactory;
50+
private readonly IOptions<InformaticsGatewayConfiguration> _options;
4851
private readonly ILogger<DataRetrievalService> _logger;
4952
private readonly IServiceScope _rootScope;
5053

@@ -63,9 +66,11 @@ internal class DataRetrievalService : IHostedService, IMonaiService, IDisposable
6366

6467
public DataRetrievalService(
6568
ILogger<DataRetrievalService> logger,
66-
IServiceScopeFactory serviceScopeFactory)
69+
IServiceScopeFactory serviceScopeFactory,
70+
IOptions<InformaticsGatewayConfiguration> options)
6771
{
6872
_serviceScopeFactory = serviceScopeFactory ?? throw new ArgumentNullException(nameof(serviceScopeFactory));
73+
_options = options ?? throw new ArgumentNullException(nameof(options));
6974
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
7075

7176
_rootScope = _serviceScopeFactory.CreateScope();
@@ -343,11 +348,8 @@ private async Task<bool> RetrieveFhirResource(string transactionId, HttpClient h
343348
request.Headers.Accept.Add(MediaTypeWithQualityHeaderValue.Parse(acceptHeader));
344349
var response = await Policy
345350
.HandleResult<HttpResponseMessage>(p => !p.IsSuccessStatusCode)
346-
.WaitAndRetryAsync(3,
347-
(retryAttempt) =>
348-
{
349-
return retryAttempt == 1 ? TimeSpan.FromMilliseconds(250) : TimeSpan.FromMilliseconds(500);
350-
},
351+
.WaitAndRetryAsync(
352+
_options.Value.Database.Retries.RetryDelays,
351353
(result, timeSpan, retryCount, context) =>
352354
{
353355
_logger.ErrorRetrievingFhirResourceWithRetry(resource.Type, resource.Id, result.Result.StatusCode, retryCount, result.Exception);

‎src/InformaticsGateway/Services/Connectors/PayloadAssembler.cs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -177,7 +177,7 @@ private async Task QueueBucketForNotification(string key, Payload payload)
177177
payload.State = Payload.PayloadState.Upload;
178178
var scope = _serviceScopeFactory.CreateScope();
179179
var repository = scope.ServiceProvider.GetRequiredService<IInformaticsGatewayRepository<Payload>>();
180-
await payload.UpdatePayload(_options.Value.Storage.Retries.RetryDelays, _logger, repository).ConfigureAwait(false);
180+
await payload.UpdatePayload(_options.Value.Database.Retries.RetryDelays, _logger, repository).ConfigureAwait(false);
181181
_workItems.Add(payload);
182182
_logger.BucketReady(key, payload.Count);
183183
}
@@ -201,7 +201,7 @@ private async Task<Payload> CreateOrGetPayload(string key, string correationId,
201201
var scope = _serviceScopeFactory.CreateScope();
202202
var repository = scope.ServiceProvider.GetRequiredService<IInformaticsGatewayRepository<Payload>>();
203203
var newPayload = new Payload(key, correationId, timeout);
204-
await newPayload.AddPayaloadToDatabase(_options.Value.Storage.Retries.RetryDelays, _logger, repository).ConfigureAwait(false);
204+
await newPayload.AddPayaloadToDatabase(_options.Value.Database.Retries.RetryDelays, _logger, repository).ConfigureAwait(false);
205205
_logger.BucketCreated(key, timeout);
206206
return newPayload;
207207
}));

‎src/InformaticsGateway/Services/Connectors/PayloadNotificationService.cs

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -182,7 +182,7 @@ private async Task UploadPayloadActionBlock(Payload payload, CancellationToken c
182182

183183
var scope = _serviceScopeFactory.CreateScope();
184184
var repository = scope.ServiceProvider.GetRequiredService<IInformaticsGatewayRepository<Payload>>();
185-
await payload.UpdatePayload(_options.Value.Storage.Retries.RetryDelays, _logger, repository).ConfigureAwait(false);
185+
await payload.UpdatePayload(_options.Value.Database.Retries.RetryDelays, _logger, repository).ConfigureAwait(false);
186186

187187
_publishQueue.Post(payload);
188188
_logger.PayloadReadyToBePublished(payload.Id);
@@ -267,7 +267,7 @@ private async Task PublishPayloadActionBlock(Payload payload)
267267

268268
var scope = _serviceScopeFactory.CreateScope();
269269
var repository = scope.ServiceProvider.GetRequiredService<IInformaticsGatewayRepository<Payload>>();
270-
await payload.DeletePayload(_options.Value.Storage.Retries.RetryDelays, _logger, repository).ConfigureAwait(false);
270+
await payload.DeletePayload(_options.Value.Database.Retries.RetryDelays, _logger, repository).ConfigureAwait(false);
271271
}
272272
catch (Exception ex)
273273
{
@@ -293,16 +293,16 @@ private async Task<PayloadAction> UpdatePayloadState(Payload payload)
293293

294294
try
295295
{
296-
if (payload.RetryCount > _options.Value.Storage.Retries.DelaysMilliseconds.Length)
296+
if (payload.RetryCount > _options.Value.Database.Retries.DelaysMilliseconds.Length)
297297
{
298298
_logger.UploadFailureStopRetry(payload.Id);
299-
await payload.DeletePayload(_options.Value.Storage.Retries.RetryDelays, _logger, repository).ConfigureAwait(false);
299+
await payload.DeletePayload(_options.Value.Database.Retries.RetryDelays, _logger, repository).ConfigureAwait(false);
300300
return PayloadAction.Deleted;
301301
}
302302
else
303303
{
304304
_logger.UploadFailureRetryLater(payload.Id, payload.State, payload.RetryCount);
305-
await payload.UpdatePayload(_options.Value.Storage.Retries.RetryDelays, _logger, repository).ConfigureAwait(false);
305+
await payload.UpdatePayload(_options.Value.Database.Retries.RetryDelays, _logger, repository).ConfigureAwait(false);
306306
return PayloadAction.Updated;
307307
}
308308
}

‎src/InformaticsGateway/Services/Storage/SpaceReclaimerService.cs

Lines changed: 3 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@ namespace Monai.Deploy.InformaticsGateway.Services.Storage
3636
internal class SpaceReclaimerService : IHostedService, IMonaiService
3737
{
3838
private readonly ILogger<SpaceReclaimerService> _logger;
39+
private readonly IOptions<InformaticsGatewayConfiguration> _configuration;
3940
private readonly IInstanceCleanupQueue _taskQueue;
4041
private readonly IFileSystem _fileSystem;
4142
private readonly string _payloadDirectory;
@@ -49,13 +50,9 @@ public SpaceReclaimerService(
4950
IOptions<InformaticsGatewayConfiguration> configuration,
5051
IFileSystem fileSystem)
5152
{
52-
if (configuration is null)
53-
{
54-
throw new ArgumentNullException(nameof(configuration));
55-
}
56-
5753
_taskQueue = taskQueue ?? throw new ArgumentNullException(nameof(taskQueue));
5854
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
55+
_configuration = configuration ?? throw new ArgumentNullException(nameof(configuration));
5956
_fileSystem = fileSystem ?? throw new ArgumentNullException(nameof(fileSystem));
6057
_payloadDirectory = configuration.Value.Storage.TemporaryDataDirFullPath;
6158
}
@@ -97,8 +94,7 @@ private void ProcessFile(FileStorageInfo file)
9794

9895
Policy.Handle<Exception>()
9996
.WaitAndRetry(
100-
3,
101-
retryAttempt => TimeSpan.FromSeconds(Math.Pow(2, retryAttempt)),
97+
_configuration.Value.Storage.Retries.RetryDelays,
10298
(exception, timespan, retryCount, context) =>
10399
{
104100
_logger.ErrorDeletingFIle(file.FilePath, retryCount, exception);

0 commit comments

Comments
 (0)