Skip to content

Update Monai.Deploy.Messaging & Monai.Deploy.Storage to 0.1 official … #96

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Jun 30, 2022
Merged

Update Monai.Deploy.Messaging & Monai.Deploy.Storage to 0.1 official … #96

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
ac1ff71
Update Monai.Deploy.Messaging & Monai.Deploy.Storage to 0.1 official …
mocsharp Jun 29, 2022
ed6b8cd
Update storage GetObjectAsync call
mocsharp Jun 29, 2022
6c31ef5
Update storage to 0.1.1
mocsharp Jun 30, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions src/Api/Monai.Deploy.InformaticsGateway.Api.csproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,8 @@ SPDX-License-Identifier: Apache License 2.0
</PackageReference>
<PackageReference Include="Macross.Json.Extensions" Version="3.0.0" />
<PackageReference Include="Microsoft.EntityFrameworkCore.Abstractions" Version="6.0.6" />
<PackageReference Include="Monai.Deploy.Messaging" Version="0.1.0-rc0046" />
<PackageReference Include="Monai.Deploy.Storage" Version="0.1.0-rc0058" />
<PackageReference Include="Monai.Deploy.Messaging" Version="0.1.2" />
<PackageReference Include="Monai.Deploy.Storage" Version="0.1.1" />
<PackageReference Include="System.ComponentModel.Annotations" Version="5.0.0" />
</ItemGroup>

Expand Down
3 changes: 1 addition & 2 deletions src/CLI/Commands/DestinationCommand.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@
using System;
using System.Collections.Generic;
using System.CommandLine;
using System.CommandLine.Invocation;
using System.CommandLine.NamingConventionBinder;
using System.CommandLine.Rendering;
using System.CommandLine.Rendering.Views;
Expand Down Expand Up @@ -131,7 +130,7 @@ private async Task<int> ListDestinationHandlerAsync(DestinationApplicationEntity
table.AddColumn(p => p.HostIp, new ContentView("Host/IP Address".Underline()));
table.AddColumn(p => p.Port, new ContentView("Port".Underline()));
table.Render(consoleRenderer, consoleRegion.GetDefaultConsoleRegion());

logger.ListedNItems(items.Count);
}
return ExitCodes.Success;
Expand Down
3 changes: 1 addition & 2 deletions src/CLI/Commands/SourceCommand.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@
using System;
using System.Collections.Generic;
using System.CommandLine;
using System.CommandLine.Invocation;
using System.CommandLine.NamingConventionBinder;
using System.CommandLine.Rendering;
using System.CommandLine.Rendering.Views;
Expand Down Expand Up @@ -127,7 +126,7 @@ private async Task<int> ListSourceHandlerAsync(SourceApplicationEntity entity, I
table.AddColumn(p => p.AeTitle, new ContentView("AE Title".Underline()));
table.AddColumn(p => p.HostIp, new ContentView("Host/IP Address".Underline()));
table.Render(consoleRenderer, consoleRegion.GetDefaultConsoleRegion());

logger.ListedNItems(items.Count);
}
return ExitCodes.Success;
Expand Down
1 change: 0 additions & 1 deletion src/CLI/Commands/StartCommand.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@
// SPDX-License-Identifier: Apache License 2.0

using System;
using System.CommandLine.Invocation;
using System.CommandLine.NamingConventionBinder;
using System.Threading;
using System.Threading.Tasks;
Expand Down
1 change: 0 additions & 1 deletion src/CLI/Commands/StatusCommand.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@
// SPDX-License-Identifier: Apache License 2.0

using System;
using System.CommandLine.Invocation;
using System.CommandLine.NamingConventionBinder;
using System.Threading;
using System.Threading.Tasks;
Expand Down
1 change: 0 additions & 1 deletion src/CLI/Commands/StopCommand.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@
// SPDX-License-Identifier: Apache License 2.0

using System;
using System.CommandLine.Invocation;
using System.CommandLine.NamingConventionBinder;
using System.Threading;
using System.Threading.Tasks;
Expand Down
1 change: 0 additions & 1 deletion src/Client/Services/HealthService.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@
// SPDX-License-Identifier: Apache License 2.0

using System.Net.Http;
using System.Net.Http.Json;
using System.Threading;
using System.Threading.Tasks;
using Ardalis.GuardClauses;
Expand Down
4 changes: 2 additions & 2 deletions src/Configuration/Monai.Deploy.InformaticsGateway.Configuration.csproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,8 @@ SPDX-License-Identifier: Apache License 2.0
</PackageReference>
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="6.0.1" />
<PackageReference Include="Microsoft.Extensions.Options" Version="6.0.0" />
<PackageReference Include="Monai.Deploy.Messaging" Version="0.1.0-rc0046" />
<PackageReference Include="Monai.Deploy.Storage" Version="0.1.0-rc0058" />
<PackageReference Include="Monai.Deploy.Messaging" Version="0.1.2" />
<PackageReference Include="Monai.Deploy.Storage" Version="0.1.1" />
<PackageReference Include="System.IO.Abstractions" Version="17.0.18" />
</ItemGroup>

Expand Down
2 changes: 1 addition & 1 deletion src/InformaticsGateway/Monai.Deploy.InformaticsGateway.csproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@
<PackageReference Include="Microsoft.Extensions.Logging" Version="6.0.0" />
<PackageReference Include="Microsoft.Extensions.Logging.Console" Version="6.0.0" />
<PackageReference Include="Microsoft.Extensions.Options" Version="6.0.0" />
<PackageReference Include="Monai.Deploy.Storage" Version="0.1.0-rc0058" />
<PackageReference Include="Monai.Deploy.Storage" Version="0.1.1" />
<PackageReference Include="Polly" Version="7.2.3" />
<PackageReference Include="Swashbuckle.AspNetCore" Version="6.3.1" />
</ItemGroup>
Expand Down
22 changes: 10 additions & 12 deletions src/InformaticsGateway/Services/Export/ExportServiceBase.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -128,7 +128,7 @@ private void OnMessageReceivedCallback(MessageReceivedEventArgs eventArgs)
};

var exportFlow = new TransformManyBlock<ExportRequestEventDetails, ExportRequestDataMessage>(
(exportRequest) => DownloadPayloadActionCallback(exportRequest, _cancellationTokenSource.Token),
exportRequest => DownloadPayloadActionCallback(exportRequest, _cancellationTokenSource.Token),
executionOptions);

var exportActionBlock = new TransformBlock<ExportRequestDataMessage, ExportRequestDataMessage>(
Expand Down Expand Up @@ -180,6 +180,8 @@ private void OnMessageReceivedCallback(MessageReceivedEventArgs eventArgs)
}
}

// TPL doesn't yet support IAsyncEnumerable
// https://github.com/dotnet/runtime/issues/30863
private IEnumerable<ExportRequestDataMessage> DownloadPayloadActionCallback(ExportRequestEventDetails exportRequest, CancellationToken cancellationToken)
{
Guard.Against.Null(exportRequest, nameof(exportRequest));
Expand All @@ -193,27 +195,23 @@ private IEnumerable<ExportRequestDataMessage> DownloadPayloadActionCallback(Expo
try
{
_logger.DownloadingFile(file);
Policy
var task = Policy
.Handle<Exception>()
.WaitAndRetry(
.WaitAndRetryAsync(
_configuration.Export.Retries.RetryDelays,
(exception, timeSpan, retryCount, context) =>
{
_logger.ErrorDownloadingPayloadWithRetry(exception, timeSpan, retryCount);
})
.Execute(() =>
.ExecuteAsync(async () =>
{
_logger.DownloadingFile(file);
var task = storageService.GetObjectAsync(_configuration.Storage.StorageServiceBucketName, file, (stream) =>
{
using var memoryStream = new MemoryStream();
stream.CopyTo(memoryStream);
exportRequestData.SetData(memoryStream.ToArray());
}, cancellationToken);

task.Wait();
var stream = await storageService.GetObjectAsync(_configuration.Storage.StorageServiceBucketName, file, cancellationToken).ConfigureAwait(false) as MemoryStream;
exportRequestData.SetData(stream.ToArray());
_logger.FileReadyForExport(file);
});

task.Wait();
}
catch (Exception ex)
{
Expand Down
4 changes: 2 additions & 2 deletions src/InformaticsGateway/Test/DummyStorageService.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,9 +36,9 @@ internal class DummyStorageService : IStorageService

public Task<Credentials> CreateTemporaryCredentialsAsync(string bucketName, string folderName, int durationSeconds = 3600, CancellationToken cancellationToken = default) => throw new NotImplementedException();

public Task GetObjectAsync(string bucketName, string objectName, Action<Stream> callback, CancellationToken cancellationToken = default) => throw new NotImplementedException();
public Task<Stream> GetObjectAsync(string bucketName, string objectName, CancellationToken cancellationToken = default) => throw new NotImplementedException();

public Task GetObjectWithCredentialsAsync(string bucketName, string objectName, Credentials credentials, Action<Stream> callback, CancellationToken cancellationToken = default) => throw new NotImplementedException();
public Task<Stream> GetObjectWithCredentialsAsync(string bucketName, string objectName, Credentials credentials, CancellationToken cancellationToken = default) => throw new NotImplementedException();

public Task<IList<VirtualFileInfo>> ListObjectsAsync(string bucketName, string prefix = "", bool recursive = false, CancellationToken cancellationToken = default) => throw new NotImplementedException();

Expand Down
28 changes: 8 additions & 20 deletions src/InformaticsGateway/Test/Services/Export/DicomWebExportServiceTest.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -122,11 +122,8 @@ public async Task ExportDataBlockCallback_ReturnsNullIfInferenceRequestCannotBeF
messageReceivedCallback(CreateMessageReceivedEventArgs(transactionId));
});

_storageService.Setup(p => p.GetObjectAsync(It.IsAny<string>(), It.IsAny<string>(), It.IsAny<Action<Stream>>(), It.IsAny<CancellationToken>()))
.Callback<string, string, Action<Stream>, CancellationToken>((bucketName, objectName, callback, cancellationToken) =>
{
callback(new MemoryStream(Encoding.UTF8.GetBytes("test")));
});
_storageService.Setup(p => p.GetObjectAsync(It.IsAny<string>(), It.IsAny<string>(), It.IsAny<CancellationToken>()))
.ReturnsAsync(new MemoryStream(Encoding.UTF8.GetBytes("test")));

_inferenceRequestStore.Setup(p => p.GetInferenceRequest(It.IsAny<string>())).Returns((InferenceRequest)null);

Expand Down Expand Up @@ -182,11 +179,8 @@ public async Task ExportDataBlockCallback_ReturnsNullIfInferenceRequestContainsN
messageReceivedCallback(CreateMessageReceivedEventArgs(transactionId));
});

_storageService.Setup(p => p.GetObjectAsync(It.IsAny<string>(), It.IsAny<string>(), It.IsAny<Action<Stream>>(), It.IsAny<CancellationToken>()))
.Callback<string, string, Action<Stream>, CancellationToken>((bucketName, objectName, callback, cancellationToken) =>
{
callback(new MemoryStream(Encoding.UTF8.GetBytes("test")));
});
_storageService.Setup(p => p.GetObjectAsync(It.IsAny<string>(), It.IsAny<string>(), It.IsAny<CancellationToken>()))
.ReturnsAsync(new MemoryStream(Encoding.UTF8.GetBytes("test")));

_inferenceRequestStore.Setup(p => p.GetInferenceRequest(It.IsAny<string>())).Returns(inferenceRequest);

Expand Down Expand Up @@ -254,11 +248,8 @@ public async Task ExportDataBlockCallback_RecordsStowFailuresAndReportFailure()
messageReceivedCallback(CreateMessageReceivedEventArgs(transactionId));
});

_storageService.Setup(p => p.GetObjectAsync(It.IsAny<string>(), It.IsAny<string>(), It.IsAny<Action<Stream>>(), It.IsAny<CancellationToken>()))
.Callback<string, string, Action<Stream>, CancellationToken>((bucketName, objectName, callback, cancellationToken) =>
{
callback(new MemoryStream(Encoding.UTF8.GetBytes("test")));
});
_storageService.Setup(p => p.GetObjectAsync(It.IsAny<string>(), It.IsAny<string>(), It.IsAny<CancellationToken>()))
.ReturnsAsync(new MemoryStream(Encoding.UTF8.GetBytes("test")));

_inferenceRequestStore.Setup(p => p.GetInferenceRequest(It.IsAny<string>())).Returns(inferenceRequest);
_dicomToolkit.Setup(p => p.Load(It.IsAny<byte[]>())).Returns(InstanceGenerator.GenerateDicomFile(sopInstanceUid: sopInstanceUid));
Expand Down Expand Up @@ -343,11 +334,8 @@ public async Task CompletesDataflow(HttpStatusCode httpStatusCode)
messageReceivedCallback(CreateMessageReceivedEventArgs(transactionId));
});

_storageService.Setup(p => p.GetObjectAsync(It.IsAny<string>(), It.IsAny<string>(), It.IsAny<Action<Stream>>(), It.IsAny<CancellationToken>()))
.Callback<string, string, Action<Stream>, CancellationToken>((bucketName, objectName, callback, cancellationToken) =>
{
callback(new MemoryStream(Encoding.UTF8.GetBytes("test")));
});
_storageService.Setup(p => p.GetObjectAsync(It.IsAny<string>(), It.IsAny<string>(), It.IsAny<CancellationToken>()))
.ReturnsAsync(new MemoryStream(Encoding.UTF8.GetBytes("test")));

_inferenceRequestStore.Setup(p => p.GetInferenceRequest(It.IsAny<string>())).Returns(inferenceRequest);
_dicomToolkit.Setup(p => p.Load(It.IsAny<byte[]>())).Returns(InstanceGenerator.GenerateDicomFile());
Expand Down
17 changes: 5 additions & 12 deletions src/InformaticsGateway/Test/Services/Export/ExportServiceBaseTest.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@
using Monai.Deploy.Messaging.Common;
using Monai.Deploy.Messaging.Events;
using Monai.Deploy.Messaging.Messages;
using Monai.Deploy.Storage;
using Monai.Deploy.Storage.API;
using Moq;
using xRetry;
Expand Down Expand Up @@ -156,12 +155,9 @@ public async Task DataflowTest_PayloadDownlaodFailure()
messageReceivedCallback(CreateMessageReceivedEventArgs());
});

_storageService.Setup(p => p.GetObjectAsync(It.IsAny<string>(), It.IsAny<string>(), It.IsAny<Action<Stream>>(), It.IsAny<CancellationToken>()))
.Callback<string, string, Action<Stream>, CancellationToken>((bucketName, objectName, callback, cancellationToken) =>
{
callback(Stream.Null);
throw new Exception("storage error");
});
_storageService.Setup(p => p.GetObjectAsync(It.IsAny<string>(), It.IsAny<string>(), It.IsAny<CancellationToken>()))
.ThrowsAsync(new Exception("storage error"));

var countdownEvent = new CountdownEvent(1);
var service = new TestExportService(_logger.Object, _configuration, _serviceScopeFactory.Object, _storageInfoProvider.Object);
service.ReportActionCompleted += (sender, e) =>
Expand Down Expand Up @@ -206,11 +202,8 @@ public async Task DataflowTest_EndToEnd()
}
});

_storageService.Setup(p => p.GetObjectAsync(It.IsAny<string>(), It.IsAny<string>(), It.IsAny<Action<Stream>>(), It.IsAny<CancellationToken>()))
.Callback<string, string, Action<Stream>, CancellationToken>((bucketName, objectName, callback, cancellationToken) =>
{
callback(new MemoryStream(Encoding.UTF8.GetBytes(testData)));
});
_storageService.Setup(p => p.GetObjectAsync(It.IsAny<string>(), It.IsAny<string>(), It.IsAny<CancellationToken>()))
.ReturnsAsync(new MemoryStream(Encoding.UTF8.GetBytes(testData)));

var countdownEvent = new CountdownEvent(5 * 3);
var service = new TestExportService(_logger.Object, _configuration, _serviceScopeFactory.Object, _storageInfoProvider.Object);
Expand Down
Loading