fixing health checks

Signed-off-by: Neil South <[email protected]>

Author: neildsouth

src/InformaticsGateway/Logging/Log.0.General.cs

@@ -59,5 +59,11 @@ public static partial class Log
 
         [LoggerMessage(EventId = 13, Level = LogLevel.Critical, Message = "Failed to start {ServiceName}.")]
         public static partial void ServiceFailedToStart(this ILogger logger, string serviceName, Exception ex);
+
+        [LoggerMessage(EventId = 14, Level = LogLevel.Critical, Message = "All services are unhealthy")]
+        public static partial void AllServiceUnheathly(this ILogger logger);
+
+        [LoggerMessage(EventId = 15, Level = LogLevel.Error, Message = "Some services are unhealthy {list}")]
+        public static partial void SomeServiceUnheathly(this ILogger logger, string list);
     }
 }

src/InformaticsGateway/Repositories/MonaiServiceLocator.cs

@@ -19,8 +19,11 @@
 using System.Linq;
 using System.Reflection;
 using Ardalis.GuardClauses;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Hosting;
 using Monai.Deploy.InformaticsGateway.Api.Rest;
 using Monai.Deploy.InformaticsGateway.Services.Common;
+using Monai.Deploy.InformaticsGateway.Services.Connectors;
 
 namespace Monai.Deploy.InformaticsGateway.Repositories
 {
@@ -51,7 +54,16 @@ public Dictionary<string, ServiceStatus> GetServiceStatus()
         {
             Guard.Against.Null(type, nameof(type));
 
-            return (_serviceProvider.GetService(type) as IMonaiService);
+            var hostedServices = _serviceProvider.GetServices<IHostedService>().ToList();
+            var t = (hostedServices[2].GetType()).Name;
+
+            var TypeInterface = type.GetInterfaces().FirstOrDefault(i => i.Name == $"I{type.Name}");
+            if (TypeInterface is null)
+            {
+                var te = _serviceProvider.GetServices<IHostedService>().FirstOrDefault(i => i.GetType().Name == type.Name);
+                return te as IMonaiService;
+            }
+            return (_serviceProvider.GetService(TypeInterface) as IMonaiService);
 
         }
 

src/InformaticsGateway/Services/Connectors/PayloadAssembler.cs

@@ -25,9 +25,11 @@
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Logging;
 using Monai.Deploy.InformaticsGateway.Api;
+using Monai.Deploy.InformaticsGateway.Api.Rest;
 using Monai.Deploy.InformaticsGateway.Api.Storage;
 using Monai.Deploy.InformaticsGateway.Database.Api.Repositories;
 using Monai.Deploy.InformaticsGateway.Logging;
+using Monai.Deploy.InformaticsGateway.Services.Common;
 using Monai.Deploy.Messaging.Events;
 
 #nullable enable
@@ -38,7 +40,7 @@ namespace Monai.Deploy.InformaticsGateway.Services.Connectors
     /// An in-memory queue for providing any files/DICOM instances received by the Informatics Gateway to
     /// other internal services.
     /// </summary>
-    internal sealed partial class PayloadAssembler : IPayloadAssembler, IDisposable
+    internal sealed partial class PayloadAssembler : IPayloadAssembler, IDisposable, IMonaiService
     {
         internal const int DEFAULT_TIMEOUT = 5;
         private readonly ILogger<PayloadAssembler> _logger;
@@ -49,13 +51,16 @@ internal sealed partial class PayloadAssembler : IPayloadAssembler, IDisposable
         private readonly Task _intializedTask;
         private readonly BlockingCollection<Payload> _workItems;
         private readonly System.Timers.Timer _timer;
+        private readonly IPayloadRepository _repository;
 
         public PayloadAssembler(
             ILogger<PayloadAssembler> logger,
             IServiceScopeFactory serviceScopeFactory)
         {
             _logger = logger ?? throw new ArgumentNullException(nameof(logger));
             _serviceScopeFactory = serviceScopeFactory ?? throw new ArgumentNullException(nameof(serviceScopeFactory));
+            var scope = _serviceScopeFactory.CreateScope();
+            _repository = scope.ServiceProvider.GetRequiredService<IPayloadRepository>();
 
             _workItems = [];
             _tokenSource = new CancellationTokenSource();
@@ -69,15 +74,18 @@ public PayloadAssembler(
             };
             _timer.Elapsed += OnTimedEvent;
             _timer.Enabled = true;
+
+            Status = ServiceStatus.Running;
         }
 
+        public string ServiceName { get => nameof(PayloadAssembler); }
+
+        public ServiceStatus Status { get; set; } = ServiceStatus.Unknown;
+
         private async Task RemovePendingPayloads()
         {
             _logger.RemovingPendingPayloads();
-            var scope = _serviceScopeFactory.CreateScope();
-            var repository = scope.ServiceProvider.GetRequiredService<IPayloadRepository>();
-
-            var removed = await repository.RemovePendingPayloadsAsync().ConfigureAwait(false);
+            var removed = await _repository.RemovePendingPayloadsAsync().ConfigureAwait(false);
 
             _logger.TotalNumberOfPayloadsRemoved(removed);
         }
@@ -192,8 +200,7 @@ private async Task QueueBucketForNotification(string key, Payload payload)
             {
                 payload.State = Payload.PayloadState.Move;
                 var scope = _serviceScopeFactory.CreateScope();
-                var repository = scope.ServiceProvider.GetRequiredService<IPayloadRepository>();
-                await repository.UpdateAsync(payload).ConfigureAwait(false);
+                await _repository.UpdateAsync(payload).ConfigureAwait(false);
                 _logger.PayloadSaved(payload.PayloadId);
                 _workItems.Add(payload);
                 _logger.BucketReady(key, payload.Count);
@@ -219,10 +226,8 @@ private async Task<Payload> CreateOrGetPayload(string key, string correlationId,
 
         private async Task<Payload> PayloadFactory(string key, string correlationId, string? workflowInstanceId, string? taskId, Messaging.Events.DataOrigin dataOrigin, uint timeout, CancellationToken cancellationToken)
         {
-            var scope = _serviceScopeFactory.CreateScope();
-            var repository = scope.ServiceProvider.GetRequiredService<IPayloadRepository>();
             var newPayload = new Payload(key, correlationId, workflowInstanceId, taskId, dataOrigin, timeout, null);
-            await repository.AddAsync(newPayload, cancellationToken).ConfigureAwait(false);
+            await _repository.AddAsync(newPayload, cancellationToken).ConfigureAwait(false);
             _logger.BucketCreated(key, timeout);
             return newPayload;
         }
@@ -232,6 +237,7 @@ public void Dispose()
             _tokenSource.Cancel();
             _payloads.Clear();
             _timer.Stop();
+            Status = ServiceStatus.Stopped;
         }
     }
 }

src/InformaticsGateway/Services/Http/MonaiHealthCheck.cs

@@ -18,18 +18,23 @@
 using System.Linq;
 using System.Threading;
 using System.Threading.Tasks;
+
 using Microsoft.Extensions.Diagnostics.HealthChecks;
+using Microsoft.Extensions.Logging;
+using Monai.Deploy.InformaticsGateway.Logging;
 using Monai.Deploy.InformaticsGateway.Repositories;
 
 namespace Monai.Deploy.InformaticsGateway.Services.Http
 {
     public class MonaiHealthCheck : IHealthCheck
     {
         private readonly IMonaiServiceLocator _monaiServiceLocator;
+        private readonly ILogger<MonaiHealthCheck> _logger;
 
-        public MonaiHealthCheck(IMonaiServiceLocator monaiServiceLocator)
+        public MonaiHealthCheck(IMonaiServiceLocator monaiServiceLocator, ILogger<MonaiHealthCheck> logger)
         {
             _monaiServiceLocator = monaiServiceLocator ?? throw new ArgumentNullException(nameof(monaiServiceLocator));
+            _logger = logger ?? throw new ArgumentNullException(nameof(logger));
         }
 
         public Task<HealthCheckResult> CheckHealthAsync(HealthCheckContext context, CancellationToken cancellationToken = default)
@@ -44,9 +49,10 @@ public Task<HealthCheckResult> CheckHealthAsync(HealthCheckContext context, Canc
 
             if (unhealthyServices.Count == services.Count)
             {
+                _logger.AllServiceUnheathly();
                 return Task.FromResult(HealthCheckResult.Unhealthy(data: unhealthyServices));
             }
-
+            _logger.SomeServiceUnheathly(string.Join(",", unhealthyServices.Keys));
             return Task.FromResult(HealthCheckResult.Degraded(data: unhealthyServices));
         }
     }