Present images in UI if retrieved

app/app.sln

@@ -27,7 +27,7 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "PrepareDocs", "prepdocs\Pre
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "EmbedFunctions", "functions\EmbedFunctions\EmbedFunctions.csproj", "{54099AF3-CFA8-4EA9-9118-A26E3E63745B}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "MinimalApi.Tests", "tests\MinimalApi.Tests\MinimalApi.Tests.csproj", "{C687997A-A569-47AA-A808-504F7DFCA97B}"
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "MinimalApi.Tests", "tests\MinimalApi.Tests\MinimalApi.Tests.csproj", "{C687997A-A569-47AA-A808-504F7DFCA97B}"
 EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution

app/backend/Extensions/ServiceCollectionExtensions.cs

@@ -55,18 +55,47 @@ internal static IServiceCollection AddAzureServices(this IServiceCollection serv
         services.AddSingleton<OpenAIClient>(sp =>
         {
             var config = sp.GetRequiredService<IConfiguration>();
-            var azureOpenAiServiceEndpoint = config["AzureOpenAiServiceEndpoint"];
-
-            ArgumentNullException.ThrowIfNullOrEmpty(azureOpenAiServiceEndpoint);
-
-            var openAIClient = new OpenAIClient(
-                new Uri(azureOpenAiServiceEndpoint), s_azureCredential);
-
-            return openAIClient;
+            var useAOAI = config["UseAOAI"] == "true";
+            if (useAOAI)
+            {
+                var azureOpenAiServiceEndpoint = config["AzureOpenAiServiceEndpoint"];
+                ArgumentNullException.ThrowIfNullOrEmpty(azureOpenAiServiceEndpoint);
+
+                var openAIClient = new OpenAIClient(new Uri(azureOpenAiServiceEndpoint), s_azureCredential);
+
+                return openAIClient;
+            }
+            else
+            {
+                var openAIApiKey = config["OpenAIApiKey"];
+                ArgumentNullException.ThrowIfNullOrEmpty(openAIApiKey);
+
+                var openAIClient = new OpenAIClient(openAIApiKey);
+                return openAIClient;
+            }
         });
 
         services.AddSingleton<AzureBlobStorageService>();
-        services.AddSingleton<ReadRetrieveReadChatService>();
+        services.AddSingleton<ReadRetrieveReadChatService>(sp =>
+        {
+            var config = sp.GetRequiredService<IConfiguration>();
+            var useGPT4V = config["UseGPT4V"] == "true";
+            var openAIClient = sp.GetRequiredService<OpenAIClient>();
+            var searchClient = sp.GetRequiredService<ISearchService>();
+            if (useGPT4V)
+            {
+                var azureComputerVisionServiceEndpoint = config["AzureComputerVisionServiceEndpoint"];
+                ArgumentNullException.ThrowIfNullOrEmpty(azureComputerVisionServiceEndpoint);
+                var httpClient = sp.GetRequiredService<IHttpClientFactory>().CreateClient();
+
+                var visionService = new AzureComputerVisionService(httpClient, azureComputerVisionServiceEndpoint, s_azureCredential);
+                return new ReadRetrieveReadChatService(searchClient, openAIClient, config, visionService, s_azureCredential);
+            }
+            else
+            {
+                return new ReadRetrieveReadChatService(searchClient, openAIClient, config, tokenCredential: s_azureCredential);
+            }
+        });
 
         return services;
     }

app/backend/Services/ReadRetrieveReadChatService.cs

@@ -26,7 +26,7 @@ public ReadRetrieveReadChatService(
         _searchClient = searchClient;
         var kernelBuilder = Kernel.CreateBuilder();
 
-        if (configuration["UseAOAI"] != "true")
+        if (configuration["UseAOAI"] == "false")
         {
             var deployment = configuration["OpenAiChatGptDeployment"];
             ArgumentNullException.ThrowIfNullOrWhiteSpace(deployment);

app/frontend/Components/Answer.razor

@@ -59,7 +59,7 @@
                          ToolTip="Show the supporting content." Disabled="@(Retort is { DataPoints: null } or { DataPoints.Length: 0 })">
                 <ChildContent>
                     <MudPaper Class="pa-2" Elevation="3">
-                        <SupportingContent DataPoints="Retort.DataPoints" />
+                        <SupportingContent DataPoints="Retort.DataPoints" Images="Retort.Images ?? []" />
                     </MudPaper>
                 </ChildContent>
             </MudTabPanel>

app/frontend/Components/ImageViewerDialog.razor

@@ -0,0 +1,23 @@
+<MudDialog DisableSidePadding="true">
+    <DialogContent>
+        <MudContainer Class="align-center justify-center mud-width-full">
+            <MudImage Src="@Src" Alt="@FileName" Height="600" />
+        </MudContainer>
+    </DialogContent>
+    <DialogActions>
+        <MudButton Color="Color.Primary" Size="Size.Large"
+                   StartIcon="@Icons.Material.Filled.Close"
+                   OnClick="@OnCloseClick">Close</MudButton>
+    </DialogActions>
+</MudDialog>
+
+@code {
+    [Parameter] public required string FileName { get; set; }
+    [Parameter] public required string Src { get; set; }
+    [CascadingParameter] public required MudDialogInstance Dialog { get; set; }
+
+    private void OnCloseClick()
+    {
+        Dialog.Cancel();
+    }
+}

app/frontend/Components/SupportingContent.razor

@@ -18,4 +18,24 @@
             <MudText Typo="Typo.body1">@((MarkupString)content!)</MudText>
         </MudListItem>
     }
+    <MudDivider />
+    @foreach (var (index, value) in Images.Select((item, i) => (i, item)))
+    {
+        var last = index == Images.Length - 1;
+        if (value is null)
+        {
+            continue;
+        }
+
+        (var title, var url) = value;
+
+        if (index is not 0 && last is false)
+        {
+            <MudDivider />
+        }
+
+        <MudListItem>
+            <MudImage Src="@url" Alt="@title" Height="400" />
+        </MudListItem>
+    }
 </MudList>

app/frontend/Components/SupportingContent.razor.cs

@@ -6,6 +6,8 @@ public sealed partial class SupportingContent
 {
     [Parameter, EditorRequired] public required SupportingContentRecord[] DataPoints { get; set; }
 
+    [Parameter, EditorRequired] public required SupportingImageRecord[] Images { get; set; }
+
     private ParsedSupportingContentItem[] _supportingContent = [];
 
     protected override void OnParametersSet()

app/frontend/Pages/Docs.razor

@@ -17,19 +17,20 @@
                 </MudCardHeader>
                 <MudCardContent>
                     <MudText Class="pb-4">
-                        Select up to ten PDF documents to upload, or explore the existing documents that have already been processed.
+                        Select up to ten documents to upload, or explore the existing documents that have already been processed.
+                        The document can be either a PDF file or an image file.
                         Each file cannot exceed a file size of @(MaxIndividualFileSize.ToHumanReadableSize())
                     </MudText>
                     <MudFileUpload @ref="_fileUpload" T="IReadOnlyList<IBrowserFile>"
-                                   Accept=".pdf" MaximumFileCount="10" FilesChanged=@(files => StateHasChanged())
-                                   Required="true" RequiredError="You must select at least one PDF file to upload.">
+                                   Accept=".pdf, .png, .jpg, .jpeg" MaximumFileCount="10" FilesChanged=@(files => StateHasChanged())
+                                   Required="true" RequiredError="You must select at least one file to upload.">
                         <ButtonTemplate>
                             <MudButton HtmlTag="label"
                                        Variant="Variant.Filled"
                                        StartIcon="@Icons.Material.Filled.FileOpen"
                                        Size="Size.Large"
                                        for="@context">
-                                Select PDF Documents
+                                Select Documents
                             </MudButton>
                         </ButtonTemplate>
                         <SelectedTemplate>
@@ -50,7 +51,7 @@
                                             }
                                             else
                                             {
-                                                <MudText>No PDF files selected.</MudText>
+                                                <MudText>No files selected.</MudText>
                                             }
                                         </div>
                                     </TitleContent>

app/frontend/Pages/Docs.razor.cs

@@ -101,7 +101,12 @@ private async Task SubmitFilesForUploadAsync()
         }
     }
 
-    private void OnShowDocument(DocumentResponse document) => Dialog.Show<PdfViewerDialog>(
+    private void OnShowDocument(DocumentResponse document)
+    {
+        var extension = Path.GetExtension(document.Name);
+        if (extension is ".pdf")
+        {
+            Dialog.Show<PdfViewerDialog>(
             $"📄 {document.Name}",
             new DialogParameters
             {
@@ -116,6 +121,36 @@ private void OnShowDocument(DocumentResponse document) => Dialog.Show<PdfViewerD
                 CloseButton = true,
                 CloseOnEscapeKey = true
             });
+        }
+        else if (extension is ".png" or ".jpg" or ".jpeg")
+        {
+            Dialog.Show<ImageViewerDialog>(
+            $"📄 {document.Name}",
+            new DialogParameters
+            {
+                [nameof(ImageViewerDialog.FileName)] = document.Name,
+                [nameof(ImageViewerDialog.Src)] = document.Url.ToString(),
+            },
+            new DialogOptions
+            {
+                MaxWidth = MaxWidth.Large,
+                FullWidth = true,
+                CloseButton = true,
+                CloseOnEscapeKey = true
+            });
+        }
+        else
+        {
+            Snackbar.Add(
+                $"Unsupported file type: '{extension}'",
+                Severity.Error,
+                static options =>
+                {
+                    options.ShowCloseIcon = true;
+                    options.VisibleStateDuration = 10_000;
+                });
+        }
+    }
 
     public void Dispose() => _cancellationTokenSource.Cancel();
 }

app/functions/EmbedFunctions/Services/MilvusEmbedService.cs

@@ -10,7 +10,7 @@ public Task CreateSearchIndexAsync(string searchIndexName, CancellationToken ct
         throw new NotImplementedException();
     }
 
-    public Task<bool> EmbedImageBlobAsync(Stream imageStream, string imageName, CancellationToken ct = default)
+    public Task<bool> EmbedImageBlobAsync(Stream imageStream, string imageUrl, string imageName, CancellationToken ct = default)
     {
         throw new NotImplementedException();
     }

app/functions/EmbedFunctions/Services/PineconeEmbedService.cs

@@ -10,7 +10,7 @@ public Task CreateSearchIndexAsync(string searchIndexName, CancellationToken ct
         throw new NotImplementedException();
     }
 
-    public Task<bool> EmbedImageBlobAsync(Stream imageStream, string imageName, CancellationToken ct = default)
+    public Task<bool> EmbedImageBlobAsync(Stream imageStream, string imageUrl, string imageName, CancellationToken ct = default)
     {
         throw new NotImplementedException();
     }

app/functions/EmbedFunctions/Services/QdrantEmbedService.cs

@@ -10,7 +10,7 @@ public Task CreateSearchIndexAsync(string searchIndexName, CancellationToken ct
         throw new NotImplementedException();
     }
 
-    public Task<bool> EmbedImageBlobAsync(Stream imageStream, string imageName, CancellationToken ct = default)
+    public Task<bool> EmbedImageBlobAsync(Stream imageStream, string imageUrl, string imageName, CancellationToken ct = default)
     {
         throw new NotImplementedException();
     }

app/prepdocs/PrepareDocs/Program.Clients.cs

@@ -24,7 +24,7 @@ private static Task<AzureSearchEmbedService> GetAzureSearchEmbedService(AppOptio
             var searchClient = await GetSearchClientAsync(o);
             var documentClient = await GetFormRecognizerClientAsync(o);
             var blobContainerClient = await GetCorpusBlobContainerClientAsync(o);
-            var openAIClient = await GetAzureOpenAIClientAsync(o);
+            var openAIClient = await GetOpenAIClientAsync(o);
             var embeddingModelName = o.EmbeddingModelName ?? throw new ArgumentNullException(nameof(o.EmbeddingModelName));
             var searchIndexName = o.SearchIndexName ?? throw new ArgumentNullException(nameof(o.SearchIndexName));
             var computerVisionService = await GetComputerVisionServiceAsync(o);
@@ -161,16 +161,26 @@ private static Task<SearchClient> GetSearchClientAsync(AppOptions options) =>
             return new AzureComputerVisionService(new HttpClient(), endpoint, DefaultCredential);
         });
 
-    private static Task<OpenAIClient> GetAzureOpenAIClientAsync(AppOptions options) =>
+    private static Task<OpenAIClient> GetOpenAIClientAsync(AppOptions options) =>
        GetLazyClientAsync<OpenAIClient>(options, s_openAILock, async o =>
        {
            if (s_openAIClient is null)
            {
-               var endpoint = o.AzureOpenAIServiceEndpoint;
-               ArgumentNullException.ThrowIfNullOrEmpty(endpoint);
-               s_openAIClient = new OpenAIClient(
-                   new Uri(endpoint),
-                   DefaultCredential);
+               var useAOAI = Environment.GetEnvironmentVariable("UseAOAI") == "true";
+               if (!useAOAI)
+               {
+                     var openAIApiKey = Environment.GetEnvironmentVariable("OPENAI_API_KEY");
+                     ArgumentNullException.ThrowIfNullOrEmpty(openAIApiKey);
+                     s_openAIClient = new OpenAIClient(openAIApiKey);
+               }
+               else
+               {
+                   var endpoint = o.AzureOpenAIServiceEndpoint;
+                   ArgumentNullException.ThrowIfNullOrEmpty(endpoint);
+                   s_openAIClient = new OpenAIClient(
+                       new Uri(endpoint),
+                       DefaultCredential);
+               }
            }
            await Task.CompletedTask;
            return s_openAIClient;

app/prepdocs/PrepareDocs/Program.cs

@@ -204,8 +204,9 @@ static async ValueTask UploadBlobsAndCreateIndexAsync(
     {
         using var stream = File.OpenRead(fileName);
         var blobName = BlobNameFromFilePage(fileName);
-        await UploadBlobAsync(fileName, blobName, container);
-        await embeddingService.EmbedImageBlobAsync(stream, fileName);
+        var imageName = Path.GetFileNameWithoutExtension(blobName);
+        var url = await UploadBlobAsync(fileName, blobName, container);
+        await embeddingService.EmbedImageBlobAsync(stream, url, imageName);
     }
     else
     {
@@ -215,12 +216,14 @@ static async ValueTask UploadBlobsAndCreateIndexAsync(
     }
 }
 
-static async Task UploadBlobAsync(string fileName, string blobName, BlobContainerClient container)
+static async Task<string> UploadBlobAsync(string fileName, string blobName, BlobContainerClient container)
 {
     var blobClient = container.GetBlobClient(blobName);
+    var url = blobClient.Uri.AbsoluteUri;
+
     if (await blobClient.ExistsAsync())
     {
-        return;
+        return url;
     }
 
     var blobHttpHeaders = new BlobHttpHeaders
@@ -230,6 +233,9 @@ static async Task UploadBlobAsync(string fileName, string blobName, BlobContaine
 
     await using var fileStream = File.OpenRead(fileName);
     await blobClient.UploadAsync(fileStream, blobHttpHeaders);
+
+
+    return url;
 }
 
 static string GetContentType(string fileName)

app/shared/Shared/Services/AzureSearchEmbedService.cs

@@ -76,12 +76,13 @@ Indexing sections from '{BlobName}' into search index '{SearchIndexName}'
     public async Task<bool> EmbedImageBlobAsync(
         Stream imageStream,
         string imageUrl,
+        string imageName,
         CancellationToken ct = default)
     {
         if (includeImageEmbeddingsField == false || computerVisionService is null)
         {
             throw new InvalidOperationException(
-                "Computer Vision service is required to include image embeddings field");
+                "Computer Vision service is required to include image embeddings field, please enable GPT_4V support");
         }
 
         var embeddings = await computerVisionService.VectorizeImageAsync(imageUrl, ct);
@@ -95,7 +96,7 @@ public async Task<bool> EmbedImageBlobAsync(
             new SearchDocument
             {
                 ["id"] = imageId,
-                ["content"] = imageUrl,
+                ["content"] = imageName,
                 ["category"] = "image",
                 ["imageEmbedding"] = embeddings.vector,
                 ["sourcefile"] = imageUrl,

app/shared/Shared/Services/IEmbedService.cs

@@ -20,6 +20,7 @@ Task<bool> EmbedPDFBlobAsync(
     /// </summary>
     Task<bool> EmbedImageBlobAsync(
         Stream imageStream,
+        string imageUrl,
         string imageName,
         CancellationToken ct = default);
 

app/tests/MinimalApi.Tests/AzureSearchEmbedServiceTest.cs

@@ -296,7 +296,7 @@ public async Task EmbedImageBlobTestAsync()
             var client = containerClient.GetBlobClient(imageBlobName);
             await client.UploadAsync(stream, true);
             var url = client.Uri.AbsoluteUri;
-            var isSucceed = await service.EmbedImageBlobAsync(stream, url);
+            var isSucceed = await service.EmbedImageBlobAsync(stream, imageBlobName, url);
             isSucceed.Should().BeTrue();
 
             // check if the image is uploaded to blob