Support GPT-4V model

README.md

@@ -45,6 +45,7 @@ description: A csharp sample app that chats with your data using OpenAI and AI S
 - [Enabling optional features](#enabling-optional-features)
   - [Enabling Application Insights](#enabling-optional-features)
   - [Enabling authentication](#enabling-authentication)
+  - [Enable GPT-4V support](#enable-gpt-4v-support)
 - [Productionizing](#productionizing)
 - [Resources](#resources)
 - [FAQ](#faq)
@@ -306,6 +307,35 @@ By default, the deployed Azure container app will have no authentication or acce
 
 To then limit access to a specific set of users or groups, you can follow the steps from [Restrict your Azure AD app to a set of users](https://learn.microsoft.com/azure/active-directory/develop/howto-restrict-your-app-to-a-set-of-users) by changing "Assignment Required?" option under the Enterprise Application, and then assigning users/groups access.  Users not granted explicit access will receive the error message -AADSTS50105: Your administrator has configured the application <app_name> to block users unless they are specifically granted ('assigned') access to the application.-
 
+### Enable GPT-4V support
+
+With GPT-4-vision-preview(GPT-4V), it's possible to support an enrichmented retrival augmented generation by providing both text and image as source content. To enable GPT-4V support, you need to enable `USE_VISION` and use `GPT-4V` model when provisioning.
+
+> [!NOTE]
+> You would need to re-indexing supporting material and re-deploy the application after enabling GPT-4V support if you have already deployed the application before. This is because enabling GPT-4V support requires new fields to be added to the search index.
+
+To enable GPT-4V support with Azure OpenAI Service, run the following commands:
+```bash
+azd env set USE_VISION true
+azd env set USE_AOAI true
+azd env set AZURE_OPENAI_CHATGPT_MODEL_NAME gpt-4
+azd env set AZURE_OPENAI_RESOURCE_LOCATION westus # gpt-4-vision-preview is only available in a few regions. Please check the model availability for more details.
+azd up
+```
+
+To enable GPT-4V support with OpenAI, run the following commands:
+```bash
+azd env set USE_VISION true
+azd env set USE_AOAI false
+azd env set OPENAI_CHATGPT_DEPLOYMENT gpt-4-vision-preview
+azd up
+```
+
+To clean up previously deployed resources, run the following command:
+```bash
+azd down --purge
+azd env set AZD_PREPDOCS_RAN false # This is to ensure that the documents are re-indexed with the new fields.
+```
 ## Productionizing
 
 This sample is designed to be a starting point for your own production application,

app/Dockerfile

@@ -9,9 +9,11 @@ FROM --platform=$BUILDPLATFORM mcr.microsoft.com/dotnet/sdk:8.0 AS build
 WORKDIR /src
 COPY ["Directory.Build.props", "."]
 COPY ["Directory.Packages.props", "."]
+COPY ["NuGet.config", "."]
 COPY ["backend/", "backend/"]
 COPY ["frontend/", "frontend/"]
 COPY ["shared/", "shared/"]
+COPY ["SharedWebComponents", "SharedWebComponents/"]
 RUN dotnet restore "backend/MinimalApi.csproj"
 
 WORKDIR "/src/backend"

app/NuGet.config

@@ -0,0 +1,14 @@
+<?xml version="1.0" encoding="utf-8"?>
+<configuration>
+  <packageSources>
+    <clear />
+    <add key="dotnet-public" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-public/nuget/v3/index.json" />
+    <add key="dotnet-tools" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json" />
+    <add key="nuget" value="https://api.nuget.org/v3/index.json" />
+  </packageSources>
+  <packageSourceMapping>
+    <packageSource key="nuget">
+      <package pattern="*" />
+    </packageSource>
+  </packageSourceMapping>
+</configuration>

app/SharedWebComponents/Components/SettingsPanel.razor

@@ -40,13 +40,10 @@
         <MudCheckBox @bind-Checked="@Settings.Overrides.SemanticCaptions" Size="Size.Large"
                      Color="Color.Primary" 
                      Label="Use query-contrextual summaries instead of whole documents" />
-
-        @if (_supportedSettings is not SupportedSettings.Chat)
-        {
-            <MudCheckBox @bind-Checked="@Settings.Overrides.SuggestFollowupQuestions" Size="Size.Large"
-                         Color="Color.Primary" Label="Suggest follow-up questions"
-                         aria-label="Suggest follow-up questions checkbox." />
-        }
+
+        <MudCheckBox @bind-Checked="@Settings.Overrides.SuggestFollowupQuestions" Size="Size.Large"
+                     Color="Color.Primary" Label="Suggest follow-up questions"
+                     aria-label="Suggest follow-up questions checkbox." />
     </div>
     <div class="d-flex align-content-end flex-wrap flex-grow-1 pa-6">
         <MudButton Variant="Variant.Filled" Color="Color.Secondary"

app/backend/Extensions/KeyVaultConfigurationBuilderExtensions.cs

@@ -6,7 +6,7 @@ internal static class KeyVaultConfigurationBuilderExtensions
 {
     internal static IConfigurationBuilder ConfigureAzureKeyVault(this IConfigurationBuilder builder)
     {
-        var azureKeyVaultEndpoint = Environment.GetEnvironmentVariable("AZURE_KEY_VAULT_ENDPOINT");
+        var azureKeyVaultEndpoint = Environment.GetEnvironmentVariable("AZURE_KEY_VAULT_ENDPOINT") ?? throw new InvalidOperationException("Azure Key Vault endpoint is not set.");
         ArgumentNullException.ThrowIfNullOrEmpty(azureKeyVaultEndpoint);
 
         builder.AddAzureKeyVault(

app/backend/Extensions/ServiceCollectionExtensions.cs

@@ -79,10 +79,10 @@ internal static IServiceCollection AddAzureServices(this IServiceCollection serv
         services.AddSingleton<ReadRetrieveReadChatService>(sp =>
         {
             var config = sp.GetRequiredService<IConfiguration>();
-            var useGPT4V = config["UseGPT4V"] == "true";
+            var useVision = config["UseVision"] == "true";
             var openAIClient = sp.GetRequiredService<OpenAIClient>();
             var searchClient = sp.GetRequiredService<ISearchService>();
-            if (useGPT4V)
+            if (useVision)
             {
                 var azureComputerVisionServiceEndpoint = config["AzureComputerVisionServiceEndpoint"];
                 ArgumentNullException.ThrowIfNullOrEmpty(azureComputerVisionServiceEndpoint);

app/backend/Services/AzureBlobStorageService.cs

@@ -17,43 +17,64 @@ internal async Task<UploadDocumentsResponse> UploadFilesAsync(IEnumerable<IFormF
 
                 await using var stream = file.OpenReadStream();
 
-                using var documents = PdfReader.Open(stream, PdfDocumentOpenMode.Import);
-                for (int i = 0; i < documents.PageCount; i++)
+                // if file is an image (end with .png, .jpg, .jpeg, .gif), upload it to blob storage
+                if (Path.GetExtension(fileName).ToLower() is ".png" or ".jpg" or ".jpeg" or ".gif")
                 {
-                    var documentName = BlobNameFromFilePage(fileName, i);
-                    var blobClient = container.GetBlobClient(documentName);
+                    var blobName = BlobNameFromFilePage(fileName);
+                    var blobClient = container.GetBlobClient(blobName);
                     if (await blobClient.ExistsAsync(cancellationToken))
                     {
                         continue;
                     }
 
-                    var tempFileName = Path.GetTempFileName();
-
-                    try
+                    var url = blobClient.Uri.AbsoluteUri;
+                    await using var fileStream = file.OpenReadStream();
+                    await blobClient.UploadAsync(fileStream, new BlobHttpHeaders
                     {
-                        using var document = new PdfDocument();
-                        document.AddPage(documents.Pages[i]);
-                        document.Save(tempFileName);
+                        ContentType = "image"
+                    }, cancellationToken: cancellationToken);
+                    uploadedFiles.Add(blobName);
+                }
+                else if (Path.GetExtension(fileName).ToLower() is ".pdf")
+                {
+                    using var documents = PdfReader.Open(stream, PdfDocumentOpenMode.Import);
+                    for (int i = 0; i < documents.PageCount; i++)
+                    {
+                        var documentName = BlobNameFromFilePage(fileName, i);
+                        var blobClient = container.GetBlobClient(documentName);
+                        if (await blobClient.ExistsAsync(cancellationToken))
+                        {
+                            continue;
+                        }
 
-                        await using var tempStream = File.OpenRead(tempFileName);
-                        await blobClient.UploadAsync(tempStream, new BlobHttpHeaders
+                        var tempFileName = Path.GetTempFileName();
+
+                        try
                         {
-                            ContentType = "application/pdf"
-                        }, cancellationToken: cancellationToken);
+                            using var document = new PdfDocument();
+                            document.AddPage(documents.Pages[i]);
+                            document.Save(tempFileName);
 
-                        uploadedFiles.Add(documentName);
-                    }
-                    finally
-                    {
-                        File.Delete(tempFileName);
+                            await using var tempStream = File.OpenRead(tempFileName);
+                            await blobClient.UploadAsync(tempStream, new BlobHttpHeaders
+                            {
+                                ContentType = "application/pdf"
+                            }, cancellationToken: cancellationToken);
+
+                            uploadedFiles.Add(documentName);
+                        }
+                        finally
+                        {
+                            File.Delete(tempFileName);
+                        }
                     }
                 }
             }
 
             if (uploadedFiles.Count is 0)
             {
                 return UploadDocumentsResponse.FromError("""
-                    No files were uploaded. Either the files already exist or the files are not PDFs.
+                    No files were uploaded. Either the files already exist or the files are not PDFs or images.
                     """);
             }
 

app/backend/Services/ReadRetrieveReadChatService.cs

@@ -177,6 +177,7 @@ You answer needs to be a json object with the following format.
         {
             MaxTokens = 1024,
             Temperature = overrides?.Temperature ?? 0.7,
+            StopSequences = [],
         };
 
         // get answer
@@ -199,7 +200,7 @@ You answer needs to be a json object with the following format.
 {ans}
 
 # Format of the response
-Return the follow-up question as a json string list.
+Return the follow-up question as a json string list. Don't put your answer between ```json and ```, return the json string directly.
 e.g.
 [
     ""What is the deductible?"",
@@ -209,6 +210,7 @@ Return the follow-up question as a json string list.
 
             var followUpQuestions = await chat.GetChatMessageContentAsync(
                 followUpQuestionChat,
+                promptExecutingSetting,
                 cancellationToken: cancellationToken);
 
             var followUpQuestionsJson = followUpQuestions.Content ?? throw new InvalidOperationException("Failed to get search query");

app/functions/EmbedFunctions/Program.cs

@@ -48,34 +48,75 @@ uri is not null
             return containerClient;
         });
 
+        services.AddSingleton<BlobServiceClient>(_ =>
+        {
+            return new BlobServiceClient(
+                GetUriFromEnvironment("AZURE_STORAGE_BLOB_ENDPOINT"), credential);
+        });
+
         services.AddSingleton<EmbedServiceFactory>();
         services.AddSingleton<EmbeddingAggregateService>();
 
         services.AddSingleton<IEmbedService, AzureSearchEmbedService>(provider =>
         {
             var searchIndexName = Environment.GetEnvironmentVariable("AZURE_SEARCH_INDEX") ?? throw new ArgumentNullException("AZURE_SEARCH_INDEX is null");
-            var embeddingModelName = Environment.GetEnvironmentVariable("AZURE_OPENAI_EMBEDDING_DEPLOYMENT") ?? throw new ArgumentNullException("AZURE_OPENAI_EMBEDDING_DEPLOYMENT is null");
-            var openaiEndPoint = Environment.GetEnvironmentVariable("AZURE_OPENAI_ENDPOINT") ?? throw new ArgumentNullException("AZURE_OPENAI_ENDPOINT is null");
-
-            var openAIClient = new OpenAIClient(new Uri(openaiEndPoint), new DefaultAzureCredential());
+            var useAOAI = Environment.GetEnvironmentVariable("USE_AOAI")?.ToLower() == "true";
+            var useVision = Environment.GetEnvironmentVariable("USE_VISION")?.ToLower() == "true";
+
+            OpenAIClient? openAIClient = null;
+            string? embeddingModelName = null;
+
+            if (useAOAI)
+            {
+                var openaiEndPoint = Environment.GetEnvironmentVariable("AZURE_OPENAI_ENDPOINT") ?? throw new ArgumentNullException("AZURE_OPENAI_ENDPOINT is null");
+                embeddingModelName = Environment.GetEnvironmentVariable("AZURE_OPENAI_EMBEDDING_DEPLOYMENT") ?? throw new ArgumentNullException("AZURE_OPENAI_EMBEDDING_DEPLOYMENT is null");
+                openAIClient = new OpenAIClient(new Uri(openaiEndPoint), new DefaultAzureCredential());
+            }
+            else
+            {
+                embeddingModelName = Environment.GetEnvironmentVariable("OPENAI_EMBEDDING_DEPLOYMENT") ?? throw new ArgumentNullException("OPENAI_EMBEDDING_DEPLOYMENT is null");
+                var openaiKey = Environment.GetEnvironmentVariable("OPENAI_API_KEY") ?? throw new ArgumentNullException("OPENAI_API_KEY is null");
+                openAIClient = new OpenAIClient(openaiKey);
+            }
 
             var searchClient = provider.GetRequiredService<SearchClient>();
             var searchIndexClient = provider.GetRequiredService<SearchIndexClient>();
-            var blobContainerClient = provider.GetRequiredService<BlobContainerClient>();
+            var corpusContainer = provider.GetRequiredService<BlobContainerClient>();
             var documentClient = provider.GetRequiredService<DocumentAnalysisClient>();
             var logger = provider.GetRequiredService<ILogger<AzureSearchEmbedService>>();
 
-            return new AzureSearchEmbedService(
+            if (useVision)
+            {
+                var visionEndpoint = Environment.GetEnvironmentVariable("AZURE_COMPUTER_VISION_ENDPOINT") ?? throw new ArgumentNullException("AZURE_COMPUTER_VISION_ENDPOINT is null");
+                var httpClient = new HttpClient();
+                var visionClient = new AzureComputerVisionService(httpClient, visionEndpoint, new DefaultAzureCredential());
+
+                return new AzureSearchEmbedService(
+                    openAIClient: openAIClient,
+                    embeddingModelName: embeddingModelName,
+                    searchClient: searchClient,
+                    searchIndexName: searchIndexName,
+                    searchIndexClient: searchIndexClient,
+                    documentAnalysisClient: documentClient,
+                    corpusContainerClient: corpusContainer,
+                    computerVisionService: visionClient,
+                    includeImageEmbeddingsField: true,
+                    logger: logger);
+            }
+            else
+            {
+                return new AzureSearchEmbedService(
                 openAIClient: openAIClient,
                 embeddingModelName: embeddingModelName,
                 searchClient: searchClient,
                 searchIndexName: searchIndexName,
                 searchIndexClient: searchIndexClient,
                 documentAnalysisClient: documentClient,
-                corpusContainerClient: blobContainerClient,
+                corpusContainerClient: corpusContainer,
                 computerVisionService: null,
                 includeImageEmbeddingsField: false,
                 logger: logger);
+            }
         });
     })
     .ConfigureFunctionsWorkerDefaults()

app/functions/EmbedFunctions/Services/EmbeddingAggregateService.cs

@@ -4,7 +4,8 @@ namespace EmbedFunctions.Services;
 
 public sealed class EmbeddingAggregateService(
     EmbedServiceFactory embedServiceFactory,
-    BlobContainerClient client,
+    BlobServiceClient blobServiceClient,
+    BlobContainerClient corpusClient,
     ILogger<EmbeddingAggregateService> logger)
 {
     internal async Task EmbedBlobAsync(Stream blobStream, string blobName)
@@ -14,23 +15,51 @@ internal async Task EmbedBlobAsync(Stream blobStream, string blobName)
             var embeddingType = GetEmbeddingType();
             var embedService = embedServiceFactory.GetEmbedService(embeddingType);
 
-            var result = await embedService.EmbedPDFBlobAsync(blobStream, blobName);
+            if (Path.GetExtension(blobName) is ".png" or ".jpg" or ".jpeg" or ".gif")
+            {
+                logger.LogInformation("Embedding image: {Name}", blobName);
+                var contentContainer = blobServiceClient.GetBlobContainerClient("content");
+                var blobClient = contentContainer.GetBlobClient(blobName);
+                var uri = blobClient.Uri.AbsoluteUri ?? throw new InvalidOperationException("Blob URI is null.");
+                var result = await embedService.EmbedImageBlobAsync(blobStream, uri, blobName);
+                var status = result switch
+                {
+                    true => DocumentProcessingStatus.Succeeded,
+                    _ => DocumentProcessingStatus.Failed
+                };
 
-            var status = result switch
+                await corpusClient.SetMetadataAsync(new Dictionary<string, string>
+                {
+                    [nameof(DocumentProcessingStatus)] = status.ToString(),
+                    [nameof(EmbeddingType)] = embeddingType.ToString()
+                });
+            }
+            else if (Path.GetExtension(blobName) is ".pdf")
             {
-                true => DocumentProcessingStatus.Succeeded,
-                _ => DocumentProcessingStatus.Failed
-            };
+                logger.LogInformation("Embedding pdf: {Name}", blobName);
+                var result = await embedService.EmbedPDFBlobAsync(blobStream, blobName);
+
+                var status = result switch
+                {
+                    true => DocumentProcessingStatus.Succeeded,
+                    _ => DocumentProcessingStatus.Failed
+                };
 
-            await client.SetMetadataAsync(new Dictionary<string, string>
+                await corpusClient.SetMetadataAsync(new Dictionary<string, string>
+                {
+                    [nameof(DocumentProcessingStatus)] = status.ToString(),
+                    [nameof(EmbeddingType)] = embeddingType.ToString()
+                });
+            }
+            else
             {
-                [nameof(DocumentProcessingStatus)] = status.ToString(),
-                [nameof(EmbeddingType)] = embeddingType.ToString()
-            });
+                throw new NotSupportedException("Unsupported file type.");
+            }
         }
         catch (Exception ex)
         {
             logger.LogError(ex, "Failed to embed: {Name}, error: {Message}", blobName, ex.Message);
+            throw;
         }
     }