Skip to content

Commit 4e6782a

Browse files
LittleLittleCloudIEvangelist
LittleLittleCloud
and
IEvangelist
authored
Present images in UI if retrieved (#274)
## Purpose <!-- Describe the intention of the changes being proposed. What problem does it solve or functionality does it add? --> * ... #257 ### Doc page ![image](https://github.com/Azure-Samples/azure-search-openai-demo-csharp/assets/16876986/392313a8-d29b-428e-a20f-4c2882014b4a) ### Chat page ![image](https://github.com/Azure-Samples/azure-search-openai-demo-csharp/assets/16876986/045d1819-cebc-4ecc-bec6-fb446f4e08ef) ![image](https://github.com/Azure-Samples/azure-search-openai-demo-csharp/assets/16876986/6d06282a-19f7-4cb0-9e24-1ec0986642c6) ![image](https://github.com/Azure-Samples/azure-search-openai-demo-csharp/assets/16876986/6cd57f12-102a-4ad3-bd7c-87d71612e5fc) ## Does this introduce a breaking change? <!-- Mark one with an "x". --> ``` [ ] Yes [ ] No ``` ## Pull Request Type What kind of change does this Pull Request introduce? <!-- Please check the one that applies to this PR using "x". --> ``` [ ] Bugfix [ ] Feature [ ] Code style update (formatting, local variables) [ ] Refactoring (no functional changes, no api changes) [ ] Documentation content changes [ ] Other... Please describe: ``` ## How to Test * Get the code ``` git clone [repo-address] cd [repo-name] git checkout [branch-name] npm install ``` * Test the code <!-- Add steps to run the tests suite and/or manually test --> ``` ``` ## What to Check Verify that the following are valid * ... ## Other Information <!-- Add any other helpful information that may be needed here. --> --------- Co-authored-by: David Pine <[email protected]>
1 parent c5a432d commit 4e6782a

20 files changed

+265
-62
lines changed

‎app/app.sln

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "PrepareDocs", "prepdocs\Pre
2727
EndProject
2828
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "EmbedFunctions", "functions\EmbedFunctions\EmbedFunctions.csproj", "{54099AF3-CFA8-4EA9-9118-A26E3E63745B}"
2929
EndProject
30-
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "MinimalApi.Tests", "tests\MinimalApi.Tests\MinimalApi.Tests.csproj", "{C687997A-A569-47AA-A808-504F7DFCA97B}"
30+
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "MinimalApi.Tests", "tests\MinimalApi.Tests\MinimalApi.Tests.csproj", "{C687997A-A569-47AA-A808-504F7DFCA97B}"
3131
EndProject
3232
Global
3333
GlobalSection(SolutionConfigurationPlatforms) = preSolution

‎app/backend/Extensions/ServiceCollectionExtensions.cs

Lines changed: 38 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -55,18 +55,47 @@ internal static IServiceCollection AddAzureServices(this IServiceCollection serv
5555
services.AddSingleton<OpenAIClient>(sp =>
5656
{
5757
var config = sp.GetRequiredService<IConfiguration>();
58-
var azureOpenAiServiceEndpoint = config["AzureOpenAiServiceEndpoint"];
59-
60-
ArgumentNullException.ThrowIfNullOrEmpty(azureOpenAiServiceEndpoint);
61-
62-
var openAIClient = new OpenAIClient(
63-
new Uri(azureOpenAiServiceEndpoint), s_azureCredential);
64-
65-
return openAIClient;
58+
var useAOAI = config["UseAOAI"] == "true";
59+
if (useAOAI)
60+
{
61+
var azureOpenAiServiceEndpoint = config["AzureOpenAiServiceEndpoint"];
62+
ArgumentNullException.ThrowIfNullOrEmpty(azureOpenAiServiceEndpoint);
63+
64+
var openAIClient = new OpenAIClient(new Uri(azureOpenAiServiceEndpoint), s_azureCredential);
65+
66+
return openAIClient;
67+
}
68+
else
69+
{
70+
var openAIApiKey = config["OpenAIApiKey"];
71+
ArgumentNullException.ThrowIfNullOrEmpty(openAIApiKey);
72+
73+
var openAIClient = new OpenAIClient(openAIApiKey);
74+
return openAIClient;
75+
}
6676
});
6777

6878
services.AddSingleton<AzureBlobStorageService>();
69-
services.AddSingleton<ReadRetrieveReadChatService>();
79+
services.AddSingleton<ReadRetrieveReadChatService>(sp =>
80+
{
81+
var config = sp.GetRequiredService<IConfiguration>();
82+
var useGPT4V = config["UseGPT4V"] == "true";
83+
var openAIClient = sp.GetRequiredService<OpenAIClient>();
84+
var searchClient = sp.GetRequiredService<ISearchService>();
85+
if (useGPT4V)
86+
{
87+
var azureComputerVisionServiceEndpoint = config["AzureComputerVisionServiceEndpoint"];
88+
ArgumentNullException.ThrowIfNullOrEmpty(azureComputerVisionServiceEndpoint);
89+
var httpClient = sp.GetRequiredService<IHttpClientFactory>().CreateClient();
90+
91+
var visionService = new AzureComputerVisionService(httpClient, azureComputerVisionServiceEndpoint, s_azureCredential);
92+
return new ReadRetrieveReadChatService(searchClient, openAIClient, config, visionService, s_azureCredential);
93+
}
94+
else
95+
{
96+
return new ReadRetrieveReadChatService(searchClient, openAIClient, config, tokenCredential: s_azureCredential);
97+
}
98+
});
7099

71100
return services;
72101
}

‎app/backend/Services/ReadRetrieveReadChatService.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ public ReadRetrieveReadChatService(
2626
_searchClient = searchClient;
2727
var kernelBuilder = Kernel.CreateBuilder();
2828

29-
if (configuration["UseAOAI"] != "true")
29+
if (configuration["UseAOAI"] == "false")
3030
{
3131
var deployment = configuration["OpenAiChatGptDeployment"];
3232
ArgumentNullException.ThrowIfNullOrWhiteSpace(deployment);

‎app/frontend/Components/Answer.razor

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,7 @@
5959
ToolTip="Show the supporting content." Disabled="@(Retort is { DataPoints: null } or { DataPoints.Length: 0 })">
6060
<ChildContent>
6161
<MudPaper Class="pa-2" Elevation="3">
62-
<SupportingContent DataPoints="Retort.DataPoints" />
62+
<SupportingContent DataPoints="Retort.DataPoints" Images="Retort.Images ?? []" />
6363
</MudPaper>
6464
</ChildContent>
6565
</MudTabPanel>

‎app/frontend/Components/ImageViewerDialog.razor

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
<MudDialog DisableSidePadding="true">
2+
<DialogContent>
3+
<MudContainer Class="align-center justify-center mud-width-full">
4+
<MudImage Src="@Src" Alt="@FileName" Height="600" />
5+
</MudContainer>
6+
</DialogContent>
7+
<DialogActions>
8+
<MudButton Color="Color.Primary" Size="Size.Large"
9+
StartIcon="@Icons.Material.Filled.Close"
10+
OnClick="@OnCloseClick">Close</MudButton>
11+
</DialogActions>
12+
</MudDialog>
13+
14+
@code {
15+
[Parameter] public required string FileName { get; set; }
16+
[Parameter] public required string Src { get; set; }
17+
[CascadingParameter] public required MudDialogInstance Dialog { get; set; }
18+
19+
private void OnCloseClick()
20+
{
21+
Dialog.Cancel();
22+
}
23+
}

‎app/frontend/Components/SupportingContent.razor

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,4 +18,24 @@
1818
<MudText Typo="Typo.body1">@((MarkupString)content!)</MudText>
1919
</MudListItem>
2020
}
21+
<MudDivider />
22+
@foreach (var (index, value) in Images.Select((item, i) => (i, item)))
23+
{
24+
var last = index == Images.Length - 1;
25+
if (value is null)
26+
{
27+
continue;
28+
}
29+
30+
(var title, var url) = value;
31+
32+
if (index is not 0 && last is false)
33+
{
34+
<MudDivider />
35+
}
36+
37+
<MudListItem>
38+
<MudImage Src="@url" Alt="@title" Height="400" />
39+
</MudListItem>
40+
}
2141
</MudList>

‎app/frontend/Components/SupportingContent.razor.cs

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,8 @@ public sealed partial class SupportingContent
66
{
77
[Parameter, EditorRequired] public required SupportingContentRecord[] DataPoints { get; set; }
88

9+
[Parameter, EditorRequired] public required SupportingImageRecord[] Images { get; set; }
10+
911
private ParsedSupportingContentItem[] _supportingContent = [];
1012

1113
protected override void OnParametersSet()

‎app/frontend/Pages/Docs.razor

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -17,19 +17,20 @@
1717
</MudCardHeader>
1818
<MudCardContent>
1919
<MudText Class="pb-4">
20-
Select up to ten PDF documents to upload, or explore the existing documents that have already been processed.
20+
Select up to ten documents to upload, or explore the existing documents that have already been processed.
21+
The document can be either a PDF file or an image file.
2122
Each file cannot exceed a file size of @(MaxIndividualFileSize.ToHumanReadableSize())
2223
</MudText>
2324
<MudFileUpload @ref="_fileUpload" T="IReadOnlyList<IBrowserFile>"
24-
Accept=".pdf" MaximumFileCount="10" FilesChanged=@(files => StateHasChanged())
25-
Required="true" RequiredError="You must select at least one PDF file to upload.">
25+
Accept=".pdf, .png, .jpg, .jpeg" MaximumFileCount="10" FilesChanged=@(files => StateHasChanged())
26+
Required="true" RequiredError="You must select at least one file to upload.">
2627
<ButtonTemplate>
2728
<MudButton HtmlTag="label"
2829
Variant="Variant.Filled"
2930
StartIcon="@Icons.Material.Filled.FileOpen"
3031
Size="Size.Large"
3132
for="@context">
32-
Select PDF Documents
33+
Select Documents
3334
</MudButton>
3435
</ButtonTemplate>
3536
<SelectedTemplate>
@@ -50,7 +51,7 @@
5051
}
5152
else
5253
{
53-
<MudText>No PDF files selected.</MudText>
54+
<MudText>No files selected.</MudText>
5455
}
5556
</div>
5657
</TitleContent>

‎app/frontend/Pages/Docs.razor.cs

Lines changed: 36 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -101,7 +101,12 @@ private async Task SubmitFilesForUploadAsync()
101101
}
102102
}
103103

104-
private void OnShowDocument(DocumentResponse document) => Dialog.Show<PdfViewerDialog>(
104+
private void OnShowDocument(DocumentResponse document)
105+
{
106+
var extension = Path.GetExtension(document.Name);
107+
if (extension is ".pdf")
108+
{
109+
Dialog.Show<PdfViewerDialog>(
105110
$"📄 {document.Name}",
106111
new DialogParameters
107112
{
@@ -116,6 +121,36 @@ private void OnShowDocument(DocumentResponse document) => Dialog.Show<PdfViewerD
116121
CloseButton = true,
117122
CloseOnEscapeKey = true
118123
});
124+
}
125+
else if (extension is ".png" or ".jpg" or ".jpeg")
126+
{
127+
Dialog.Show<ImageViewerDialog>(
128+
$"📄 {document.Name}",
129+
new DialogParameters
130+
{
131+
[nameof(ImageViewerDialog.FileName)] = document.Name,
132+
[nameof(ImageViewerDialog.Src)] = document.Url.ToString(),
133+
},
134+
new DialogOptions
135+
{
136+
MaxWidth = MaxWidth.Large,
137+
FullWidth = true,
138+
CloseButton = true,
139+
CloseOnEscapeKey = true
140+
});
141+
}
142+
else
143+
{
144+
Snackbar.Add(
145+
$"Unsupported file type: '{extension}'",
146+
Severity.Error,
147+
static options =>
148+
{
149+
options.ShowCloseIcon = true;
150+
options.VisibleStateDuration = 10_000;
151+
});
152+
}
153+
}
119154

120155
public void Dispose() => _cancellationTokenSource.Cancel();
121156
}

‎app/functions/EmbedFunctions/Services/MilvusEmbedService.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ public Task CreateSearchIndexAsync(string searchIndexName, CancellationToken ct
1010
throw new NotImplementedException();
1111
}
1212

13-
public Task<bool> EmbedImageBlobAsync(Stream imageStream, string imageName, CancellationToken ct = default)
13+
public Task<bool> EmbedImageBlobAsync(Stream imageStream, string imageUrl, string imageName, CancellationToken ct = default)
1414
{
1515
throw new NotImplementedException();
1616
}

‎app/functions/EmbedFunctions/Services/PineconeEmbedService.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ public Task CreateSearchIndexAsync(string searchIndexName, CancellationToken ct
1010
throw new NotImplementedException();
1111
}
1212

13-
public Task<bool> EmbedImageBlobAsync(Stream imageStream, string imageName, CancellationToken ct = default)
13+
public Task<bool> EmbedImageBlobAsync(Stream imageStream, string imageUrl, string imageName, CancellationToken ct = default)
1414
{
1515
throw new NotImplementedException();
1616
}

‎app/functions/EmbedFunctions/Services/QdrantEmbedService.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ public Task CreateSearchIndexAsync(string searchIndexName, CancellationToken ct
1010
throw new NotImplementedException();
1111
}
1212

13-
public Task<bool> EmbedImageBlobAsync(Stream imageStream, string imageName, CancellationToken ct = default)
13+
public Task<bool> EmbedImageBlobAsync(Stream imageStream, string imageUrl, string imageName, CancellationToken ct = default)
1414
{
1515
throw new NotImplementedException();
1616
}

‎app/prepdocs/PrepareDocs/Program.Clients.cs

Lines changed: 17 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ private static Task<AzureSearchEmbedService> GetAzureSearchEmbedService(AppOptio
2424
var searchClient = await GetSearchClientAsync(o);
2525
var documentClient = await GetFormRecognizerClientAsync(o);
2626
var blobContainerClient = await GetCorpusBlobContainerClientAsync(o);
27-
var openAIClient = await GetAzureOpenAIClientAsync(o);
27+
var openAIClient = await GetOpenAIClientAsync(o);
2828
var embeddingModelName = o.EmbeddingModelName ?? throw new ArgumentNullException(nameof(o.EmbeddingModelName));
2929
var searchIndexName = o.SearchIndexName ?? throw new ArgumentNullException(nameof(o.SearchIndexName));
3030
var computerVisionService = await GetComputerVisionServiceAsync(o);
@@ -161,16 +161,26 @@ private static Task<SearchClient> GetSearchClientAsync(AppOptions options) =>
161161
return new AzureComputerVisionService(new HttpClient(), endpoint, DefaultCredential);
162162
});
163163

164-
private static Task<OpenAIClient> GetAzureOpenAIClientAsync(AppOptions options) =>
164+
private static Task<OpenAIClient> GetOpenAIClientAsync(AppOptions options) =>
165165
GetLazyClientAsync<OpenAIClient>(options, s_openAILock, async o =>
166166
{
167167
if (s_openAIClient is null)
168168
{
169-
var endpoint = o.AzureOpenAIServiceEndpoint;
170-
ArgumentNullException.ThrowIfNullOrEmpty(endpoint);
171-
s_openAIClient = new OpenAIClient(
172-
new Uri(endpoint),
173-
DefaultCredential);
169+
var useAOAI = Environment.GetEnvironmentVariable("UseAOAI") == "true";
170+
if (!useAOAI)
171+
{
172+
var openAIApiKey = Environment.GetEnvironmentVariable("OPENAI_API_KEY");
173+
ArgumentNullException.ThrowIfNullOrEmpty(openAIApiKey);
174+
s_openAIClient = new OpenAIClient(openAIApiKey);
175+
}
176+
else
177+
{
178+
var endpoint = o.AzureOpenAIServiceEndpoint;
179+
ArgumentNullException.ThrowIfNullOrEmpty(endpoint);
180+
s_openAIClient = new OpenAIClient(
181+
new Uri(endpoint),
182+
DefaultCredential);
183+
}
174184
}
175185
await Task.CompletedTask;
176186
return s_openAIClient;

‎app/prepdocs/PrepareDocs/Program.cs

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -204,8 +204,9 @@ static async ValueTask UploadBlobsAndCreateIndexAsync(
204204
{
205205
using var stream = File.OpenRead(fileName);
206206
var blobName = BlobNameFromFilePage(fileName);
207-
await UploadBlobAsync(fileName, blobName, container);
208-
await embeddingService.EmbedImageBlobAsync(stream, fileName);
207+
var imageName = Path.GetFileNameWithoutExtension(blobName);
208+
var url = await UploadBlobAsync(fileName, blobName, container);
209+
await embeddingService.EmbedImageBlobAsync(stream, url, imageName);
209210
}
210211
else
211212
{
@@ -215,12 +216,14 @@ static async ValueTask UploadBlobsAndCreateIndexAsync(
215216
}
216217
}
217218

218-
static async Task UploadBlobAsync(string fileName, string blobName, BlobContainerClient container)
219+
static async Task<string> UploadBlobAsync(string fileName, string blobName, BlobContainerClient container)
219220
{
220221
var blobClient = container.GetBlobClient(blobName);
222+
var url = blobClient.Uri.AbsoluteUri;
223+
221224
if (await blobClient.ExistsAsync())
222225
{
223-
return;
226+
return url;
224227
}
225228

226229
var blobHttpHeaders = new BlobHttpHeaders
@@ -230,6 +233,9 @@ static async Task UploadBlobAsync(string fileName, string blobName, BlobContaine
230233

231234
await using var fileStream = File.OpenRead(fileName);
232235
await blobClient.UploadAsync(fileStream, blobHttpHeaders);
236+
237+
238+
return url;
233239
}
234240

235241
static string GetContentType(string fileName)

‎app/shared/Shared/Services/AzureSearchEmbedService.cs

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -76,12 +76,13 @@ Indexing sections from '{BlobName}' into search index '{SearchIndexName}'
7676
public async Task<bool> EmbedImageBlobAsync(
7777
Stream imageStream,
7878
string imageUrl,
79+
string imageName,
7980
CancellationToken ct = default)
8081
{
8182
if (includeImageEmbeddingsField == false || computerVisionService is null)
8283
{
8384
throw new InvalidOperationException(
84-
"Computer Vision service is required to include image embeddings field");
85+
"Computer Vision service is required to include image embeddings field, please enable GPT_4V support");
8586
}
8687

8788
var embeddings = await computerVisionService.VectorizeImageAsync(imageUrl, ct);
@@ -95,7 +96,7 @@ public async Task<bool> EmbedImageBlobAsync(
9596
new SearchDocument
9697
{
9798
["id"] = imageId,
98-
["content"] = imageUrl,
99+
["content"] = imageName,
99100
["category"] = "image",
100101
["imageEmbedding"] = embeddings.vector,
101102
["sourcefile"] = imageUrl,

‎app/shared/Shared/Services/IEmbedService.cs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ Task<bool> EmbedPDFBlobAsync(
2020
/// </summary>
2121
Task<bool> EmbedImageBlobAsync(
2222
Stream imageStream,
23+
string imageUrl,
2324
string imageName,
2425
CancellationToken ct = default);
2526

‎app/tests/MinimalApi.Tests/AzureSearchEmbedServiceTest.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -296,7 +296,7 @@ public async Task EmbedImageBlobTestAsync()
296296
var client = containerClient.GetBlobClient(imageBlobName);
297297
await client.UploadAsync(stream, true);
298298
var url = client.Uri.AbsoluteUri;
299-
var isSucceed = await service.EmbedImageBlobAsync(stream, url);
299+
var isSucceed = await service.EmbedImageBlobAsync(stream, imageBlobName, url);
300300
isSucceed.Should().BeTrue();
301301

302302
// check if the image is uploaded to blob

0 commit comments

Comments
 (0)