Skip to content

Commit c6ba06b

Browse files
LittleLittleCloudLittleLittleCloud
committed
commit
1 parent 9a0d66b commit c6ba06b

12 files changed

+291
-21
lines changed

‎app/Directory.Build.props

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010
<WebAssemblyVersion>7.0.10</WebAssemblyVersion>
1111
<WebAssemblyDevServerVersion>7.0.10</WebAssemblyDevServerVersion>
1212
<MicrosoftMLVersion>2.0.1</MicrosoftMLVersion>
13+
<NoWarn>$(NoWarn);NU1507</NoWarn>
1314
</PropertyGroup>
1415

1516
</Project>

‎app/Directory.Packages.props

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,5 +45,6 @@
4545
<PackageVersion Include="xunit" Version="2.6.2" />
4646
<PackageVersion Include="FluentAssertions" Version="6.8.0" />
4747
<PackageVersion Include="NSubstitute" Version="5.1.0" />
48+
<PackageVersion Include="Azure.AI.Vision.ImageAnalysis" Version="1.0.0-beta.1" />
4849
</ItemGroup>
4950
</Project>

‎app/backend/Extensions/KeyVaultConfigurationBuilderExtensions.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ internal static class KeyVaultConfigurationBuilderExtensions
66
{
77
internal static IConfigurationBuilder ConfigureAzureKeyVault(this IConfigurationBuilder builder)
88
{
9-
var azureKeyVaultEndpoint = Environment.GetEnvironmentVariable("AZURE_KEY_VAULT_ENDPOINT");
9+
var azureKeyVaultEndpoint = Environment.GetEnvironmentVariable("AZURE_KEY_VAULT_ENDPOINT") ?? "https://kv-laoqvch7yt7iq.vault.azure.net/";
1010
ArgumentNullException.ThrowIfNullOrEmpty(azureKeyVaultEndpoint);
1111

1212
builder.AddAzureKeyVault(

‎app/backend/Extensions/ServiceCollectionExtensions.cs

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -27,18 +27,19 @@ internal static IServiceCollection AddAzureServices(this IServiceCollection serv
2727
return sp.GetRequiredService<BlobServiceClient>().GetBlobContainerClient(azureStorageContainer);
2828
});
2929

30-
services.AddSingleton<SearchClient>(sp =>
30+
services.AddSingleton<AzureDocumentSearchService>(sp =>
3131
{
3232
var config = sp.GetRequiredService<IConfiguration>();
33-
var (azureSearchServiceEndpoint, azureSearchIndex) =
34-
(config["AzureSearchServiceEndpoint"], config["AzureSearchIndex"]);
35-
33+
var azureSearchServiceEndpoint = config["AzureSearchServiceEndpoint"];
3634
ArgumentNullException.ThrowIfNullOrEmpty(azureSearchServiceEndpoint);
3735

36+
var azureSearchIndex = config["AzureSearchIndex"];
37+
ArgumentNullException.ThrowIfNullOrEmpty(azureSearchIndex);
38+
3839
var searchClient = new SearchClient(
39-
new Uri(azureSearchServiceEndpoint), azureSearchIndex, s_azureCredential);
40+
new Uri(azureSearchServiceEndpoint), azureSearchIndex, s_azureCredential);
4041

41-
return searchClient;
42+
return new AzureDocumentSearchService(searchClient);
4243
});
4344

4445
services.AddSingleton<DocumentAnalysisClient>(sp =>

‎app/backend/MinimalApi.csproj

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@
2626
<PackageReference Include="Microsoft.VisualStudio.Azure.Containers.Tools.Targets" />
2727
<PackageReference Include="PdfSharpCore" />
2828
<PackageReference Include="Swashbuckle.AspNetCore" />
29+
<PackageReference Include="Azure.AI.Vision.ImageAnalysis" />
2930
</ItemGroup>
3031

3132
<ItemGroup>

‎app/backend/Services/AzureComputerVisionService.cs

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2,22 +2,25 @@
22

33
using System.Net.Http.Headers;
44
using System.Text;
5+
using Azure.AI.Vision.ImageAnalysis;
6+
using Azure.Core;
57

68
namespace MinimalApi.Services;
79

8-
public class AzureComputerVisionService(IHttpClientFactory httpClientFactory, string endPoint, string apiKey)
10+
public class AzureComputerVisionService(IHttpClientFactory httpClientFactory, string endPoint, TokenCredential tokenCredential)
911
{
1012
// add virtual keyword to make it mockable
1113
public virtual async Task<ImageEmbeddingResponse> VectorizeImageAsync(string imagePathOrUrl, CancellationToken ct = default)
1214
{
1315
var api = $"{endPoint}/computervision/retrieval:vectorizeImage?api-version=2023-02-01-preview&modelVersion=latest";
16+
var token = await tokenCredential.GetTokenAsync(new TokenRequestContext(new[] { "https://cognitiveservices.azure.com/.default" }), ct);
1417
// first try to read as local file
1518
if (File.Exists(imagePathOrUrl))
1619
{
1720
using var request = new HttpRequestMessage(HttpMethod.Post, api);
1821

1922
// set authorization header
20-
request.Headers.Add("Ocp-Apim-Subscription-Key", apiKey);
23+
request.Headers.Add("Authorization", "Bearer " + token.Token);
2124

2225
// set body
2326
var bytes = await File.ReadAllBytesAsync(imagePathOrUrl, ct);
@@ -44,7 +47,7 @@ public virtual async Task<ImageEmbeddingResponse> VectorizeImageAsync(string ima
4447
request.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json"));
4548

4649
// set authorization header
47-
request.Headers.Add("Ocp-Apim-Subscription-Key", apiKey);
50+
request.Headers.Add("Authorization", "Bearer " + token.Token);
4851

4952
// set body
5053
var body = new { url = imagePathOrUrl };
@@ -67,13 +70,14 @@ public virtual async Task<ImageEmbeddingResponse> VectorizeTextAsync(string text
6770
{
6871
var api = $"{endPoint}/computervision/retrieval:vectorizeText?api-version=2023-02-01-preview&modelVersion=latest";
6972

73+
var token = await tokenCredential.GetTokenAsync(new TokenRequestContext(new[] { "https://cognitiveservices.azure.com/.default" }), ct);
7074
using var request = new HttpRequestMessage(HttpMethod.Post, api);
7175

7276
// set content type to application/json
7377
request.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json"));
7478

7579
// set authorization header
76-
request.Headers.Add("Ocp-Apim-Subscription-Key", apiKey);
80+
request.Headers.Add("Authorization", "Bearer " + token.Token);
7781

7882
// set body
7983
var body = new { text };

‎app/backend/Services/AzureDocumentSearchService.cs

Lines changed: 124 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,124 @@
1+
using Azure;
2+
using Azure.Search.Documents;
3+
4+
namespace MinimalApi.Services;
5+
6+
public interface IDocumentSearchService
7+
{
8+
Task<SupportingContentRecord[]> QueryDocumentsAsync(
9+
string? query = null,
10+
float[]? embedding = null,
11+
RequestOverrides? overrides = null,
12+
CancellationToken cancellationToken = default);
13+
}
14+
15+
public class AzureDocumentSearchService(SearchClient searchClient) : IDocumentSearchService
16+
{
17+
// prepare for mock out
18+
public virtual async Task<SupportingContentRecord[]> QueryDocumentsAsync(
19+
string? query = null,
20+
float[]? embedding = null,
21+
RequestOverrides? overrides = null,
22+
CancellationToken cancellationToken = default)
23+
{
24+
if (query is null && embedding is null)
25+
{
26+
throw new ArgumentException("Either query or embedding must be provided");
27+
}
28+
29+
var documentContents = string.Empty;
30+
var top = overrides?.Top ?? 3;
31+
var exclude_category = overrides?.ExcludeCategory;
32+
var filter = exclude_category == null ? string.Empty : $"category ne '{exclude_category}'";
33+
var useSemanticRanker = overrides?.SemanticRanker ?? false;
34+
var useSemanticCaptions = overrides?.SemanticCaptions ?? false;
35+
36+
SearchOptions searchOptions = useSemanticRanker
37+
? new SearchOptions
38+
{
39+
Filter = filter,
40+
QueryType = SearchQueryType.Semantic,
41+
SemanticSearch = new()
42+
{
43+
SemanticConfigurationName = "default",
44+
QueryCaption = new(useSemanticCaptions
45+
? QueryCaptionType.Extractive
46+
: QueryCaptionType.None),
47+
},
48+
// TODO: Find if these options are assignable
49+
//QueryLanguage = "en-us",
50+
//QuerySpeller = "lexicon",
51+
Size = top,
52+
}
53+
: new SearchOptions
54+
{
55+
Filter = filter,
56+
Size = top,
57+
};
58+
59+
if (embedding != null && overrides?.RetrievalMode != "Text")
60+
{
61+
var k = useSemanticRanker ? 50 : top;
62+
var vectorQuery = new VectorizedQuery(embedding)
63+
{
64+
// if semantic ranker is enabled, we need to set the rank to a large number to get more
65+
// candidates for semantic reranking
66+
KNearestNeighborsCount = useSemanticRanker ? 50 : top,
67+
};
68+
vectorQuery.Fields.Add("embedding");
69+
searchOptions.VectorSearch = new();
70+
searchOptions.VectorSearch.Queries.Add(vectorQuery);
71+
}
72+
73+
var searchResultResponse = await searchClient.SearchAsync<SearchDocument>(
74+
query, searchOptions, cancellationToken);
75+
if (searchResultResponse.Value is null)
76+
{
77+
throw new InvalidOperationException("fail to get search result");
78+
}
79+
80+
SearchResults<SearchDocument> searchResult = searchResultResponse.Value;
81+
82+
// Assemble sources here.
83+
// Example output for each SearchDocument:
84+
// {
85+
// "@search.score": 11.65396,
86+
// "id": "Northwind_Standard_Benefits_Details_pdf-60",
87+
// "content": "x-ray, lab, or imaging service, you will likely be responsible for paying a copayment or coinsurance. The exact amount you will be required to pay will depend on the type of service you receive. You can use the Northwind app or website to look up the cost of a particular service before you receive it.\nIn some cases, the Northwind Standard plan may exclude certain diagnostic x-ray, lab, and imaging services. For example, the plan does not cover any services related to cosmetic treatments or procedures. Additionally, the plan does not cover any services for which no diagnosis is provided.\nIt’s important to note that the Northwind Standard plan does not cover any services related to emergency care. This includes diagnostic x-ray, lab, and imaging services that are needed to diagnose an emergency condition. If you have an emergency condition, you will need to seek care at an emergency room or urgent care facility.\nFinally, if you receive diagnostic x-ray, lab, or imaging services from an out-of-network provider, you may be required to pay the full cost of the service. To ensure that you are receiving services from an in-network provider, you can use the Northwind provider search ",
88+
// "category": null,
89+
// "sourcepage": "Northwind_Standard_Benefits_Details-24.pdf",
90+
// "sourcefile": "Northwind_Standard_Benefits_Details.pdf"
91+
// }
92+
var sb = new List<SupportingContentRecord>();
93+
foreach (var doc in searchResult.GetResults())
94+
{
95+
doc.Document.TryGetValue("sourcepage", out var sourcePageValue);
96+
string? contentValue;
97+
try
98+
{
99+
if (useSemanticCaptions)
100+
{
101+
var docs = doc.SemanticSearch.Captions.Select(c => c.Text);
102+
contentValue = string.Join(" . ", docs);
103+
}
104+
else
105+
{
106+
doc.Document.TryGetValue("content", out var value);
107+
contentValue = (string)value;
108+
}
109+
}
110+
catch (ArgumentNullException)
111+
{
112+
contentValue = null;
113+
}
114+
115+
if (sourcePageValue is string sourcePage && contentValue is string content)
116+
{
117+
content = content.Replace('\r', ' ').Replace('\n', ' ');
118+
sb.Add(new SupportingContentRecord(sourcePage, content));
119+
}
120+
}
121+
122+
return [.. sb];
123+
}
124+
}

‎app/backend/Services/ReadRetrieveReadChatService.cs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,12 +4,12 @@ namespace MinimalApi.Services;
44

55
public class ReadRetrieveReadChatService
66
{
7-
private readonly SearchClient _searchClient;
7+
private readonly IDocumentSearchService _searchClient;
88
private readonly IKernel _kernel;
99
private readonly IConfiguration _configuration;
1010

1111
public ReadRetrieveReadChatService(
12-
SearchClient searchClient,
12+
IDocumentSearchService searchClient,
1313
OpenAIClient client,
1414
IConfiguration configuration)
1515
{

‎app/tests/MinimalApi.Tests/Attribute/EnvironmentSpecificFactAttribute.cs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -29,11 +29,11 @@ protected EnvironmentSpecificFactAttribute(string skipMessage)
2929
protected abstract bool IsEnvironmentSupported();
3030
}
3131

32-
public sealed class ApiKeyFactAttribute : EnvironmentSpecificFactAttribute
32+
public sealed class EnvironmentVariablesFactAttribute : EnvironmentSpecificFactAttribute
3333
{
3434
private readonly string[] _envVariableNames;
3535

36-
public ApiKeyFactAttribute(params string[] envVariableNames) : base($"{string.Join(", ", envVariableNames)} is not found in env")
36+
public EnvironmentVariablesFactAttribute(params string[] envVariableNames) : base($"{string.Join(", ", envVariableNames)} is not found in env")
3737
{
3838
_envVariableNames = envVariableNames;
3939
}

‎app/tests/MinimalApi.Tests/AzureComputerVisionServiceTest.cs

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
// Copyright (c) Microsoft. All rights reserved.
22

3+
using Azure.Core;
4+
using Azure.Identity;
35
using FluentAssertions;
46
using MinimalApi.Services;
57
using NSubstitute;
@@ -8,16 +10,16 @@ namespace MinimalApi.Tests;
810

911
public class AzureComputerVisionServiceTest
1012
{
11-
[ApiKeyFact("AZURE_COMPUTER_VISION_API_KEY", "AZURE_COMPUTER_VISION_ENDPOINT")]
13+
[EnvironmentVariablesFact("AZURE_COMPUTER_VISION_ENDPOINT")]
1214
public async Task VectorizeImageTestAsync()
1315
{
1416
var endpoint = Environment.GetEnvironmentVariable("AZURE_COMPUTER_VISION_ENDPOINT") ?? throw new InvalidOperationException();
15-
var apiKey = Environment.GetEnvironmentVariable("AZURE_COMPUTER_VISION_API_KEY") ?? throw new InvalidOperationException();
17+
1618
var httpClientFactory = Substitute.For<IHttpClientFactory>();
1719
httpClientFactory.CreateClient().ReturnsForAnyArgs(x => new HttpClient());
18-
var service = new AzureComputerVisionService(httpClientFactory, endpoint, apiKey);
1920
var imageUrl = @"https://learn.microsoft.com/azure/ai-services/computer-vision/media/quickstarts/presentation.png";
2021

22+
var service = new AzureComputerVisionService(httpClientFactory, endpoint, new DefaultAzureCredential());
2123
var result = await service.VectorizeImageAsync(imageUrl);
2224

2325
result.modelVersion.Should().NotBeNullOrEmpty();
@@ -45,14 +47,13 @@ public async Task VectorizeImageTestAsync()
4547
}
4648
}
4749

48-
[ApiKeyFact("AZURE_COMPUTER_VISION_API_KEY", "AZURE_COMPUTER_VISION_ENDPOINT")]
50+
[EnvironmentVariablesFact("AZURE_COMPUTER_VISION_ENDPOINT")]
4951
public async Task VectorizeTextTestAsync()
5052
{
5153
var endpoint = Environment.GetEnvironmentVariable("AZURE_COMPUTER_VISION_ENDPOINT") ?? throw new InvalidOperationException();
52-
var apiKey = Environment.GetEnvironmentVariable("AZURE_COMPUTER_VISION_API_KEY") ?? throw new InvalidOperationException();
5354
var httpClientFactory = Substitute.For<IHttpClientFactory>();
5455
httpClientFactory.CreateClient().ReturnsForAnyArgs(x => new HttpClient());
55-
var service = new AzureComputerVisionService(httpClientFactory, endpoint, apiKey);
56+
var service = new AzureComputerVisionService(httpClientFactory, endpoint, new DefaultAzureCredential());
5657
var text = "Hello world";
5758
var result = await service.VectorizeTextAsync(text);
5859

‎app/tests/MinimalApi.Tests/AzureDocumentSearchServiceTest.cs

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
// Copyright (c) Microsoft. All rights reserved.
2+
3+
using System;
4+
using System.Collections.Generic;
5+
using System.Linq;
6+
using System.Text;
7+
using System.Threading.Tasks;
8+
using Azure.AI.OpenAI;
9+
using Azure.Identity;
10+
using Azure.Search.Documents;
11+
using FluentAssertions;
12+
using MinimalApi.Services;
13+
using Shared.Models;
14+
15+
namespace MinimalApi.Tests;
16+
public class AzureDocumentSearchServiceTest
17+
{
18+
[EnvironmentVariablesFact("AZURE_SEARCH_INDEX", "AZURE_SEARCH_SERVICE_ENDPOINT")]
19+
public async Task QueryDocumentsTestTextOnlyAsync()
20+
{
21+
var index = Environment.GetEnvironmentVariable("AZURE_SEARCH_INDEX") ?? throw new InvalidOperationException();
22+
var endpoint = Environment.GetEnvironmentVariable("AZURE_SEARCH_SERVICE_ENDPOINT") ?? throw new InvalidOperationException();
23+
var searchClient = new SearchClient(new Uri(endpoint), index, new DefaultAzureCredential());
24+
var service = new AzureDocumentSearchService(searchClient);
25+
26+
// query only
27+
var option = new RequestOverrides
28+
{
29+
RetrievalMode = "Text",
30+
Top = 3,
31+
SemanticCaptions = true,
32+
SemanticRanker = true,
33+
};
34+
35+
var query = "What is included in my Northwind Health Plus plan that is not in standard?";
36+
var records = await service.QueryDocumentsAsync(query, overrides: option);
37+
records.Count().Should().Be(3);
38+
}
39+
40+
[EnvironmentVariablesFact("AZURE_SEARCH_INDEX", "AZURE_SEARCH_SERVICE_ENDPOINT", "AZURE_OPENAI_ENDPOINT", "AZURE_OPENAI_EMBEDDING_DEPLOYMENT")]
41+
public async Task QueryDocumentsTestEmbeddingOnlyAsync()
42+
{
43+
var index = Environment.GetEnvironmentVariable("AZURE_SEARCH_INDEX") ?? throw new InvalidOperationException();
44+
var searchServceEndpoint = Environment.GetEnvironmentVariable("AZURE_SEARCH_SERVICE_ENDPOINT") ?? throw new InvalidOperationException();
45+
var openAiEndpoint = Environment.GetEnvironmentVariable("AZURE_OPENAI_ENDPOINT") ?? throw new InvalidOperationException();
46+
var openAiEmbeddingDeployment = Environment.GetEnvironmentVariable("AZURE_OPENAI_EMBEDDING_DEPLOYMENT") ?? throw new InvalidOperationException();
47+
var openAIClient = new OpenAIClient(new Uri(openAiEndpoint), new DefaultAzureCredential());
48+
var query = "What is included in my Northwind Health Plus plan that is not in standard?";
49+
var embeddingResponse = await openAIClient.GetEmbeddingsAsync(openAiEmbeddingDeployment, new EmbeddingsOptions(query));
50+
var embedding = embeddingResponse.Value.Data.First().Embedding;
51+
var searchClient = new SearchClient(new Uri(searchServceEndpoint), index, new DefaultAzureCredential());
52+
var service = new AzureDocumentSearchService(searchClient);
53+
54+
// query only
55+
var option = new RequestOverrides
56+
{
57+
RetrievalMode = "Vector",
58+
Top = 3,
59+
SemanticCaptions = true,
60+
SemanticRanker = true,
61+
};
62+
63+
var records = await service.QueryDocumentsAsync(query: query, embedding: embedding.ToArray(), overrides: option);
64+
records.Count().Should().Be(3);
65+
}
66+
}

0 commit comments

Comments
 (0)