Skip to content

Commit ac2bf51

Browse files
author
awstools
committed
feat(client-bedrock): Add support for Knowledge Base Evaluations & LLM as a judge
1 parent 1bea595 commit ac2bf51

File tree

8 files changed

+3598
-377
lines changed

8 files changed

+3598
-377
lines changed

‎clients/client-bedrock/src/commands/BatchDeleteEvaluationJobCommand.ts

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,9 @@ export interface BatchDeleteEvaluationJobCommandInput extends BatchDeleteEvaluat
3333
export interface BatchDeleteEvaluationJobCommandOutput extends BatchDeleteEvaluationJobResponse, __MetadataBearer {}
3434

3535
/**
36-
* <p>Creates a batch deletion job. A model evaluation job can only be deleted if it has following status <code>FAILED</code>, <code>COMPLETED</code>, and <code>STOPPED</code>. You can request up to 25 model evaluation jobs be deleted in a single request.</p>
36+
* <p>Deletes a batch of evaluation jobs. An evaluation job can only be deleted if it has
37+
* following status <code>FAILED</code>, <code>COMPLETED</code>, and <code>STOPPED</code>.
38+
* You can request up to 25 model evaluation jobs be deleted in a single request.</p>
3739
* @example
3840
* Use a bare-bones client and the command you need to make an API call.
3941
* ```javascript

‎clients/client-bedrock/src/commands/CreateEvaluationJobCommand.ts

Lines changed: 156 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@ export interface CreateEvaluationJobCommandInput extends CreateEvaluationJobRequ
3232
export interface CreateEvaluationJobCommandOutput extends CreateEvaluationJobResponse, __MetadataBearer {}
3333

3434
/**
35-
* <p>API operation for creating and managing Amazon Bedrock automatic model evaluation jobs and model evaluation jobs that use human workers. To learn more about the requirements for creating a model evaluation job see, <a href="https://docs.aws.amazon.com/bedrock/latest/userguide/model-evaluation.html">Model evaluation</a>.</p>
35+
* <p>Creates an evaluation job.</p>
3636
* @example
3737
* Use a bare-bones client and the command you need to make an API call.
3838
* ```javascript
@@ -51,6 +51,7 @@ export interface CreateEvaluationJobCommandOutput extends CreateEvaluationJobRes
5151
* value: "STRING_VALUE", // required
5252
* },
5353
* ],
54+
* applicationType: "ModelEvaluation" || "RagEvaluation",
5455
* evaluationConfig: { // EvaluationConfig Union: only one key present
5556
* automated: { // AutomatedEvaluationConfig
5657
* datasetMetricConfigs: [ // EvaluationDatasetMetricConfigs // required
@@ -67,6 +68,13 @@ export interface CreateEvaluationJobCommandOutput extends CreateEvaluationJobRes
6768
* ],
6869
* },
6970
* ],
71+
* evaluatorModelConfig: { // EvaluatorModelConfig Union: only one key present
72+
* bedrockEvaluatorModels: [ // BedrockEvaluatorModels
73+
* { // BedrockEvaluatorModel
74+
* modelIdentifier: "STRING_VALUE", // required
75+
* },
76+
* ],
77+
* },
7078
* },
7179
* human: { // HumanEvaluationConfig
7280
* humanWorkflowConfig: { // HumanWorkflowConfig
@@ -101,7 +109,153 @@ export interface CreateEvaluationJobCommandOutput extends CreateEvaluationJobRes
101109
* { // EvaluationModelConfig Union: only one key present
102110
* bedrockModel: { // EvaluationBedrockModel
103111
* modelIdentifier: "STRING_VALUE", // required
104-
* inferenceParams: "STRING_VALUE", // required
112+
* inferenceParams: "STRING_VALUE",
113+
* },
114+
* },
115+
* ],
116+
* ragConfigs: [ // RagConfigs
117+
* { // RAGConfig Union: only one key present
118+
* knowledgeBaseConfig: { // KnowledgeBaseConfig Union: only one key present
119+
* retrieveConfig: { // RetrieveConfig
120+
* knowledgeBaseId: "STRING_VALUE", // required
121+
* knowledgeBaseRetrievalConfiguration: { // KnowledgeBaseRetrievalConfiguration
122+
* vectorSearchConfiguration: { // KnowledgeBaseVectorSearchConfiguration
123+
* numberOfResults: Number("int"),
124+
* overrideSearchType: "HYBRID" || "SEMANTIC",
125+
* filter: { // RetrievalFilter Union: only one key present
126+
* equals: { // FilterAttribute
127+
* key: "STRING_VALUE", // required
128+
* value: "DOCUMENT_VALUE", // required
129+
* },
130+
* notEquals: {
131+
* key: "STRING_VALUE", // required
132+
* value: "DOCUMENT_VALUE", // required
133+
* },
134+
* greaterThan: {
135+
* key: "STRING_VALUE", // required
136+
* value: "DOCUMENT_VALUE", // required
137+
* },
138+
* greaterThanOrEquals: {
139+
* key: "STRING_VALUE", // required
140+
* value: "DOCUMENT_VALUE", // required
141+
* },
142+
* lessThan: {
143+
* key: "STRING_VALUE", // required
144+
* value: "DOCUMENT_VALUE", // required
145+
* },
146+
* lessThanOrEquals: "<FilterAttribute>",
147+
* in: "<FilterAttribute>",
148+
* notIn: "<FilterAttribute>",
149+
* startsWith: "<FilterAttribute>",
150+
* listContains: "<FilterAttribute>",
151+
* stringContains: "<FilterAttribute>",
152+
* andAll: [ // RetrievalFilterList
153+
* {// Union: only one key present
154+
* equals: "<FilterAttribute>",
155+
* notEquals: "<FilterAttribute>",
156+
* greaterThan: "<FilterAttribute>",
157+
* greaterThanOrEquals: "<FilterAttribute>",
158+
* lessThan: "<FilterAttribute>",
159+
* lessThanOrEquals: "<FilterAttribute>",
160+
* in: "<FilterAttribute>",
161+
* notIn: "<FilterAttribute>",
162+
* startsWith: "<FilterAttribute>",
163+
* listContains: "<FilterAttribute>",
164+
* stringContains: "<FilterAttribute>",
165+
* andAll: [
166+
* "<RetrievalFilter>",
167+
* ],
168+
* orAll: [
169+
* "<RetrievalFilter>",
170+
* ],
171+
* },
172+
* ],
173+
* orAll: [
174+
* "<RetrievalFilter>",
175+
* ],
176+
* },
177+
* },
178+
* },
179+
* },
180+
* retrieveAndGenerateConfig: { // RetrieveAndGenerateConfiguration
181+
* type: "KNOWLEDGE_BASE" || "EXTERNAL_SOURCES", // required
182+
* knowledgeBaseConfiguration: { // KnowledgeBaseRetrieveAndGenerateConfiguration
183+
* knowledgeBaseId: "STRING_VALUE", // required
184+
* modelArn: "STRING_VALUE", // required
185+
* retrievalConfiguration: {
186+
* vectorSearchConfiguration: {
187+
* numberOfResults: Number("int"),
188+
* overrideSearchType: "HYBRID" || "SEMANTIC",
189+
* filter: "<RetrievalFilter>",
190+
* },
191+
* },
192+
* generationConfiguration: { // GenerationConfiguration
193+
* promptTemplate: { // PromptTemplate
194+
* textPromptTemplate: "STRING_VALUE",
195+
* },
196+
* guardrailConfiguration: { // GuardrailConfiguration
197+
* guardrailId: "STRING_VALUE", // required
198+
* guardrailVersion: "STRING_VALUE", // required
199+
* },
200+
* kbInferenceConfig: { // KbInferenceConfig
201+
* textInferenceConfig: { // TextInferenceConfig
202+
* temperature: Number("float"),
203+
* topP: Number("float"),
204+
* maxTokens: Number("int"),
205+
* stopSequences: [ // RAGStopSequences
206+
* "STRING_VALUE",
207+
* ],
208+
* },
209+
* },
210+
* additionalModelRequestFields: { // AdditionalModelRequestFields
211+
* "<keys>": "DOCUMENT_VALUE",
212+
* },
213+
* },
214+
* orchestrationConfiguration: { // OrchestrationConfiguration
215+
* queryTransformationConfiguration: { // QueryTransformationConfiguration
216+
* type: "QUERY_DECOMPOSITION", // required
217+
* },
218+
* },
219+
* },
220+
* externalSourcesConfiguration: { // ExternalSourcesRetrieveAndGenerateConfiguration
221+
* modelArn: "STRING_VALUE", // required
222+
* sources: [ // ExternalSources // required
223+
* { // ExternalSource
224+
* sourceType: "S3" || "BYTE_CONTENT", // required
225+
* s3Location: { // S3ObjectDoc
226+
* uri: "STRING_VALUE", // required
227+
* },
228+
* byteContent: { // ByteContentDoc
229+
* identifier: "STRING_VALUE", // required
230+
* contentType: "STRING_VALUE", // required
231+
* data: new Uint8Array(), // e.g. Buffer.from("") or new TextEncoder().encode("") // required
232+
* },
233+
* },
234+
* ],
235+
* generationConfiguration: { // ExternalSourcesGenerationConfiguration
236+
* promptTemplate: {
237+
* textPromptTemplate: "STRING_VALUE",
238+
* },
239+
* guardrailConfiguration: {
240+
* guardrailId: "STRING_VALUE", // required
241+
* guardrailVersion: "STRING_VALUE", // required
242+
* },
243+
* kbInferenceConfig: {
244+
* textInferenceConfig: {
245+
* temperature: Number("float"),
246+
* topP: Number("float"),
247+
* maxTokens: Number("int"),
248+
* stopSequences: [
249+
* "STRING_VALUE",
250+
* ],
251+
* },
252+
* },
253+
* additionalModelRequestFields: {
254+
* "<keys>": "DOCUMENT_VALUE",
255+
* },
256+
* },
257+
* },
258+
* },
105259
* },
106260
* },
107261
* ],

‎clients/client-bedrock/src/commands/GetEvaluationJobCommand.ts

Lines changed: 156 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -33,8 +33,7 @@ export interface GetEvaluationJobCommandInput extends GetEvaluationJobRequest {}
3333
export interface GetEvaluationJobCommandOutput extends GetEvaluationJobResponse, __MetadataBearer {}
3434

3535
/**
36-
* <p>Retrieves the properties associated with a model evaluation job, including the
37-
* status of the job. For more information, see <a href="https://docs.aws.amazon.com/bedrock/latest/userguide/model-evaluation.html">Model evaluation</a>.</p>
36+
* <p>Gets information about an evaluation job, such as the status of the job.</p>
3837
* @example
3938
* Use a bare-bones client and the command you need to make an API call.
4039
* ```javascript
@@ -54,6 +53,7 @@ export interface GetEvaluationJobCommandOutput extends GetEvaluationJobResponse,
5453
* // roleArn: "STRING_VALUE", // required
5554
* // customerEncryptionKeyId: "STRING_VALUE",
5655
* // jobType: "Human" || "Automated", // required
56+
* // applicationType: "ModelEvaluation" || "RagEvaluation",
5757
* // evaluationConfig: { // EvaluationConfig Union: only one key present
5858
* // automated: { // AutomatedEvaluationConfig
5959
* // datasetMetricConfigs: [ // EvaluationDatasetMetricConfigs // required
@@ -70,6 +70,13 @@ export interface GetEvaluationJobCommandOutput extends GetEvaluationJobResponse,
7070
* // ],
7171
* // },
7272
* // ],
73+
* // evaluatorModelConfig: { // EvaluatorModelConfig Union: only one key present
74+
* // bedrockEvaluatorModels: [ // BedrockEvaluatorModels
75+
* // { // BedrockEvaluatorModel
76+
* // modelIdentifier: "STRING_VALUE", // required
77+
* // },
78+
* // ],
79+
* // },
7380
* // },
7481
* // human: { // HumanEvaluationConfig
7582
* // humanWorkflowConfig: { // HumanWorkflowConfig
@@ -104,7 +111,153 @@ export interface GetEvaluationJobCommandOutput extends GetEvaluationJobResponse,
104111
* // { // EvaluationModelConfig Union: only one key present
105112
* // bedrockModel: { // EvaluationBedrockModel
106113
* // modelIdentifier: "STRING_VALUE", // required
107-
* // inferenceParams: "STRING_VALUE", // required
114+
* // inferenceParams: "STRING_VALUE",
115+
* // },
116+
* // },
117+
* // ],
118+
* // ragConfigs: [ // RagConfigs
119+
* // { // RAGConfig Union: only one key present
120+
* // knowledgeBaseConfig: { // KnowledgeBaseConfig Union: only one key present
121+
* // retrieveConfig: { // RetrieveConfig
122+
* // knowledgeBaseId: "STRING_VALUE", // required
123+
* // knowledgeBaseRetrievalConfiguration: { // KnowledgeBaseRetrievalConfiguration
124+
* // vectorSearchConfiguration: { // KnowledgeBaseVectorSearchConfiguration
125+
* // numberOfResults: Number("int"),
126+
* // overrideSearchType: "HYBRID" || "SEMANTIC",
127+
* // filter: { // RetrievalFilter Union: only one key present
128+
* // equals: { // FilterAttribute
129+
* // key: "STRING_VALUE", // required
130+
* // value: "DOCUMENT_VALUE", // required
131+
* // },
132+
* // notEquals: {
133+
* // key: "STRING_VALUE", // required
134+
* // value: "DOCUMENT_VALUE", // required
135+
* // },
136+
* // greaterThan: {
137+
* // key: "STRING_VALUE", // required
138+
* // value: "DOCUMENT_VALUE", // required
139+
* // },
140+
* // greaterThanOrEquals: {
141+
* // key: "STRING_VALUE", // required
142+
* // value: "DOCUMENT_VALUE", // required
143+
* // },
144+
* // lessThan: {
145+
* // key: "STRING_VALUE", // required
146+
* // value: "DOCUMENT_VALUE", // required
147+
* // },
148+
* // lessThanOrEquals: "<FilterAttribute>",
149+
* // in: "<FilterAttribute>",
150+
* // notIn: "<FilterAttribute>",
151+
* // startsWith: "<FilterAttribute>",
152+
* // listContains: "<FilterAttribute>",
153+
* // stringContains: "<FilterAttribute>",
154+
* // andAll: [ // RetrievalFilterList
155+
* // {// Union: only one key present
156+
* // equals: "<FilterAttribute>",
157+
* // notEquals: "<FilterAttribute>",
158+
* // greaterThan: "<FilterAttribute>",
159+
* // greaterThanOrEquals: "<FilterAttribute>",
160+
* // lessThan: "<FilterAttribute>",
161+
* // lessThanOrEquals: "<FilterAttribute>",
162+
* // in: "<FilterAttribute>",
163+
* // notIn: "<FilterAttribute>",
164+
* // startsWith: "<FilterAttribute>",
165+
* // listContains: "<FilterAttribute>",
166+
* // stringContains: "<FilterAttribute>",
167+
* // andAll: [
168+
* // "<RetrievalFilter>",
169+
* // ],
170+
* // orAll: [
171+
* // "<RetrievalFilter>",
172+
* // ],
173+
* // },
174+
* // ],
175+
* // orAll: [
176+
* // "<RetrievalFilter>",
177+
* // ],
178+
* // },
179+
* // },
180+
* // },
181+
* // },
182+
* // retrieveAndGenerateConfig: { // RetrieveAndGenerateConfiguration
183+
* // type: "KNOWLEDGE_BASE" || "EXTERNAL_SOURCES", // required
184+
* // knowledgeBaseConfiguration: { // KnowledgeBaseRetrieveAndGenerateConfiguration
185+
* // knowledgeBaseId: "STRING_VALUE", // required
186+
* // modelArn: "STRING_VALUE", // required
187+
* // retrievalConfiguration: {
188+
* // vectorSearchConfiguration: {
189+
* // numberOfResults: Number("int"),
190+
* // overrideSearchType: "HYBRID" || "SEMANTIC",
191+
* // filter: "<RetrievalFilter>",
192+
* // },
193+
* // },
194+
* // generationConfiguration: { // GenerationConfiguration
195+
* // promptTemplate: { // PromptTemplate
196+
* // textPromptTemplate: "STRING_VALUE",
197+
* // },
198+
* // guardrailConfiguration: { // GuardrailConfiguration
199+
* // guardrailId: "STRING_VALUE", // required
200+
* // guardrailVersion: "STRING_VALUE", // required
201+
* // },
202+
* // kbInferenceConfig: { // KbInferenceConfig
203+
* // textInferenceConfig: { // TextInferenceConfig
204+
* // temperature: Number("float"),
205+
* // topP: Number("float"),
206+
* // maxTokens: Number("int"),
207+
* // stopSequences: [ // RAGStopSequences
208+
* // "STRING_VALUE",
209+
* // ],
210+
* // },
211+
* // },
212+
* // additionalModelRequestFields: { // AdditionalModelRequestFields
213+
* // "<keys>": "DOCUMENT_VALUE",
214+
* // },
215+
* // },
216+
* // orchestrationConfiguration: { // OrchestrationConfiguration
217+
* // queryTransformationConfiguration: { // QueryTransformationConfiguration
218+
* // type: "QUERY_DECOMPOSITION", // required
219+
* // },
220+
* // },
221+
* // },
222+
* // externalSourcesConfiguration: { // ExternalSourcesRetrieveAndGenerateConfiguration
223+
* // modelArn: "STRING_VALUE", // required
224+
* // sources: [ // ExternalSources // required
225+
* // { // ExternalSource
226+
* // sourceType: "S3" || "BYTE_CONTENT", // required
227+
* // s3Location: { // S3ObjectDoc
228+
* // uri: "STRING_VALUE", // required
229+
* // },
230+
* // byteContent: { // ByteContentDoc
231+
* // identifier: "STRING_VALUE", // required
232+
* // contentType: "STRING_VALUE", // required
233+
* // data: new Uint8Array(), // required
234+
* // },
235+
* // },
236+
* // ],
237+
* // generationConfiguration: { // ExternalSourcesGenerationConfiguration
238+
* // promptTemplate: {
239+
* // textPromptTemplate: "STRING_VALUE",
240+
* // },
241+
* // guardrailConfiguration: {
242+
* // guardrailId: "STRING_VALUE", // required
243+
* // guardrailVersion: "STRING_VALUE", // required
244+
* // },
245+
* // kbInferenceConfig: {
246+
* // textInferenceConfig: {
247+
* // temperature: Number("float"),
248+
* // topP: Number("float"),
249+
* // maxTokens: Number("int"),
250+
* // stopSequences: [
251+
* // "STRING_VALUE",
252+
* // ],
253+
* // },
254+
* // },
255+
* // additionalModelRequestFields: {
256+
* // "<keys>": "DOCUMENT_VALUE",
257+
* // },
258+
* // },
259+
* // },
260+
* // },
108261
* // },
109262
* // },
110263
* // ],

0 commit comments

Comments
 (0)