(EAI-1046): better handle tracing errs (#746)

* better handle tracing errs

* clarify err msg

---------

Co-authored-by: Ben Perlmutter <[email protected]>

Author: mongodben

packages/chatbot-server-mongodb-public/src/tracing/routesUpdateTraceHandlers.ts

@@ -79,6 +79,7 @@ export function makeAddMessageToConversationUpdateTrace({
           model: analyzerModel,
         },
         embeddingModelName,
+        reqId,
       });
       await scrubbedMessageStore.insertScrubbedMessages({
         messages: scrubbedMessages,
@@ -149,14 +150,25 @@ export function makeAddMessageToConversationUpdateTrace({
     }
 
     try {
+      const judgeScores = shouldJudge
+        ? await getLlmAsAJudgeScores(llmAsAJudge, tracingData).catch(
+            (error) => {
+              logRequest({
+                reqId,
+                message: `Error getting LLM as a judge scores in addMessageToConversationUpdateTrace: ${error}`,
+                type: "error",
+              });
+              return undefined;
+            }
+          )
+        : undefined;
+
       braintrustLogger.updateSpan({
         id: traceId,
         tags: tracingData.tags,
         scores: {
           ...getTracingScores(tracingData, k),
-          ...(shouldJudge
-            ? await getLlmAsAJudgeScores(llmAsAJudge, tracingData)
-            : undefined),
+          ...(judgeScores ?? {}),
         },
         metadata: {
           authUser: maybeAuthUser ?? null,

packages/chatbot-server-mongodb-public/src/tracing/scrubbedMessages/makeScrubbedMessagesFromTracingData.ts

@@ -1,3 +1,4 @@
+import { logRequest } from "../../utils";
 import { extractTracingData } from "../extractTracingData";
 import { analyzeMessage, MessageAnalysis } from "./analyzeMessage";
 import { redactPii } from "./redactPii";
@@ -8,19 +9,32 @@ export async function makeScrubbedMessagesFromTracingData({
   tracingData,
   analysis,
   embeddingModelName,
+  reqId,
 }: {
   tracingData: ReturnType<typeof extractTracingData>;
   analysis?: {
     model: LanguageModel;
   };
   embeddingModelName: string;
+  reqId: string;
 }): Promise<ScrubbedMessage<MessageAnalysis>[]> {
   const { userMessage, assistantMessage } = tracingData;
 
-  // User message scrubbing
   const userAnalysis = analysis
-    ? await analyzeMessage(userMessage.content, analysis.model)
+    ? await analyzeMessage(userMessage.content, analysis.model).catch(
+        (error) => {
+          logRequest({
+            reqId,
+            message: `Error analyzing scrubbed user message in tracing: ${JSON.stringify(
+              error
+            )}`,
+            type: "error",
+          });
+          return undefined;
+        }
+      )
     : undefined;
+
   const { redactedText: redactedUserContent, piiFound: userMessagePii } =
     redactPii(userMessage.content);
 

packages/mongodb-chatbot-server/src/routes/conversations/UpdateTraceFunc.ts

@@ -1,5 +1,6 @@
 import { Conversation, ConversationsService } from "mongodb-rag-core";
 import { ObjectId } from "mongodb-rag-core/mongodb";
+import { logRequest } from "../../utils";
 
 export type UpdateTraceFuncParams = {
   reqId: string;
@@ -32,6 +33,14 @@ export async function updateTraceIfExists({
         reqId,
         traceId: assistantResponseMessageId.toHexString(),
         conversation: updatedConversationForTrace,
+      }).catch((error) => {
+        logRequest({
+          reqId,
+          type: "error",
+          message: `Failed to update trace with Error: ${JSON.stringify(
+            error
+          )}`,
+        });
       });
     }
   }