Skip to content

Commit 3c29ebe

Browse files
ibuildthecloudibuildthecloud
authored
Merge pull request #647 from ibuildthecloud/toolmiss
chore: return missing tool call to LLM, don't fail
2 parents 3055632 + 19a5189 commit 3c29ebe

File tree

9 files changed

+181
-11
lines changed

9 files changed

+181
-11
lines changed

‎pkg/engine/engine.go

Lines changed: 14 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -45,8 +45,9 @@ type Return struct {
4545
}
4646

4747
type Call struct {
48-
ToolID string `json:"toolID,omitempty"`
49-
Input string `json:"input,omitempty"`
48+
Missing bool `json:"missing,omitempty"`
49+
ToolID string `json:"toolID,omitempty"`
50+
Input string `json:"input,omitempty"`
5051
}
5152

5253
type CallResult struct {
@@ -216,10 +217,7 @@ func NewContext(ctx context.Context, prg *types.Program, input string) (Context,
216217
}
217218

218219
func (c *Context) SubCallContext(ctx context.Context, input, toolID, callID string, toolCategory ToolCategory) (Context, error) {
219-
tool, ok := c.Program.ToolSet[toolID]
220-
if !ok {
221-
return Context{}, fmt.Errorf("failed to file tool for id [%s]", toolID)
222-
}
220+
tool := c.Program.ToolSet[toolID]
223221

224222
if callID == "" {
225223
callID = counter.Next()
@@ -387,19 +385,25 @@ func (e *Engine) complete(ctx context.Context, state *State) (*Return, error) {
387385
state.Pending = map[string]types.CompletionToolCall{}
388386
for _, content := range resp.Content {
389387
if content.ToolCall != nil {
390-
var toolID string
388+
var (
389+
toolID string
390+
missing bool
391+
)
391392
for _, tool := range state.Completion.Tools {
392393
if strings.EqualFold(tool.Function.Name, content.ToolCall.Function.Name) {
393394
toolID = tool.Function.ToolID
394395
}
395396
}
396397
if toolID == "" {
397-
return nil, fmt.Errorf("failed to find tool id for tool %s in tool_call result", content.ToolCall.Function.Name)
398+
log.Debugf("failed to find tool id for tool %s in tool_call result", content.ToolCall.Function.Name)
399+
toolID = content.ToolCall.Function.Name
400+
missing = true
398401
}
399402
state.Pending[content.ToolCall.ID] = *content.ToolCall
400403
ret.Calls[content.ToolCall.ID] = Call{
401-
ToolID: toolID,
402-
Input: content.ToolCall.Function.Arguments,
404+
ToolID: toolID,
405+
Missing: missing,
406+
Input: content.ToolCall.Function.Arguments,
403407
}
404408
} else {
405409
cp := content.Text

‎pkg/runner/runner.go

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -802,6 +802,18 @@ func (r *Runner) subCalls(callCtx engine.Context, monitor Monitor, env []string,
802802

803803
for _, id := range ids {
804804
call := state.Continuation.Calls[id]
805+
if call.Missing {
806+
resultLock.Lock()
807+
callResults = append(callResults, SubCallResult{
808+
ToolID: call.ToolID,
809+
CallID: id,
810+
State: &State{
811+
Result: &[]string{fmt.Sprintf("ERROR: can not call unknown tool named [%s]", call.ToolID)}[0],
812+
},
813+
})
814+
resultLock.Unlock()
815+
continue
816+
}
805817
d.Run(func(ctx context.Context) error {
806818
result, err := r.subCall(ctx, callCtx, monitor, env, call.ToolID, call.Input, id, toolCategory)
807819
if err != nil {

‎pkg/tests/runner_test.go

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -948,3 +948,18 @@ func TestSysContext(t *testing.T) {
948948
require.Len(t, context.Call.AgentGroup, 1)
949949
assert.Equal(t, context.Call.AgentGroup[0].Named, "iAmSuperman")
950950
}
951+
952+
func TestMissingTool(t *testing.T) {
953+
r := tester.NewRunner(t)
954+
955+
r.RespondWith(tester.Result{
956+
Func: types.CompletionFunctionCall{
957+
Name: "not bob",
958+
},
959+
})
960+
961+
resp, err := r.Run("", "Input 1")
962+
require.NoError(t, err)
963+
r.AssertResponded(t)
964+
autogold.Expect("TEST RESULT CALL: 2").Equal(t, resp)
965+
}

‎pkg/tests/testdata/TestMissingTool/call1-resp.golden

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
`{
2+
"role": "assistant",
3+
"content": [
4+
{
5+
"toolCall": {
6+
"id": "call_1",
7+
"function": {
8+
"name": "not bob"
9+
}
10+
}
11+
}
12+
],
13+
"usage": {}
14+
}`

‎pkg/tests/testdata/TestMissingTool/call1.golden

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
`{
2+
"model": "gpt-4o",
3+
"tools": [
4+
{
5+
"function": {
6+
"toolID": "testdata/TestMissingTool/test.gpt:Bob",
7+
"name": "Bob",
8+
"parameters": null
9+
}
10+
}
11+
],
12+
"messages": [
13+
{
14+
"role": "system",
15+
"content": [
16+
{
17+
"text": "Call tool Bob"
18+
}
19+
],
20+
"usage": {}
21+
},
22+
{
23+
"role": "user",
24+
"content": [
25+
{
26+
"text": "Input 1"
27+
}
28+
],
29+
"usage": {}
30+
}
31+
]
32+
}`

‎pkg/tests/testdata/TestMissingTool/call2-resp.golden

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
`{
2+
"role": "assistant",
3+
"content": [
4+
{
5+
"text": "TEST RESULT CALL: 2"
6+
}
7+
],
8+
"usage": {}
9+
}`

‎pkg/tests/testdata/TestMissingTool/call2.golden

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
`{
2+
"model": "gpt-4o",
3+
"tools": [
4+
{
5+
"function": {
6+
"toolID": "testdata/TestMissingTool/test.gpt:Bob",
7+
"name": "Bob",
8+
"parameters": null
9+
}
10+
}
11+
],
12+
"messages": [
13+
{
14+
"role": "system",
15+
"content": [
16+
{
17+
"text": "Call tool Bob"
18+
}
19+
],
20+
"usage": {}
21+
},
22+
{
23+
"role": "user",
24+
"content": [
25+
{
26+
"text": "Input 1"
27+
}
28+
],
29+
"usage": {}
30+
},
31+
{
32+
"role": "assistant",
33+
"content": [
34+
{
35+
"toolCall": {
36+
"id": "call_1",
37+
"function": {
38+
"name": "not bob"
39+
}
40+
}
41+
}
42+
],
43+
"usage": {}
44+
},
45+
{
46+
"role": "tool",
47+
"content": [
48+
{
49+
"text": "ERROR: can not call unknown tool named [not bob]"
50+
}
51+
],
52+
"toolCall": {
53+
"id": "call_1",
54+
"function": {
55+
"name": "not bob"
56+
}
57+
},
58+
"usage": {}
59+
}
60+
]
61+
}`

‎pkg/tests/testdata/TestMissingTool/test.gpt

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
tools: Bob
2+
3+
Call tool Bob
4+
5+
---
6+
name: Bob
7+
8+
#!sys.echo
9+
10+
You called?

‎pkg/tests/tester/runner.go

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -104,7 +104,20 @@ func (c *Client) Call(_ context.Context, messageRequest types.CompletionRequest,
104104
}
105105

106106
if result.Func.Name != "" {
107-
c.t.Fatalf("failed to find tool %s", result.Func.Name)
107+
return &types.CompletionMessage{
108+
Role: types.CompletionMessageRoleTypeAssistant,
109+
Content: []types.ContentPart{
110+
{
111+
ToolCall: &types.CompletionToolCall{
112+
ID: fmt.Sprintf("call_%d", c.id),
113+
Function: types.CompletionFunctionCall{
114+
Name: result.Func.Name,
115+
Arguments: result.Func.Arguments,
116+
},
117+
},
118+
},
119+
},
120+
}, nil
108121
}
109122

110123
return &types.CompletionMessage{

0 commit comments

Comments
 (0)