feat: chat-completion-client retries by default - to disable via env (#917)

iwilltry42 · web-flow · commit eb036809105c · 2024-12-16T22:07:44.000+01:00
* feat: chat-completion-client retries by default - to disable via env
diff --git a/go.mod b/go.mod
@@ -15,7 +15,7 @@ require (
 	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510
 	github.com/google/uuid v1.6.0
 	github.com/gptscript-ai/broadcaster v0.0.0-20240625175512-c43682019b86
-	github.com/gptscript-ai/chat-completion-client v0.0.0-20241104122544-5fe75f07c131
+	github.com/gptscript-ai/chat-completion-client v0.0.0-20241216203633-5c0178fb89ed
 	github.com/gptscript-ai/cmd v0.0.0-20240802230653-326b7baf6fcb
 	github.com/gptscript-ai/go-gptscript v0.9.5-rc5.0.20240927213153-2af51434b93e
 	github.com/gptscript-ai/tui v0.0.0-20240923192013-172e51ccf1d6
diff --git a/go.sum b/go.sum
@@ -200,8 +200,8 @@ github.com/gorilla/css v1.0.0 h1:BQqNyPTi50JCFMTw/b67hByjMVXZRwGha6wxVGkeihY=
 github.com/gorilla/css v1.0.0/go.mod h1:Dn721qIggHpt4+EFCcTLTU/vk5ySda2ReITrtgBl60c=
 github.com/gptscript-ai/broadcaster v0.0.0-20240625175512-c43682019b86 h1:m9yLtIEd0z1ia8qFjq3u0Ozb6QKwidyL856JLJp6nbA=
 github.com/gptscript-ai/broadcaster v0.0.0-20240625175512-c43682019b86/go.mod h1:lK3K5EZx4dyT24UG3yCt0wmspkYqrj4D/8kxdN3relk=
-github.com/gptscript-ai/chat-completion-client v0.0.0-20241104122544-5fe75f07c131 h1:y2FcmT4X8U606gUS0teX5+JWX9K/NclsLEhHiyrd+EU=
-github.com/gptscript-ai/chat-completion-client v0.0.0-20241104122544-5fe75f07c131/go.mod h1:7P/o6/IWa1KqsntVf68hSnLKuu3+xuqm6lYhch1w4jo=
+github.com/gptscript-ai/chat-completion-client v0.0.0-20241216203633-5c0178fb89ed h1:qMHm0IYpKgmw4KHX76RMB/duSICxo7IZuimPCKb0qG4=
+github.com/gptscript-ai/chat-completion-client v0.0.0-20241216203633-5c0178fb89ed/go.mod h1:7P/o6/IWa1KqsntVf68hSnLKuu3+xuqm6lYhch1w4jo=
 github.com/gptscript-ai/cmd v0.0.0-20240802230653-326b7baf6fcb h1:ky2J2CzBOskC7Jgm2VJAQi2x3p7FVGa+2/PcywkFJuc=
 github.com/gptscript-ai/cmd v0.0.0-20240802230653-326b7baf6fcb/go.mod h1:DJAo1xTht1LDkNYFNydVjTHd576TC7MlpsVRl3oloVw=
 github.com/gptscript-ai/go-gptscript v0.9.5-rc5.0.20240927213153-2af51434b93e h1:WpNae0NBx+Ri8RB3SxF8DhadDKU7h+jfWPQterDpbJA=
diff --git a/pkg/openai/client.go b/pkg/openai/client.go
@@ -555,10 +555,19 @@ func (c *Client) call(ctx context.Context, request openai.ChatCompletionRequest,
 	var (
 		headers          map[string]string
 		modelProviderEnv []string
+		retryOpts        = []openai.RetryOptions{
+			{
+				Retries:        5,
+				RetryAboveCode: 499,        // 5xx errors
+				RetryCodes:     []int{429}, // 429 Too Many Requests (ratelimit)
+			},
+		}
 	)
 	for _, e := range env {
 		if strings.HasPrefix(e, "GPTSCRIPT_MODEL_PROVIDER_") {
 			modelProviderEnv = append(modelProviderEnv, e)
+		} else if strings.HasPrefix(e, "GPTSCRIPT_DISABLE_RETRIES") {
+			retryOpts = nil
 		}
 	}
 
@@ -572,7 +581,7 @@ func (c *Client) call(ctx context.Context, request openai.ChatCompletionRequest,
 
 	if !streamResponse {
 		request.StreamOptions = nil
-		resp, err := c.c.CreateChatCompletion(ctx, request, headers)
+		resp, err := c.c.CreateChatCompletion(ctx, request, headers, retryOpts...)
 		if err != nil {
 			return types.CompletionMessage{}, err
 		}
@@ -597,7 +606,7 @@ func (c *Client) call(ctx context.Context, request openai.ChatCompletionRequest,
 		}), nil
 	}
 
-	stream, err := c.c.CreateChatCompletionStream(ctx, request, headers)
+	stream, err := c.c.CreateChatCompletionStream(ctx, request, headers, retryOpts...)
 	if err != nil {
 		return types.CompletionMessage{}, err
 	}

Original file line number	Diff line number	Diff line change
`@@ -555,10 +555,19 @@ func (c *Client) call(ctx context.Context, request openai.ChatCompletionRequest,`
`555`	`555`	`var (`
`556`	`556`	`headers map[string]string`
`557`	`557`	`modelProviderEnv []string`
	`558`	`+ retryOpts = []openai.RetryOptions{`
	`559`	`+ {`
	`560`	`+ Retries: 5,`
	`561`	`+ RetryAboveCode: 499, // 5xx errors`
	`562`	`+ RetryCodes: []int{429}, // 429 Too Many Requests (ratelimit)`
	`563`	`+ },`
	`564`	`+ }`
`558`	`565`	`)`
`559`	`566`	`for _, e := range env {`
`560`	`567`	`if strings.HasPrefix(e, "GPTSCRIPT_MODEL_PROVIDER_") {`
`561`	`568`	`modelProviderEnv = append(modelProviderEnv, e)`
	`569`	`+ } else if strings.HasPrefix(e, "GPTSCRIPT_DISABLE_RETRIES") {`
	`570`	`+ retryOpts = nil`
`562`	`571`	`}`
`563`	`572`	`}`
`564`	`573`
`@@ -572,7 +581,7 @@ func (c *Client) call(ctx context.Context, request openai.ChatCompletionRequest,`
`572`	`581`
`573`	`582`	`if !streamResponse {`
`574`	`583`	`request.StreamOptions = nil`
`575`		`- resp, err := c.c.CreateChatCompletion(ctx, request, headers)`
	`584`	`+ resp, err := c.c.CreateChatCompletion(ctx, request, headers, retryOpts...)`
`576`	`585`	`if err != nil {`
`577`	`586`	`return types.CompletionMessage{}, err`
`578`	`587`	`}`
`@@ -597,7 +606,7 @@ func (c *Client) call(ctx context.Context, request openai.ChatCompletionRequest,`
`597`	`606`	`}), nil`
`598`	`607`	`}`
`599`	`608`
`600`		`- stream, err := c.c.CreateChatCompletionStream(ctx, request, headers)`
	`609`	`+ stream, err := c.c.CreateChatCompletionStream(ctx, request, headers, retryOpts...)`
`601`	`610`	`if err != nil {`
`602`	`611`	`return types.CompletionMessage{}, err`
`603`	`612`	`}`