From efeabd5aa615a0cff12d6ea0c892e9d68f330122 Mon Sep 17 00:00:00 2001 From: Aiden Cline Date: Fri, 30 Oct 2026 14:51:46 -0500 Subject: [PATCH 1/3] fix(ai): recognize LiteLLM context overflow errors --- packages/ai/src/provider-error.ts | 5 ++++- packages/ai/test/provider-error.test.ts | 16 ++++++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/packages/ai/src/provider-error.ts b/packages/ai/src/provider-error.ts index 3b3817d4f802..a9d7e86cd944 100644 --- a/packages/ai/src/provider-error.ts +++ b/packages/ai/src/provider-error.ts @@ -42,6 +42,8 @@ const patterns = [ // Hugging Face Text Generation Inference, e.g. Together /`inputs` tokens \+ `max_new_tokens` must be <= \d+/i, /context window exceeds limit/i, + // LiteLLM's ContextWindowExceededError + /context window exceeded/i, /exceeded model token limit/i, /context[_ ]length[_ ]exceeded/i, /context length is only \d+ tokens/i, @@ -259,7 +261,8 @@ export function classifyProviderFailure(input: ProviderFailure): AIError["reason const clientScoped = input.status === undefined || (input.status >= 400 && input.status < 500) if ( - clientScoped && + // Gateways can pass an upstream overflow on as a 5xx, e.g. LiteLLM's mid-stream fallback. + (input.status === undefined || input.status >= 400) && (codes.includes("context_length_exceeded") || codes.includes("model_context_window_exceeded") || codes.includes("request_too_large") || diff --git a/packages/ai/test/provider-error.test.ts b/packages/ai/test/provider-error.test.ts index 9f1e46ad37de..0a4d54f04672 100644 --- a/packages/ai/test/provider-error.test.ts +++ b/packages/ai/test/provider-error.test.ts @@ -21,6 +21,8 @@ describe("provider error classification", () => { "Prompt exceeds max length", "The input is too long for the selected model.", "This text request is too large for the model's configured input limit.", + // LiteLLM's own check, before it calls the model. + "litellm.ContextWindowExceededError: litellm.BadRequestError: litellm._pre_call_checks: Context Window exceeded for given call. No models have context window large enough for this call.", ] expect(messages.every(isContextOverflow)).toBe(true) @@ -47,6 +49,20 @@ describe("provider error classification", () => { ) }) + test("classifies an overflow a gateway reports as a server error as context overflow", () => { + // LiteLLM wraps an upstream overflow in a 503 when it falls back mid-stream. + expect( + classifyProviderFailure({ + message: + "litellm.ServiceUnavailableError: litellm.MidStreamFallbackError: litellm.APIConnectionError: APIConnectionError: OpenAIException - Requested token count exceeds the model's maximum context length of 131072 tokens.", + status: 503, + }), + ).toMatchObject({ _tag: "InvalidRequest", classification: "context-overflow" }) + expect(classifyProviderFailure({ message: "Service temporarily unavailable", status: 503 })._tag).toBe( + "ProviderInternal", + ) + }) + test("does not classify rate limits as context overflow", () => { const messages = [ "Throttling error: Too many tokens, please wait before trying again.", From 182d32608b8f561e7256222d62fbb797cb52247e Mon Sep 17 00:00:00 2001 From: Aiden Cline Date: Fri, 30 Oct 2026 14:54:59 -0500 Subject: [PATCH 2/3] test(core): cover a provider cutting an oversized prompt to its window --- packages/core/test/session-runner.test.ts | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/packages/core/test/session-runner.test.ts b/packages/core/test/session-runner.test.ts index 948d3d861768..d91c8d18dd6c 100644 --- a/packages/core/test/session-runner.test.ts +++ b/packages/core/test/session-runner.test.ts @@ -5952,6 +5952,26 @@ describe("SessionRunnerLLM", () => { }) } + scenario("compacts when the provider cuts an oversized prompt to its window and replies with nothing", function* (s) { + // Xiaomi MiMo answers an oversized prompt with HTTP 200, finish "length", no output, and a prompt count that fills + // the window instead of rejecting it. + s.currentModel = recoveryModel + yield* s.llm.push( + TestLLM.complete({ reason: { normalized: "length" }, usage: { inputTokens: 199_994, nonCachedInputTokens: 199_994, outputTokens: 0 } }), + TestLLM.text("## Objective\n- summary", "summary"), + TestLLM.text("Done", "done"), + ) + + yield* s.runPrompt("Large prompt") + + expect(s.requests).toHaveLength(3) + expect(userTexts(s.requests[1]).at(-1)).toContain("Summarize only what the user and the assistant said and did") + expect((yield* s.messages).filter((message) => message.type === "compaction")).toMatchObject([ + { status: "completed" }, + ]) + expect((yield* s.context).at(-1)).toMatchObject(Expected.assistant({ finish: "stop" }, [Expected.text("Done")])) + }) + for (const tools of ["hosted", "unfinished"] as const) { scenario(`caps output token limit continuations and resets for new input (${tools})`, function* (s) { const toolEvents = From 27a943ed2aba14f07aa7bf92ee69aa4bda18659e Mon Sep 17 00:00:00 2001 From: Aiden Cline Date: Fri, 30 Oct 2026 15:29:53 -0500 Subject: [PATCH 3/3] fix(ai): keep overflow matching to client errors --- packages/ai/src/provider-error.ts | 3 +-- packages/ai/test/provider-error.test.ts | 14 -------------- 2 files changed, 1 insertion(+), 16 deletions(-) diff --git a/packages/ai/src/provider-error.ts b/packages/ai/src/provider-error.ts index a9d7e86cd944..c34160b6065f 100644 --- a/packages/ai/src/provider-error.ts +++ b/packages/ai/src/provider-error.ts @@ -261,8 +261,7 @@ export function classifyProviderFailure(input: ProviderFailure): AIError["reason const clientScoped = input.status === undefined || (input.status >= 400 && input.status < 500) if ( - // Gateways can pass an upstream overflow on as a 5xx, e.g. LiteLLM's mid-stream fallback. - (input.status === undefined || input.status >= 400) && + clientScoped && (codes.includes("context_length_exceeded") || codes.includes("model_context_window_exceeded") || codes.includes("request_too_large") || diff --git a/packages/ai/test/provider-error.test.ts b/packages/ai/test/provider-error.test.ts index 0a4d54f04672..84c38fa16057 100644 --- a/packages/ai/test/provider-error.test.ts +++ b/packages/ai/test/provider-error.test.ts @@ -49,20 +49,6 @@ describe("provider error classification", () => { ) }) - test("classifies an overflow a gateway reports as a server error as context overflow", () => { - // LiteLLM wraps an upstream overflow in a 503 when it falls back mid-stream. - expect( - classifyProviderFailure({ - message: - "litellm.ServiceUnavailableError: litellm.MidStreamFallbackError: litellm.APIConnectionError: APIConnectionError: OpenAIException - Requested token count exceeds the model's maximum context length of 131072 tokens.", - status: 503, - }), - ).toMatchObject({ _tag: "InvalidRequest", classification: "context-overflow" }) - expect(classifyProviderFailure({ message: "Service temporarily unavailable", status: 503 })._tag).toBe( - "ProviderInternal", - ) - }) - test("does not classify rate limits as context overflow", () => { const messages = [ "Throttling error: Too many tokens, please wait before trying again.",