Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions packages/ai/src/provider-error.ts
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,8 @@ const patterns = [
// Hugging Face Text Generation Inference, e.g. Together
/`inputs` tokens \+ `max_new_tokens` must be <= \d+/i,
/context window exceeds limit/i,
// LiteLLM's ContextWindowExceededError
/context window exceeded/i,
/exceeded model token limit/i,
/context[_ ]length[_ ]exceeded/i,
/context length is only \d+ tokens/i,
Expand Down
2 changes: 2 additions & 0 deletions packages/ai/test/provider-error.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,8 @@ describe("provider error classification", () => {
"Prompt exceeds max length",
"The input is too long for the selected model.",
"This text request is too large for the model's configured input limit.",
// LiteLLM's own check, before it calls the model.
"litellm.ContextWindowExceededError: litellm.BadRequestError: litellm._pre_call_checks: Context Window exceeded for given call. No models have context window large enough for this call.",
]

expect(messages.every(isContextOverflow)).toBe(true)
Expand Down
20 changes: 20 additions & 0 deletions packages/core/test/session-runner.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5952,6 +5952,26 @@ describe("SessionRunnerLLM", () => {
})
}

scenario("compacts when the provider cuts an oversized prompt to its window and replies with nothing", function* (s) {
// Xiaomi MiMo answers an oversized prompt with HTTP 200, finish "length", no output, and a prompt count that fills
// the window instead of rejecting it.
s.currentModel = recoveryModel
yield* s.llm.push(
TestLLM.complete({ reason: { normalized: "length" }, usage: { inputTokens: 199_994, nonCachedInputTokens: 199_994, outputTokens: 0 } }),
TestLLM.text("## Objective\n- summary", "summary"),
TestLLM.text("Done", "done"),
)

yield* s.runPrompt("Large prompt")

expect(s.requests).toHaveLength(3)
expect(userTexts(s.requests[1]).at(-1)).toContain("Summarize only what the user and the assistant said and did")
expect((yield* s.messages).filter((message) => message.type === "compaction")).toMatchObject([
{ status: "completed" },
])
expect((yield* s.context).at(-1)).toMatchObject(Expected.assistant({ finish: "stop" }, [Expected.text("Done")]))
})

for (const tools of ["hosted", "unfinished"] as const) {
scenario(`caps output token limit continuations and resets for new input (${tools})`, function* (s) {
const toolEvents =
Expand Down
Loading