|
| 1 | +import { emptyUsage } from '#human/llm/usage'; |
| 2 | +import { afterEach, describe, expect, it } from 'vitest'; |
| 3 | + |
| 4 | +import { IEventBus } from '#/app/event/eventBus'; |
| 5 | +import { IModelService } from '#/llm-adapter/model/model'; |
| 6 | +import { IAgentLLMRequesterService } from '#/agent/llmRequester/llmRequester'; |
| 7 | +import { IAgentProfileService } from '#/agent/profile/profile'; |
| 8 | +import { ModelFallbackSwitched, WarningIssued } from '#/agent/profile/profileOps'; |
| 9 | + |
| 10 | +import { recordingTelemetry, type TelemetryRecord } from '../../app/telemetry/stubs'; |
| 11 | +import { |
| 12 | + createTestAgent, |
| 13 | + llmGenerateServices, |
| 14 | + requesterFromGenerateFn, |
| 15 | + telemetryServices, |
| 16 | + type TestAgentContext, |
| 17 | +} from '../../harness'; |
| 18 | + |
| 19 | +function okGenerate() { |
| 20 | + let calls = 0; |
| 21 | + return { |
| 22 | + services: llmGenerateServices( |
| 23 | + requesterFromGenerateFn(async () => { |
| 24 | + calls += 1; |
| 25 | + return { |
| 26 | + id: `mf-${String(calls)}`, |
| 27 | + message: { |
| 28 | + role: 'assistant' as const, |
| 29 | + content: [{ type: 'text' as const, text: 'ok' }], |
| 30 | + toolCalls: [], |
| 31 | + }, |
| 32 | + usage: emptyUsage(), |
| 33 | + finishReason: 'completed' as const, |
| 34 | + rawFinishReason: 'stop', |
| 35 | + }; |
| 36 | + }), |
| 37 | + ), |
| 38 | + calls: () => calls, |
| 39 | + }; |
| 40 | +} |
| 41 | + |
| 42 | +describe('turn-start model fallback', () => { |
| 43 | + let ctx: TestAgentContext; |
| 44 | + |
| 45 | + afterEach(async () => { |
| 46 | + try { |
| 47 | + await ctx.expectResumeMatches(); |
| 48 | + } finally { |
| 49 | + await ctx.dispose(); |
| 50 | + } |
| 51 | + }); |
| 52 | + |
| 53 | + function requestTurn(turnId: number): Promise<unknown> { |
| 54 | + return ctx.get(IAgentLLMRequesterService).request({ source: { type: 'turn', turnId, step: 1 } }); |
| 55 | + } |
| 56 | + |
| 57 | + function captureSwitches(): ModelFallbackSwitched[] { |
| 58 | + const switched: ModelFallbackSwitched[] = []; |
| 59 | + ctx.get(IEventBus).subscribe(ModelFallbackSwitched, (event) => switched.push(event)); |
| 60 | + return switched; |
| 61 | + } |
| 62 | + |
| 63 | + it('switches a dead bound model to the best-ranked ready model', async () => { |
| 64 | + const generate = okGenerate(); |
| 65 | + const records: TelemetryRecord[] = []; |
| 66 | + ctx = createTestAgent(generate.services, telemetryServices(recordingTelemetry(records))); |
| 67 | + const switched = captureSwitches(); |
| 68 | + const notices: WarningIssued[] = []; |
| 69 | + ctx.get(IEventBus).subscribe(WarningIssued, (event) => notices.push(event)); |
| 70 | + ctx.get(IAgentProfileService).update({ modelAlias: 'dead-model' }); |
| 71 | + |
| 72 | + await requestTurn(1); |
| 73 | + |
| 74 | + expect(generate.calls()).toBe(1); |
| 75 | + expect(ctx.get(IAgentProfileService).getModel()).toBe('mock-model'); |
| 76 | + expect(switched).toHaveLength(1); |
| 77 | + expect(switched[0]).toMatchObject({ turnId: 1, fromModel: 'dead-model', toModel: 'mock-model' }); |
| 78 | + expect(notices).toHaveLength(1); |
| 79 | + expect(notices[0]?.code).toBe('model-fallback'); |
| 80 | + expect(notices[0]?.message).toContain('dead-model'); |
| 81 | + expect(notices[0]?.message).toContain('mock-model'); |
| 82 | + const fallbackEvents = records.filter((record) => record.event === 'model_fallback_triggered'); |
| 83 | + expect(fallbackEvents).toHaveLength(1); |
| 84 | + expect(fallbackEvents[0]?.properties).toMatchObject({ |
| 85 | + turn_id: 1, |
| 86 | + from_model: 'dead-model', |
| 87 | + to_model: 'mock-model', |
| 88 | + }); |
| 89 | + }); |
| 90 | + |
| 91 | + it('switches a model whose provider was deleted', async () => { |
| 92 | + ctx = createTestAgent(okGenerate().services, { |
| 93 | + initialConfig: { |
| 94 | + models: { |
| 95 | + 'orphan-model': { provider: 'no-such-provider', model: 'orphan', maxContextSize: 1_000 }, |
| 96 | + }, |
| 97 | + }, |
| 98 | + }); |
| 99 | + const switched = captureSwitches(); |
| 100 | + ctx.get(IAgentProfileService).update({ modelAlias: 'orphan-model' }); |
| 101 | + |
| 102 | + await requestTurn(1); |
| 103 | + |
| 104 | + expect(ctx.get(IAgentProfileService).getModel()).toBe('mock-model'); |
| 105 | + expect(switched).toHaveLength(1); |
| 106 | + expect(switched[0]).toMatchObject({ fromModel: 'orphan-model', toModel: 'mock-model' }); |
| 107 | + }); |
| 108 | + |
| 109 | + it('skips an unready candidate that would win the ranking', async () => { |
| 110 | + ctx = createTestAgent(okGenerate().services, { |
| 111 | + initialConfig: { |
| 112 | + models: { |
| 113 | + 'zombie-model': { provider: 'no-such-provider', model: 'zombie', maxContextSize: 100_000_000 }, |
| 114 | + }, |
| 115 | + }, |
| 116 | + }); |
| 117 | + const switched = captureSwitches(); |
| 118 | + ctx.get(IAgentProfileService).update({ modelAlias: 'dead-model' }); |
| 119 | + |
| 120 | + await requestTurn(1); |
| 121 | + |
| 122 | + expect(ctx.get(IAgentProfileService).getModel()).toBe('mock-model'); |
| 123 | + expect(switched).toHaveLength(1); |
| 124 | + expect(switched[0]).toMatchObject({ fromModel: 'dead-model', toModel: 'mock-model' }); |
| 125 | + }); |
| 126 | + |
| 127 | + it('keeps the dead alias when no candidate is ready', async () => { |
| 128 | + ctx = createTestAgent(okGenerate().services, { |
| 129 | + initialConfig: { |
| 130 | + models: { |
| 131 | + 'aaa-model': { provider: 'no-such-provider', model: 'aaa', maxContextSize: 5_000 }, |
| 132 | + 'orphan-model': { provider: 'no-such-provider', model: 'orphan', maxContextSize: 1_000 }, |
| 133 | + }, |
| 134 | + }, |
| 135 | + }); |
| 136 | + const switched = captureSwitches(); |
| 137 | + await ctx.get(IModelService).delete('mock-model'); |
| 138 | + ctx.get(IAgentProfileService).update({ modelAlias: 'orphan-model' }); |
| 139 | + |
| 140 | + await requestTurn(1).catch(() => undefined); |
| 141 | + |
| 142 | + expect(ctx.get(IAgentProfileService).getModel()).toBe('orphan-model'); |
| 143 | + expect(switched).toHaveLength(0); |
| 144 | + }); |
| 145 | + |
| 146 | + it('leaves a healthy bound model untouched', async () => { |
| 147 | + const records: TelemetryRecord[] = []; |
| 148 | + ctx = createTestAgent(okGenerate().services, telemetryServices(recordingTelemetry(records))); |
| 149 | + const switched = captureSwitches(); |
| 150 | + |
| 151 | + await requestTurn(1); |
| 152 | + |
| 153 | + expect(ctx.get(IAgentProfileService).getModel()).toBe('mock-model'); |
| 154 | + expect(switched).toHaveLength(0); |
| 155 | + expect(records.filter((record) => record.event === 'model_fallback_triggered')).toHaveLength(0); |
| 156 | + }); |
| 157 | +}); |
0 commit comments