From 241650f271c06a7b95296f2878ed42e2560a13dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Manso?= Date: Sat, 5 Sep 2026 16:18:29 +0000 Subject: [PATCH] fix(web): stop a stale session snapshot from reverting a model switch (#3379) Co-authored-by: Joao Manso Co-authored-by: Carlos Florencio --- ...model-selector-consecutive-switch.test.tsx | 187 +++++++++++++++ .../model-selector-consecutive-switch.spec.ts | 158 +++++++++++++ .../session-models-user-selection.test.ts | 216 ++++++++++++++++++ apps/web/lib/ws/handlers/session-models.ts | 29 ++- 4 files changed, 588 insertions(+), 2 deletions(-) create mode 100644 apps/web/components/task/model-selector-consecutive-switch.test.tsx create mode 100644 apps/web/e2e/tests/chat/model-selector-consecutive-switch.spec.ts create mode 100644 apps/web/lib/ws/handlers/session-models-user-selection.test.ts diff --git a/apps/web/components/task/model-selector-consecutive-switch.test.tsx b/apps/web/components/task/model-selector-consecutive-switch.test.tsx new file mode 100644 index 0000000000..0107553706 --- /dev/null +++ b/apps/web/components/task/model-selector-consecutive-switch.test.tsx @@ -0,0 +1,187 @@ +import { act, cleanup, fireEvent, render, screen } from "@testing-library/react"; +import { TooltipProvider } from "@kandev/ui/tooltip"; +import { afterEach, describe, expect, it, vi } from "vitest"; + +import { ModelSelector } from "@/components/task/model-selector"; +import { setSessionConfigOption } from "@/lib/api/domains/session-api"; + +const mocks = vi.hoisted(() => { + const SESSION_ID = "session-1"; + const OPUS = "opus[1m]"; + const listeners = new Set<() => void>(); + const modelOptions = [ + { value: "opus", name: "Opus" }, + { value: "opus[1m]", name: "Opus 1M" }, + { value: "sonnet", name: "Sonnet" }, + { value: "claude-fable-5", name: "Fable" }, + { value: "claude-fable-5[1m]", name: "Fable 1M" }, + ]; + const appState = { + activeModel: { bySessionId: {} as Record }, + sessionModels: { + bySessionId: { + [SESSION_ID]: { + currentModelId: OPUS, + models: modelOptions.map((option) => ({ + modelId: option.value, + name: option.name, + })), + configOptions: [ + { + type: "select", + id: "mode", + name: "Mode", + currentValue: "agent", + options: [{ value: "agent", name: "Agent" }], + }, + { + type: "select", + id: "model", + name: "Model", + currentValue: OPUS, + category: "model", + options: modelOptions, + }, + { + type: "select", + id: "effort", + name: "Effort", + currentValue: "high", + options: [ + { value: "high", name: "High" }, + { value: "medium", name: "Medium" }, + ], + }, + { + type: "select", + id: "fast", + name: "Fast", + currentValue: "false", + options: [ + { value: "false", name: "Off" }, + { value: "true", name: "On" }, + ], + }, + ], + configOptionsSettled: true, + configBaseline: { effort: "high", fast: "false", mode: "agent" }, + }, + } as Record, + }, + settingsAgents: { items: [] }, + taskSessions: { + items: { + [SESSION_ID]: { + agent_profile_id: "profile-1", + agent_profile_snapshot: { config_options: { fast: "false" } }, + metadata: { + runtime_config: { + config_options: { effort: "high", fast: "false", mode: "agent", model: OPUS }, + }, + }, + }, + }, + }, + setActiveModel: vi.fn((sessionId: string, modelId: string) => { + appState.activeModel.bySessionId[sessionId] = modelId; + notify(); + }), + setSessionModels: vi.fn((sessionId: string, data: unknown) => { + appState.sessionModels.bySessionId[sessionId] = data; + notify(); + }), + }; + function notify() { + for (const listener of listeners) listener(); + } + return { appState, listeners, modelOptions, SESSION_ID }; +}); + +const SESSION_ID = mocks.SESSION_ID; +const OPUS = "opus[1m]"; +const FABLE = "claude-fable-5[1m]"; + +vi.mock("@/components/state-provider", async () => { + const { useEffect, useReducer } = await import("react"); + return { + useAppStore: (selector: (state: typeof mocks.appState) => unknown) => { + const [, force] = useReducer((count: number) => count + 1, 0); + useEffect(() => { + mocks.listeners.add(force); + return () => { + mocks.listeners.delete(force); + }; + }, [force]); + return selector(mocks.appState); + }, + }; +}); + +vi.mock("@/components/toast-provider", () => ({ + useToast: () => ({ toast: vi.fn() }), +})); + +vi.mock("@/hooks/domains/settings/use-available-agents", () => ({ + useAvailableAgents: () => ({ items: [] }), +})); + +vi.mock("@/hooks/domains/settings/use-settings-data", () => ({ + useSettingsData: vi.fn(), +})); + +vi.mock("@/lib/api/domains/session-api", () => ({ + setSessionConfigOption: vi.fn(() => Promise.resolve({ ok: true })), + setSessionMode: vi.fn(() => Promise.resolve({ ok: true })), + setSessionModel: vi.fn(() => Promise.resolve({ ok: true })), +})); + +afterEach(() => { + cleanup(); + vi.mocked(setSessionConfigOption).mockClear(); +}); + +function pickModel(label: string) { + const trigger = screen.getByRole("button", { name: "Session model settings" }); + // Picking a model leaves the picker open when the agent advertises extra + // config options, so only toggle it when it is actually closed. + if (trigger.getAttribute("aria-expanded") !== "true") { + fireEvent.pointerDown(trigger); + fireEvent.click(trigger); + } + fireEvent.click(screen.getByRole("option", { name: new RegExp(label) })); +} + +// Mirrors the convergence event the backend emits after a model switch: the +// provider re-advertises the option catalog for the newly selected model, which +// for Fable no longer carries the Opus-only "fast" toggle. +function applyConvergenceEvent(modelId: string, optionIds: string[]) { + const previous = mocks.appState.sessionModels.bySessionId[SESSION_ID] as { + configOptions: { id: string; currentValue: string }[]; + }; + act(() => { + mocks.appState.setSessionModels(SESSION_ID, { + ...previous, + currentModelId: modelId, + configOptions: previous.configOptions + .filter((option) => optionIds.includes(option.id)) + .map((option) => (option.id === "model" ? { ...option, currentValue: modelId } : option)), + }); + }); +} + +describe("consecutive session model switches", () => { + it("sends every switch to the backend, not just the first", () => { + render( + + + , + ); + + pickModel("Fable 1M"); + expect(setSessionConfigOption).toHaveBeenNthCalledWith(1, SESSION_ID, "model", FABLE); + applyConvergenceEvent(FABLE, ["mode", "model", "effort"]); + + pickModel("Opus 1M"); + expect(setSessionConfigOption).toHaveBeenNthCalledWith(2, SESSION_ID, "model", OPUS); + }); +}); diff --git a/apps/web/e2e/tests/chat/model-selector-consecutive-switch.spec.ts b/apps/web/e2e/tests/chat/model-selector-consecutive-switch.spec.ts new file mode 100644 index 0000000000..4150a58fcd --- /dev/null +++ b/apps/web/e2e/tests/chat/model-selector-consecutive-switch.spec.ts @@ -0,0 +1,158 @@ +import type { Locator, Page } from "@playwright/test"; +import { test, expect } from "../../fixtures/test-base"; +import { SessionPage } from "../../pages/session-page"; +import { watchWs, type WsWatcher } from "../../helpers/causal-waits"; + +/** + * Guards consecutive model switches from the chat-input model selector. + * + * mock-agent re-advertises its whole config catalog on a model change (the + * smart model publishes a different effort option set), which is what a real + * ACP provider does. The regression this covers: after the first switch the + * picker stopped accepting further selections — the trigger no longer opened + * the list — until a prompt refreshed the session. + */ + +type ModelPicker = { + trigger: Locator; + /** + * Opens the picker from a known-closed state, selects a model, and returns + * once the provider has converged on it. The label cannot settle before that + * session.models_updated notification lands, so waiting on the frame removes + * the round trip from the assertions that follow. + */ + pick: (name: RegExp, modelId: string) => Promise; +}; + +function modelPicker(page: Page, ws: WsWatcher, sessionId: string): ModelPicker { + const trigger = page.getByRole("button", { name: "Session model settings" }); + const listbox = page.getByRole("listbox"); + return { + trigger, + pick: async (name, modelId) => { + if (await listbox.isVisible()) { + await page.keyboard.press("Escape"); + await expect(listbox).toBeHidden(); + } + await trigger.click(); + await expect(listbox).toBeVisible(); + + const converged = ws.waitForEvent("session.models_updated", { + where: (payload) => + payload.session_id === sessionId && payload.current_model_id === modelId, + }); + await listbox.getByRole("option", { name }).click(); + await converged; + }, + }; +} + +test.describe("Chat model selector — consecutive switches", () => { + test("applies a second and third model switch after the first", async ({ + testPage, + apiClient, + seedData, + }) => { + const ws = watchWs(testPage); + const task = await apiClient.createTaskWithAgent( + seedData.workspaceId, + "Consecutive Model Switch Test", + seedData.agentProfileId, + { + description: "/e2e:simple-message", + workflow_id: seedData.workflowId, + workflow_step_id: seedData.startStepId, + repository_ids: [seedData.repositoryId], + }, + ); + const sessionId = task.session_id; + if (!sessionId) throw new Error("expected an auto-started session"); + + await testPage.goto(`/t/${task.id}`); + const session = new SessionPage(testPage); + await session.waitForLoad(); + await session.waitForChatIdle({ timeout: 30_000 }); + + // The startup model event can precede the page's subscription, so read the + // settled model from the backend rather than waiting on a frame that may + // already be gone. + await expect + .poll(async () => { + const { sessions } = await apiClient.listTaskSessions(task.id); + const metadata = sessions.find((item) => item.id === sessionId)?.metadata; + return (metadata?.runtime_config as { model?: string } | undefined)?.model; + }) + .toBe("mock-fast"); + const { trigger, pick } = modelPicker(testPage, ws, sessionId); + await expect(trigger).toContainText("Mock Fast"); + + await pick(/Mock Smart/, "mock-smart"); + await expect(trigger).toContainText("Mock Smart"); + + await pick(/Mock Slow/, "mock-slow"); + await expect(trigger).toContainText("Mock Slow"); + + // Back to the model the session started on. + await pick(/Mock Fast/, "mock-fast"); + await expect(trigger).toContainText("Mock Fast"); + + await expect + .poll(async () => { + const { sessions } = await apiClient.listTaskSessions(task.id); + const metadata = sessions.find((item) => item.id === sessionId)?.metadata; + const overrides = metadata?.runtime_config_overrides as { model?: string } | undefined; + return overrides?.model; + }) + .toBe("mock-fast"); + }); + + test("applies a switch made right after a page reload", async ({ + testPage, + apiClient, + seedData, + }) => { + const ws = watchWs(testPage); + const task = await apiClient.createTaskWithAgent( + seedData.workspaceId, + "Reloaded Model Switch Test", + seedData.agentProfileId, + { + description: "/e2e:simple-message", + workflow_id: seedData.workflowId, + workflow_step_id: seedData.startStepId, + repository_ids: [seedData.repositoryId], + }, + ); + const sessionId = task.session_id; + if (!sessionId) throw new Error("expected an auto-started session"); + + await testPage.goto(`/t/${task.id}`); + const session = new SessionPage(testPage); + await session.waitForLoad(); + await session.waitForChatIdle({ timeout: 30_000 }); + + await expect + .poll(async () => { + const { sessions } = await apiClient.listTaskSessions(task.id); + const metadata = sessions.find((item) => item.id === sessionId)?.metadata; + return (metadata?.runtime_config as { model?: string } | undefined)?.model; + }) + .toBe("mock-fast"); + const { trigger, pick } = modelPicker(testPage, ws, sessionId); + await expect(trigger).toContainText("Mock Fast"); + + await pick(/Mock Smart/, "mock-smart"); + await expect(trigger).toContainText("Mock Smart"); + + // A reload drops the client-side hydration bookkeeping while the persisted + // session metadata still names the pre-switch model, so the next switch + // must not be reverted by that stale copy. watchWs survives page.reload(). + await testPage.reload(); + await session.waitForLoad(); + await expect(trigger).toContainText("Mock Smart"); + + await pick(/Mock Fast/, "mock-fast"); + await expect(trigger).toContainText("Mock Fast"); + await expect(trigger).not.toContainText("Mock Smart"); + }); +}); diff --git a/apps/web/lib/ws/handlers/session-models-user-selection.test.ts b/apps/web/lib/ws/handlers/session-models-user-selection.test.ts new file mode 100644 index 0000000000..ecce31cba9 --- /dev/null +++ b/apps/web/lib/ws/handlers/session-models-user-selection.test.ts @@ -0,0 +1,216 @@ +import { describe, expect, it, vi } from "vitest"; +import type { StoreApi } from "zustand"; +import type { AppState } from "@/lib/state/store"; +import type { BackendMessageMap } from "@/lib/types/backend"; +import type { TaskSession } from "@/lib/types/http"; +import type { SessionModelsPayload } from "@/lib/types/session-runtime-payloads"; +import { registerSessionModelsHandlers } from "./session-models"; + +const providerModelId = "gpt-5.6-sol"; +const providerModelName = "GPT-5.6 Sol"; + +function makeStore(overrides: Partial = {}): StoreApi { + const state = { + activeModel: { bySessionId: {} }, + contextWindow: { bySessionId: {} }, + sessionModels: { bySessionId: {} }, + taskSessions: { items: {} }, + setActiveModel: vi.fn(), + ...overrides, + } as unknown as AppState; + state.setSessionModels = vi.fn((sessionId, data) => { + state.sessionModels.bySessionId[sessionId] = data; + }); + state.clearContextWindow = vi.fn((sessionId) => { + delete state.contextWindow.bySessionId[sessionId]; + }); + + return { + getState: () => state, + setState: vi.fn(), + subscribe: vi.fn(), + destroy: vi.fn(), + getInitialState: vi.fn(), + } as unknown as StoreApi; +} + +function makePayload( + currentModelId: string, + overrides: Partial = {}, +): SessionModelsPayload { + return { + task_id: "task-1", + session_id: "session-1", + agent_id: "agent-1", + current_model_id: currentModelId, + models: [], + config_options: [], + timestamp: "2026-06-11T00:00:00.000Z", + ...overrides, + }; +} + +function makeMessage(payload: SessionModelsPayload): BackendMessageMap["session.models_updated"] { + return { + id: "message-1", + type: "notification", + action: "session.models_updated", + payload, + }; +} + +function makeTaskSession(metadata: Record): TaskSession { + return { + id: "session-1", + task_id: "task-1", + state: "WAITING_FOR_INPUT", + metadata, + started_at: "2026-06-11T00:00:00.000Z", + updated_at: "2026-06-11T00:00:00.000Z", + } as TaskSession; +} + +describe("session.models_updated user selection convergence", () => { + const pickedModelId = "gpt-5.3-codex-spark"; + const laterModelId = "gpt-5.4-mini"; + + // After a page reload the store has no hydration bookkeeping, so persisted + // runtime metadata is authoritative until a payload confirms it. A model the + // user just picked in this store is such a confirmation: without it, the + // convergence event for the user's own selection is overwritten by the + // stale cached override and the picker snaps back to the previous model. + it("lets a convergence event for the user's own selection win over stale persisted metadata", () => { + const store = makeStore({ + activeModel: { bySessionId: { "session-1": pickedModelId } } as AppState["activeModel"], + taskSessions: { + items: { + "session-1": makeTaskSession({ + runtime_config: { model: providerModelId }, + runtime_config_overrides: { model: providerModelId }, + }), + }, + }, + sessionModels: { + bySessionId: { + "session-1": { + currentModelId: providerModelId, + models: [], + configOptions: [], + configOptionsSettled: true, + }, + }, + } as AppState["sessionModels"], + }); + const handler = registerSessionModelsHandlers(store)["session.models_updated"]!; + + handler( + makeMessage( + makePayload(pickedModelId, { + config_options: [ + { + type: "select", + id: "model", + name: "Model", + category: "model", + current_value: pickedModelId, + options: [ + { value: providerModelId, name: providerModelName }, + { value: pickedModelId, name: "Codex Spark" }, + ], + }, + ], + }), + ), + ); + + const entry = store.getState().sessionModels.bySessionId["session-1"]; + expect(entry.currentModelId).toBe(pickedModelId); + expect(entry.configOptions[0]?.currentValue).toBe(pickedModelId); + }); + + it("keeps trusting the payload for the switch after the user's selection converged", () => { + const store = makeStore({ + activeModel: { bySessionId: { "session-1": pickedModelId } } as AppState["activeModel"], + taskSessions: { + items: { + "session-1": makeTaskSession({ + runtime_config: { model: providerModelId }, + runtime_config_overrides: { model: providerModelId }, + }), + }, + }, + sessionModels: { + bySessionId: { + "session-1": { + currentModelId: providerModelId, + models: [], + configOptions: [], + configOptionsSettled: true, + }, + }, + } as AppState["sessionModels"], + }); + const handler = registerSessionModelsHandlers(store)["session.models_updated"]!; + + handler(makeMessage(makePayload(pickedModelId))); + // The provider re-reports the same session a moment later; the stale + // override must not resurrect the pre-switch model. + handler(makeMessage(makePayload(laterModelId))); + + expect(store.getState().sessionModels.bySessionId["session-1"].currentModelId).toBe( + laterModelId, + ); + }); +}); + +describe("session.models_updated startup guard", () => { + const pickedModelId = "gpt-5.3-codex-spark"; + + it("keeps persisted state during unsettled startup when active model matches", () => { + const store = makeStore({ + activeModel: { bySessionId: { "session-1": pickedModelId } } as AppState["activeModel"], + taskSessions: { + items: { + "session-1": { + ...makeTaskSession({ runtime_config: { model: providerModelId } }), + state: "STARTING", + }, + }, + }, + sessionModels: { + bySessionId: { + "session-1": { + currentModelId: providerModelId, + models: [], + configOptions: [], + configOptionsSettled: true, + }, + }, + } as AppState["sessionModels"], + }); + const handler = registerSessionModelsHandlers(store)["session.models_updated"]!; + + handler( + makeMessage( + makePayload(pickedModelId, { + config_options_settled: false, + config_options: [ + { + type: "select", + id: "model", + name: "Model", + category: "model", + current_value: pickedModelId, + options: [{ value: pickedModelId, name: "Codex Spark" }], + }, + ], + }), + ), + ); + + expect(store.getState().sessionModels.bySessionId["session-1"]).toMatchObject({ + currentModelId: providerModelId, + configOptions: [expect.objectContaining({ id: "model", currentValue: providerModelId })], + }); + }); +}); diff --git a/apps/web/lib/ws/handlers/session-models.ts b/apps/web/lib/ws/handlers/session-models.ts index ae4da2ded9..70f1a997f5 100644 --- a/apps/web/lib/ws/handlers/session-models.ts +++ b/apps/web/lib/ws/handlers/session-models.ts @@ -98,14 +98,31 @@ function isUnsettledStartupModelsPayload( ); } +// activeModel only ever records a selection the user made in this store, so a +// payload reporting that model is the provider confirming that selection rather +// than stale resume state. The cached session metadata still names the +// pre-switch model until a session update lands, so it cannot arbitrate here. +function payloadConfirmsUserSelection( + state: AppState, + sessionId: string, + payload: SessionModelsPayload, +): boolean { + const selected = state.activeModel?.bySessionId?.[sessionId]; + return !!selected && resolveCurrentModelId(payload) === selected; +} + function shouldHydrateSessionModelsPayload( payload: SessionModelsPayload, matchesPersisted: boolean, + confirmsUserSelection: boolean, unsettledStartup: boolean, ): boolean { // Two-layer defense: the backend gates unsettled startup events, while this // barrier protects reconnects where the client session state lags. - return payload.config_options_settled === true || (matchesPersisted && !unsettledStartup); + return ( + payload.config_options_settled === true || + ((matchesPersisted || confirmsUserSelection) && !unsettledStartup) + ); } function shouldUsePersistedRuntimeConfig(hydrated: boolean, unsettledStartup: boolean): boolean { @@ -345,7 +362,15 @@ export function registerSessionModelsHandlers(store: StoreApi): WsHand ); const persisted = usePersistedRuntime ? persistedRuntimeConfig(state, sessionId) : {}; const matchesPersisted = payloadMatchesPersistedRuntime(payload, persisted); - if (shouldHydrateSessionModelsPayload(payload, matchesPersisted, unsettledStartup)) { + const confirmsUserSelection = payloadConfirmsUserSelection(state, sessionId, payload); + if ( + shouldHydrateSessionModelsPayload( + payload, + matchesPersisted, + confirmsUserSelection, + unsettledStartup, + ) + ) { hydrated.add(sessionId); } const pendingRuntime = shouldUsePersistedRuntimeConfig(