diff --git a/apps/web/components/task/chat/voice-input-button.test.tsx b/apps/web/components/task/chat/voice-input-button.test.tsx new file mode 100644 index 0000000000..e69c8a4620 --- /dev/null +++ b/apps/web/components/task/chat/voice-input-button.test.tsx @@ -0,0 +1,245 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { cleanup, fireEvent, render, screen } from "@testing-library/react"; + +// ── Hoisted mocks (must be set before importing the component under test) ── + +const voicePrefs = vi.hoisted(() => ({ + value: { + enabled: true, + engine: "auto" as "auto" | "webSpeech" | "whisperWeb" | "whisperServer", + language: "auto", + mode: "hold" as "hold" | "toggle", + autoSend: false, + whisperWebModel: "base" as "tiny" | "base" | "small", + }, +})); + +const voiceInputResult = vi.hoisted(() => ({ + supported: true as boolean, + state: "idle" as "idle" | "requesting" | "recording" | "processing", + modelLoad: { state: "idle" as const, progress: 0 }, + start: vi.fn(async () => undefined), + stop: vi.fn(async () => undefined), + cancel: vi.fn(() => undefined), + engine: "webSpeech" as const, + error: null as null | { code: string; message: string }, +})); + +const coarsePointer = vi.hoisted(() => ({ value: false })); + +vi.mock("@/components/state-provider", () => ({ + useAppStore: ( + selector: (state: { + userSettings: { + voiceMode: typeof voicePrefs.value; + keyboardShortcuts: Record; + }; + }) => unknown, + ) => + selector({ + userSettings: { voiceMode: voicePrefs.value, keyboardShortcuts: {} }, + }), +})); + +vi.mock("@/hooks/use-voice-input", () => ({ + useVoiceInput: () => voiceInputResult, +})); + +vi.mock("@/hooks/use-keyboard-shortcut", () => ({ + useKeyboardShortcut: () => undefined, +})); + +vi.mock("@/components/toast-provider", () => ({ + useToast: () => ({ toast: vi.fn() }), +})); + +vi.mock("@kandev/ui/tooltip", () => ({ + Tooltip: ({ children }: { children: React.ReactNode }) => <>{children}, + TooltipTrigger: ({ children }: { children: React.ReactNode }) => <>{children}, + TooltipContent: ({ children }: { children: React.ReactNode }) => <>{children}, +})); + +// ── matchMedia stub honouring the hoisted `coarsePointer.value` flag ── + +beforeEach(() => { + voicePrefs.value = { + enabled: true, + engine: "auto", + language: "auto", + mode: "hold", + autoSend: false, + whisperWebModel: "base", + }; + voiceInputResult.supported = true; + voiceInputResult.state = "idle"; + voiceInputResult.modelLoad = { state: "idle", progress: 0 }; + voiceInputResult.start.mockClear(); + voiceInputResult.stop.mockClear(); + voiceInputResult.cancel.mockClear(); + coarsePointer.value = false; + + Object.defineProperty(window, "matchMedia", { + configurable: true, + writable: true, + value: (query: string) => ({ + matches: query.includes("coarse") ? coarsePointer.value : false, + media: query, + onchange: null, + addEventListener: () => undefined, + removeEventListener: () => undefined, + addListener: () => undefined, + removeListener: () => undefined, + dispatchEvent: () => false, + }), + }); +}); + +afterEach(() => { + cleanup(); +}); + +// Re-import after the mocks so the module under test sees the mocked deps. +import { VoiceInputButton } from "./voice-input-button"; + +const BUTTON_TESTID = "voice-input-button"; + +function renderButton() { + return render( {}} onAutoSend={() => {}} />); +} + +describe("VoiceInputButton — hold-mode pointer handlers", () => { + it("captures the pointer on pointerdown and starts recording", () => { + renderButton(); + const button = screen.getByTestId(BUTTON_TESTID); + const setCap = vi.fn(); + (button as unknown as { setPointerCapture: (id: number) => void }).setPointerCapture = setCap; + + fireEvent.pointerDown(button, { pointerId: 7 }); + + expect(setCap).toHaveBeenCalledWith(7); + expect(voiceInputResult.start).toHaveBeenCalledTimes(1); + }); + + it("releases the pointer and stops on pointerup", () => { + renderButton(); + const button = screen.getByTestId(BUTTON_TESTID); + const releaseCap = vi.fn(); + (button as unknown as { hasPointerCapture: () => boolean }).hasPointerCapture = () => true; + (button as unknown as { releasePointerCapture: (id: number) => void }).releasePointerCapture = + releaseCap; + + fireEvent.pointerUp(button, { pointerId: 7 }); + + expect(releaseCap).toHaveBeenCalledWith(7); + expect(voiceInputResult.stop).toHaveBeenCalledTimes(1); + }); + + it("stops on pointercancel without triggering on pointerleave", () => { + renderButton(); + const button = screen.getByTestId(BUTTON_TESTID); + (button as unknown as { hasPointerCapture: () => boolean }).hasPointerCapture = () => false; + (button as unknown as { releasePointerCapture: (id: number) => void }).releasePointerCapture = + () => undefined; + + // pointerleave used to abort recording. Confirm it no longer does. + fireEvent.pointerLeave(button, { pointerId: 7 }); + expect(voiceInputResult.stop).not.toHaveBeenCalled(); + + fireEvent.pointerCancel(button, { pointerId: 7 }); + expect(voiceInputResult.stop).toHaveBeenCalledTimes(1); + }); + + it("does not throw if setPointerCapture is not present on the element", () => { + renderButton(); + const button = screen.getByTestId(BUTTON_TESTID); + // Some test/UA combinations strip setPointerCapture — handler must swallow. + expect(() => fireEvent.pointerDown(button, { pointerId: 1 })).not.toThrow(); + }); +}); + +describe("VoiceInputButton — coarse-pointer override", () => { + it("reports stored hold mode but effective toggle when matchMedia(coarse) matches", () => { + coarsePointer.value = true; + renderButton(); + const button = screen.getByTestId(BUTTON_TESTID); + expect(button.getAttribute("data-mode")).toBe("hold"); + expect(button.getAttribute("data-effective-mode")).toBe("toggle"); + }); + + it("does not attach hold pointer handlers in coarse-pointer toggle fallback", () => { + coarsePointer.value = true; + renderButton(); + const button = screen.getByTestId(BUTTON_TESTID); + fireEvent.pointerDown(button, { pointerId: 1 }); + fireEvent.pointerUp(button, { pointerId: 1 }); + // toggle handler is bound to onClick; pointer events alone should be inert. + expect(voiceInputResult.start).not.toHaveBeenCalled(); + expect(voiceInputResult.stop).not.toHaveBeenCalled(); + + // Click is what drives toggle. + fireEvent.click(button); + expect(voiceInputResult.start).toHaveBeenCalledTimes(1); + }); + + it("preserves hold behaviour on fine-pointer devices", () => { + coarsePointer.value = false; + renderButton(); + const button = screen.getByTestId(BUTTON_TESTID); + expect(button.getAttribute("data-effective-mode")).toBe("hold"); + }); + + it("applies the 40px touch-target class on coarse-pointer devices", () => { + coarsePointer.value = true; + renderButton(); + const button = screen.getByTestId(BUTTON_TESTID); + expect(button.className).toContain("h-10"); + expect(button.className).toContain("w-10"); + expect(button.className).not.toContain("h-7"); + }); +}); + +describe("VoiceInputButton — stored toggle mode is unaffected by pointer kind", () => { + it("renders effective toggle and ignores pointer events when stored mode is toggle", () => { + voicePrefs.value = { ...voicePrefs.value, mode: "toggle" }; + coarsePointer.value = false; + renderButton(); + const button = screen.getByTestId(BUTTON_TESTID); + expect(button.getAttribute("data-mode")).toBe("toggle"); + expect(button.getAttribute("data-effective-mode")).toBe("toggle"); + + // Pointer handlers must not be attached in toggle mode; pointerdown alone + // should not start a recording. Click is what drives toggle. + fireEvent.pointerDown(button, { pointerId: 4 }); + expect(voiceInputResult.start).not.toHaveBeenCalled(); + fireEvent.click(button); + expect(voiceInputResult.start).toHaveBeenCalledTimes(1); + }); +}); + +describe("VoiceInputButton — recording-state stop transition", () => { + it("invokes stop() on click when state is recording (toggle path)", () => { + voicePrefs.value = { ...voicePrefs.value, mode: "toggle" }; + voiceInputResult.state = "recording"; + renderButton(); + const button = screen.getByTestId(BUTTON_TESTID); + fireEvent.click(button); + expect(voiceInputResult.stop).toHaveBeenCalledTimes(1); + expect(voiceInputResult.start).not.toHaveBeenCalled(); + }); +}); + +describe("VoiceInputButton — disabled feature & unsupported fallback", () => { + it("renders nothing when voice mode is disabled in settings", () => { + voicePrefs.value = { ...voicePrefs.value, enabled: false }; + const { container } = renderButton(); + expect(container.firstChild).toBeNull(); + }); + + it("renders the unsupported fallback button when no engine is available", () => { + voiceInputResult.supported = false; + renderButton(); + const button = screen.getByTestId(BUTTON_TESTID); + expect(button.getAttribute("data-state")).toBe("unsupported"); + expect(button.getAttribute("aria-label")).toBe("Voice input unavailable"); + }); +}); diff --git a/apps/web/components/task/chat/voice-input-button.tsx b/apps/web/components/task/chat/voice-input-button.tsx index 96fc0bf75d..93bdd31fdf 100644 --- a/apps/web/components/task/chat/voice-input-button.tsx +++ b/apps/web/components/task/chat/voice-input-button.tsx @@ -1,6 +1,6 @@ "use client"; -import { useCallback, useEffect, useRef } from "react"; +import { useCallback, useEffect, useRef, useSyncExternalStore } from "react"; import { IconLoader2, IconMicrophone, IconPlayerStopFilled } from "@tabler/icons-react"; import { Button } from "@kandev/ui/button"; @@ -68,18 +68,54 @@ function toastForError(toast: ReturnType["toast"], err: VoiceEr // ── Activation handlers ────────────────────────────────────────────────── +// Hold mode survives finger drift by claiming the pointer with +// setPointerCapture on pointerdown. Without capture, any movement that crosses +// the button's bounds (a common occurrence on touch — small finger shift, +// soft-keyboard reflow, OS gesture handoff) fires pointerleave and would stop +// recording mid-utterance. pointerleave is intentionally NOT wired here for +// the same reason. Capture is released on pointerup/pointercancel; both also +// trigger stop(). +function safePointerCapture(target: Element, pointerId: number): void { + try { + target.setPointerCapture(pointerId); + } catch { + // setPointerCapture can throw InvalidPointerId on browsers that have + // already released the implicit capture (older WebKit). Safe to ignore — + // the worst case is the prior behaviour where hold could stop on drift. + } +} + +function safePointerRelease(target: Element, pointerId: number): void { + try { + // `hasPointerCapture` is in lib.dom but the runtime check guards against + // ancient/headless environments that strip it (some jsdom + happy-dom + // versions). If the method is present and reports no capture, skip the + // release to avoid the no-op throw Safari occasionally fires. + if (typeof target.hasPointerCapture === "function" && !target.hasPointerCapture(pointerId)) { + return; + } + target.releasePointerCapture(pointerId); + } catch { + // ignore — Safari can throw if capture was already released. + } +} + function buildHoldHandlers(start: () => Promise, stop: () => Promise) { return { onPointerDown: (e: React.PointerEvent) => { e.preventDefault(); + safePointerCapture(e.currentTarget, e.pointerId); void start(); }, onPointerUp: (e: React.PointerEvent) => { e.preventDefault(); + safePointerRelease(e.currentTarget, e.pointerId); + void stop(); + }, + onPointerCancel: (e: React.PointerEvent) => { + safePointerRelease(e.currentTarget, e.pointerId); void stop(); }, - onPointerLeave: () => void stop(), - onPointerCancel: () => void stop(), }; } @@ -114,6 +150,34 @@ function useAutoSendOnTranscript( ); } +// Subscribes to `(pointer: coarse)` so the component re-renders when the user +// docks/undocks an external pointer (Surface, iPad with Magic Keyboard). SSR +// returns false, matching desktop default — mobile-first hydration corrects on +// first client paint. Pattern mirrors `useResponsiveBreakpoint`. +function subscribeCoarsePointer(callback: () => void): () => void { + if (typeof window === "undefined" || typeof window.matchMedia !== "function") return () => {}; + const mql = window.matchMedia("(pointer: coarse)"); + mql.addEventListener("change", callback); + return () => mql.removeEventListener("change", callback); +} + +function getCoarsePointerSnapshot(): boolean { + if (typeof window === "undefined" || typeof window.matchMedia !== "function") return false; + return window.matchMedia("(pointer: coarse)").matches; +} + +function getCoarsePointerServerSnapshot(): boolean { + return false; +} + +function useIsCoarsePointer(): boolean { + return useSyncExternalStore( + subscribeCoarsePointer, + getCoarsePointerSnapshot, + getCoarsePointerServerSnapshot, + ); +} + function useVoiceShortcut( enabled: boolean, state: VoiceInputState, @@ -192,11 +256,99 @@ export function VoiceInputButton({ onTranscript, onAutoSend, disabled }: VoiceIn ); } +// Hold-to-talk is unreliable on touch even with pointer capture: the platform +// reclaims the pointer for system gestures (back swipes, scroll-chains) and the +// user's stored preference becomes a trap. Coarse-pointer devices silently get +// toggle behaviour; persisted `voiceMode.mode` is left alone so docking a +// keyboard restores the user's choice without a save. +function resolveEffectiveMode( + prefMode: "hold" | "toggle", + isCoarsePointer: boolean, +): "hold" | "toggle" { + return prefMode === "hold" && !isCoarsePointer ? "hold" : "toggle"; +} + +function resolveTooltip(args: { + modelLoad: VoiceModelLoadState; + modelLabel: string; + state: VoiceInputState; + holdMode: boolean; +}): string { + const { modelLoad, modelLabel, state, holdMode } = args; + if (modelLoad.state === "loading") { + const pct = Number.isFinite(modelLoad.progress) + ? Math.min(100, Math.max(0, Math.round(modelLoad.progress * 100))) + : 0; + return `Downloading ${modelLabel}… ${pct}%`; + } + return `${TOOLTIP_BY_STATE[state]}${holdMode && state === "idle" ? " (hold)" : ""}`; +} + +type VoiceMicButtonProps = { + state: VoiceInputState; + modelLoad: VoiceModelLoadState; + disabled?: boolean; + storedMode: "hold" | "toggle"; + effectiveMode: "hold" | "toggle"; + isCoarsePointer: boolean; + pointerHandlers: ReturnType | Record; + onClick: (() => void) | undefined; +}; + +// Styled to mirror SubmitButton (h-7 w-7 rounded-full primary fill) so the two +// prominent input actions read as a pair on the right of the toolbar. +// Recording flips to a destructive fill with a pulsing ring so the active +// state is unmistakable even on mobile. Touch-input sizing bumps to 10×10 +// (40px) for an easier finger target without disturbing the desktop pair. +function VoiceMicButton({ + state, + modelLoad, + disabled, + storedMode, + effectiveMode, + isCoarsePointer, + pointerHandlers, + onClick, +}: VoiceMicButtonProps) { + const isRecording = state === "recording"; + const isBusy = state === "requesting" || state === "processing" || modelLoad.state === "loading"; + return ( + + ); +} + function EnabledVoiceInputButton({ onTranscript, onAutoSend, disabled }: VoiceInputButtonProps) { const { toast } = useToast(); const voiceMode = useAppStore((s) => s.userSettings.voiceMode); const handleError = useCallback((err: VoiceError) => toastForError(toast, err), [toast]); const wrappedTranscript = useAutoSendOnTranscript(onTranscript, onAutoSend, voiceMode.autoSend); + const isCoarsePointer = useIsCoarsePointer(); const { supported, state, modelLoad, start, stop, cancel } = useVoiceInput({ onTranscript: wrappedTranscript, @@ -217,19 +369,13 @@ function EnabledVoiceInputButton({ onTranscript, onAutoSend, disabled }: VoiceIn // the button silently left mobile users with no discoverable feedback. if (!supported) return ; - const isRecording = state === "recording"; - const isBusy = state === "requesting" || state === "processing" || modelLoad.state === "loading"; - const holdMode = voiceMode.mode === "hold"; - + const effectiveMode = resolveEffectiveMode(voiceMode.mode, isCoarsePointer); + const holdMode = effectiveMode === "hold"; const pointerHandlers = holdMode ? buildHoldHandlers(start, stop) : {}; const onClick = holdMode ? undefined : buildToggleHandler(state, start, stop); - const modelLabel = whisperModelConfig(voiceMode.whisperWebModel).label; + const tooltipText = resolveTooltip({ modelLoad, modelLabel, state, holdMode }); - // Styled to mirror SubmitButton (h-7 w-7 rounded-full primary fill) so the - // two prominent input actions read as a pair on the right of the toolbar. - // Recording flips to a destructive fill with a pulsing ring so the active - // state is unmistakable even on mobile. return (
- + /> - - {modelLoad.state === "loading" - ? `Downloading ${modelLabel}… ${Number.isFinite(modelLoad.progress) ? Math.min(100, Math.max(0, Math.round(modelLoad.progress * 100))) : 0}%` - : `${TOOLTIP_BY_STATE[state]}${holdMode && state === "idle" ? " (hold)" : ""}`} - + {tooltipText}
); diff --git a/apps/web/e2e/helpers/api-client.ts b/apps/web/e2e/helpers/api-client.ts index 572ac6fd30..688973baad 100644 --- a/apps/web/e2e/helpers/api-client.ts +++ b/apps/web/e2e/helpers/api-client.ts @@ -7,6 +7,7 @@ import type { TaskSessionState, } from "../../lib/types/http"; import type { Agent, AgentProfile } from "../../lib/types/http-agents"; +import type { VoiceModeSettings } from "../../lib/types/http-voice"; import type { SSHAgentReadinessResponse, SSHProbeShellsResponse, @@ -609,6 +610,7 @@ export class ApiClient { default_utility_model?: string; sidebar_views?: unknown[]; kanban_view_mode?: string; + voice_mode?: VoiceModeSettings; }): Promise { await this.request("PATCH", "/api/v1/user/settings", settings); } diff --git a/apps/web/e2e/tests/chat/mobile-voice-mode.spec.ts b/apps/web/e2e/tests/chat/mobile-voice-mode.spec.ts new file mode 100644 index 0000000000..4ed1389cd7 --- /dev/null +++ b/apps/web/e2e/tests/chat/mobile-voice-mode.spec.ts @@ -0,0 +1,90 @@ +import { test, expect } from "../../fixtures/test-base"; +import type { SeedData } from "../../fixtures/test-base"; +import type { ApiClient } from "../../helpers/api-client"; +import { SessionPage } from "../../pages/session-page"; + +// Mobile (Pixel 5) regression coverage for the voice-mode entry point and +// the chat composer's mic button. Filename matches /mobile-.*\.spec\.ts/ so +// the `mobile-chrome` Playwright project picks it up. +// +// Two assertions: +// 1. The Voice Mode settings page renders all cards at mobile width when +// navigated to directly — guards against the "setting missing" symptom. +// 2. The mic button mounts on the mobile chat composer with the coarse-pointer +// override: a user with stored `voiceMode.mode = "hold"` gets toggle +// behaviour (data-effective-mode = "toggle") and a 40px touch target. + +async function seedTask(apiClient: ApiClient, seedData: SeedData, title: string) { + return apiClient.createTaskWithAgent(seedData.workspaceId, title, seedData.agentProfileId, { + description: "/e2e:simple-message", + workflow_id: seedData.workflowId, + workflow_step_id: seedData.startStepId, + repository_ids: [seedData.repositoryId], + }); +} + +test.describe("Mobile voice mode", () => { + test.describe.configure({ retries: 1, timeout: 60_000 }); + + test("Voice Mode settings page renders every card at mobile width", async ({ testPage }) => { + await testPage.goto("/settings/voice-mode"); + + // Page header confirms route resolved to the right shell. The page title + // is rendered by SettingsSection — match the heading text exactly so we + // don't pick up the sidebar entry that shares the label. + await expect(testPage.getByRole("heading", { name: "Voice Mode", exact: true })).toBeVisible({ + timeout: 15_000, + }); + + // The five cards rendered by `voice-mode-settings.tsx` at mobile width. + await expect(testPage.getByText("Enable Voice Input")).toBeVisible(); + await expect(testPage.getByText("Transcription Engine")).toBeVisible(); + await expect(testPage.getByText("Behavior")).toBeVisible(); + await expect(testPage.getByText("Whisper Web Model")).toBeVisible(); + await expect(testPage.getByText(/Shortcut$/)).toBeVisible(); + + // The mobile sidebar trigger (md:hidden) lives in the topbar; assert it + // exists so we know the sidebar entry chain that exposed this page is + // not silently removed by a future layout change. + const sidebarTrigger = testPage.locator('button[data-sidebar="trigger"]'); + await expect(sidebarTrigger).toBeVisible(); + }); + + test("voice input mic button mounts on the mobile chat composer with the coarse-pointer touch target and effective toggle mode", async ({ + testPage, + apiClient, + seedData, + }) => { + // Seed hold mode so the coarse-pointer override assertion is non-vacuous: + // data-mode="hold" (stored pref) vs data-effective-mode="toggle" (override) + // proves the coarse-pointer branch fired, not that the default was toggle. + await apiClient.saveUserSettings({ + voice_mode: { + enabled: true, + engine: "auto", + language: "auto", + mode: "hold", + auto_send: false, + whisper_web_model: "base", + }, + }); + + const task = await seedTask(apiClient, seedData, "Mobile voice button"); + await testPage.goto(`/t/${task.id}`); + const session = new SessionPage(testPage); + await session.waitForLoad(); + + const micButton = testPage.getByTestId("voice-input-button"); + await expect(micButton).toBeVisible({ timeout: 15_000 }); + + // Pixel 5 advertises `(pointer: coarse)`. Three signals that the coarse- + // pointer override fired: + // - data-mode="hold" (the stored user preference) + // - data-effective-mode="toggle" (the override: hold→toggle on coarse) + // - h-10 w-10 (40px touch target, only set when isCoarsePointer=true) + await expect(micButton).toHaveAttribute("data-mode", "hold"); + await expect(micButton).toHaveAttribute("data-effective-mode", "toggle"); + await expect(micButton).toHaveClass(/(^|\s)h-10(\s|$)/); + await expect(micButton).toHaveClass(/(^|\s)w-10(\s|$)/); + }); +});