From 181a64a5fb851e755dd721c563d0d95ddeedfa48 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Flor=C3=AAncio?= Date: Fri, 17 Apr 2026 14:22:34 +0100 Subject: [PATCH] fix: prevent utility agents settings page crash on null models (#602) --- apps/backend/cmd/mock-agent/main.go | 23 +- apps/backend/internal/agent/agents/mock.go | 16 + .../agentctl/server/utility/acp_executor.go | 7 +- apps/backend/internal/utility/dto/dto.go | 5 + .../internal/utility/handlers/handlers.go | 38 ++- .../utility/handlers/handlers_test.go | 283 ++++++++++++++++++ .../settings/utility-agent-dialog.tsx | 23 +- .../settings/utility-agents-section.tsx | 2 +- .../components/settings/utility-sections.tsx | 24 +- apps/web/e2e/fixtures/backend.ts | 6 +- .../e2e/tests/settings/utility-agents.spec.ts | 195 ++++++++++++ apps/web/lib/api/domains/utility-api.ts | 7 + 12 files changed, 591 insertions(+), 38 deletions(-) create mode 100644 apps/backend/internal/utility/handlers/handlers_test.go create mode 100644 apps/web/e2e/tests/settings/utility-agents.spec.ts diff --git a/apps/backend/cmd/mock-agent/main.go b/apps/backend/cmd/mock-agent/main.go index a1a780aae2..fce8808a29 100644 --- a/apps/backend/cmd/mock-agent/main.go +++ b/apps/backend/cmd/mock-agent/main.go @@ -75,6 +75,9 @@ func (a *mockAgent) Initialize(_ context.Context, _ acp.InitializeRequest) (acp. // NewSession creates a new conversation session. // MCP servers from the ACP request are registered so callMCPTool can use them. +// The Models field advertises the available models so the host utility +// capability probe can populate them in the cache — this is what makes +// the utility-agents settings page show model options for mock-agent in E2E. func (a *mockAgent) NewSession(_ context.Context, req acp.NewSessionRequest) (acp.NewSessionResponse, error) { sid := acp.SessionId(fmt.Sprintf("mock-session-%d", os.Getpid())) a.mu.Lock() @@ -85,7 +88,25 @@ func (a *mockAgent) NewSession(_ context.Context, req acp.NewSessionRequest) (ac // This bridges ACP protocol MCP config to the mock agent's MCP client. registerACPMcpServers(req.McpServers) - return acp.NewSessionResponse{SessionId: sid}, nil + return acp.NewSessionResponse{ + SessionId: sid, + Models: mockSessionModels(), + }, nil +} + +// mockSessionModels returns the mock agent's advertised model list for ACP +// session responses. Two models are exposed so tests can verify both +// selection and default behavior (mock-fast is the default). +func mockSessionModels() *acp.UnstableSessionModelState { + fastDesc := "Fast mock model for testing" + smartDesc := "Smart mock model for testing" + return &acp.UnstableSessionModelState{ + CurrentModelId: "mock-fast", + AvailableModels: []acp.UnstableModelInfo{ + {ModelId: "mock-fast", Name: "Mock Fast", Description: &fastDesc}, + {ModelId: "mock-smart", Name: "Mock Smart", Description: &smartDesc}, + }, + } } // LoadSession restores a previous session for resume. diff --git a/apps/backend/internal/agent/agents/mock.go b/apps/backend/internal/agent/agents/mock.go index fc471bf9e3..8c4aaa1e13 100644 --- a/apps/backend/internal/agent/agents/mock.go +++ b/apps/backend/internal/agent/agents/mock.go @@ -17,6 +17,7 @@ var mockLogoDark []byte var ( _ Agent = (*MockAgent)(nil) _ PassthroughAgent = (*MockAgent)(nil) + _ InferenceAgent = (*MockAgent)(nil) ) type MockAgent struct { @@ -113,3 +114,18 @@ func (a *MockAgent) InstallScript() string { return "" } func (a *MockAgent) PermissionSettings() map[string]PermissionSetting { return emptyPermSettings } + +// InferenceConfig enables one-shot inference via ACP. The mock-agent binary +// advertises its available models in the session/new response, so the host +// utility capability probe populates them into the cache without any static +// model list here. +func (a *MockAgent) InferenceConfig() *InferenceConfig { + binary := "mock-agent" + if a.binaryPath != "" { + binary = a.binaryPath + } + return &InferenceConfig{ + Supported: true, + Command: NewCommand(binary), + } +} diff --git a/apps/backend/internal/agentctl/server/utility/acp_executor.go b/apps/backend/internal/agentctl/server/utility/acp_executor.go index 0b1267756d..34ad017734 100644 --- a/apps/backend/internal/agentctl/server/utility/acp_executor.go +++ b/apps/backend/internal/agentctl/server/utility/acp_executor.go @@ -403,9 +403,10 @@ func derefString(p *string) string { // is not derived from untrusted input — even though the value is // semantically the same as the base name taken from InferenceConfig.Command. var allowedProbeCommands = map[string]string{ - "npx": "npx", - "auggie": "auggie", - "opencode": "opencode", + "npx": "npx", + "auggie": "auggie", + "opencode": "opencode", + "mock-agent": "mock-agent", } // resolveProbeCommand validates and returns a hard-coded executable name for diff --git a/apps/backend/internal/utility/dto/dto.go b/apps/backend/internal/utility/dto/dto.go index 37be4c277a..427bc29ee3 100644 --- a/apps/backend/internal/utility/dto/dto.go +++ b/apps/backend/internal/utility/dto/dto.go @@ -162,6 +162,11 @@ type InferenceModelDTO struct { Name string `json:"name"` Description string `json:"description"` IsDefault bool `json:"is_default"` + // Meta carries agent-specific extras from ACP's `_meta` field. For + // GitHub Copilot this includes `copilotUsage` (e.g. "1x", "0.33x", + // "0x" — the premium-request multiplier) which the UI renders as a + // cost badge next to the model name. + Meta map[string]any `json:"meta,omitempty"` } // InferenceAgentDTO represents an agent that supports inference. diff --git a/apps/backend/internal/utility/handlers/handlers.go b/apps/backend/internal/utility/handlers/handlers.go index 59af0ba36e..f87b7c03ba 100644 --- a/apps/backend/internal/utility/handlers/handlers.go +++ b/apps/backend/internal/utility/handlers/handlers.go @@ -27,9 +27,11 @@ type InferenceExecutor interface { } // HostUtilityExecutor runs sessionless utility prompts via the long-lived -// per-agent-type host agentctl instances. +// per-agent-type host agentctl instances and exposes the cached per-agent +// capabilities (models, modes) populated by the boot-time ACP probe. type HostUtilityExecutor interface { ExecutePrompt(ctx context.Context, agentType, model, mode, prompt string) (*hostutility.PromptResult, error) + Get(agentType string) (hostutility.AgentCapabilities, bool) } // UserSettingsProvider provides user settings for default utility agent/model. @@ -310,15 +312,43 @@ func (h *Handlers) httpListCalls(c *gin.Context) { func (h *Handlers) httpListInferenceAgents(c *gin.Context) { inferenceAgents := h.executor.ListInferenceAgentsWithContext(c.Request.Context()) - // Convert to DTO. Models are no longer listed per-agent here — the - // frontend should read them from the host utility capability cache via - // GET /api/v1/agents/:type/capabilities. + // Build the response from the host utility capability cache (boot-time + // ACP probe). Only include agents whose probe reached StatusOK — an + // agent that isn't authenticated, not installed, or still probing + // can't actually run a utility prompt, so showing it in the picker + // just leads the user into a dead end. + // + // hostExecutor is an optional dependency (see executeSessionless); + // without it we can't check health or surface models, so the list is + // empty by design rather than showing unusable options. result := make([]dto.InferenceAgentDTO, 0, len(inferenceAgents)) + if h.hostExecutor == nil { + c.JSON(http.StatusOK, dto.InferenceAgentsResponse{Agents: result}) + return + } for _, ia := range inferenceAgents { + // The cache is keyed by ag.ID() (see bootstrapAgent), not + // ag.Name() — built-in ACP agents return distinct strings for the + // two (e.g. "claude-acp" vs "Claude ACP Agent"). + caps, ok := h.hostExecutor.Get(ia.ID) + if !ok || caps.Status != hostutility.StatusOK { + continue + } + models := make([]dto.InferenceModelDTO, 0, len(caps.Models)) + for _, m := range caps.Models { + models = append(models, dto.InferenceModelDTO{ + ID: m.ID, + Name: m.Name, + Description: m.Description, + IsDefault: m.ID == caps.CurrentModelID, + Meta: m.Meta, + }) + } result = append(result, dto.InferenceAgentDTO{ ID: ia.ID, Name: ia.Name, DisplayName: ia.DisplayName, + Models: models, }) } c.JSON(http.StatusOK, dto.InferenceAgentsResponse{Agents: result}) diff --git a/apps/backend/internal/utility/handlers/handlers_test.go b/apps/backend/internal/utility/handlers/handlers_test.go new file mode 100644 index 0000000000..38e84d9d3d --- /dev/null +++ b/apps/backend/internal/utility/handlers/handlers_test.go @@ -0,0 +1,283 @@ +package handlers + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/gin-gonic/gin" + + "github.com/kandev/kandev/internal/agent/hostutility" + "github.com/kandev/kandev/internal/agent/lifecycle" + agentctlutil "github.com/kandev/kandev/internal/agentctl/server/utility" + "github.com/kandev/kandev/internal/common/logger" +) + +type stubInferenceExecutor struct { + agents []lifecycle.InferenceAgentInfo +} + +func (s *stubInferenceExecutor) ExecuteInferencePrompt(_ context.Context, _, _, _, _ string) (*agentctlutil.PromptResponse, error) { + return nil, nil +} + +func (s *stubInferenceExecutor) ListInferenceAgentsWithContext(_ context.Context) []lifecycle.InferenceAgentInfo { + return s.agents +} + +type stubHostUtility struct { + caps map[string]hostutility.AgentCapabilities +} + +func (s *stubHostUtility) ExecutePrompt(_ context.Context, _, _, _, _ string) (*hostutility.PromptResult, error) { + return nil, nil +} + +func (s *stubHostUtility) Get(agentType string) (hostutility.AgentCapabilities, bool) { + c, ok := s.caps[agentType] + return c, ok +} + +// wantModel is the expected shape of a single model in the response for a +// given agent. +type wantModel struct { + id string + isDefault bool + meta map[string]any +} + +// TestHttpListInferenceAgents covers the full /api/v1/utility/inference-agents +// response contract: +// +// - Only agents whose host utility probe reached StatusOK are included — +// an agent that needs auth, isn't installed, or is still probing can't +// actually run a utility prompt, so it must be filtered out of the +// picker rather than leading the user into a dead end. +// - `models` is always a JSON array, never null — a null slice would +// crash the frontend's flatMap over `ia.models`. +// - `is_default` is set on the model matching CurrentModelID. +// - `meta` (e.g. Copilot's `copilotUsage` cost multiplier) propagates +// through to the DTO so the model combobox can render cost badges. +func TestHttpListInferenceAgents(t *testing.T) { + gin.SetMode(gin.TestMode) + + // Fixtures use the real built-in ACP agent shape (distinct ID/Name) so + // a future ID/Name mixup in the cache lookup fails the test. + claude := lifecycle.InferenceAgentInfo{ID: "claude-acp", Name: "Claude ACP Agent", DisplayName: "Claude"} + codex := lifecycle.InferenceAgentInfo{ID: "codex-acp", Name: "Codex ACP Agent", DisplayName: "Codex"} + copilot := lifecycle.InferenceAgentInfo{ID: "copilot-acp", Name: "Copilot ACP Agent", DisplayName: "Copilot"} + + tests := []struct { + name string + agents []lifecycle.InferenceAgentInfo + caps map[string]hostutility.AgentCapabilities + nilHost bool + wantByAgent map[string][]wantModel + }{ + { + name: "healthy agent with cached models is included with is_default", + agents: []lifecycle.InferenceAgentInfo{claude}, + caps: map[string]hostutility.AgentCapabilities{ + "claude-acp": { + AgentType: "claude-acp", + Status: hostutility.StatusOK, + CurrentModelID: "sonnet", + Models: []hostutility.Model{ + {ID: "sonnet", Name: "Sonnet"}, + {ID: "opus", Name: "Opus"}, + }, + }, + }, + wantByAgent: map[string][]wantModel{ + "Claude ACP Agent": { + {id: "sonnet", isDefault: true}, + {id: "opus", isDefault: false}, + }, + }, + }, + { + // Primary UX bug this filter guards against: Codex/Auggie with + // lock icons (auth_required) appearing in the utility picker. + name: "auth_required agent is filtered out", + agents: []lifecycle.InferenceAgentInfo{claude, codex}, + caps: map[string]hostutility.AgentCapabilities{ + "claude-acp": { + AgentType: "claude-acp", + Status: hostutility.StatusOK, + Models: []hostutility.Model{{ID: "sonnet", Name: "Sonnet"}}, + }, + "codex-acp": { + AgentType: "codex-acp", + Status: hostutility.StatusAuthRequired, + }, + }, + wantByAgent: map[string][]wantModel{ + "Claude ACP Agent": {{id: "sonnet", isDefault: false}}, + }, + }, + { + name: "failed and probing agents are filtered out", + agents: []lifecycle.InferenceAgentInfo{claude, codex, copilot}, + caps: map[string]hostutility.AgentCapabilities{ + "claude-acp": {Status: hostutility.StatusFailed}, + "codex-acp": {Status: hostutility.StatusProbing}, + "copilot-acp": {Status: hostutility.StatusOK, Models: []hostutility.Model{{ID: "gpt-5", Name: "GPT-5"}}}, + }, + wantByAgent: map[string][]wantModel{ + "Copilot ACP Agent": {{id: "gpt-5", isDefault: false}}, + }, + }, + { + name: "agent with no cache entry is filtered out", + agents: []lifecycle.InferenceAgentInfo{claude}, + caps: nil, + wantByAgent: map[string][]wantModel{}, + }, + { + name: "model meta (copilot cost) propagates to DTO", + agents: []lifecycle.InferenceAgentInfo{copilot}, + caps: map[string]hostutility.AgentCapabilities{ + "copilot-acp": { + AgentType: "copilot-acp", + Status: hostutility.StatusOK, + CurrentModelID: "gpt-5", + Models: []hostutility.Model{ + {ID: "gpt-5", Name: "GPT-5", Meta: map[string]any{"copilotUsage": "1x"}}, + {ID: "gpt-5-mini", Name: "GPT-5 Mini", Meta: map[string]any{"copilotUsage": "0.33x"}}, + }, + }, + }, + wantByAgent: map[string][]wantModel{ + "Copilot ACP Agent": { + {id: "gpt-5", isDefault: true, meta: map[string]any{"copilotUsage": "1x"}}, + {id: "gpt-5-mini", isDefault: false, meta: map[string]any{"copilotUsage": "0.33x"}}, + }, + }, + }, + { + name: "no inference agents returns empty list", + agents: nil, + caps: nil, + wantByAgent: map[string][]wantModel{}, + }, + { + // hostExecutor is optional throughout the package (see the nil + // guard in executeSessionless). Without it we can't check + // health, so the list is empty — never a panic. + name: "nil hostExecutor yields empty list, no panic", + agents: []lifecycle.InferenceAgentInfo{claude}, + nilHost: true, + wantByAgent: map[string][]wantModel{}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + log, _ := logger.NewLogger(logger.LoggingConfig{Level: "error", Format: "json"}) + h := &Handlers{ + executor: &stubInferenceExecutor{agents: tt.agents}, + logger: log, + } + if !tt.nilHost { + h.hostExecutor = &stubHostUtility{caps: tt.caps} + } + + router := gin.New() + router.GET("/api/v1/utility/inference-agents", h.httpListInferenceAgents) + + req := httptest.NewRequest(http.MethodGet, "/api/v1/utility/inference-agents", nil) + rec := httptest.NewRecorder() + router.ServeHTTP(rec, req) + + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want 200", rec.Code) + } + + // The raw body must never contain "models":null — that shape + // crashed the frontend before the original fix. + body := rec.Body.String() + if strings.Contains(body, `"models":null`) { + t.Fatalf("response must never contain \"models\":null, got body: %s", body) + } + + var resp struct { + Agents []struct { + Name string `json:"name"` + Models []struct { + ID string `json:"id"` + IsDefault bool `json:"is_default"` + Meta map[string]any `json:"meta,omitempty"` + } `json:"models"` + } `json:"agents"` + } + if err := json.Unmarshal(rec.Body.Bytes(), &resp); err != nil { + t.Fatalf("unmarshal response: %v", err) + } + + if len(resp.Agents) != len(tt.wantByAgent) { + t.Fatalf("got %d agents (%v), want %d", len(resp.Agents), agentNames(resp.Agents), len(tt.wantByAgent)) + } + + for _, a := range resp.Agents { + if a.Models == nil { + t.Errorf("agent %q: models slice decoded as nil (should be empty slice, never nil)", a.Name) + } + want, ok := tt.wantByAgent[a.Name] + if !ok { + t.Errorf("unexpected agent %q in response", a.Name) + continue + } + if len(a.Models) != len(want) { + t.Errorf("agent %q: got %d models, want %d", a.Name, len(a.Models), len(want)) + continue + } + for i, m := range a.Models { + if m.ID != want[i].id { + t.Errorf("agent %q model[%d]: got id %q, want %q", a.Name, i, m.ID, want[i].id) + } + if m.IsDefault != want[i].isDefault { + t.Errorf("agent %q model %q: got is_default=%v, want %v", a.Name, m.ID, m.IsDefault, want[i].isDefault) + } + if !equalMeta(m.Meta, want[i].meta) { + t.Errorf("agent %q model %q: got meta=%v, want %v", a.Name, m.ID, m.Meta, want[i].meta) + } + } + } + }) + } +} + +func agentNames(agents []struct { + Name string `json:"name"` + Models []struct { + ID string `json:"id"` + IsDefault bool `json:"is_default"` + Meta map[string]any `json:"meta,omitempty"` + } `json:"models"` +}) []string { + names := make([]string, 0, len(agents)) + for _, a := range agents { + names = append(names, a.Name) + } + return names +} + +// equalMeta does a shallow equality check on the meta maps. A nil want map +// matches a nil or empty got map (JSON omits empty maps via omitempty). +func equalMeta(got, want map[string]any) bool { + if len(want) == 0 { + return len(got) == 0 + } + if len(got) != len(want) { + return false + } + for k, v := range want { + if got[k] != v { + return false + } + } + return true +} diff --git a/apps/web/components/settings/utility-agent-dialog.tsx b/apps/web/components/settings/utility-agent-dialog.tsx index 2ba6f4ff00..f2cc4c132f 100644 --- a/apps/web/components/settings/utility-agent-dialog.tsx +++ b/apps/web/components/settings/utility-agent-dialog.tsx @@ -6,6 +6,7 @@ import { Button } from "@kandev/ui/button"; import { Input } from "@kandev/ui/input"; import { Label } from "@kandev/ui/label"; import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@kandev/ui/select"; +import { ModelCombobox } from "@/components/settings/model-combobox"; import { type UtilityAgent, createUtilityAgent, @@ -87,18 +88,14 @@ function AgentModelSelect({
- + m.is_default)?.id} + placeholder="Select model..." + disabled={availableModels.length === 0} + />
); @@ -199,7 +196,7 @@ export function UtilityAgentDialog({ open, onOpenChange, agent, onSuccess }: Pro // Auto-select default model when agent changes useEffect(() => { if (selectedAgent && !form.model) { - const defaultModel = selectedAgent.models.find((m) => m.is_default); + const defaultModel = (selectedAgent.models ?? []).find((m) => m.is_default); if (defaultModel) { setForm((f) => ({ ...f, model: defaultModel.id })); } diff --git a/apps/web/components/settings/utility-agents-section.tsx b/apps/web/components/settings/utility-agents-section.tsx index 1545dda454..38cf345622 100644 --- a/apps/web/components/settings/utility-agents-section.tsx +++ b/apps/web/components/settings/utility-agents-section.tsx @@ -20,7 +20,7 @@ import { function buildAllModels(inferenceAgents: InferenceAgent[]) { return inferenceAgents.flatMap((ia) => - ia.models.map((m) => ({ + (ia.models ?? []).map((m) => ({ value: `${ia.id}|${m.id}`, label: `${ia.display_name} / ${m.name}`, agentName: ia.display_name, diff --git a/apps/web/components/settings/utility-sections.tsx b/apps/web/components/settings/utility-sections.tsx index 23e1efa73a..72d51c3fec 100644 --- a/apps/web/components/settings/utility-sections.tsx +++ b/apps/web/components/settings/utility-sections.tsx @@ -5,6 +5,7 @@ import { Button } from "@kandev/ui/button"; import { Label } from "@kandev/ui/label"; import { Separator } from "@kandev/ui/separator"; import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@kandev/ui/select"; +import { ModelCombobox } from "@/components/settings/model-combobox"; import type { UtilityAgent, InferenceAgent } from "@/lib/api/domains/utility-api"; const USE_DEFAULT = "__USE_DEFAULT__"; @@ -27,6 +28,7 @@ export function DefaultModelSection({ }: DefaultModelSectionProps) { const selectedAgent = inferenceAgents.find((a) => a.id === defaultAgentId); const modelOptions = selectedAgent?.models ?? []; + const currentModelId = modelOptions.find((m) => m.is_default)?.id; return (
@@ -52,24 +54,16 @@ export function DefaultModelSection({
-
+
- + />
diff --git a/apps/web/e2e/fixtures/backend.ts b/apps/web/e2e/fixtures/backend.ts index e2978245ab..6eba0558df 100644 --- a/apps/web/e2e/fixtures/backend.ts +++ b/apps/web/e2e/fixtures/backend.ts @@ -223,7 +223,11 @@ exec git "$@" const backendEnv = { ...stripGitHubTokens(process.env as Record), - PATH: `${shimDir}:${originalPath}`, + // Prepend the kandev bin dir so the host utility probe can locate + // the `mock-agent` binary via PATH. In production that dir is the + // same as the running kandev binary's dir, but e2e spawns via an + // absolute path and doesn't inherit that location. + PATH: `${shimDir}:${path.join(BACKEND_DIR, "bin")}:${originalPath}`, KANDEV_E2E_ORIGINAL_PATH: originalPath, KANDEV_E2E_GIT_DELAY_FILE: shimDelayFile, HOME: tmpDir, diff --git a/apps/web/e2e/tests/settings/utility-agents.spec.ts b/apps/web/e2e/tests/settings/utility-agents.spec.ts new file mode 100644 index 0000000000..14927479d0 --- /dev/null +++ b/apps/web/e2e/tests/settings/utility-agents.spec.ts @@ -0,0 +1,195 @@ +import { test, expect } from "../../fixtures/test-base"; + +/** + * Covers the Utility Agents settings page. + * + * The first test is a regression guard for a bug where the backend emitted + * `models: null` on /api/v1/utility/inference-agents. The frontend's flatMap + * over `ia.models` blew up and crashed the whole settings page during render. + * The other tests smoke-check the page loads and walk through the main + * interactions (open the page, inspect sections, open the create dialog). + */ +test.describe("Utility Agents settings page", () => { + test("does not crash when backend returns models: null", async ({ testPage }) => { + // Simulate the exact shape the backend used to emit. Guards against a + // regression where frontend null-deref would take the whole page down. + await testPage.route("**/api/v1/utility/inference-agents", (route) => + route.fulfill({ + status: 200, + contentType: "application/json", + body: JSON.stringify({ + agents: [ + { + id: "broken-agent", + name: "broken-agent", + display_name: "Broken Agent", + models: null, + }, + ], + }), + }), + ); + + const pageErrors: Error[] = []; + testPage.on("pageerror", (err) => pageErrors.push(err)); + + await testPage.goto("/settings/utility-agents"); + + await expect( + testPage.getByRole("heading", { name: "Utility Agents", exact: true }), + ).toBeVisible({ timeout: 15_000 }); + + expect(pageErrors, `uncaught errors: ${pageErrors.map((e) => e.message).join("; ")}`).toEqual( + [], + ); + }); + + test("renders all sections with seeded built-in utility agents", async ({ testPage }) => { + const pageErrors: Error[] = []; + testPage.on("pageerror", (err) => pageErrors.push(err)); + + await testPage.goto("/settings/utility-agents"); + + // Top-level heading + subtitle. + await expect( + testPage.getByRole("heading", { name: "Utility Agents", exact: true }), + ).toBeVisible({ timeout: 15_000 }); + await expect( + testPage.getByText("One-shot AI helpers for commits, PRs, and prompts."), + ).toBeVisible(); + + // Default-model section. + await expect( + testPage.getByRole("heading", { name: "Default utility agent model", exact: true }), + ).toBeVisible(); + + // Built-in actions (seeded on first boot — see builtins.go). + // Assert a representative subset; the full list lives server-side. + await expect(testPage.getByText("commit-message", { exact: true })).toBeVisible(); + await expect(testPage.getByText("pr-title", { exact: true })).toBeVisible(); + await expect(testPage.getByText("enhance-prompt", { exact: true })).toBeVisible(); + + // Custom agents section + empty state. + await expect( + testPage.getByRole("heading", { name: "Custom utility agents", exact: true }), + ).toBeVisible(); + await expect(testPage.getByText("No custom utility agents.")).toBeVisible(); + + expect(pageErrors, `uncaught errors: ${pageErrors.map((e) => e.message).join("; ")}`).toEqual( + [], + ); + }); + + test("opens the create-agent dialog from the Add button", async ({ testPage }) => { + const pageErrors: Error[] = []; + testPage.on("pageerror", (err) => pageErrors.push(err)); + + await testPage.goto("/settings/utility-agents"); + + await expect( + testPage.getByRole("heading", { name: "Utility Agents", exact: true }), + ).toBeVisible({ timeout: 15_000 }); + + await testPage.getByRole("button", { name: "Add", exact: true }).click(); + + // The dialog is rendered by UtilityAgentDialog; title differs between + // create and edit mode. We're in create mode here. + const dialog = testPage.getByRole("dialog"); + await expect(dialog).toBeVisible({ timeout: 5_000 }); + await expect(dialog.getByText("Create Utility Agent")).toBeVisible(); + + // Close the dialog — nothing should explode. + await dialog.getByRole("button", { name: "Cancel" }).click(); + await expect(dialog).not.toBeVisible(); + + expect(pageErrors, `uncaught errors: ${pageErrors.map((e) => e.message).join("; ")}`).toEqual( + [], + ); + }); + + test("selecting an agent populates the model combobox (ACP probe)", async ({ + testPage, + backend, + }) => { + // Regression guard for "I select an agent but can't select a model". + // The mock-agent binary advertises `mock-fast` (default) and `mock-smart` + // in its session/new response, so the boot-time ACP probe populates the + // host utility capability cache. The backend filters out agents whose + // probe didn't reach StatusOK, so in E2E (KANDEV_MOCK_AGENT=only) the + // Agent dropdown should show exactly one option: Mock. + const pageErrors: Error[] = []; + testPage.on("pageerror", (err) => pageErrors.push(err)); + + // The probe runs in a goroutine at boot, so the first page load may + // land before the cache is populated. Poll the backend directly until + // mock-agent is reported with its models so the UI assertions below + // aren't racing the probe. + await expect + .poll( + async () => { + const resp = await testPage.request.get( + `${backend.baseUrl}/api/v1/utility/inference-agents`, + ); + if (!resp.ok()) return 0; + const data = (await resp.json()) as { + agents: { id: string; models?: { id: string }[] | null }[]; + }; + const mock = data.agents.find((a) => a.id === "mock-agent"); + return mock?.models?.length ?? 0; + }, + { timeout: 15_000, intervals: [250, 500, 1000] }, + ) + .toBeGreaterThanOrEqual(2); + + await testPage.goto("/settings/utility-agents"); + await expect( + testPage.getByRole("heading", { name: "Utility Agents", exact: true }), + ).toBeVisible({ timeout: 15_000 }); + + // The default-model section has an Agent select (shadcn) and a Model + // combobox (the shared ModelCombobox from the profile page). Each is + // scoped by the Label above it (no `htmlFor`). + const agentSelect = testPage + .locator('div:has(> label:text-is("Agent"))') + .first() + .getByRole("combobox"); + const modelCombobox = testPage + .locator('div:has(> label:text-is("Model"))') + .first() + .getByRole("combobox"); + + // Model combobox starts disabled until an agent is picked. + await expect(modelCombobox).toBeDisabled(); + + // Open the Agent dropdown: the only healthy option in E2E is Mock. + // This implicitly guards the backend filter — if an auth_required or + // still-probing agent had leaked through, it would show up here too. + await agentSelect.click(); + const agentListbox = testPage.getByRole("listbox"); + await expect(agentListbox).toBeVisible(); + await expect(agentListbox.getByRole("option")).toHaveCount(1); + await expect(agentListbox.getByRole("option", { name: "Mock", exact: true })).toBeVisible(); + await agentListbox.getByRole("option", { name: "Mock", exact: true }).click(); + await expect(agentListbox).not.toBeVisible(); + + // Model combobox is now enabled. Open the popover and verify both + // probed models are listed as command items, along with the "(default)" + // badge on mock-fast. + await expect(modelCombobox).toBeEnabled(); + await modelCombobox.click(); + await expect(testPage.getByRole("option", { name: /Mock Fast.*\(default\)/ })).toBeVisible(); + await expect(testPage.getByRole("option", { name: /Mock Smart/ })).toBeVisible(); + + // Search input is part of the ModelCombobox — filtering narrows the list. + await testPage.getByPlaceholder("Search models...").fill("smart"); + await expect(testPage.getByRole("option", { name: /Mock Fast/ })).toHaveCount(0); + + // Pick Mock Smart and verify the trigger reflects the selection. + await testPage.getByRole("option", { name: /Mock Smart/ }).click(); + await expect(modelCombobox).toContainText("Mock Smart"); + + expect(pageErrors, `uncaught errors: ${pageErrors.map((e) => e.message).join("; ")}`).toEqual( + [], + ); + }); +}); diff --git a/apps/web/lib/api/domains/utility-api.ts b/apps/web/lib/api/domains/utility-api.ts index 2144b0edb8..7dce107446 100644 --- a/apps/web/lib/api/domains/utility-api.ts +++ b/apps/web/lib/api/domains/utility-api.ts @@ -42,6 +42,13 @@ export type InferenceModel = { name: string; description: string; is_default: boolean; + /** + * Agent-specific extras from ACP's `_meta` field. GitHub Copilot exposes + * `copilotUsage` (e.g. "1x", "0.33x", "0x" — the premium-request + * multiplier) which the model combobox renders as a cost badge. Shape + * matches `ModelEntry.meta` so the same `` accepts this. + */ + meta?: Record; }; export type InferenceAgent = {