diff --git a/.github/scripts/pr-walkthrough-workflow-contract_test.py b/.github/scripts/pr-walkthrough-workflow-contract_test.py index 9316ae2d49..65b325d49a 100644 --- a/.github/scripts/pr-walkthrough-workflow-contract_test.py +++ b/.github/scripts/pr-walkthrough-workflow-contract_test.py @@ -129,7 +129,7 @@ class PRWalkthroughWorkflowContractTest(unittest.TestCase): 'PUBLIC_URL="${WALKTHROUGH_BASE_URL%/}/pr/${PR_NUMBER}/${SHORT_HEAD_SHA}.html"', "RESPONSE_META_PATH=pr-walkthrough-public.response", "curl --fail", - "--write-out '%{http_code}\\t%{content_type}\\t%{url_effective}'", + "--write-out '%{http_code}\\t%{content_type}\\t%{url_effective}\\n'", "response_status", 'response_effective_url" != "$PUBLIC_URL"', "response_content_type", diff --git a/.github/workflows/pr-walkthrough-reconcile.yml b/.github/workflows/pr-walkthrough-reconcile.yml index d1ec8cbe68..91cdac8750 100644 --- a/.github/workflows/pr-walkthrough-reconcile.yml +++ b/.github/workflows/pr-walkthrough-reconcile.yml @@ -69,7 +69,7 @@ jobs: if ! curl --fail --silent --show-error --location --max-time 30 \ --retry 3 --retry-all-errors --retry-delay 2 --retry-max-time 30 \ --output "$PUBLIC_HTML_PATH" \ - --write-out '%{http_code}\t%{content_type}\t%{url_effective}' \ + --write-out '%{http_code}\t%{content_type}\t%{url_effective}\n' \ > "$RESPONSE_META_PATH" \ "$PUBLIC_URL"; then echo "Canonical walkthrough is not publicly available; skipping repair." >> "$GITHUB_STEP_SUMMARY" diff --git a/apps/backend/cmd/mock-agent/main.go b/apps/backend/cmd/mock-agent/main.go index bb75fd41ff..ef8ae26333 100644 --- a/apps/backend/cmd/mock-agent/main.go +++ b/apps/backend/cmd/mock-agent/main.go @@ -24,6 +24,9 @@ const ( modelFast = "mock-fast" modelSmart = "mock-smart" modelSlow = "mock-slow" + modelUnique = "opus[1m]" + modelAmbiguousFirst = "opus[270k]" + modelAmbiguousLast = "opus[1m, fast]" reasoningEffortLow = "low" reasoningEffortMed = "medium" reasoningEffortHigh = "high" @@ -207,22 +210,24 @@ func mockSessionConfigOptionsForModel(model string) []acp.SessionConfigOption { {Value: "max", Name: "Max", Description: ptr("Use maximum reasoning")}, } } + modelOptions := acp.SessionConfigSelectOptionsUngrouped{ + {Value: modelFast, Name: "Mock Fast", Description: ptr("Fast mock model for testing")}, + {Value: modelSmart, Name: "Mock Smart", Description: ptr("Smart mock model for testing")}, + // E2E fixtures use "mock-slow" for the slow-response delay tier. + // It must be advertised so the no-silent-model-fallback strict + // policy (which fails session start when the profile model is + // absent from the advertised list) does not reject it. + {Value: modelSlow, Name: "Mock Slow", Description: ptr("Slow mock model for testing")}, + } + modelOptions = append(modelOptions, mockModelVariationOptions()...) return []acp.SessionConfigOption{ {Select: &acp.SessionConfigOptionSelect{ Category: &modelCat, CurrentValue: acp.SessionConfigValueId(model), Id: "model", Name: "Model", - Options: acp.SessionConfigSelectOptions{Ungrouped: &acp.SessionConfigSelectOptionsUngrouped{ - {Value: modelFast, Name: "Mock Fast", Description: ptr("Fast mock model for testing")}, - {Value: modelSmart, Name: "Mock Smart", Description: ptr("Smart mock model for testing")}, - // E2E fixtures use "mock-slow" for the slow-response delay tier. - // It must be advertised so the no-silent-model-fallback strict - // policy (which fails session start when the profile model is - // absent from the advertised list) does not reject it. - {Value: modelSlow, Name: "Mock Slow", Description: ptr("Slow mock model for testing")}, - }}, - Type: "select", + Options: acp.SessionConfigSelectOptions{Ungrouped: &modelOptions}, + Type: "select", }}, {Select: &acp.SessionConfigOptionSelect{ Category: &modeCat, @@ -247,6 +252,22 @@ func mockSessionConfigOptionsForModel(model string) []acp.SessionConfigOption { } } +func mockModelVariationOptions() []acp.SessionConfigSelectOption { + catalog := strings.ToLower(strings.TrimSpace(os.Getenv("MOCK_AGENT_MODEL_CATALOG"))) + if catalog == "ambiguous" { + return []acp.SessionConfigSelectOption{ + {Value: modelAmbiguousFirst, Name: "Opus (270k)", Description: ptr("Ambiguous variation fixture")}, + {Value: modelAmbiguousLast, Name: "Opus (1m, fast)", Description: ptr("Ambiguous variation fixture")}, + } + } + if catalog == "unique" || (catalog == "" && strings.EqualFold(os.Getenv("KANDEV_E2E_MOCK"), "true")) { + return []acp.SessionConfigSelectOption{ + {Value: modelUnique, Name: "Opus (1m)", Description: ptr("Unique variation fixture")}, + } + } + return nil +} + func cloneSessionConfigOptions(options []acp.SessionConfigOption) []acp.SessionConfigOption { cloned := make([]acp.SessionConfigOption, len(options)) copy(cloned, options) diff --git a/apps/backend/cmd/mock-agent/session_config_test.go b/apps/backend/cmd/mock-agent/session_config_test.go index 8d0dbf681f..fb84eb6239 100644 --- a/apps/backend/cmd/mock-agent/session_config_test.go +++ b/apps/backend/cmd/mock-agent/session_config_test.go @@ -224,6 +224,26 @@ func TestMockSessionConfigOptionsForModelAdvertisesSlowModel(t *testing.T) { } } +func TestMockSessionConfigOptionsVariationCatalogs(t *testing.T) { + t.Setenv("KANDEV_E2E_MOCK", "true") + t.Setenv("MOCK_AGENT_MODEL_CATALOG", "") + unique := modelIDs(findModelOption(t)) + if !containsString(unique, modelUnique) { + t.Fatalf("unique catalog = %v, missing %s", unique, modelUnique) + } + if containsString(unique, modelAmbiguousFirst) || containsString(unique, modelAmbiguousLast) { + t.Fatalf("unique catalog = %v, must not include ambiguous variations", unique) + } + + t.Setenv("MOCK_AGENT_MODEL_CATALOG", "ambiguous") + ambiguous := modelIDs(findModelOption(t)) + for _, want := range []string{modelAmbiguousFirst, modelAmbiguousLast} { + if !containsString(ambiguous, want) { + t.Errorf("ambiguous catalog = %v, missing %s", ambiguous, want) + } + } +} + func findModelOption(t *testing.T) acp.SessionConfigOption { t.Helper() for _, option := range mockSessionConfigOptionsForModel(modelFast) { @@ -235,6 +255,17 @@ func findModelOption(t *testing.T) acp.SessionConfigOption { return acp.SessionConfigOption{} } +func modelIDs(option acp.SessionConfigOption) []string { + if option.Select == nil || option.Select.Options.Ungrouped == nil { + return nil + } + ids := make([]string, 0, len(*option.Select.Options.Ungrouped)) + for _, model := range *option.Select.Options.Ungrouped { + ids = append(ids, string(model.Value)) + } + return ids +} + func containsString(values []string, want string) bool { for _, v := range values { if v == want { diff --git a/apps/backend/internal/agent/runtime/lifecycle/manager_interaction.go b/apps/backend/internal/agent/runtime/lifecycle/manager_interaction.go index dc4a5e6022..9937d6c8ca 100644 --- a/apps/backend/internal/agent/runtime/lifecycle/manager_interaction.go +++ b/apps/backend/internal/agent/runtime/lifecycle/manager_interaction.go @@ -543,7 +543,9 @@ func (m *Manager) reapplySessionModelAfterReset( m.sessionManager.publishModelSelectionWarningEvent(execution, newSessionID, decision) } if decision.EffectiveModel != "" && - (decision.Outcome == ModelSelectionOutcomeApplied || decision.Outcome == ModelSelectionOutcomeExplicitFallback) { + (decision.Outcome == ModelSelectionOutcomeApplied || + decision.Outcome == ModelSelectionOutcomeExplicitFallback || + decision.Outcome == ModelSelectionOutcomeUniqueVariation) { m.logger.Info("re-applied session model after context reset", zap.String("execution_id", execution.ID), zap.String("session_id", execution.SessionID), diff --git a/apps/backend/internal/agent/runtime/lifecycle/manager_interaction_runtime_config_reset_test.go b/apps/backend/internal/agent/runtime/lifecycle/manager_interaction_runtime_config_reset_test.go index 44f72d4fbc..873e28f25f 100644 --- a/apps/backend/internal/agent/runtime/lifecycle/manager_interaction_runtime_config_reset_test.go +++ b/apps/backend/internal/agent/runtime/lifecycle/manager_interaction_runtime_config_reset_test.go @@ -99,6 +99,51 @@ func TestManager_ResetAgentContext_ReappliesSessionRuntimeConfig(t *testing.T) { require.Equal(t, v1.AgentStatusReady, exec.Status) } +func TestReapplySessionModel_UsesUniqueAdvertisedVariation(t *testing.T) { + mgr := newTestManager(t) + mock := newRestartMockAgentctlServer(t, false, false) + client := createTestClient(t, mock.server.URL) + t.Cleanup(client.Close) + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) + t.Cleanup(cancel) + require.NoError(t, client.StreamUpdates(ctx, func(agentctl.AgentEvent) {}, nil, nil)) + + exec := runtimeConfigResetExecution(client, true) + exec.SetModelState(&CachedModelState{ + CurrentModelID: "provider-default", + Models: []streams.SessionModelInfo{{ModelID: "opus[1m]"}}, + }) + + require.NoError(t, mgr.reapplySessionModelAfterReset(ctx, exec, "reset-session", "opus")) + require.Equal(t, []string{"opus[1m]"}, mock.getSetModelIDs()) +} + +func TestWorkspaceRebindModel_UsesUniqueAdvertisedVariation(t *testing.T) { + mgr := newTestManager(t) + mock := newRestartMockAgentctlServer(t, false, false) + client := createTestClient(t, mock.server.URL) + t.Cleanup(client.Close) + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) + t.Cleanup(cancel) + require.NoError(t, client.StreamUpdates(ctx, func(agentctl.AgentEvent) {}, nil, nil)) + + exec := runtimeConfigResetExecution(client, true) + exec.SetModelState(&CachedModelState{ + CurrentModelID: "provider-default", + Models: []streams.SessionModelInfo{{ModelID: "opus[1m]"}}, + }) + + require.NoError(t, mgr.reapplyReboundSessionConfig( + ctx, + exec, + "rebind-session", + &CachedModelState{}, + "opus", + nil, + )) + require.Equal(t, []string{"opus[1m]"}, mock.getSetModelIDs()) +} + func TestManager_RestartAgentProcess_ReappliesSessionRuntimeConfig(t *testing.T) { mgr := newTestManager(t) provider := runtimeConfigResetWorkspaceInfo() diff --git a/apps/backend/internal/agent/runtime/lifecycle/session.go b/apps/backend/internal/agent/runtime/lifecycle/session.go index fb89a7f757..210a09c600 100644 --- a/apps/backend/internal/agent/runtime/lifecycle/session.go +++ b/apps/backend/internal/agent/runtime/lifecycle/session.go @@ -513,10 +513,12 @@ func (sm *SessionManager) applyStartModelPolicyToEffectiveModel( } effective.decision = decision if decision.Outcome == ModelSelectionOutcomeApplied || - decision.Outcome == ModelSelectionOutcomeExplicitFallback { + decision.Outcome == ModelSelectionOutcomeExplicitFallback || + decision.Outcome == ModelSelectionOutcomeUniqueVariation { effective.appliedModel = decision.EffectiveModel } - if decision.Outcome == ModelSelectionOutcomeExplicitFallback { + if decision.Outcome == ModelSelectionOutcomeExplicitFallback || + decision.Outcome == ModelSelectionOutcomeUniqueVariation { effective.model = decision.EffectiveModel } return effective diff --git a/apps/backend/internal/agent/runtime/lifecycle/session_model_selection_warning_test.go b/apps/backend/internal/agent/runtime/lifecycle/session_model_selection_warning_test.go new file mode 100644 index 0000000000..0c10f33782 --- /dev/null +++ b/apps/backend/internal/agent/runtime/lifecycle/session_model_selection_warning_test.go @@ -0,0 +1,100 @@ +package lifecycle + +import ( + "context" + "testing" + "time" + + "github.com/kandev/kandev/internal/agent/agents" + agentctl "github.com/kandev/kandev/internal/agent/runtime/agentctl" + "github.com/kandev/kandev/internal/agentctl/types/streams" + "github.com/kandev/kandev/pkg/agent" +) + +func TestInitializeAndPrompt_UniqueVariationPublishesProviderNeutralWarning(t *testing.T) { + mock := newMockAgentServer(t) + defer mock.Close() + + log := newSessionTestLogger() + stopCh := newTestStopCh(t) + sm := NewSessionManager(log, stopCh) + streamMgr := NewStreamManager(log, StreamCallbacks{ + OnAgentEvent: func(execution *AgentExecution, event agentctl.AgentEvent) {}, + }, nil, stopCh) + cleanupStreamManager(t, stopCh, streamMgr) + eventBus := &MockEventBusWithTracking{} + sm.SetDependencies(NewEventPublisher(eventBus, log), streamMgr, nil, nil) + + client := createTestClient(t, mock.server.URL) + defer client.Close() + execution := &AgentExecution{ + ID: "exec-unique-variation", + TaskID: "task-1", + SessionID: "session-1", + WorkspacePath: "/workspace", + agentctl: client, + promptDoneCh: make(chan PromptCompletionSignal, 1), + } + execution.SetModelState(modelState("opus[1m]")) + agentConfig := &testAgent{ + id: "test-agent", + enabled: true, + runtimeConfig: &agents.RuntimeConfig{ + Cmd: agents.NewCommand("test-agent"), + Protocol: agent.ProtocolACP, + SessionConfig: agents.SessionConfig{}, + ResourceLimits: agents.ResourceLimits{MemoryMB: 512, CPUCores: 0.5, Timeout: time.Hour}, + }, + } + + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + defer cancel() + err := sm.InitializeAndPrompt(ctx, execution, agentConfig, "", nil, nil, + func(executionID string) error { return nil }, + StartModelPolicy{Model: "opus"}, "plan", nil) + if err != nil { + t.Fatalf("InitializeAndPrompt failed: %v", err) + } + + setModelCalls := 0 + for _, action := range mock.getActionLog() { + if action == "agent.session.set_model" { + setModelCalls++ + } + } + if setModelCalls != 1 { + t.Fatalf("set_model calls = %d, want exactly 1", setModelCalls) + } + + var warning *streams.ModelSelectionWarning + legacyFallbackEvents := 0 + for _, event := range eventBus.getStreamEvents() { + if event.Data == nil { + continue + } + if event.Data.ModelSelectionWarning != nil { + warning = event.Data.ModelSelectionWarning + } + if event.Data.Type == streams.EventTypeSessionModelFallback { + legacyFallbackEvents++ + } + } + if warning == nil { + t.Fatal("expected a model-selection warning event") + } + if warning.RequestedModel != "opus" { + t.Errorf("warning requested model = %q, want opus", warning.RequestedModel) + } + if warning.EffectiveModel != "opus[1m]" { + t.Errorf("warning effective model = %q, want opus[1m]", warning.EffectiveModel) + } + if warning.FallbackModel != "" { + t.Errorf("warning fallback model = %q, want empty", warning.FallbackModel) + } + if warning.Reason != ModelSelectionReasonUniqueVariationApplied { + t.Errorf("warning reason = %q, want %q", warning.Reason, ModelSelectionReasonUniqueVariationApplied) + } + if legacyFallbackEvents != 0 { + t.Fatalf("legacy fallback events = %d, want zero for unique variation", legacyFallbackEvents) + } +} diff --git a/apps/backend/internal/agent/runtime/lifecycle/start_model.go b/apps/backend/internal/agent/runtime/lifecycle/start_model.go index 4606acec5f..9d8099f8d7 100644 --- a/apps/backend/internal/agent/runtime/lifecycle/start_model.go +++ b/apps/backend/internal/agent/runtime/lifecycle/start_model.go @@ -3,6 +3,7 @@ package lifecycle import ( "context" "fmt" + "strings" "github.com/kandev/kandev/internal/agentctl/sessionmodel" "github.com/kandev/kandev/internal/common/logger" @@ -31,6 +32,7 @@ const ( ModelSelectionOutcomeNone ModelSelectionOutcome = "" ModelSelectionOutcomeApplied ModelSelectionOutcome = "applied" ModelSelectionOutcomeExplicitFallback ModelSelectionOutcome = "explicit_fallback" + ModelSelectionOutcomeUniqueVariation ModelSelectionOutcome = "unique_variation" ModelSelectionOutcomeProviderDefault ModelSelectionOutcome = "provider_default" ) @@ -42,6 +44,7 @@ const ( ModelSelectionReasonCatalogEmpty = "catalog_empty" ModelSelectionReasonSelectionUnsupported = "selection_unsupported" ModelSelectionReasonSelectionFailedAutoFallback = "selection_failed_auto_fallback" + ModelSelectionReasonUniqueVariationApplied = "unique_variation_applied" ) // ModelSelectionDecision is the single model-selection result shared by @@ -84,6 +87,32 @@ func containsModel(ids []string, id string) bool { return false } +func uniqueAdvertisedModelVariation(requested string, advertised []string) string { + if requested == "" || strings.ContainsAny(requested, "[]") { + return "" + } + + prefix := requested + "[" + candidates := make(map[string]struct{}) + for _, id := range advertised { + if !strings.HasPrefix(id, prefix) || !strings.HasSuffix(id, "]") { + continue + } + variant := strings.TrimSuffix(strings.TrimPrefix(id, prefix), "]") + if variant == "" || strings.ContainsAny(variant, "[]") { + continue + } + candidates[id] = struct{}{} + if len(candidates) > 1 { + return "" + } + } + for candidate := range candidates { + return candidate + } + return "" +} + func providerDefaultDecision(state *CachedModelState, policy StartModelPolicy, reason string) ModelSelectionDecision { decision := ModelSelectionDecision{ RequestedModel: policy.Model, @@ -114,6 +143,9 @@ func applyStartModelPolicy( if policy.Model == "" { return ModelSelectionDecision{Outcome: ModelSelectionOutcomeNone}, nil } + if policy.AutoFallback { + policy.FallbackModel = "" + } decision := ModelSelectionDecision{ RequestedModel: policy.Model, @@ -125,9 +157,15 @@ func applyStartModelPolicy( } if !containsModel(advertised, policy.Model) { + if policy.AutoFallback { + return providerDefaultDecision(state, policy, ModelSelectionReasonRequestedNotAdvertised), nil + } if policy.FallbackModel != "" && containsModel(advertised, policy.FallbackModel) { return applyAdvertisedFallback(ctx, log, applier, state, policy, decision) } + if variation := uniqueAdvertisedModelVariation(policy.Model, advertised); variation != "" { + return applyUniqueAdvertisedVariation(ctx, log, applier, state, policy, decision, variation) + } reason := ModelSelectionReasonRequestedNotAdvertised if policy.FallbackModel != "" { reason = ModelSelectionReasonFallbackNotAdvertised @@ -159,6 +197,41 @@ func applyStartModelPolicy( return decision, nil } +func applyUniqueAdvertisedVariation( + ctx context.Context, + log *logger.Logger, + applier modelApplier, + state *CachedModelState, + policy StartModelPolicy, + decision ModelSelectionDecision, + variation string, +) (ModelSelectionDecision, error) { + decision.FallbackModel = "" + policy.FallbackModel = "" + decision.SetModelCalled = true + if err := applier.SetModel(ctx, variation); err != nil { + if sessionmodel.IsMethodNotFound(err) { + decision = providerDefaultDecision(state, policy, ModelSelectionReasonSelectionUnsupported) + decision.SetModelCalled = true + return decision, nil + } + if policy.AutoFallback { + decision = providerDefaultDecision(state, policy, ModelSelectionReasonSelectionFailedAutoFallback) + decision.SetModelCalled = true + return decision, nil + } + return decision, fmt.Errorf("failed to set unique model variation %q: %w", variation, err) + } + decision.EffectiveModel = variation + decision.Outcome = ModelSelectionOutcomeUniqueVariation + decision.Reason = ModelSelectionReasonUniqueVariationApplied + decision.Warning = true + log.Info("start model unavailable, using unique advertised variation", + zap.String("start_model", policy.Model), + zap.String("variation_model", variation)) + return decision, nil +} + func applyAdvertisedFallback( ctx context.Context, log *logger.Logger, diff --git a/apps/backend/internal/agent/runtime/lifecycle/start_model_executor_authority_test.go b/apps/backend/internal/agent/runtime/lifecycle/start_model_executor_authority_test.go index 02a8514769..405c9b679b 100644 --- a/apps/backend/internal/agent/runtime/lifecycle/start_model_executor_authority_test.go +++ b/apps/backend/internal/agent/runtime/lifecycle/start_model_executor_authority_test.go @@ -47,6 +47,104 @@ func TestApplyStartModelPolicyExecutorAuthority(t *testing.T) { wantEffective: "fallback", wantWarning: true, }, + { + name: "unique variation is applied when fallback is absent", + state: modelState("opus[1m]"), + policy: StartModelPolicy{Model: "opus"}, + wantCalls: []string{"opus[1m]"}, + wantOutcome: ModelSelectionOutcomeUniqueVariation, + wantReason: ModelSelectionReasonUniqueVariationApplied, + wantEffective: "opus[1m]", + wantWarning: true, + }, + { + name: "advertised fallback takes precedence over unique variation", + state: modelState("fallback", "opus[1m]"), + policy: StartModelPolicy{Model: "opus", FallbackModel: "fallback"}, + wantCalls: []string{"fallback"}, + wantOutcome: ModelSelectionOutcomeExplicitFallback, + wantReason: ModelSelectionReasonRequestedNotAdvertised, + wantEffective: "fallback", + wantWarning: true, + }, + { + name: "unadvertised fallback does not block unique variation", + state: modelState("opus[1m]"), + policy: StartModelPolicy{Model: "opus", FallbackModel: "fallback"}, + wantCalls: []string{"opus[1m]"}, + wantOutcome: ModelSelectionOutcomeUniqueVariation, + wantReason: ModelSelectionReasonUniqueVariationApplied, + wantEffective: "opus[1m]", + wantWarning: true, + }, + { + name: "auto fallback keeps unavailable model on provider default", + state: modelState("fallback", "opus[1m]"), + policy: StartModelPolicy{Model: "opus", FallbackModel: "fallback", AutoFallback: true}, + wantOutcome: ModelSelectionOutcomeProviderDefault, + wantReason: ModelSelectionReasonRequestedNotAdvertised, + wantWarning: true, + }, + { + name: "duplicate unique variation is one candidate", + state: modelState("opus[1m]", "opus[1m]"), + policy: StartModelPolicy{Model: "opus"}, + wantCalls: []string{"opus[1m]"}, + wantOutcome: ModelSelectionOutcomeUniqueVariation, + wantReason: ModelSelectionReasonUniqueVariationApplied, + wantEffective: "opus[1m]", + wantWarning: true, + }, + { + name: "multiple variations keep provider default", + state: modelState("opus[1m]", "opus[270k]"), + policy: StartModelPolicy{Model: "opus"}, + wantOutcome: ModelSelectionOutcomeProviderDefault, + wantReason: ModelSelectionReasonRequestedNotAdvertised, + wantWarning: true, + }, + { + name: "malformed variations keep provider default", + state: modelState("opus[]", "opus[1m", "opus[1m][fast]", "opus-pro[1m]"), + policy: StartModelPolicy{Model: "opus"}, + wantOutcome: ModelSelectionOutcomeProviderDefault, + wantReason: ModelSelectionReasonRequestedNotAdvertised, + wantWarning: true, + }, + { + name: "variation text remains opaque", + state: modelState("opus[1m, fast]"), + policy: StartModelPolicy{Model: "opus"}, + wantCalls: []string{"opus[1m, fast]"}, + wantOutcome: ModelSelectionOutcomeUniqueVariation, + wantReason: ModelSelectionReasonUniqueVariationApplied, + wantEffective: "opus[1m, fast]", + wantWarning: true, + }, + { + name: "case-sensitive variation matching keeps provider default", + state: modelState("Opus[1m]"), + policy: StartModelPolicy{Model: "opus"}, + wantOutcome: ModelSelectionOutcomeProviderDefault, + wantReason: ModelSelectionReasonRequestedNotAdvertised, + wantWarning: true, + }, + { + name: "bracketed request does not infer another variation", + state: modelState("opus[2m]"), + policy: StartModelPolicy{Model: "opus[1m]"}, + wantOutcome: ModelSelectionOutcomeProviderDefault, + wantReason: ModelSelectionReasonRequestedNotAdvertised, + wantWarning: true, + }, + { + name: "exact bracketed request remains exact", + state: modelState("opus[1m]", "opus[2m]"), + policy: StartModelPolicy{Model: "opus[1m]"}, + wantCalls: []string{"opus[1m]"}, + wantOutcome: ModelSelectionOutcomeApplied, + wantEffective: "opus[1m]", + }, { name: "advertised fallback method not supported keeps provider default", state: modelState("fallback"), @@ -57,6 +155,16 @@ func TestApplyStartModelPolicyExecutorAuthority(t *testing.T) { wantReason: ModelSelectionReasonSelectionUnsupported, wantWarning: true, }, + { + name: "unique variation method not supported keeps provider default", + state: modelState("opus[1m]"), + policy: StartModelPolicy{Model: "opus", FallbackModel: "fallback"}, + applierErrors: []error{methodNotFoundErr()}, + wantCalls: []string{"opus[1m]"}, + wantOutcome: ModelSelectionOutcomeProviderDefault, + wantReason: ModelSelectionReasonSelectionUnsupported, + wantWarning: true, + }, { name: "unadvertised fallback keeps provider default", state: modelState("executor-default"), @@ -132,6 +240,12 @@ func TestApplyStartModelPolicyExecutorAuthority(t *testing.T) { if decision.Warning != tt.wantWarning { t.Errorf("warning = %v, want %v", decision.Warning, tt.wantWarning) } + if decision.Outcome == ModelSelectionOutcomeUniqueVariation && decision.FallbackModel != "" { + t.Errorf("unique variation fallback model = %q, want empty", decision.FallbackModel) + } + if tt.policy.AutoFallback && decision.FallbackModel != "" { + t.Errorf("auto-fallback fallback model = %q, want empty", decision.FallbackModel) + } }) } } diff --git a/apps/web/components/settings/profile-form-fields.test.tsx b/apps/web/components/settings/profile-form-fields.test.tsx index 431b6984d0..ed55dc62b2 100644 --- a/apps/web/components/settings/profile-form-fields.test.tsx +++ b/apps/web/components/settings/profile-form-fields.test.tsx @@ -150,6 +150,20 @@ describe("ProfileFormFields no-silent-model-fallback rows", () => { expect(trigger.textContent).toContain("claude-gone"); }); + it("names a unique advertised variation while preserving the saved model", () => { + renderForm(formData({ model: "opus" }), { + ...modelConfig, + available_models: [{ id: "opus[1m]", name: "Opus (1m)" }], + }); + + const advisory = screen.getByTestId("profile-model-variation-advisory"); + expect(advisory.textContent).toContain("opus[1m]"); + expect(advisory.textContent).toContain("opus"); + expect( + screen.getByRole("button", { name: profileStartModelSettingsLabel }).textContent, + ).toContain("opus"); + }); + it("shows the agent fallback row when auto-fallback is off", () => { renderForm(formData({ auto_fallback: false })); expandFallbackSettings(); diff --git a/apps/web/components/settings/profile-model-fields.tsx b/apps/web/components/settings/profile-model-fields.tsx index 65295fa896..c558d7059e 100644 --- a/apps/web/components/settings/profile-model-fields.tsx +++ b/apps/web/components/settings/profile-model-fields.tsx @@ -23,6 +23,7 @@ import { import type { ModelConfig, ModeEntry, ModelEntry } from "@/lib/types/http"; import type { PermissionKey } from "@/lib/agent-permissions"; import type { CLIFlag } from "@/lib/types/http"; +import { findUniqueModelVariation } from "@/lib/model-variation"; import { SettingsFieldDescription, SettingsFieldLabel, @@ -83,6 +84,12 @@ export function ModelPicker({ // in the list, greyed out and unselectable, so the user sees what was // configured instead of it silently vanishing. const modelIsGone = Boolean(profile.model && !modelOptions.some((m) => m.id === profile.model)); + const uniqueVariation = modelIsGone + ? findUniqueModelVariation( + profile.model, + modelOptions.map((model) => model.id), + ) + : null; if (modelIsGone) { modelOptions.unshift({ id: profile.model!, @@ -99,22 +106,32 @@ export function ModelPicker({ })); return ( - onChange({ model: value })} - onConfigChange={(configId, value) => - onChange({ config_options: { ...(profile.config_options ?? {}), [configId]: value } }) - } - placeholder={placeholder ?? t("settings:selectAModel")} - ariaLabel={ariaLabel} - popoverAlign="start" - disabled={disabled} - configOptionsLoading={configOptionsLoading} - keepOpenOnModelChange={keepOpenOnModelChange} - triggerClassName={modelIsGone ? "text-destructive" : undefined} - /> +
+ onChange({ model: value })} + onConfigChange={(configId, value) => + onChange({ config_options: { ...(profile.config_options ?? {}), [configId]: value } }) + } + placeholder={placeholder ?? t("settings:selectAModel")} + ariaLabel={ariaLabel} + popoverAlign="start" + disabled={disabled} + configOptionsLoading={configOptionsLoading} + keepOpenOnModelChange={keepOpenOnModelChange} + triggerClassName={modelIsGone ? "text-destructive" : undefined} + /> + {uniqueVariation && ( +

+ {t("settings:modelVariationAdvisory", { + model: profile.model, + variation: uniqueVariation, + })} +

+ )} +
); } diff --git a/apps/web/components/task-create-dialog-options.test.tsx b/apps/web/components/task-create-dialog-options.test.tsx index 79a1537e3c..20cd143607 100644 --- a/apps/web/components/task-create-dialog-options.test.tsx +++ b/apps/web/components/task-create-dialog-options.test.tsx @@ -58,6 +58,19 @@ const AGENT_WITH_GPT: AvailableAgent = { }, } as unknown as AvailableAgent; +const AGENT_WITH_OPUS_VARIATION: AvailableAgent = { + name: "omp-acp", + available: true, + model_config: { + default_model: "opus[1m]", + available_models: [{ id: "opus[1m]", name: "Opus (1m)" }], + current_model_id: "opus[1m]", + available_modes: [], + supports_dynamic_models: false, + status: "ok", + }, +} as unknown as AvailableAgent; + const GONE_MODEL = "claude-gone"; const DATA_DISABLED = "data-disabled"; const MODEL_PROBE_WARNING_TEST_ID = "agent-profile-model-probe-warning"; @@ -225,6 +238,14 @@ describe("useAgentProfileOptions executor-authoritative model hint", () => { expect(option.getAttribute(DATA_DISABLED)).toBeNull(); }); + it("names a unique advertised variation without disabling the profile", () => { + setAvailableAgents([AGENT_WITH_OPUS_VARIATION]); + const option = renderOptions([profileOption({ model: "opus" })]); + + expect(option.getAttribute(DATA_DISABLED)).toBeNull(); + expect(getModelProbeWarningLabel()).toContain("opus[1m]"); + }); + it("keeps a profile with an empty (agent default) model selectable", () => { const option = renderOptions([profileOption({ model: "" })]); expect(option.getAttribute(DATA_DISABLED)).toBeNull(); diff --git a/apps/web/components/task-create-dialog-options.tsx b/apps/web/components/task-create-dialog-options.tsx index dda5b710c7..509ec43606 100644 --- a/apps/web/components/task-create-dialog-options.tsx +++ b/apps/web/components/task-create-dialog-options.tsx @@ -32,6 +32,7 @@ import { formatUserHomePath, truncateRepoPath } from "@/lib/utils"; import { getExecutorIcon } from "@/lib/executor-icons"; import { AgentLogo } from "@/components/agent-logo"; import { getCapabilityWarning } from "@/lib/capability-warning"; +import { findUniqueModelVariation } from "@/lib/model-variation"; import { useTouchDrawer } from "@/hooks/use-compact-task-chrome"; import { buildBranchKeywords } from "./branch-picker-options"; import { @@ -229,9 +230,18 @@ export function useAgentProfileOptions( const startModelGone = Boolean( profile.model && advertised.length > 0 && !advertised.includes(profile.model), ); - const modelProbeNote = startModelGone - ? t("settings:profileStartModelNotAdvertisedOnHost", { model: profile.model }) - : undefined; + const uniqueVariation = startModelGone + ? findUniqueModelVariation(profile.model ?? "", advertised) + : null; + let modelProbeNote: string | undefined; + if (startModelGone) { + modelProbeNote = uniqueVariation + ? t("settings:profileStartModelUniqueVariationOnHost", { + model: profile.model, + variation: uniqueVariation, + }) + : t("settings:profileStartModelNotAdvertisedOnHost", { model: profile.model }); + } const renderProfileLabel = (modelProbeWarning: React.ReactNode) => ( diff --git a/apps/web/components/task/chat/messages/status-message.test.tsx b/apps/web/components/task/chat/messages/status-message.test.tsx index 84435f79a9..430d3a7083 100644 --- a/apps/web/components/task/chat/messages/status-message.test.tsx +++ b/apps/web/components/task/chat/messages/status-message.test.tsx @@ -42,6 +42,23 @@ describe("StatusMessage model selection warnings", () => { expect(screen.getByText("Check the agent version in the executor.")).toBeTruthy(); expect(screen.getByText("The saved model was not advertised by the executor.")).toBeTruthy(); }); + + it("explains when the executor applied the only advertised variation", () => { + const comment = modelSelectionWarningMessage(); + comment.metadata = { + ...comment.metadata, + reason: "unique_variation_applied", + effective_model: "opus[1m]", + fallback_model: undefined, + }; + + render(); + + expect( + screen.getByText("The executor applied the only advertised variation of the saved model."), + ).toBeTruthy(); + expect(screen.queryByText(/fallback model/i)).toBeNull(); + }); }); describe("StatusMessage branch replacement warnings", () => { diff --git a/apps/web/components/task/chat/messages/status-message.tsx b/apps/web/components/task/chat/messages/status-message.tsx index 3a5b7e245f..afc8e37440 100644 --- a/apps/web/components/task/chat/messages/status-message.tsx +++ b/apps/web/components/task/chat/messages/status-message.tsx @@ -180,6 +180,7 @@ function modelSelectionReasonLabel(reason: string | undefined): string { const keyByReason: Record = { requested_not_advertised: "task:modelSelectionReasonRequestedNotAdvertised", fallback_not_advertised: "task:modelSelectionReasonFallbackNotAdvertised", + unique_variation_applied: "task:modelSelectionReasonUniqueVariation", catalog_empty: "task:modelSelectionReasonCatalogEmpty", selection_unsupported: "task:modelSelectionReasonUnsupported", selection_failed_auto_fallback: "task:modelSelectionReasonAutoFallback", diff --git a/apps/web/e2e/tests/session/model-mismatch-warning-helpers.ts b/apps/web/e2e/tests/session/model-mismatch-warning-helpers.ts index 0fbe860ee9..67a15a9eba 100644 --- a/apps/web/e2e/tests/session/model-mismatch-warning-helpers.ts +++ b/apps/web/e2e/tests/session/model-mismatch-warning-helpers.ts @@ -2,6 +2,10 @@ import type { AgentProfile } from "../../../lib/types/http-agents"; import type { ApiClient } from "../../helpers/api-client"; export const UNADVERTISED_MODEL = "host-only-model"; +export const MODEL_VARIATION_BASE = "opus"; +export const UNIQUE_MODEL_VARIATION = "opus[1m]"; +export const AMBIGUOUS_MODEL_VARIATIONS = ["opus[270k]", "opus[1m, fast]"] as const; +const MODEL_CATALOG_ENV = "MOCK_AGENT_MODEL_CATALOG"; export async function createMismatchedProfile( apiClient: ApiClient, @@ -13,6 +17,20 @@ export async function createMismatchedProfile( return apiClient.createAgentProfile(agent.id, name, { model: UNADVERTISED_MODEL }); } +export async function createModelVariationProfile( + apiClient: ApiClient, + name: string, + catalog: "unique" | "ambiguous", +): Promise { + const { agents } = await apiClient.listAgents(); + const agent = agents.find((item) => item.name === "mock-agent") ?? agents[0]; + if (!agent) throw new Error("The E2E fixture must provide a mock agent"); + return apiClient.createAgentProfile(agent.id, name, { + model: MODEL_VARIATION_BASE, + env_vars: [{ key: MODEL_CATALOG_ENV, value: catalog }], + }); +} + export async function readModelSelectionWarnings( apiClient: ApiClient, sessionId: string, diff --git a/apps/web/e2e/tests/session/model-mismatch-warning.spec.ts b/apps/web/e2e/tests/session/model-mismatch-warning.spec.ts index 0be82cf950..6f5f961f9c 100644 --- a/apps/web/e2e/tests/session/model-mismatch-warning.spec.ts +++ b/apps/web/e2e/tests/session/model-mismatch-warning.spec.ts @@ -2,7 +2,10 @@ import { expect, test } from "../../fixtures/test-base"; import { waitForSessionDone } from "../../helpers/session"; import { SessionPage } from "../../pages/session-page"; import { + createModelVariationProfile, createMismatchedProfile, + MODEL_VARIATION_BASE, + UNIQUE_MODEL_VARIATION, readModelSelectionWarnings, UNADVERTISED_MODEL, } from "./model-mismatch-warning-helpers"; @@ -73,4 +76,124 @@ test.describe("model mismatch warning", () => { await apiClient.deleteAgentProfile(profile.id, true).catch(() => {}); } }); + + test("applies one unique advertised variation and persists its warning", async ({ + testPage, + apiClient, + seedData, + }) => { + test.setTimeout(120_000); + const profile = await createModelVariationProfile( + apiClient, + "Unique model variation profile", + "unique", + ); + try { + const task = await apiClient.createTaskWithAgent( + seedData.workspaceId, + "Unique model variation warning", + profile.id, + { + description: "/e2e:simple-message", + workflow_id: seedData.workflowId, + workflow_step_id: seedData.startStepId, + repository_ids: [seedData.repositoryId], + }, + ); + expect(task.session_id).toBeTruthy(); + await waitForSessionDone( + apiClient, + task.id, + task.session_id!, + "Waiting for unique model variation task continuation", + ); + + await expect + .poll(() => readModelSelectionWarnings(apiClient, task.session_id!), { + timeout: 15_000, + message: "Waiting for the unique model variation warning", + }) + .toHaveLength(1); + const warnings = await readModelSelectionWarnings(apiClient, task.session_id!); + expect(warnings[0]?.metadata).toMatchObject({ + kind: "model_selection_warning", + reason: "unique_variation_applied", + requested_model: MODEL_VARIATION_BASE, + effective_model: UNIQUE_MODEL_VARIATION, + agent_id: "mock-agent", + }); + expect(warnings[0]?.metadata?.fallback_model).toBeUndefined(); + expect((await apiClient.getAgentProfile(profile.id)).model).toBe(MODEL_VARIATION_BASE); + + await testPage.goto(`/t/${task.id}`); + const session = new SessionPage(testPage); + await session.waitForLoad(); + await expect( + session.activeChat().getByText("The executor could not use the saved model selection."), + ).toBeVisible({ timeout: 15_000 }); + await expect(session.activeChat()).toContainText(UNIQUE_MODEL_VARIATION); + + await testPage.reload(); + await session.waitForLoad(); + await expect( + session.activeChat().getByText("The executor could not use the saved model selection."), + ).toBeVisible({ timeout: 15_000 }); + await expect(session.activeChat()).toContainText(UNIQUE_MODEL_VARIATION); + await expect + .poll(() => readModelSelectionWarnings(apiClient, task.session_id!)) + .toHaveLength(1); + } finally { + await apiClient.deleteAgentProfile(profile.id, true).catch(() => {}); + } + }); + + test("does not infer a model when the executor advertises multiple variations", async ({ + apiClient, + seedData, + }) => { + test.setTimeout(120_000); + const profile = await createModelVariationProfile( + apiClient, + "Ambiguous model variation profile", + "ambiguous", + ); + try { + const task = await apiClient.createTaskWithAgent( + seedData.workspaceId, + "Ambiguous model variation warning", + profile.id, + { + description: "/e2e:simple-message", + workflow_id: seedData.workflowId, + workflow_step_id: seedData.startStepId, + repository_ids: [seedData.repositoryId], + }, + ); + expect(task.session_id).toBeTruthy(); + await waitForSessionDone( + apiClient, + task.id, + task.session_id!, + "Waiting for ambiguous model variation task continuation", + ); + + await expect + .poll(() => readModelSelectionWarnings(apiClient, task.session_id!), { + timeout: 15_000, + message: "Waiting for the ambiguous model variation warning", + }) + .toHaveLength(1); + const warnings = await readModelSelectionWarnings(apiClient, task.session_id!); + expect(warnings[0]?.metadata).toMatchObject({ + kind: "model_selection_warning", + reason: "requested_not_advertised", + requested_model: MODEL_VARIATION_BASE, + effective_model: "mock-fast", + agent_id: "mock-agent", + }); + expect((await apiClient.getAgentProfile(profile.id)).model).toBe(MODEL_VARIATION_BASE); + } finally { + await apiClient.deleteAgentProfile(profile.id, true).catch(() => {}); + } + }); }); diff --git a/apps/web/e2e/tests/settings/mobile-no-silent-model-fallback.spec.ts b/apps/web/e2e/tests/settings/mobile-no-silent-model-fallback.spec.ts index 1a9a0e3d2e..9b77ccbdaa 100644 --- a/apps/web/e2e/tests/settings/mobile-no-silent-model-fallback.spec.ts +++ b/apps/web/e2e/tests/settings/mobile-no-silent-model-fallback.spec.ts @@ -2,7 +2,10 @@ import { expect, test } from "../../fixtures/test-base"; import { KanbanPage } from "../../pages/kanban-page"; import { assertNoDocumentHorizontalOverflow } from "../../helpers/layout-assertions"; import { + createModelVariationProfile, createMismatchedProfile, + MODEL_VARIATION_BASE, + UNIQUE_MODEL_VARIATION, UNADVERTISED_MODEL, } from "../session/model-mismatch-warning-helpers"; @@ -41,4 +44,44 @@ test.describe("executor-authoritative model selection on mobile", () => { await apiClient.deleteAgentProfile(profile.id, true).catch(() => {}); } }); + + test("opens the unique variation advisory by touch without overflow", async ({ + testPage, + apiClient, + }) => { + const profile = await createModelVariationProfile( + apiClient, + "Mobile unique host variation profile", + "unique", + ); + try { + const kanban = new KanbanPage(testPage); + await kanban.goto(); + await testPage.reload({ waitUntil: "networkidle" }); + await testPage.getByRole("button", { name: "Add task" }).tap(); + + const dialog = testPage.getByTestId("create-task-dialog"); + await expect(dialog).toBeVisible(); + const selector = dialog.getByTestId("agent-profile-selector"); + await selector.tap(); + + const option = testPage + .getByRole("listbox") + .getByRole("option", { name: profile.name, exact: false }); + await expect(option).toBeVisible(); + await expect(option).toBeEnabled(); + const warning = option.getByTestId("agent-profile-model-probe-warning"); + await expect(warning).toBeVisible(); + const warningText = `The host probe found one possible variation of ${MODEL_VARIATION_BASE}: ${UNIQUE_MODEL_VARIATION}. The selected executor will decide the model at launch.`; + await warning.tap(); + await expect( + testPage + .locator('[data-slot="drawer-content"][data-state="open"]') + .filter({ hasText: warningText }), + ).toBeVisible(); + await assertNoDocumentHorizontalOverflow(testPage, "mobile unique variation advisory"); + } finally { + await apiClient.deleteAgentProfile(profile.id, true).catch(() => {}); + } + }); }); diff --git a/apps/web/e2e/tests/settings/no-silent-model-fallback.spec.ts b/apps/web/e2e/tests/settings/no-silent-model-fallback.spec.ts index 382ab9224b..852ce3d30a 100644 --- a/apps/web/e2e/tests/settings/no-silent-model-fallback.spec.ts +++ b/apps/web/e2e/tests/settings/no-silent-model-fallback.spec.ts @@ -1,7 +1,10 @@ import { expect, test } from "../../fixtures/test-base"; import { KanbanPage } from "../../pages/kanban-page"; import { + createModelVariationProfile, createMismatchedProfile, + MODEL_VARIATION_BASE, + UNIQUE_MODEL_VARIATION, UNADVERTISED_MODEL, } from "../session/model-mismatch-warning-helpers"; @@ -41,4 +44,44 @@ test.describe("executor-authoritative model selection", () => { await apiClient.deleteAgentProfile(profile.id, true).catch(() => {}); } }); + + test("names one host-advertised variation while keeping the profile selectable", async ({ + testPage, + apiClient, + }) => { + const profile = await createModelVariationProfile( + apiClient, + "Unique host variation selectable profile", + "unique", + ); + try { + const kanban = new KanbanPage(testPage); + await kanban.goto(); + await testPage.reload({ waitUntil: "networkidle" }); + await kanban.createTaskButton.first().click(); + + const dialog = testPage.getByTestId("create-task-dialog"); + await expect(dialog).toBeVisible(); + const selector = dialog.getByTestId("agent-profile-selector"); + await selector.click(); + + const option = testPage + .getByRole("listbox") + .getByRole("option", { name: profile.name, exact: false }); + await expect(option).toBeVisible(); + await expect(option).toBeEnabled(); + const warning = option.getByTestId("agent-profile-model-probe-warning"); + await expect(warning).toBeVisible(); + const warningText = `The host probe found one possible variation of ${MODEL_VARIATION_BASE}: ${UNIQUE_MODEL_VARIATION}. The selected executor will decide the model at launch.`; + await warning.hover(); + await expect( + testPage + .locator('[data-slot="tooltip-content"]:not([data-state="closed"])') + .filter({ hasText: warningText }), + ).toBeVisible(); + await option.click(); + } finally { + await apiClient.deleteAgentProfile(profile.id, true).catch(() => {}); + } + }); }); diff --git a/apps/web/lib/model-variation.test.ts b/apps/web/lib/model-variation.test.ts new file mode 100644 index 0000000000..b9aaa1655e --- /dev/null +++ b/apps/web/lib/model-variation.test.ts @@ -0,0 +1,57 @@ +import { describe, expect, it } from "vitest"; +import { findUniqueModelVariation } from "./model-variation"; + +describe("findUniqueModelVariation", () => { + it.each([ + { + name: "finds one bracketed variation", + requested: "opus", + advertised: ["opus[1m]"], + expected: "opus[1m]", + }, + { + name: "deduplicates the same variation", + requested: "opus", + advertised: ["opus[1m]", "opus[1m]"], + expected: "opus[1m]", + }, + { + name: "keeps opaque variation text", + requested: "opus", + advertised: ["opus[1m, fast]"], + expected: "opus[1m, fast]", + }, + ])("$name", ({ requested, advertised, expected }) => { + expect(findUniqueModelVariation(requested, advertised)).toBe(expected); + }); + + it.each([ + { + name: "returns no candidate for an empty catalog", + requested: "opus", + advertised: [], + }, + { + name: "returns no candidate for ambiguous variations", + requested: "opus", + advertised: ["opus[1m]", "opus[270k]"], + }, + { + name: "returns no candidate for case differences", + requested: "opus", + advertised: ["Opus[1m]"], + }, + { + name: "returns no candidate for a bracketed request", + requested: "opus[1m]", + advertised: ["opus[2m]"], + }, + { + name: "returns no candidate for malformed IDs", + requested: "opus", + advertised: ["opus[]", "opus[1m", "opus[1m][fast]", "opus-pro[1m]"], + }, + ])("$name", ({ requested, advertised }) => { + expect(findUniqueModelVariation(requested, advertised)).toBeNull(); + }); +}); diff --git a/apps/web/lib/model-variation.ts b/apps/web/lib/model-variation.ts new file mode 100644 index 0000000000..e9ce9066c4 --- /dev/null +++ b/apps/web/lib/model-variation.ts @@ -0,0 +1,17 @@ +function isModelVariation(requested: string, advertised: string): boolean { + const prefix = `${requested}[`; + if (!advertised.startsWith(prefix) || !advertised.endsWith("]")) return false; + + const variant = advertised.slice(prefix.length, -1); + return variant.length > 0 && !variant.includes("[") && !variant.includes("]"); +} + +export function findUniqueModelVariation( + requested: string, + advertised: readonly string[], +): string | null { + if (!requested || requested.includes("[") || requested.includes("]")) return null; + + const candidates = new Set(advertised.filter((model) => isModelVariation(requested, model))); + return candidates.size === 1 ? ([...candidates][0] ?? null) : null; +} diff --git a/apps/web/src/locales/en/settings.json b/apps/web/src/locales/en/settings.json index 67e30cf50f..54b2dc5a11 100644 --- a/apps/web/src/locales/en/settings.json +++ b/apps/web/src/locales/en/settings.json @@ -504,6 +504,8 @@ "profileForTasksCreatedByAgents": "Profile for Tasks Created by Agents", "programming": "Programming", "profileStartModelNotAdvertisedOnHost": "The host probe did not advertise {{model}}. The selected executor will decide the model at launch.", + "profileStartModelUniqueVariationOnHost": "The host probe found one possible variation of {{model}}: {{variation}}. The selected executor will decide the model at launch.", + "modelVariationAdvisory": "The host catalog has one possible variation of {{model}}: {{variation}}. The saved model remains unchanged until launch.", "promptAdd": "Add prompt", "promptEditorAriaLabel": "Prompt editor", "promptEditorBothHint": "Type {{placeholder}} for available placeholders or {{mention}} for saved prompts.", diff --git a/apps/web/src/locales/en/task.json b/apps/web/src/locales/en/task.json index c409169883..8df29a2f2c 100644 --- a/apps/web/src/locales/en/task.json +++ b/apps/web/src/locales/en/task.json @@ -883,6 +883,7 @@ "modelSelectionReasonCatalogEmpty": "The executor did not advertise a model catalog.", "modelSelectionReasonFallbackNotAdvertised": "The saved fallback model was not advertised by the executor.", "modelSelectionReasonRequestedNotAdvertised": "The saved model was not advertised by the executor.", + "modelSelectionReasonUniqueVariation": "The executor applied the only advertised variation of the saved model.", "modelSelectionReasonUnknown": "{{reason}}", "modelSelectionReasonUnsupported": "The executor does not support model selection.", "modelSelectionRemediationAgentVersion": "Check the agent version in the executor.", diff --git a/apps/web/src/locales/pseudo/settings.json b/apps/web/src/locales/pseudo/settings.json index b2b81d858d..854dc35322 100644 --- a/apps/web/src/locales/pseudo/settings.json +++ b/apps/web/src/locales/pseudo/settings.json @@ -504,6 +504,8 @@ "profileForTasksCreatedByAgents": "Ƥŕōƒĩĺē ƒōŕ Ţàśķś Ćŕēàţēď ƀŷ Àĝēńţś", "programming": "Ƥŕōĝŕàḿḿĩńĝ", "profileStartModelNotAdvertisedOnHost": "Ţĥē ĥōśţ ƥŕōƀē ďĩď ńōţ àďvēŕţĩśē {{model}}. Ţĥē śēĺēćţēď ēxēćũţōŕ ŵĩĺĺ ďēćĩďē ţĥē ḿōďēĺ àţ ĺàũńćĥ.", + "profileStartModelUniqueVariationOnHost": "Ţĥē ĥōśţ ƥŕōƀē ƒōũńď ōńē ƥōśśĩƀĺē vàŕĩàţĩōń ōƒ {{model}}: {{variation}}. Ţĥē śēĺēćţēď ēxēćũţōŕ ŵĩĺĺ ďēćĩďē ţĥē ḿōďēĺ àţ ĺàũńćĥ.", + "modelVariationAdvisory": "Ţĥē ĥōśţ ćàţàĺōĝ ĥàś ōńē ƥōśśĩƀĺē vàŕĩàţĩōń ōƒ {{model}}: {{variation}}. Ţĥē śàvēď ḿōďēĺ ŕēḿàĩńś ũńćĥàńĝēď ũńţĩĺ ĺàũńćĥ.", "promptAdd": "Àďď ƥŕōḿƥţ", "promptEditorAriaLabel": "Ƥŕōḿƥţ ēďĩţōŕ", "promptEditorBothHint": "Ţŷƥē {{placeholder}} ƒōŕ àvàĩĺàƀĺē ƥĺàćēĥōĺďēŕś ōŕ {{mention}} ƒōŕ śàvēď ƥŕōḿƥţś.", diff --git a/apps/web/src/locales/pseudo/task.json b/apps/web/src/locales/pseudo/task.json index 440ee95bcd..a6a12b2841 100644 --- a/apps/web/src/locales/pseudo/task.json +++ b/apps/web/src/locales/pseudo/task.json @@ -883,6 +883,7 @@ "modelSelectionReasonCatalogEmpty": "Ţĥē ēxēćũţōŕ ďĩď ńōţ àďvēŕţĩśē à ḿōďēĺ ćàţàĺōĝ.", "modelSelectionReasonFallbackNotAdvertised": "Ţĥē śàvēď ƒàĺĺƀàćķ ḿōďēĺ ŵàś ńōţ àďvēŕţĩśēď ƀŷ ţĥē ēxēćũţōŕ.", "modelSelectionReasonRequestedNotAdvertised": "Ţĥē śàvēď ḿōďēĺ ŵàś ńōţ àďvēŕţĩśēď ƀŷ ţĥē ēxēćũţōŕ.", + "modelSelectionReasonUniqueVariation": "Ţĥē ēxēćũţōŕ àƥƥĺĩēď ţĥē ōńĺŷ àďvēŕţĩśēď vàŕĩàţĩōń ōƒ ţĥē śàvēď ḿōďēĺ.", "modelSelectionReasonUnknown": "{{reason}}", "modelSelectionReasonUnsupported": "Ţĥē ēxēćũţōŕ ďōēś ńōţ śũƥƥōŕţ ḿōďēĺ śēĺēćţĩōń.", "modelSelectionRemediationAgentVersion": "Ćĥēćķ ţĥē àĝēńţ vēŕśĩōń ĩń ţĥē ēxēćũţōŕ.", diff --git a/apps/web/src/locales/pt-pt/settings.json b/apps/web/src/locales/pt-pt/settings.json index c4e52d21f6..3e665b9d50 100644 --- a/apps/web/src/locales/pt-pt/settings.json +++ b/apps/web/src/locales/pt-pt/settings.json @@ -751,6 +751,8 @@ "terminalAndEditors": "Terminal e editores", "workspacesAndAccess": "Workspaces e acesso", "profileStartModelNotAdvertisedOnHost": "A sondagem do anfitrião não anunciou {{model}}. O executor selecionado decidirá o modelo no arranque.", + "profileStartModelUniqueVariationOnHost": "A sondagem do anfitrião encontrou uma única variação possível de {{model}}: {{variation}}. O executor selecionado decidirá o modelo no arranque.", + "modelVariationAdvisory": "O catálogo do anfitrião tem uma única variação possível de {{model}}: {{variation}}. O modelo guardado permanece inalterado até ao arranque.", "navUnits": "Unidades da organização", "unitsTitle": "Unidades da organização", "unitsDescription": "Uma unidade contém equipas, áreas de trabalho e pessoas. Pertencer a uma unidade dá acesso a todas as áreas de trabalho abaixo dela, incluindo as criadas depois.", diff --git a/apps/web/src/locales/pt-pt/task.json b/apps/web/src/locales/pt-pt/task.json index 2bb658f9ee..6d47570952 100644 --- a/apps/web/src/locales/pt-pt/task.json +++ b/apps/web/src/locales/pt-pt/task.json @@ -1958,6 +1958,7 @@ "modelSelectionReasonCatalogEmpty": "O executor não anunciou um catálogo de modelos.", "modelSelectionReasonFallbackNotAdvertised": "O modelo alternativo guardado não foi anunciado pelo executor.", "modelSelectionReasonRequestedNotAdvertised": "O modelo guardado não foi anunciado pelo executor.", + "modelSelectionReasonUniqueVariation": "O executor aplicou a única variação anunciada do modelo guardado.", "modelSelectionReasonUnknown": "{{reason}}", "modelSelectionReasonUnsupported": "O executor não suporta seleção de modelo.", "modelSelectionRemediationAgentVersion": "Verifique a versão do agente no executor.", diff --git a/apps/web/src/locales/zh-cn/settings.json b/apps/web/src/locales/zh-cn/settings.json index bc779fc291..91cc1fd4c9 100644 --- a/apps/web/src/locales/zh-cn/settings.json +++ b/apps/web/src/locales/zh-cn/settings.json @@ -751,6 +751,8 @@ "keyValueKeyPlaceholder": "键", "keyValueValuePlaceholder": "值", "profileStartModelNotAdvertisedOnHost": "主机探测未公布 {{model}}。所选执行器将在启动时决定使用的模型。", + "profileStartModelUniqueVariationOnHost": "主机探测发现 {{model}} 的一个可能变体：{{variation}}。所选执行器将在启动时决定使用的模型。", + "modelVariationAdvisory": "主机目录中有一个可能的 {{model}} 变体：{{variation}}。保存的模型在启动前保持不变。", "navUnits": "组织单元", "unitsTitle": "组织单元", "unitsDescription": "单元包含团队、工作区和成员。加入某个单元即可访问其下的所有工作区，包括之后创建的。", diff --git a/apps/web/src/locales/zh-cn/task.json b/apps/web/src/locales/zh-cn/task.json index a057f80f45..5b5e452978 100644 --- a/apps/web/src/locales/zh-cn/task.json +++ b/apps/web/src/locales/zh-cn/task.json @@ -1958,6 +1958,7 @@ "modelSelectionReasonCatalogEmpty": "执行器未公布模型目录。", "modelSelectionReasonFallbackNotAdvertised": "执行器未公布已保存的备选模型。", "modelSelectionReasonRequestedNotAdvertised": "执行器未公布已保存的模型。", + "modelSelectionReasonUniqueVariation": "执行器应用了已保存模型唯一公布的变体。", "modelSelectionReasonUnknown": "{{reason}}", "modelSelectionReasonUnsupported": "该执行器不支持选择模型。", "modelSelectionRemediationAgentVersion": "请检查执行器中的智能体版本。", diff --git a/apps/web/src/locales/zh-hk/settings.json b/apps/web/src/locales/zh-hk/settings.json index fb696accbb..a4ca7d3ed3 100644 --- a/apps/web/src/locales/zh-hk/settings.json +++ b/apps/web/src/locales/zh-hk/settings.json @@ -751,6 +751,8 @@ "keyValueKeyPlaceholder": "鍵", "keyValueValuePlaceholder": "值", "profileStartModelNotAdvertisedOnHost": "主機探測未公佈 {{model}}。所選執行器將在啓動時決定使用的模型。", + "profileStartModelUniqueVariationOnHost": "主機探測發現 {{model}} 的一個可能變體：{{variation}}。所選執行器將在啓動時決定使用的模型。", + "modelVariationAdvisory": "主機目錄中有一個可能的 {{model}} 變體：{{variation}}。已儲存的模型在啟動前保持不變。", "navUnits": "組織單元", "unitsTitle": "組織單元", "unitsDescription": "單元包含團隊、工作區和成員。加入某個單元即可存取其下的所有工作區，包括之後建立的。", diff --git a/apps/web/src/locales/zh-hk/task.json b/apps/web/src/locales/zh-hk/task.json index ed53af8973..bd96ef4f6b 100644 --- a/apps/web/src/locales/zh-hk/task.json +++ b/apps/web/src/locales/zh-hk/task.json @@ -1958,6 +1958,7 @@ "modelSelectionReasonCatalogEmpty": "執行器未公佈模型目錄。", "modelSelectionReasonFallbackNotAdvertised": "執行器未公佈已儲存的備選模型。", "modelSelectionReasonRequestedNotAdvertised": "執行器未公佈已儲存的模型。", + "modelSelectionReasonUniqueVariation": "執行器套用了已儲存模型唯一公佈的變體。", "modelSelectionReasonUnknown": "{{reason}}", "modelSelectionReasonUnsupported": "該執行器不支援選擇模型。", "modelSelectionRemediationAgentVersion": "請檢查執行器中的代理程式版本。", diff --git a/apps/web/src/locales/zh-tw/settings.json b/apps/web/src/locales/zh-tw/settings.json index d9b0199227..ea271fe70f 100644 --- a/apps/web/src/locales/zh-tw/settings.json +++ b/apps/web/src/locales/zh-tw/settings.json @@ -751,6 +751,8 @@ "keyValueKeyPlaceholder": "鍵", "keyValueValuePlaceholder": "值", "profileStartModelNotAdvertisedOnHost": "主機探測未公佈 {{model}}。所選執行器將在啟動時決定使用的模型。", + "profileStartModelUniqueVariationOnHost": "主機探測發現 {{model}} 的一個可能變體：{{variation}}。所選執行器將在啟動時決定使用的模型。", + "modelVariationAdvisory": "主機目錄中有一個可能的 {{model}} 變體：{{variation}}。已儲存的模型在啟動前保持不變。", "navUnits": "組織單元", "unitsTitle": "組織單元", "unitsDescription": "單元包含團隊、工作區和成員。加入某個單元即可存取其下的所有工作區，包括之後建立的。", diff --git a/apps/web/src/locales/zh-tw/task.json b/apps/web/src/locales/zh-tw/task.json index e9b18c5382..e4f0d5b8b3 100644 --- a/apps/web/src/locales/zh-tw/task.json +++ b/apps/web/src/locales/zh-tw/task.json @@ -1958,6 +1958,7 @@ "modelSelectionReasonCatalogEmpty": "執行器未公佈模型目錄。", "modelSelectionReasonFallbackNotAdvertised": "執行器未公佈已儲存的備選模型。", "modelSelectionReasonRequestedNotAdvertised": "執行器未公佈已儲存的模型。", + "modelSelectionReasonUniqueVariation": "執行器套用了已儲存模型唯一公佈的變體。", "modelSelectionReasonUnknown": "{{reason}}", "modelSelectionReasonUnsupported": "該執行器不支援選擇模型。", "modelSelectionRemediationAgentVersion": "請檢查執行器中的代理程式版本。", diff --git a/docs/decisions/2026-09-07-unique-model-variation-resolution.md b/docs/decisions/2026-09-07-unique-model-variation-resolution.md new file mode 100644 index 0000000000..e089a2a1bd --- /dev/null +++ b/docs/decisions/2026-09-07-unique-model-variation-resolution.md @@ -0,0 +1,84 @@ +# ADR-2026-09-07: Resolve Only One Advertised Model Variation + +**Status:** proposed +**Date:** 2026-09-07 +**Area:** backend, frontend, protocol +**Amends:** `2026-08-15-executor-authoritative-model-selection` + +## Context + +An agent CLI can replace a bare model ID with bracketed variations. A profile +can keep `opus` while the executor later advertises only `opus[1m]`. + +The current executor-authoritative policy treats the saved ID as absent. It +uses an explicit fallback when available. Otherwise, it continues with the +provider current or default model and records a warning. This result can move a +session to an unrelated model even when the catalog has one clear variation of +the requested model. + +Variation text is provider-owned. Kandev cannot safely rank `270k`, `1m`, +`fast`, or future labels. Catalog order and the current-model marker do not +express user intent. + +## Decision + +Kandev recognizes a model variation only by a narrow bracketed ID shape. A bare +requested ID `x` matches an advertised ID `x[v]` when `v` is non-empty and +contains no bracket. Matching is exact and case-sensitive. Kandev treats `v` as +opaque text. + +For profiles with `auto_fallback = false`, the executor-authoritative resolution +order is: + +1. Exact requested model. +2. Advertised explicit fallback from the profile. +3. One distinct advertised variation of the requested bare model. +4. Provider current or default model, with no speculative model-selection call. + +Kandev applies step 3 only when exactly one distinct candidate exists. Zero or +multiple candidates produce no inferred selection. An unadvertised explicit +fallback does not block step 3. + +A requested ID that contains a bracket is an exact ID only. Kandev does not +infer another variation from it. + +Profiles with `auto_fallback = true` retain the legacy path when the requested +model is absent. Kandev ignores the configured explicit fallback, does not +infer a variation, and continues with the provider current or default model. +Apply errors for an advertised model remain best-effort in that mode. + +The lifecycle stores `unique_variation` as the decision outcome and +`unique_variation_applied` as the warning reason. The effective model contains +the advertised variation. The explicit fallback field keeps its existing +meaning and stays empty when Kandev infers a variation. + +The saved profile model does not change. The rule runs for initial launch, +context reset, and workspace rebind. The executor catalog remains authoritative. +Any host-catalog result remains an advisory. + +## Consequences + +- A profile with `opus` follows `opus[1m]` when it is the only advertised + variation. +- A catalog with `opus[270k]` and `opus[1m, fast]` remains ambiguous and uses + the existing provider-default path. +- An explicit advertised fallback remains stronger than inference. +- Kandev never sends a model ID that the executor did not advertise. +- Users receive a durable warning because the effective model differs from the + saved request. +- Provider-specific ranking logic does not enter the lifecycle policy. +- Backend and frontend helpers need mirrored contract tests. The backend result + remains authoritative when their input catalogs differ. + +## Alternatives Considered + +1. **Always choose the first variation.** Rejected because catalog order is not + a user preference or a stable provider contract. +2. **Prefer the provider current model.** Rejected because current state can + reflect a default or an earlier session, not the saved profile intent. +3. **Rank known labels such as context size or fast mode.** Rejected because + labels are provider-owned and can change without a Kandev release. +4. **Rewrite the saved profile to the inferred model.** Rejected because one + executor observation must not change behavior for every executor. +5. **Infer from an already bracketed request.** Rejected because it can replace + one explicit variation with another without user intent. diff --git a/docs/decisions/INDEX.md b/docs/decisions/INDEX.md index c7e7e20b3f..17ff4e25df 100644 --- a/docs/decisions/INDEX.md +++ b/docs/decisions/INDEX.md @@ -95,6 +95,7 @@ Read individual ADRs for full context. Create new ones via `/record decision` or | 2026-08-15-subagent-context-schema-boundary | [Start with the execution-aware subagent schema](2026-08-15-subagent-context-schema-boundary.md) | accepted | backend | 2026-08-15 | | 2026-08-15-portable-agent-configuration-bundles | [Use Explicit Portable Agent Configuration Bundles](2026-08-15-portable-agent-configuration-bundles.md) | accepted | backend, frontend, protocol, security | 2026-08-15 | | 2026-08-15-executor-authoritative-model-selection | [Let the Executor Own Model Selection](2026-08-15-executor-authoritative-model-selection.md) | accepted | backend, frontend, protocol, persistence | 2026-08-15 | +| 2026-09-07-unique-model-variation-resolution | [Resolve Only One Advertised Model Variation](2026-09-07-unique-model-variation-resolution.md) | proposed | backend, frontend, protocol | 2026-09-07 | | 2026-08-18-context-reset-preserves-runtime-configuration | [Preserve ACP Runtime Configuration Across Context Reset](2026-08-18-context-reset-preserves-runtime-configuration.md) | accepted | backend, protocol, workflow | 2026-08-18 | | 2026-08-19-parent-authorized-child-workspace-sources | [Parent-authorized child workspace sources](2026-08-19-parent-authorized-child-workspace-sources.md) | accepted | backend, protocol, security | 2026-08-19 | | 2026-08-24-task-snapshotted-branch-policies | [Snapshot Repository Branch Policies on Tasks](2026-08-24-task-snapshotted-branch-policies.md) | proposed | backend, frontend, persistence | 2026-08-24 | diff --git a/docs/plans/unique-model-variation-fallback/plan.md b/docs/plans/unique-model-variation-fallback/plan.md new file mode 100644 index 0000000000..42d08d2ea8 --- /dev/null +++ b/docs/plans/unique-model-variation-fallback/plan.md @@ -0,0 +1,129 @@ +--- +created: 2026-09-07 +status: complete +requirements: + - REQ-AGENTS-NO-SILENT-MODEL-FALLBACK-001 + - REQ-AGENTS-NO-SILENT-MODEL-FALLBACK-002 +system_design: + - ../../specs/agents/system-design/no-silent-model-fallback-01.md + - ../../specs/agents/system-design/no-silent-model-fallback-02.md +legacy_specs: [] +--- + +# Implementation Plan: Unique Model Variation Fallback + +## Overview + +Extend the central executor-authoritative model policy with one deterministic +inference step. Implement the runtime contract first, expose its stable decision +in the existing frontend surfaces, then prove and document the user outcome. + +## Scope + +### In scope + +- Resolve a missing bare model to its only distinct advertised bracketed + variation. +- Keep exact models and advertised explicit fallbacks at higher precedence. +- Persist a distinct warning outcome and reason without rewriting the profile. +- Present host results as advisory on desktop and touch interfaces. +- Test and document unique and ambiguous catalogs. + +### Out of scope + +- Provider-specific ranking of variation labels. +- Selection among multiple variations. +- Profile schema changes or saved-model migration. +- Office post-start route changes or mid-turn model switching. + +## Technical approach + +### Runtime policy + +Add a pure unique-candidate helper in +`apps/backend/internal/agent/runtime/lifecycle/start_model.go`. It accepts the +requested ID and executor model IDs. It returns one distinct advertised ID only +when the structural contract matches. + +For profiles with automatic fallback disabled, extend `applyStartModelPolicy` +with this order: exact request, advertised explicit fallback, unique variation, +provider current or default. Automatic-fallback profiles retain their legacy +no-selection behavior when the requested model is absent. Add +`unique_variation` to `ModelSelectionOutcome` and +`unique_variation_applied` to stable warning reasons. + +Keep `EffectiveModel` as the inferred ID. Do not set `FallbackModel` for this +outcome. `publishModelSelectionWarningEvent` must therefore emit the new reason +without the legacy `session_model_fallback` compatibility event. + +Initial launch, context reset, and workspace rebind already use this policy. +Targeted tests must prove that no separate path bypasses the new order. + +### Frontend advisory + +Add the mirrored pure matcher under `apps/web/lib/`. Its table tests use the +same contract cases as the Go helper. Backend lifecycle data remains the launch +authority. + +Use the helper in `task-create-dialog-options.tsx` and +`settings/profile-model-fields.tsx`. One host variation names the possible +target while keeping the saved value and selectable profile. Zero or multiple +variations keep the current missing-model presentation. + +Map `unique_variation_applied` in `status-message.tsx`. Add localized copy to +all required `settings` and `task` catalogs. Reuse the existing desktop tooltip, +touch drawer, responsive model selector, and task-chat status layout. + +### Documentation + +Update `docs/public/agents-and-profiles.md` and `docs/public/executors.md` with +the four-step order, one unique example, one ambiguous example, and the +unchanged saved-profile rule. + +## Tests + +- AC-002.1 and AC-002.2: Go policy tests prove exact and explicit-fallback + precedence before one inferred variation, including the legacy + auto-fallback path. +- AC-002.3 through AC-002.5: mirrored Go and TypeScript table tests prove zero, + duplicate, multiple, case-different, bracketed, prefix, and malformed cases. +- AC-002.6: lifecycle event and React status-message tests prove the distinct + outcome, effective model, absent explicit-fallback field, and durable copy. +- AC-002.7: task picker and profile editor tests prove advisory host behavior. +- AC-002.8: lifecycle integration tests prove launch, reset, and rebind parity. + +## E2E tests + +- AC-002.2 and AC-002.6: extend + `apps/web/e2e/tests/session/model-mismatch-warning.spec.ts` to launch `opus` + against one `opus[1m]`, show the effective model, and retain one warning after + reload. +- AC-002.3: add an ambiguous catalog with `opus[270k]` and + `opus[1m, fast]`; prove that Kandev does not select either variation. +- AC-002.7: extend the desktop and mobile settings mismatch specs. Prove that + the profile remains selectable, the advisory is keyboard or tap accessible, + and mobile has no horizontal document overflow. + +## Work orders + +- [x] [Task 01: Resolve one executor variation](task-01-runtime-resolution.md) +- [x] [Task 02: Present variation decisions](task-02-frontend-advisory.md) +- [x] [Task 03: Prove variation workflows](task-03-e2e-verification.md) +- [x] [Task 04: Document variation fallback](task-04-public-docs.md) + +## Verification results + +Backend verification passed: 2,364 lifecycle tests, 29 focused model-policy +tests, and 212 mock-agent tests. Frontend verification passed: 51 focused +unit tests, typecheck, lint, `i18n:check`, and `i18n:ratchet`. Desktop E2E +passed 5 tests and mobile E2E passed 3 tests. Public-document validation +passed 61 tests and validated 46 published pages. Specification lint passed. + +## Risks + +- Provider variation labels can change. Kandev must keep their content opaque. +- Host and executor catalogs can differ. Host preview logic cannot become a + launch gate. +- Older frontends can render the new reason as unknown during mixed-version + use. Requested and effective fields must remain useful. +- Duplicate or malformed catalog rows can make a naive candidate count unstable. diff --git a/docs/plans/unique-model-variation-fallback/task-01-runtime-resolution.md b/docs/plans/unique-model-variation-fallback/task-01-runtime-resolution.md new file mode 100644 index 0000000000..b33720e2fc --- /dev/null +++ b/docs/plans/unique-model-variation-fallback/task-01-runtime-resolution.md @@ -0,0 +1,100 @@ +--- +id: "01-runtime-resolution" +title: "Resolve one executor variation" +status: done +wave: 1 +depends_on: [] +plan: "plan.md" +requirements: + - REQ-AGENTS-NO-SILENT-MODEL-FALLBACK-001 + - REQ-AGENTS-NO-SILENT-MODEL-FALLBACK-002 +acceptance_criteria: + - AC-AGENTS-NO-SILENT-MODEL-FALLBACK-002.1 + - AC-AGENTS-NO-SILENT-MODEL-FALLBACK-002.2 + - AC-AGENTS-NO-SILENT-MODEL-FALLBACK-002.3 + - AC-AGENTS-NO-SILENT-MODEL-FALLBACK-002.4 + - AC-AGENTS-NO-SILENT-MODEL-FALLBACK-002.5 + - AC-AGENTS-NO-SILENT-MODEL-FALLBACK-002.6 + - AC-AGENTS-NO-SILENT-MODEL-FALLBACK-002.8 +system_design: + - ../../specs/agents/system-design/no-silent-model-fallback-01.md + - ../../specs/agents/system-design/no-silent-model-fallback-02.md +--- + +# Task 01: Resolve One Executor Variation + +## Summary + +Extend the single lifecycle model policy with safe unique-variation inference. +Expose a distinct decision without changing explicit fallback semantics. + +## In scope + +- Add the pure backend candidate matcher. +- Apply the defined four-step precedence order. +- Extend decision, logging, and warning metadata behavior. +- Cover initial launch, context reset, and workspace rebind. + +## Out of scope + +- Host advisory UI and translations. +- Provider-specific variation parsing or ranking. +- Profile persistence changes. + +## Acceptance + +- One valid distinct variation is the only inferred model-selection call. +- Exact and advertised explicit fallback models remain higher priority; zero or + multiple variations cause no inference. +- Automatic-fallback profiles keep the legacy no-selection behavior when the + requested model is absent. +- The warning records the unique outcome and effective model without setting + the explicit fallback field or compatibility fallback event. + +## Verification + +```bash +( + cd apps/backend + go test -tags fts5 -run 'TestApplyStartModelPolicy|TestInitializeAndPrompt.*Model|TestReapplySessionModel|TestWorkspaceRebind.*Model|Test.*ModelSelectionWarning' ./internal/agent/runtime/lifecycle + go test -tags fts5 ./internal/agent/runtime/lifecycle +) +git diff --check +``` + +## Files likely touched + +- `apps/backend/internal/agent/runtime/lifecycle/start_model.go` +- `apps/backend/internal/agent/runtime/lifecycle/start_model_executor_authority_test.go` +- `apps/backend/internal/agent/runtime/lifecycle/session.go` +- `apps/backend/internal/agent/runtime/lifecycle/session_model_selection_warning_test.go` +- `apps/backend/internal/agentctl/types/streams/agent.go` + +## Dependencies + +None. + +## Risks + +- The Cursor-specific configuration parser rejects labels such as `1m`. This + matcher must remain a separate model-ID helper. +- A special inferred-model apply path can weaken existing error handling. + +## Parallelism + +`sequential` + +## Inputs + +- Requirement 002 and its matching examples. +- The unique-variation ADR. +- Existing `applyStartModelPolicy` and warning publication tests. + +## Results + +Implemented the executor-authoritative unique-variation policy in +`start_model.go`, including explicit-fallback precedence, opaque and +case-sensitive matching, the legacy automatic-fallback path, and distinct +warning metadata without a legacy fallback event. Initial launch, context +reset, and workspace rebind all use the same policy. Focused model-policy +tests passed (29), and the full lifecycle suite passed (2,364). diff --git a/docs/plans/unique-model-variation-fallback/task-02-frontend-advisory.md b/docs/plans/unique-model-variation-fallback/task-02-frontend-advisory.md new file mode 100644 index 0000000000..25c0932acf --- /dev/null +++ b/docs/plans/unique-model-variation-fallback/task-02-frontend-advisory.md @@ -0,0 +1,102 @@ +--- +id: "02-frontend-advisory" +title: "Present variation decisions" +status: done +wave: 2 +depends_on: ["01-runtime-resolution"] +plan: "plan.md" +requirements: + - REQ-AGENTS-NO-SILENT-MODEL-FALLBACK-002 +acceptance_criteria: + - AC-AGENTS-NO-SILENT-MODEL-FALLBACK-002.3 + - AC-AGENTS-NO-SILENT-MODEL-FALLBACK-002.4 + - AC-AGENTS-NO-SILENT-MODEL-FALLBACK-002.5 + - AC-AGENTS-NO-SILENT-MODEL-FALLBACK-002.6 + - AC-AGENTS-NO-SILENT-MODEL-FALLBACK-002.7 +system_design: + - ../../specs/agents/system-design/no-silent-model-fallback-01.md +--- + +# Task 02: Present Variation Decisions + +## Summary + +Present one host variation as an advisory and render the runtime decision in +task chat. Keep the host probe non-authoritative on every viewport. + +## In scope + +- Add the mirrored frontend matcher and table tests. +- Update task-create and profile-editor missing-model states. +- Render the new warning reason with requested and effective models. +- Add all required localized copy. + +## Out of scope + +- Runtime model selection. +- New dialog, drawer, or navigation patterns. +- Changes to the saved profile model. + +## Acceptance + +- One host variation names a possible target and keeps the profile selectable. +- Zero or multiple host variations keep the existing missing-model state. +- Desktop, keyboard, and touch users can reach the advisory and durable warning. + +## Verification + +```bash +( + cd apps + pnpm --filter @kandev/web test -- --run components/task-create-dialog-options components/settings/profile-form-fields components/task/chat/messages/status-message lib/model-variation + pnpm --filter @kandev/web lint +) +( + cd apps/web + pnpm run typecheck + pnpm run i18n:check + pnpm run i18n:ratchet +) +git diff --check +``` + +## Files likely touched + +- `apps/web/lib/model-variation.ts` +- `apps/web/lib/model-variation.test.ts` +- `apps/web/components/task-create-dialog-options.tsx` +- `apps/web/components/task-create-dialog-options.test.tsx` +- `apps/web/components/settings/profile-model-fields.tsx` +- `apps/web/components/settings/profile-form-fields.test.tsx` +- `apps/web/components/task/chat/messages/status-message.tsx` +- `apps/web/components/task/chat/messages/status-message.test.tsx` +- `apps/web/src/locales/*/settings.json` +- `apps/web/src/locales/*/task.json` + +## Dependencies + +Task 01. + +## Risks + +- Host preview text can sound authoritative when another executor can advertise + a different catalog. +- Missing and resolvable states must be mutually exclusive in the profile row. + +## Parallelism + +`sequential` + +## Inputs + +- Task 01 warning reason and payload fields. +- Existing host warning tooltip, touch drawer, profile selector, and status + message patterns. + +## Results + +Added the mirrored structural matcher, host advisory states, and the task-chat +warning reason while preserving the saved profile model. Added localized copy +for all required catalogs and retained the existing desktop tooltip and mobile +touch-drawer patterns. Focused frontend tests passed (51), typecheck and lint +passed, and both i18n checks passed. diff --git a/docs/plans/unique-model-variation-fallback/task-03-e2e-verification.md b/docs/plans/unique-model-variation-fallback/task-03-e2e-verification.md new file mode 100644 index 0000000000..ef9bcd7e39 --- /dev/null +++ b/docs/plans/unique-model-variation-fallback/task-03-e2e-verification.md @@ -0,0 +1,84 @@ +--- +id: "03-e2e-verification" +title: "Prove variation workflows" +status: done +wave: 3 +depends_on: ["01-runtime-resolution", "02-frontend-advisory"] +plan: "plan.md" +requirements: + - REQ-AGENTS-NO-SILENT-MODEL-FALLBACK-002 +acceptance_criteria: + - AC-AGENTS-NO-SILENT-MODEL-FALLBACK-002.2 + - AC-AGENTS-NO-SILENT-MODEL-FALLBACK-002.3 + - AC-AGENTS-NO-SILENT-MODEL-FALLBACK-002.6 + - AC-AGENTS-NO-SILENT-MODEL-FALLBACK-002.7 +system_design: + - ../../specs/agents/system-design/no-silent-model-fallback-01.md +--- + +# Task 03: Prove Variation Workflows + +## Summary + +Extend the existing model-mismatch browser flows with unique and ambiguous +catalogs. Prove the launch result, durable warning, and touch interaction. + +## In scope + +- Desktop launch and reload coverage for one variation. +- Desktop no-inference coverage for multiple variations. +- Mobile advisory and warning access through tap. +- Horizontal overflow checks for the changed mobile states. + +## Out of scope + +- New browser fixtures unrelated to model catalogs. +- Full E2E suite execution. +- Public documentation. + +## Acceptance + +- `opus` launches as `opus[1m]` only when that is the sole distinct variation. +- Two variations do not select either candidate. +- The warning survives reload and the mobile flow needs no hover interaction. + +## Verification + +```bash +cd apps/web && pnpm e2e:run tests/settings/no-silent-model-fallback.spec.ts tests/session/model-mismatch-warning.spec.ts +cd apps/web && pnpm e2e:run --project mobile-chrome --no-build tests/settings/mobile-no-silent-model-fallback.spec.ts tests/session/mobile-model-mismatch-warning.spec.ts +git diff --check +``` + +## Files likely touched + +- `apps/web/e2e/tests/settings/no-silent-model-fallback.spec.ts` +- `apps/web/e2e/tests/settings/mobile-no-silent-model-fallback.spec.ts` +- `apps/web/e2e/tests/session/model-mismatch-warning.spec.ts` +- `apps/web/e2e/tests/session/mobile-model-mismatch-warning.spec.ts` +- `apps/web/e2e/tests/session/model-mismatch-warning-helpers.ts` + +## Dependencies + +Tasks 01 and 02. + +## Risks + +- Mock catalogs must prove the executor decision, not only the host advisory. +- Each scenario needs isolated profile data so another catalog does not change + the distinct candidate count. + +## Parallelism + +`parallel-safe` with Task 04 after Tasks 01 and 02. + +## Inputs + +- Completed runtime and frontend work. +- Existing desktop and mobile model-mismatch fixtures. + +## Results + +Added unique and ambiguous mock-agent catalogs and covered launch, reload, +profile persistence, host advisory access, and mobile overflow. The focused +desktop E2E run passed 5 tests; the focused mobile E2E run passed 3 tests. diff --git a/docs/plans/unique-model-variation-fallback/task-04-public-docs.md b/docs/plans/unique-model-variation-fallback/task-04-public-docs.md new file mode 100644 index 0000000000..fa80b4a4b5 --- /dev/null +++ b/docs/plans/unique-model-variation-fallback/task-04-public-docs.md @@ -0,0 +1,83 @@ +--- +id: "04-public-docs" +title: "Document variation fallback" +status: done +wave: 3 +depends_on: ["01-runtime-resolution", "02-frontend-advisory"] +plan: "plan.md" +requirements: + - REQ-AGENTS-NO-SILENT-MODEL-FALLBACK-002 +acceptance_criteria: + - AC-AGENTS-NO-SILENT-MODEL-FALLBACK-002.1 + - AC-AGENTS-NO-SILENT-MODEL-FALLBACK-002.2 + - AC-AGENTS-NO-SILENT-MODEL-FALLBACK-002.3 + - AC-AGENTS-NO-SILENT-MODEL-FALLBACK-002.5 + - AC-AGENTS-NO-SILENT-MODEL-FALLBACK-002.6 +system_design: + - ../../specs/agents/system-design/no-silent-model-fallback-01.md +--- + +# Task 04: Document Variation Fallback + +## Summary + +Update the public profile and executor guides with the unique-variation rule. +Give users enough detail to predict unique and ambiguous results. + +## In scope + +- Document the four-step launch order. +- Add one unique and one ambiguous example. +- State that labels remain opaque and saved profiles do not change. +- Run the public-doc validation commands. + +## Out of scope + +- Release notes and changelog generation. +- Internal implementation detail beyond user-visible behavior. +- New screenshots or media. + +## Acceptance + +- Both guides describe the same precedence and ambiguity rule. +- The docs state that the executor is authoritative and the profile stays + unchanged. +- Public-doc validators pass. + +## Verification + +```bash +node --test scripts/validate-public-docs.test.mjs +node scripts/validate-public-docs.mjs +git diff --check +``` + +## Files likely touched + +- `docs/public/agents-and-profiles.md` +- `docs/public/executors.md` + +## Dependencies + +Tasks 01 and 02. + +## Risks + +- Examples can imply that Kandev understands `1m` or `fast`. The docs must state + that it compares only the ID shape and candidate count. + +## Parallelism + +`parallel-safe` with Task 03 after Tasks 01 and 02. + +## Inputs + +- Final runtime precedence and warning behavior. +- Existing host-probe and remote-executor guide sections. + +## Results + +Updated the public agent-profile and executor guides with the four-step +resolution order, unique and ambiguous examples, opaque variation matching, +and the unchanged saved-profile rule. Public-document tests passed (61) and +the validator checked 46 published pages. diff --git a/docs/public/agents-and-profiles.md b/docs/public/agents-and-profiles.md index 3c5130af0e..19a316678a 100644 --- a/docs/public/agents-and-profiles.md +++ b/docs/public/agents-and-profiles.md @@ -171,6 +171,31 @@ change makes a saved option value unsupported, Kandev removes that value after a successful resolution; a failed resolution keeps the draft unchanged so you can retry it. +### Model IDs and executor catalogs + +The host model probe is an editing hint. The selected executor owns the model +catalog at launch. Kandev resolves a saved model in this order: + +1. Use the exact requested model when the executor advertises it. +2. Use the advertised explicit fallback when the requested model is absent. +3. Use one unique bracketed variation when the requested model is absent and + the executor advertises exactly one matching ID. +4. Use the agent's current or default model when no earlier choice applies. + +Profiles with automatic fallback enabled keep the legacy behavior when the +saved model is absent: Kandev ignores the explicit fallback and does not infer +a variation. It uses the agent's current or default model instead. + +For example, a saved `opus` model can launch as `opus[1m]` when that is the +only advertised `opus[...]` ID. If the executor advertises both +`opus[270k]` and `opus[1m, fast]`, Kandev does not choose either variation. +Variation text is opaque and model IDs remain case-sensitive. + +Kandev shows a warning when the launch result differs from the saved model. +The saved profile remains `opus`; Kandev does not rewrite it after applying a +unique variation or using the agent default. Recheck the executor catalog when +the warning repeats after credentials, copied configuration, or agent updates. + ### Use a dynamic profile > [!EXPERIMENTAL] @@ -228,13 +253,13 @@ The model list shown while editing a profile comes from a host probe. It is an editing hint, not a launch gate. A profile remains selectable when its saved model is missing from that host list. -At task launch, the selected executor's ACP catalog is authoritative. Kandev -sends the requested model only when the executor advertises it. If it does -not, Kandev uses an advertised fallback when available, or sends no model -request and continues with the agent's current or default model. Kandev stores -one warning in task chat with the requested model and the effective model when -known. The warning also identifies the agent and executor and asks you to -check credentials, copied configuration, and the agent version. +At task launch, the selected executor's ACP catalog is authoritative. For +profiles without automatic fallback, Kandev follows the four-step order above. +For profiles with automatic fallback enabled, an absent saved model causes no +model request, and Kandev ignores the explicit fallback and any variation. +Kandev stores one warning in task chat with the requested model and the +effective model when known. The warning also identifies the agent and executor +and asks you to check credentials, copied configuration, and the agent version. The saved profile model is not changed. Optional portable configuration can copy selected allowlisted files into a remote executor, but it cannot guarantee diff --git a/docs/public/executors.md b/docs/public/executors.md index 6c88167ecf..b5787be4c2 100644 --- a/docs/public/executors.md +++ b/docs/public/executors.md @@ -101,15 +101,30 @@ Review the selected bundles before saving the profile. ### Model selection in remote executors The host model probe helps edit a profile, but it is not the launch authority. -At launch, the selected executor's advertised ACP catalog decides whether -Kandev sends the saved model. If the executor does not advertise that model, -Kandev sends no request for it. It uses an advertised fallback only when one -exists; otherwise the agent uses its current or default model. +At launch, the selected executor's advertised ACP catalog decides the model. +For profiles without automatic fallback, Kandev applies this deterministic +order: -Kandev writes one warning to task chat when this happens. The warning can list -the requested model, effective model, agent, executor, and executor profile. -It also tells you to check executor credentials, copied agent configuration, -and the agent version. Kandev does not rewrite the saved profile model. +1. An exact advertised model ID. +2. An advertised explicit fallback. +3. One unique advertised bracketed variation of the saved model. +4. The agent's current or default model. + +For profiles with automatic fallback enabled, an absent saved model keeps the +legacy behavior: Kandev ignores the explicit fallback and does not infer a +variation. It uses the agent's current or default model instead. + +For example, a saved `opus` model uses `opus[1m]` when that is the only +advertised variation. With both `opus[270k]` and `opus[1m, fast]`, Kandev +does not infer a choice and uses the agent's current or default model instead. +It treats model IDs as case-sensitive and variation text as opaque. + +Kandev writes one warning to task chat when the effective model differs from +the saved model. The warning can list the requested model, effective model, +agent, executor, and executor profile. It also tells you to check executor +credentials, copied agent configuration, and the agent version. Kandev does +not rewrite the saved profile model, including after applying a unique +variation. Portable configuration can improve parity, but it does not guarantee equal host and executor model catalogs. diff --git a/docs/specs/agents/requirements/no-silent-model-fallback.md b/docs/specs/agents/requirements/no-silent-model-fallback.md index dbf0cae06f..2cc1b853e5 100644 --- a/docs/specs/agents/requirements/no-silent-model-fallback.md +++ b/docs/specs/agents/requirements/no-silent-model-fallback.md @@ -1,26 +1,101 @@ --- -status: draft +status: active system: agents created: 2026-08-23 owners: - kandev --- + # No Silent Model Fallback Requirements ## Overview -**Status**: implemented (executor-authority amendment 2026-08-15) **Date**: 2026-08-04 **Slug**: `no-silent-model-fallback` +Kandev applies an agent profile model against the model catalog from the +selected executor. A different effective model must be deterministic and +visible to the user. The agent system owns this contract because it owns +profiles, ACP session startup, and model-selection warnings. + +## Terminology + +- **Requested model:** The model ID stored in the agent profile. +- **Explicit fallback:** The optional fallback model ID stored in the profile. +- **Bare model ID:** A model ID that contains no `[` or `]` character. +- **Model variation:** An advertised model ID with the exact form + `[]`. The variant contains no bracket + character. Its content is otherwise opaque. +- **Unique variation:** The only distinct advertised variation for a requested + bare model ID. ## Requirements -### REQ-AGENTS-NO-SILENT-MODEL-FALLBACK-001: No Silent Model Fallback +### REQ-AGENTS-NO-SILENT-MODEL-FALLBACK-001: Use executor-authoritative model selection -**Intent:** **Status**: implemented (executor-authority amendment 2026-08-15) **Date**: 2026-08-04 **Slug**: `no-silent-model-fallback` +**Intent:** Keep task launch operational without hiding a difference between +the saved model and the effective executor model. + +**User story:** As a user, I want Kandev to use the selected executor's model +catalog and report any fallback, so that I can understand which model runs. #### Acceptance criteria -- **AC-AGENTS-NO-SILENT-MODEL-FALLBACK-001.1:** When a consumer uses **Status**: implemented (executor-authority amendment 2026-08-15), the system shall provide the documented behavior. +- **AC-AGENTS-NO-SILENT-MODEL-FALLBACK-001.1:** When the executor advertises + the requested model, the system shall apply that exact model. +- **AC-AGENTS-NO-SILENT-MODEL-FALLBACK-001.2:** When the requested model is + absent and the executor advertises the configured explicit fallback, the + system shall apply the explicit fallback. +- **AC-AGENTS-NO-SILENT-MODEL-FALLBACK-001.3:** When no permitted advertised + model applies, the system shall make no speculative model-selection call and + shall continue with the provider current or default model. +- **AC-AGENTS-NO-SILENT-MODEL-FALLBACK-001.4:** When the effective model differs + from the requested model, the system shall persist one structured warning + that identifies the requested and effective models when known. +- **AC-AGENTS-NO-SILENT-MODEL-FALLBACK-001.5:** The system shall not rewrite the + saved profile model from an executor model-selection decision. + +### REQ-AGENTS-NO-SILENT-MODEL-FALLBACK-002: Resolve one advertised model variation + +**Intent:** Preserve a user's bare model choice when a CLI replaces that model +ID with one unambiguous bracketed variation. + +**User story:** As a user, I want `opus` to resolve to `opus[1m]` when that is +the only advertised variation, so that a provider catalog change does not move +my session to an unrelated default model. + +#### Acceptance criteria + +- **AC-AGENTS-NO-SILENT-MODEL-FALLBACK-002.1:** When automatic fallback is + disabled, the exact requested model is absent, and an advertised explicit + fallback exists, the system shall apply the explicit fallback before it + considers an inferred variation. Automatic-fallback profiles shall retain + their legacy no-selection behavior for an absent requested model. +- **AC-AGENTS-NO-SILENT-MODEL-FALLBACK-002.2:** When no advertised explicit + fallback exists and the catalog contains exactly one distinct variation of a + requested bare model ID, the system shall apply that advertised variation. +- **AC-AGENTS-NO-SILENT-MODEL-FALLBACK-002.3:** When the catalog contains zero + or more than one distinct variation, the system shall not infer a model. +- **AC-AGENTS-NO-SILENT-MODEL-FALLBACK-002.4:** When the requested ID contains a + bracket, the system shall treat it as an exact ID and shall not infer another + variation. +- **AC-AGENTS-NO-SILENT-MODEL-FALLBACK-002.5:** Matching shall be case-sensitive + and shall not parse, rank, or assign meaning to variation text. +- **AC-AGENTS-NO-SILENT-MODEL-FALLBACK-002.6:** When Kandev applies a unique + variation, it shall keep the saved model unchanged and shall persist a + warning that identifies the inferred effective model. +- **AC-AGENTS-NO-SILENT-MODEL-FALLBACK-002.7:** The host profile probe shall + present unique-variation resolution as an advisory. It shall not become the + launch authority or disable the profile. +- **AC-AGENTS-NO-SILENT-MODEL-FALLBACK-002.8:** The same resolution order shall + apply at initial launch, context reset, and workspace rebind. + +## Out of scope + +- Ranking multiple variations by context size, speed, price, or display order. +- Parsing provider-specific variation syntax beyond the bracketed ID shape. +- Inferring a base model from an already bracketed request. +- Rewriting the profile model to the current executor's advertised variation. +- Changing Office post-start provider routing or mid-turn model switching. ## System design -The migrated technical source is split into [part 1](../system-design/no-silent-model-fallback-01.md), [part 2](../system-design/no-silent-model-fallback-02.md). +The technical source is split into [part 1](../system-design/no-silent-model-fallback-01.md) +and [part 2](../system-design/no-silent-model-fallback-02.md). diff --git a/docs/specs/agents/system-design/no-silent-model-fallback-01.md b/docs/specs/agents/system-design/no-silent-model-fallback-01.md index 2c20168e4c..a3dc936212 100644 --- a/docs/specs/agents/system-design/no-silent-model-fallback-01.md +++ b/docs/specs/agents/system-design/no-silent-model-fallback-01.md @@ -1,8 +1,9 @@ --- -status: draft +status: current system: agents requirements: - REQ-AGENTS-NO-SILENT-MODEL-FALLBACK-001 + - REQ-AGENTS-NO-SILENT-MODEL-FALLBACK-002 created: 2026-08-23 owners: - kandev @@ -11,13 +12,15 @@ owners: ## Purpose and boundaries -This design preserves the technical source detail for `REQ-AGENTS-NO-SILENT-MODEL-FALLBACK-001` during migration. +This design defines executor-authoritative model fallback, including safe +resolution from a bare requested model to one advertised bracketed variation. ## Requirement mapping | Requirement | Design section | | --- | --- | | `REQ-AGENTS-NO-SILENT-MODEL-FALLBACK-001` | [Migrated source detail](#migrated-source-detail) | +| `REQ-AGENTS-NO-SILENT-MODEL-FALLBACK-002` | [Unique model-variation resolution](#unique-model-variation-resolution) | ## Migrated source detail @@ -146,6 +149,121 @@ the launch. The agent uses its default and Kandev persists a warning. The executor ACP catalog is authoritative for launch. The host probe remains an editing hint and does not block profile selection. +## Unique model-variation resolution + +### Matching contract + +ACP model IDs remain opaque and case-sensitive. Kandev recognizes only this +structural relationship: + +```text +[] +``` + +The requested ID must contain no `[` or `]` character. The advertised ID must +start with the complete requested ID followed by `[`, end with `]`, and contain +a non-empty variant with no bracket character. Kandev does not trim, parse, or +rank the variant. + +Examples: + +| Requested | Advertised candidates | Result | +| --- | --- | --- | +| `opus` | `opus[1m]` | Apply `opus[1m]`. | +| `opus` | `opus[270k]`, `opus[1m, fast]` | Do not infer. | +| `opus[1m]` | `opus[270k]` | Do not infer. | +| `opus` | `myopus[1m]`, `opus-extra[1m]` | Do not infer. | +| `opus` | `opus[]`, `opus[1m][fast]` | Do not infer. | + +Duplicate catalog rows with the same model ID count as one distinct candidate. +This rule keeps the outcome stable when an agent repeats a catalog entry. + +### Resolution order + +For profiles with `auto_fallback = false`, the lifecycle applies this order +after ACP initialization: + +1. Apply the exact requested model when advertised. +2. Apply the configured explicit fallback when advertised. +3. Apply the requested model's unique advertised variation when one exists. +4. Otherwise, make no model-selection call and keep the provider current or + default model. + +An unadvertised explicit fallback does not block step 3. More than one distinct +variation blocks inference even when one candidate appears first or is the +provider current model. + +When `auto_fallback = true`, the legacy path takes precedence when the +requested model is absent: Kandev ignores the explicit fallback and does not +infer a variation. It makes no model-selection call and continues on the +provider current or default model with a warning. Apply errors for an +advertised model remain best-effort in that mode. + +`applyStartModelPolicy` remains the single runtime owner. A pure helper returns +the unique candidate or no candidate. Initial launch, context reset, and +workspace rebind continue to call the same policy. + +### Decision and warning contract + +The typed decision adds the outcome `unique_variation` and the stable reason +`unique_variation_applied`. `EffectiveModel` contains the advertised variation. +`FallbackModel` remains reserved for a configured explicit fallback and is not +overloaded with an inferred value. + +Kandev calls `SetModel` only with the candidate's exact advertised ID. A failed +call follows the existing advertised-model rule: fail the operation unless +legacy `auto_fallback` permits provider-default continuation. A +method-not-supported response also follows the current provider-default path. + +The persisted `model_selection_warning` event uses the new reason and existing +requested and effective model fields. The profile remains unchanged. The +decision ID continues to include the requested model, reason, and effective +model, so the warning stays idempotent for one session decision. + +### Host advisory and mobile behavior + +The frontend uses a small pure helper with the same case-sensitive matching +table. This calculation is advisory because the host catalog can differ from +the selected executor catalog. + +When the host advertises one variation, the task profile option remains +selectable and identifies the possible target in its existing amber advisory. +The profile editor keeps the saved bare ID visible and describes the unique +host variation instead of reporting that the model is simply gone. Selecting +the advertised variation remains an explicit way to update the profile. + +When the host advertises zero or multiple variations, the current missing-model +presentation remains. The runtime can still reach a different decision from +the selected executor catalog. + +The task-create advisory keeps its existing desktop tooltip and coarse-pointer +drawer. The profile editor uses its existing responsive selector. The durable +chat warning uses the existing status-message layout on desktop and mobile. +This change adds no nested scroll area or hover-only information. + +### Tests and operational evidence + +Backend table tests cover exact precedence, explicit-fallback precedence, +one variation, duplicate rows, multiple variations, bracketed requests, +case differences, malformed IDs, method-not-supported, and apply errors. +Existing launch, reset, rebind, warning persistence, and reload tests cover the +new decision outcome. + +Frontend unit tests mirror the matching table and cover the task picker, +profile editor, and localized status warning. Existing desktop and mobile model +mismatch E2E flows add the unique and ambiguous catalog cases. + +Structured lifecycle logs record the requested model, effective model, outcome, +and distinct candidate count. They do not record credentials or configuration +file contents. + +No database migration is required. + +## Related decisions + +- [Let the Executor Own Model Selection](../../../decisions/2026-08-15-executor-authoritative-model-selection.md) +- [Resolve Only One Advertised Model Variation](../../../decisions/2026-09-07-unique-model-variation-resolution.md) + ## Backend Changes ### 1. Agent profile schema (new columns) diff --git a/docs/specs/agents/system-design/no-silent-model-fallback-02.md b/docs/specs/agents/system-design/no-silent-model-fallback-02.md index 5ac456c6c7..2a712c9fa8 100644 --- a/docs/specs/agents/system-design/no-silent-model-fallback-02.md +++ b/docs/specs/agents/system-design/no-silent-model-fallback-02.md @@ -1,8 +1,9 @@ --- -status: draft +status: current system: agents requirements: - REQ-AGENTS-NO-SILENT-MODEL-FALLBACK-001 + - REQ-AGENTS-NO-SILENT-MODEL-FALLBACK-002 created: 2026-08-23 owners: - kandev @@ -11,13 +12,15 @@ owners: ## Purpose and boundaries -This design preserves the technical source detail for `REQ-AGENTS-NO-SILENT-MODEL-FALLBACK-001` during migration. +This design records risks for executor-authoritative model fallback and unique +variation resolution. ## Requirement mapping | Requirement | Design section | | --- | --- | | `REQ-AGENTS-NO-SILENT-MODEL-FALLBACK-001` | [Migrated source detail](#migrated-source-detail) | +| `REQ-AGENTS-NO-SILENT-MODEL-FALLBACK-002` | [Risks & Open Questions](#risks--open-questions) | ## Migrated source detail @@ -51,3 +54,14 @@ This design preserves the technical source detail for `REQ-AGENTS-NO-SILENT-MODE - **Hover is supplementary**: every info icon is focusable, and coarse-pointer devices receive the same content in a drawer. The visible option helper copy remains the baseline explanation. +- **Provider-specific labels remain opaque**: `1m`, `270k`, and `fast` have no + Kandev-defined order. More than one variation therefore fails closed to the + existing provider-default path. +- **Host and executor decisions can differ**: a unique host variation is only + an advisory. The selected executor can advertise zero, one, or multiple + variations at launch. +- **Bracketed requests do not drift sideways**: a request such as `opus[1m]` + does not infer `opus[270k]`. The user must select another explicit model. +- **Legacy automatic fallback remains unchanged**: when `auto_fallback` is + enabled and the requested model is absent, Kandev does not apply an explicit + fallback or infer a variation. It continues with the provider default.