feat(agents): resolve unique advertised model variations (#3472)

This commit is contained in:
Carlos Florêncio
2026-09-07 19:23:17 +01:00
committed by GitHub
parent 410a274629
commit cb1ffd0f58
46 changed files with 1676 additions and 59 deletions
@@ -129,7 +129,7 @@ class PRWalkthroughWorkflowContractTest(unittest.TestCase):
'PUBLIC_URL="${WALKTHROUGH_BASE_URL%/}/pr/${PR_NUMBER}/${SHORT_HEAD_SHA}.html"',
"RESPONSE_META_PATH=pr-walkthrough-public.response",
"curl --fail",
"--write-out '%{http_code}\\t%{content_type}\\t%{url_effective}'",
"--write-out '%{http_code}\\t%{content_type}\\t%{url_effective}\\n'",
"response_status",
'response_effective_url" != "$PUBLIC_URL"',
"response_content_type",
@@ -69,7 +69,7 @@ jobs:
if ! curl --fail --silent --show-error --location --max-time 30 \
--retry 3 --retry-all-errors --retry-delay 2 --retry-max-time 30 \
--output "$PUBLIC_HTML_PATH" \
--write-out '%{http_code}\t%{content_type}\t%{url_effective}' \
--write-out '%{http_code}\t%{content_type}\t%{url_effective}\n' \
> "$RESPONSE_META_PATH" \
"$PUBLIC_URL"; then
echo "Canonical walkthrough is not publicly available; skipping repair." >> "$GITHUB_STEP_SUMMARY"
+31 -10
View File
@@ -24,6 +24,9 @@ const (
modelFast = "mock-fast"
modelSmart = "mock-smart"
modelSlow = "mock-slow"
modelUnique = "opus[1m]"
modelAmbiguousFirst = "opus[270k]"
modelAmbiguousLast = "opus[1m, fast]"
reasoningEffortLow = "low"
reasoningEffortMed = "medium"
reasoningEffortHigh = "high"
@@ -207,22 +210,24 @@ func mockSessionConfigOptionsForModel(model string) []acp.SessionConfigOption {
{Value: "max", Name: "Max", Description: ptr("Use maximum reasoning")},
}
}
modelOptions := acp.SessionConfigSelectOptionsUngrouped{
{Value: modelFast, Name: "Mock Fast", Description: ptr("Fast mock model for testing")},
{Value: modelSmart, Name: "Mock Smart", Description: ptr("Smart mock model for testing")},
// E2E fixtures use "mock-slow" for the slow-response delay tier.
// It must be advertised so the no-silent-model-fallback strict
// policy (which fails session start when the profile model is
// absent from the advertised list) does not reject it.
{Value: modelSlow, Name: "Mock Slow", Description: ptr("Slow mock model for testing")},
}
modelOptions = append(modelOptions, mockModelVariationOptions()...)
return []acp.SessionConfigOption{
{Select: &acp.SessionConfigOptionSelect{
Category: &modelCat,
CurrentValue: acp.SessionConfigValueId(model),
Id: "model",
Name: "Model",
Options: acp.SessionConfigSelectOptions{Ungrouped: &acp.SessionConfigSelectOptionsUngrouped{
{Value: modelFast, Name: "Mock Fast", Description: ptr("Fast mock model for testing")},
{Value: modelSmart, Name: "Mock Smart", Description: ptr("Smart mock model for testing")},
// E2E fixtures use "mock-slow" for the slow-response delay tier.
// It must be advertised so the no-silent-model-fallback strict
// policy (which fails session start when the profile model is
// absent from the advertised list) does not reject it.
{Value: modelSlow, Name: "Mock Slow", Description: ptr("Slow mock model for testing")},
}},
Type: "select",
Options: acp.SessionConfigSelectOptions{Ungrouped: &modelOptions},
Type: "select",
}},
{Select: &acp.SessionConfigOptionSelect{
Category: &modeCat,
@@ -247,6 +252,22 @@ func mockSessionConfigOptionsForModel(model string) []acp.SessionConfigOption {
}
}
func mockModelVariationOptions() []acp.SessionConfigSelectOption {
catalog := strings.ToLower(strings.TrimSpace(os.Getenv("MOCK_AGENT_MODEL_CATALOG")))
if catalog == "ambiguous" {
return []acp.SessionConfigSelectOption{
{Value: modelAmbiguousFirst, Name: "Opus (270k)", Description: ptr("Ambiguous variation fixture")},
{Value: modelAmbiguousLast, Name: "Opus (1m, fast)", Description: ptr("Ambiguous variation fixture")},
}
}
if catalog == "unique" || (catalog == "" && strings.EqualFold(os.Getenv("KANDEV_E2E_MOCK"), "true")) {
return []acp.SessionConfigSelectOption{
{Value: modelUnique, Name: "Opus (1m)", Description: ptr("Unique variation fixture")},
}
}
return nil
}
func cloneSessionConfigOptions(options []acp.SessionConfigOption) []acp.SessionConfigOption {
cloned := make([]acp.SessionConfigOption, len(options))
copy(cloned, options)
@@ -224,6 +224,26 @@ func TestMockSessionConfigOptionsForModelAdvertisesSlowModel(t *testing.T) {
}
}
func TestMockSessionConfigOptionsVariationCatalogs(t *testing.T) {
t.Setenv("KANDEV_E2E_MOCK", "true")
t.Setenv("MOCK_AGENT_MODEL_CATALOG", "")
unique := modelIDs(findModelOption(t))
if !containsString(unique, modelUnique) {
t.Fatalf("unique catalog = %v, missing %s", unique, modelUnique)
}
if containsString(unique, modelAmbiguousFirst) || containsString(unique, modelAmbiguousLast) {
t.Fatalf("unique catalog = %v, must not include ambiguous variations", unique)
}
t.Setenv("MOCK_AGENT_MODEL_CATALOG", "ambiguous")
ambiguous := modelIDs(findModelOption(t))
for _, want := range []string{modelAmbiguousFirst, modelAmbiguousLast} {
if !containsString(ambiguous, want) {
t.Errorf("ambiguous catalog = %v, missing %s", ambiguous, want)
}
}
}
func findModelOption(t *testing.T) acp.SessionConfigOption {
t.Helper()
for _, option := range mockSessionConfigOptionsForModel(modelFast) {
@@ -235,6 +255,17 @@ func findModelOption(t *testing.T) acp.SessionConfigOption {
return acp.SessionConfigOption{}
}
func modelIDs(option acp.SessionConfigOption) []string {
if option.Select == nil || option.Select.Options.Ungrouped == nil {
return nil
}
ids := make([]string, 0, len(*option.Select.Options.Ungrouped))
for _, model := range *option.Select.Options.Ungrouped {
ids = append(ids, string(model.Value))
}
return ids
}
func containsString(values []string, want string) bool {
for _, v := range values {
if v == want {
@@ -543,7 +543,9 @@ func (m *Manager) reapplySessionModelAfterReset(
m.sessionManager.publishModelSelectionWarningEvent(execution, newSessionID, decision)
}
if decision.EffectiveModel != "" &&
(decision.Outcome == ModelSelectionOutcomeApplied || decision.Outcome == ModelSelectionOutcomeExplicitFallback) {
(decision.Outcome == ModelSelectionOutcomeApplied ||
decision.Outcome == ModelSelectionOutcomeExplicitFallback ||
decision.Outcome == ModelSelectionOutcomeUniqueVariation) {
m.logger.Info("re-applied session model after context reset",
zap.String("execution_id", execution.ID),
zap.String("session_id", execution.SessionID),
@@ -99,6 +99,51 @@ func TestManager_ResetAgentContext_ReappliesSessionRuntimeConfig(t *testing.T) {
require.Equal(t, v1.AgentStatusReady, exec.Status)
}
func TestReapplySessionModel_UsesUniqueAdvertisedVariation(t *testing.T) {
mgr := newTestManager(t)
mock := newRestartMockAgentctlServer(t, false, false)
client := createTestClient(t, mock.server.URL)
t.Cleanup(client.Close)
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
t.Cleanup(cancel)
require.NoError(t, client.StreamUpdates(ctx, func(agentctl.AgentEvent) {}, nil, nil))
exec := runtimeConfigResetExecution(client, true)
exec.SetModelState(&CachedModelState{
CurrentModelID: "provider-default",
Models: []streams.SessionModelInfo{{ModelID: "opus[1m]"}},
})
require.NoError(t, mgr.reapplySessionModelAfterReset(ctx, exec, "reset-session", "opus"))
require.Equal(t, []string{"opus[1m]"}, mock.getSetModelIDs())
}
func TestWorkspaceRebindModel_UsesUniqueAdvertisedVariation(t *testing.T) {
mgr := newTestManager(t)
mock := newRestartMockAgentctlServer(t, false, false)
client := createTestClient(t, mock.server.URL)
t.Cleanup(client.Close)
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
t.Cleanup(cancel)
require.NoError(t, client.StreamUpdates(ctx, func(agentctl.AgentEvent) {}, nil, nil))
exec := runtimeConfigResetExecution(client, true)
exec.SetModelState(&CachedModelState{
CurrentModelID: "provider-default",
Models: []streams.SessionModelInfo{{ModelID: "opus[1m]"}},
})
require.NoError(t, mgr.reapplyReboundSessionConfig(
ctx,
exec,
"rebind-session",
&CachedModelState{},
"opus",
nil,
))
require.Equal(t, []string{"opus[1m]"}, mock.getSetModelIDs())
}
func TestManager_RestartAgentProcess_ReappliesSessionRuntimeConfig(t *testing.T) {
mgr := newTestManager(t)
provider := runtimeConfigResetWorkspaceInfo()
@@ -513,10 +513,12 @@ func (sm *SessionManager) applyStartModelPolicyToEffectiveModel(
}
effective.decision = decision
if decision.Outcome == ModelSelectionOutcomeApplied ||
decision.Outcome == ModelSelectionOutcomeExplicitFallback {
decision.Outcome == ModelSelectionOutcomeExplicitFallback ||
decision.Outcome == ModelSelectionOutcomeUniqueVariation {
effective.appliedModel = decision.EffectiveModel
}
if decision.Outcome == ModelSelectionOutcomeExplicitFallback {
if decision.Outcome == ModelSelectionOutcomeExplicitFallback ||
decision.Outcome == ModelSelectionOutcomeUniqueVariation {
effective.model = decision.EffectiveModel
}
return effective
@@ -0,0 +1,100 @@
package lifecycle
import (
"context"
"testing"
"time"
"github.com/kandev/kandev/internal/agent/agents"
agentctl "github.com/kandev/kandev/internal/agent/runtime/agentctl"
"github.com/kandev/kandev/internal/agentctl/types/streams"
"github.com/kandev/kandev/pkg/agent"
)
func TestInitializeAndPrompt_UniqueVariationPublishesProviderNeutralWarning(t *testing.T) {
mock := newMockAgentServer(t)
defer mock.Close()
log := newSessionTestLogger()
stopCh := newTestStopCh(t)
sm := NewSessionManager(log, stopCh)
streamMgr := NewStreamManager(log, StreamCallbacks{
OnAgentEvent: func(execution *AgentExecution, event agentctl.AgentEvent) {},
}, nil, stopCh)
cleanupStreamManager(t, stopCh, streamMgr)
eventBus := &MockEventBusWithTracking{}
sm.SetDependencies(NewEventPublisher(eventBus, log), streamMgr, nil, nil)
client := createTestClient(t, mock.server.URL)
defer client.Close()
execution := &AgentExecution{
ID: "exec-unique-variation",
TaskID: "task-1",
SessionID: "session-1",
WorkspacePath: "/workspace",
agentctl: client,
promptDoneCh: make(chan PromptCompletionSignal, 1),
}
execution.SetModelState(modelState("opus[1m]"))
agentConfig := &testAgent{
id: "test-agent",
enabled: true,
runtimeConfig: &agents.RuntimeConfig{
Cmd: agents.NewCommand("test-agent"),
Protocol: agent.ProtocolACP,
SessionConfig: agents.SessionConfig{},
ResourceLimits: agents.ResourceLimits{MemoryMB: 512, CPUCores: 0.5, Timeout: time.Hour},
},
}
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
defer cancel()
err := sm.InitializeAndPrompt(ctx, execution, agentConfig, "", nil, nil,
func(executionID string) error { return nil },
StartModelPolicy{Model: "opus"}, "plan", nil)
if err != nil {
t.Fatalf("InitializeAndPrompt failed: %v", err)
}
setModelCalls := 0
for _, action := range mock.getActionLog() {
if action == "agent.session.set_model" {
setModelCalls++
}
}
if setModelCalls != 1 {
t.Fatalf("set_model calls = %d, want exactly 1", setModelCalls)
}
var warning *streams.ModelSelectionWarning
legacyFallbackEvents := 0
for _, event := range eventBus.getStreamEvents() {
if event.Data == nil {
continue
}
if event.Data.ModelSelectionWarning != nil {
warning = event.Data.ModelSelectionWarning
}
if event.Data.Type == streams.EventTypeSessionModelFallback {
legacyFallbackEvents++
}
}
if warning == nil {
t.Fatal("expected a model-selection warning event")
}
if warning.RequestedModel != "opus" {
t.Errorf("warning requested model = %q, want opus", warning.RequestedModel)
}
if warning.EffectiveModel != "opus[1m]" {
t.Errorf("warning effective model = %q, want opus[1m]", warning.EffectiveModel)
}
if warning.FallbackModel != "" {
t.Errorf("warning fallback model = %q, want empty", warning.FallbackModel)
}
if warning.Reason != ModelSelectionReasonUniqueVariationApplied {
t.Errorf("warning reason = %q, want %q", warning.Reason, ModelSelectionReasonUniqueVariationApplied)
}
if legacyFallbackEvents != 0 {
t.Fatalf("legacy fallback events = %d, want zero for unique variation", legacyFallbackEvents)
}
}
@@ -3,6 +3,7 @@ package lifecycle
import (
"context"
"fmt"
"strings"
"github.com/kandev/kandev/internal/agentctl/sessionmodel"
"github.com/kandev/kandev/internal/common/logger"
@@ -31,6 +32,7 @@ const (
ModelSelectionOutcomeNone ModelSelectionOutcome = ""
ModelSelectionOutcomeApplied ModelSelectionOutcome = "applied"
ModelSelectionOutcomeExplicitFallback ModelSelectionOutcome = "explicit_fallback"
ModelSelectionOutcomeUniqueVariation ModelSelectionOutcome = "unique_variation"
ModelSelectionOutcomeProviderDefault ModelSelectionOutcome = "provider_default"
)
@@ -42,6 +44,7 @@ const (
ModelSelectionReasonCatalogEmpty = "catalog_empty"
ModelSelectionReasonSelectionUnsupported = "selection_unsupported"
ModelSelectionReasonSelectionFailedAutoFallback = "selection_failed_auto_fallback"
ModelSelectionReasonUniqueVariationApplied = "unique_variation_applied"
)
// ModelSelectionDecision is the single model-selection result shared by
@@ -84,6 +87,32 @@ func containsModel(ids []string, id string) bool {
return false
}
func uniqueAdvertisedModelVariation(requested string, advertised []string) string {
if requested == "" || strings.ContainsAny(requested, "[]") {
return ""
}
prefix := requested + "["
candidates := make(map[string]struct{})
for _, id := range advertised {
if !strings.HasPrefix(id, prefix) || !strings.HasSuffix(id, "]") {
continue
}
variant := strings.TrimSuffix(strings.TrimPrefix(id, prefix), "]")
if variant == "" || strings.ContainsAny(variant, "[]") {
continue
}
candidates[id] = struct{}{}
if len(candidates) > 1 {
return ""
}
}
for candidate := range candidates {
return candidate
}
return ""
}
func providerDefaultDecision(state *CachedModelState, policy StartModelPolicy, reason string) ModelSelectionDecision {
decision := ModelSelectionDecision{
RequestedModel: policy.Model,
@@ -114,6 +143,9 @@ func applyStartModelPolicy(
if policy.Model == "" {
return ModelSelectionDecision{Outcome: ModelSelectionOutcomeNone}, nil
}
if policy.AutoFallback {
policy.FallbackModel = ""
}
decision := ModelSelectionDecision{
RequestedModel: policy.Model,
@@ -125,9 +157,15 @@ func applyStartModelPolicy(
}
if !containsModel(advertised, policy.Model) {
if policy.AutoFallback {
return providerDefaultDecision(state, policy, ModelSelectionReasonRequestedNotAdvertised), nil
}
if policy.FallbackModel != "" && containsModel(advertised, policy.FallbackModel) {
return applyAdvertisedFallback(ctx, log, applier, state, policy, decision)
}
if variation := uniqueAdvertisedModelVariation(policy.Model, advertised); variation != "" {
return applyUniqueAdvertisedVariation(ctx, log, applier, state, policy, decision, variation)
}
reason := ModelSelectionReasonRequestedNotAdvertised
if policy.FallbackModel != "" {
reason = ModelSelectionReasonFallbackNotAdvertised
@@ -159,6 +197,41 @@ func applyStartModelPolicy(
return decision, nil
}
func applyUniqueAdvertisedVariation(
ctx context.Context,
log *logger.Logger,
applier modelApplier,
state *CachedModelState,
policy StartModelPolicy,
decision ModelSelectionDecision,
variation string,
) (ModelSelectionDecision, error) {
decision.FallbackModel = ""
policy.FallbackModel = ""
decision.SetModelCalled = true
if err := applier.SetModel(ctx, variation); err != nil {
if sessionmodel.IsMethodNotFound(err) {
decision = providerDefaultDecision(state, policy, ModelSelectionReasonSelectionUnsupported)
decision.SetModelCalled = true
return decision, nil
}
if policy.AutoFallback {
decision = providerDefaultDecision(state, policy, ModelSelectionReasonSelectionFailedAutoFallback)
decision.SetModelCalled = true
return decision, nil
}
return decision, fmt.Errorf("failed to set unique model variation %q: %w", variation, err)
}
decision.EffectiveModel = variation
decision.Outcome = ModelSelectionOutcomeUniqueVariation
decision.Reason = ModelSelectionReasonUniqueVariationApplied
decision.Warning = true
log.Info("start model unavailable, using unique advertised variation",
zap.String("start_model", policy.Model),
zap.String("variation_model", variation))
return decision, nil
}
func applyAdvertisedFallback(
ctx context.Context,
log *logger.Logger,
@@ -47,6 +47,104 @@ func TestApplyStartModelPolicyExecutorAuthority(t *testing.T) {
wantEffective: "fallback",
wantWarning: true,
},
{
name: "unique variation is applied when fallback is absent",
state: modelState("opus[1m]"),
policy: StartModelPolicy{Model: "opus"},
wantCalls: []string{"opus[1m]"},
wantOutcome: ModelSelectionOutcomeUniqueVariation,
wantReason: ModelSelectionReasonUniqueVariationApplied,
wantEffective: "opus[1m]",
wantWarning: true,
},
{
name: "advertised fallback takes precedence over unique variation",
state: modelState("fallback", "opus[1m]"),
policy: StartModelPolicy{Model: "opus", FallbackModel: "fallback"},
wantCalls: []string{"fallback"},
wantOutcome: ModelSelectionOutcomeExplicitFallback,
wantReason: ModelSelectionReasonRequestedNotAdvertised,
wantEffective: "fallback",
wantWarning: true,
},
{
name: "unadvertised fallback does not block unique variation",
state: modelState("opus[1m]"),
policy: StartModelPolicy{Model: "opus", FallbackModel: "fallback"},
wantCalls: []string{"opus[1m]"},
wantOutcome: ModelSelectionOutcomeUniqueVariation,
wantReason: ModelSelectionReasonUniqueVariationApplied,
wantEffective: "opus[1m]",
wantWarning: true,
},
{
name: "auto fallback keeps unavailable model on provider default",
state: modelState("fallback", "opus[1m]"),
policy: StartModelPolicy{Model: "opus", FallbackModel: "fallback", AutoFallback: true},
wantOutcome: ModelSelectionOutcomeProviderDefault,
wantReason: ModelSelectionReasonRequestedNotAdvertised,
wantWarning: true,
},
{
name: "duplicate unique variation is one candidate",
state: modelState("opus[1m]", "opus[1m]"),
policy: StartModelPolicy{Model: "opus"},
wantCalls: []string{"opus[1m]"},
wantOutcome: ModelSelectionOutcomeUniqueVariation,
wantReason: ModelSelectionReasonUniqueVariationApplied,
wantEffective: "opus[1m]",
wantWarning: true,
},
{
name: "multiple variations keep provider default",
state: modelState("opus[1m]", "opus[270k]"),
policy: StartModelPolicy{Model: "opus"},
wantOutcome: ModelSelectionOutcomeProviderDefault,
wantReason: ModelSelectionReasonRequestedNotAdvertised,
wantWarning: true,
},
{
name: "malformed variations keep provider default",
state: modelState("opus[]", "opus[1m", "opus[1m][fast]", "opus-pro[1m]"),
policy: StartModelPolicy{Model: "opus"},
wantOutcome: ModelSelectionOutcomeProviderDefault,
wantReason: ModelSelectionReasonRequestedNotAdvertised,
wantWarning: true,
},
{
name: "variation text remains opaque",
state: modelState("opus[1m, fast]"),
policy: StartModelPolicy{Model: "opus"},
wantCalls: []string{"opus[1m, fast]"},
wantOutcome: ModelSelectionOutcomeUniqueVariation,
wantReason: ModelSelectionReasonUniqueVariationApplied,
wantEffective: "opus[1m, fast]",
wantWarning: true,
},
{
name: "case-sensitive variation matching keeps provider default",
state: modelState("Opus[1m]"),
policy: StartModelPolicy{Model: "opus"},
wantOutcome: ModelSelectionOutcomeProviderDefault,
wantReason: ModelSelectionReasonRequestedNotAdvertised,
wantWarning: true,
},
{
name: "bracketed request does not infer another variation",
state: modelState("opus[2m]"),
policy: StartModelPolicy{Model: "opus[1m]"},
wantOutcome: ModelSelectionOutcomeProviderDefault,
wantReason: ModelSelectionReasonRequestedNotAdvertised,
wantWarning: true,
},
{
name: "exact bracketed request remains exact",
state: modelState("opus[1m]", "opus[2m]"),
policy: StartModelPolicy{Model: "opus[1m]"},
wantCalls: []string{"opus[1m]"},
wantOutcome: ModelSelectionOutcomeApplied,
wantEffective: "opus[1m]",
},
{
name: "advertised fallback method not supported keeps provider default",
state: modelState("fallback"),
@@ -57,6 +155,16 @@ func TestApplyStartModelPolicyExecutorAuthority(t *testing.T) {
wantReason: ModelSelectionReasonSelectionUnsupported,
wantWarning: true,
},
{
name: "unique variation method not supported keeps provider default",
state: modelState("opus[1m]"),
policy: StartModelPolicy{Model: "opus", FallbackModel: "fallback"},
applierErrors: []error{methodNotFoundErr()},
wantCalls: []string{"opus[1m]"},
wantOutcome: ModelSelectionOutcomeProviderDefault,
wantReason: ModelSelectionReasonSelectionUnsupported,
wantWarning: true,
},
{
name: "unadvertised fallback keeps provider default",
state: modelState("executor-default"),
@@ -132,6 +240,12 @@ func TestApplyStartModelPolicyExecutorAuthority(t *testing.T) {
if decision.Warning != tt.wantWarning {
t.Errorf("warning = %v, want %v", decision.Warning, tt.wantWarning)
}
if decision.Outcome == ModelSelectionOutcomeUniqueVariation && decision.FallbackModel != "" {
t.Errorf("unique variation fallback model = %q, want empty", decision.FallbackModel)
}
if tt.policy.AutoFallback && decision.FallbackModel != "" {
t.Errorf("auto-fallback fallback model = %q, want empty", decision.FallbackModel)
}
})
}
}
@@ -150,6 +150,20 @@ describe("ProfileFormFields no-silent-model-fallback rows", () => {
expect(trigger.textContent).toContain("claude-gone");
});
it("names a unique advertised variation while preserving the saved model", () => {
renderForm(formData({ model: "opus" }), {
...modelConfig,
available_models: [{ id: "opus[1m]", name: "Opus (1m)" }],
});
const advisory = screen.getByTestId("profile-model-variation-advisory");
expect(advisory.textContent).toContain("opus[1m]");
expect(advisory.textContent).toContain("opus");
expect(
screen.getByRole("button", { name: profileStartModelSettingsLabel }).textContent,
).toContain("opus");
});
it("shows the agent fallback row when auto-fallback is off", () => {
renderForm(formData({ auto_fallback: false }));
expandFallbackSettings();
@@ -23,6 +23,7 @@ import {
import type { ModelConfig, ModeEntry, ModelEntry } from "@/lib/types/http";
import type { PermissionKey } from "@/lib/agent-permissions";
import type { CLIFlag } from "@/lib/types/http";
import { findUniqueModelVariation } from "@/lib/model-variation";
import {
SettingsFieldDescription,
SettingsFieldLabel,
@@ -83,6 +84,12 @@ export function ModelPicker({
// in the list, greyed out and unselectable, so the user sees what was
// configured instead of it silently vanishing.
const modelIsGone = Boolean(profile.model && !modelOptions.some((m) => m.id === profile.model));
const uniqueVariation = modelIsGone
? findUniqueModelVariation(
profile.model,
modelOptions.map((model) => model.id),
)
: null;
if (modelIsGone) {
modelOptions.unshift({
id: profile.model!,
@@ -99,22 +106,32 @@ export function ModelPicker({
}));
return (
<ModelConfigSelector
modelOptions={modelOptions}
currentModel={currentModel}
configOptions={selectedConfigOptions}
onModelChange={(value) => onChange({ model: value })}
onConfigChange={(configId, value) =>
onChange({ config_options: { ...(profile.config_options ?? {}), [configId]: value } })
}
placeholder={placeholder ?? t("settings:selectAModel")}
ariaLabel={ariaLabel}
popoverAlign="start"
disabled={disabled}
configOptionsLoading={configOptionsLoading}
keepOpenOnModelChange={keepOpenOnModelChange}
triggerClassName={modelIsGone ? "text-destructive" : undefined}
/>
<div className="space-y-1.5">
<ModelConfigSelector
modelOptions={modelOptions}
currentModel={currentModel}
configOptions={selectedConfigOptions}
onModelChange={(value) => onChange({ model: value })}
onConfigChange={(configId, value) =>
onChange({ config_options: { ...(profile.config_options ?? {}), [configId]: value } })
}
placeholder={placeholder ?? t("settings:selectAModel")}
ariaLabel={ariaLabel}
popoverAlign="start"
disabled={disabled}
configOptionsLoading={configOptionsLoading}
keepOpenOnModelChange={keepOpenOnModelChange}
triggerClassName={modelIsGone ? "text-destructive" : undefined}
/>
{uniqueVariation && (
<p className="text-xs text-muted-foreground" data-testid="profile-model-variation-advisory">
{t("settings:modelVariationAdvisory", {
model: profile.model,
variation: uniqueVariation,
})}
</p>
)}
</div>
);
}
@@ -58,6 +58,19 @@ const AGENT_WITH_GPT: AvailableAgent = {
},
} as unknown as AvailableAgent;
const AGENT_WITH_OPUS_VARIATION: AvailableAgent = {
name: "omp-acp",
available: true,
model_config: {
default_model: "opus[1m]",
available_models: [{ id: "opus[1m]", name: "Opus (1m)" }],
current_model_id: "opus[1m]",
available_modes: [],
supports_dynamic_models: false,
status: "ok",
},
} as unknown as AvailableAgent;
const GONE_MODEL = "claude-gone";
const DATA_DISABLED = "data-disabled";
const MODEL_PROBE_WARNING_TEST_ID = "agent-profile-model-probe-warning";
@@ -225,6 +238,14 @@ describe("useAgentProfileOptions executor-authoritative model hint", () => {
expect(option.getAttribute(DATA_DISABLED)).toBeNull();
});
it("names a unique advertised variation without disabling the profile", () => {
setAvailableAgents([AGENT_WITH_OPUS_VARIATION]);
const option = renderOptions([profileOption({ model: "opus" })]);
expect(option.getAttribute(DATA_DISABLED)).toBeNull();
expect(getModelProbeWarningLabel()).toContain("opus[1m]");
});
it("keeps a profile with an empty (agent default) model selectable", () => {
const option = renderOptions([profileOption({ model: "" })]);
expect(option.getAttribute(DATA_DISABLED)).toBeNull();
@@ -32,6 +32,7 @@ import { formatUserHomePath, truncateRepoPath } from "@/lib/utils";
import { getExecutorIcon } from "@/lib/executor-icons";
import { AgentLogo } from "@/components/agent-logo";
import { getCapabilityWarning } from "@/lib/capability-warning";
import { findUniqueModelVariation } from "@/lib/model-variation";
import { useTouchDrawer } from "@/hooks/use-compact-task-chrome";
import { buildBranchKeywords } from "./branch-picker-options";
import {
@@ -229,9 +230,18 @@ export function useAgentProfileOptions(
const startModelGone = Boolean(
profile.model && advertised.length > 0 && !advertised.includes(profile.model),
);
const modelProbeNote = startModelGone
? t("settings:profileStartModelNotAdvertisedOnHost", { model: profile.model })
: undefined;
const uniqueVariation = startModelGone
? findUniqueModelVariation(profile.model ?? "", advertised)
: null;
let modelProbeNote: string | undefined;
if (startModelGone) {
modelProbeNote = uniqueVariation
? t("settings:profileStartModelUniqueVariationOnHost", {
model: profile.model,
variation: uniqueVariation,
})
: t("settings:profileStartModelNotAdvertisedOnHost", { model: profile.model });
}
const renderProfileLabel = (modelProbeWarning: React.ReactNode) => (
<span className="flex min-w-0 flex-1 flex-col gap-1">
<span className="flex shrink-0 items-center justify-between gap-2">
@@ -42,6 +42,23 @@ describe("StatusMessage model selection warnings", () => {
expect(screen.getByText("Check the agent version in the executor.")).toBeTruthy();
expect(screen.getByText("The saved model was not advertised by the executor.")).toBeTruthy();
});
it("explains when the executor applied the only advertised variation", () => {
const comment = modelSelectionWarningMessage();
comment.metadata = {
...comment.metadata,
reason: "unique_variation_applied",
effective_model: "opus[1m]",
fallback_model: undefined,
};
render(<StatusMessage comment={comment} />);
expect(
screen.getByText("The executor applied the only advertised variation of the saved model."),
).toBeTruthy();
expect(screen.queryByText(/fallback model/i)).toBeNull();
});
});
describe("StatusMessage branch replacement warnings", () => {
@@ -180,6 +180,7 @@ function modelSelectionReasonLabel(reason: string | undefined): string {
const keyByReason: Record<string, string> = {
requested_not_advertised: "task:modelSelectionReasonRequestedNotAdvertised",
fallback_not_advertised: "task:modelSelectionReasonFallbackNotAdvertised",
unique_variation_applied: "task:modelSelectionReasonUniqueVariation",
catalog_empty: "task:modelSelectionReasonCatalogEmpty",
selection_unsupported: "task:modelSelectionReasonUnsupported",
selection_failed_auto_fallback: "task:modelSelectionReasonAutoFallback",
@@ -2,6 +2,10 @@ import type { AgentProfile } from "../../../lib/types/http-agents";
import type { ApiClient } from "../../helpers/api-client";
export const UNADVERTISED_MODEL = "host-only-model";
export const MODEL_VARIATION_BASE = "opus";
export const UNIQUE_MODEL_VARIATION = "opus[1m]";
export const AMBIGUOUS_MODEL_VARIATIONS = ["opus[270k]", "opus[1m, fast]"] as const;
const MODEL_CATALOG_ENV = "MOCK_AGENT_MODEL_CATALOG";
export async function createMismatchedProfile(
apiClient: ApiClient,
@@ -13,6 +17,20 @@ export async function createMismatchedProfile(
return apiClient.createAgentProfile(agent.id, name, { model: UNADVERTISED_MODEL });
}
export async function createModelVariationProfile(
apiClient: ApiClient,
name: string,
catalog: "unique" | "ambiguous",
): Promise<AgentProfile> {
const { agents } = await apiClient.listAgents();
const agent = agents.find((item) => item.name === "mock-agent") ?? agents[0];
if (!agent) throw new Error("The E2E fixture must provide a mock agent");
return apiClient.createAgentProfile(agent.id, name, {
model: MODEL_VARIATION_BASE,
env_vars: [{ key: MODEL_CATALOG_ENV, value: catalog }],
});
}
export async function readModelSelectionWarnings(
apiClient: ApiClient,
sessionId: string,
@@ -2,7 +2,10 @@ import { expect, test } from "../../fixtures/test-base";
import { waitForSessionDone } from "../../helpers/session";
import { SessionPage } from "../../pages/session-page";
import {
createModelVariationProfile,
createMismatchedProfile,
MODEL_VARIATION_BASE,
UNIQUE_MODEL_VARIATION,
readModelSelectionWarnings,
UNADVERTISED_MODEL,
} from "./model-mismatch-warning-helpers";
@@ -73,4 +76,124 @@ test.describe("model mismatch warning", () => {
await apiClient.deleteAgentProfile(profile.id, true).catch(() => {});
}
});
test("applies one unique advertised variation and persists its warning", async ({
testPage,
apiClient,
seedData,
}) => {
test.setTimeout(120_000);
const profile = await createModelVariationProfile(
apiClient,
"Unique model variation profile",
"unique",
);
try {
const task = await apiClient.createTaskWithAgent(
seedData.workspaceId,
"Unique model variation warning",
profile.id,
{
description: "/e2e:simple-message",
workflow_id: seedData.workflowId,
workflow_step_id: seedData.startStepId,
repository_ids: [seedData.repositoryId],
},
);
expect(task.session_id).toBeTruthy();
await waitForSessionDone(
apiClient,
task.id,
task.session_id!,
"Waiting for unique model variation task continuation",
);
await expect
.poll(() => readModelSelectionWarnings(apiClient, task.session_id!), {
timeout: 15_000,
message: "Waiting for the unique model variation warning",
})
.toHaveLength(1);
const warnings = await readModelSelectionWarnings(apiClient, task.session_id!);
expect(warnings[0]?.metadata).toMatchObject({
kind: "model_selection_warning",
reason: "unique_variation_applied",
requested_model: MODEL_VARIATION_BASE,
effective_model: UNIQUE_MODEL_VARIATION,
agent_id: "mock-agent",
});
expect(warnings[0]?.metadata?.fallback_model).toBeUndefined();
expect((await apiClient.getAgentProfile(profile.id)).model).toBe(MODEL_VARIATION_BASE);
await testPage.goto(`/t/${task.id}`);
const session = new SessionPage(testPage);
await session.waitForLoad();
await expect(
session.activeChat().getByText("The executor could not use the saved model selection."),
).toBeVisible({ timeout: 15_000 });
await expect(session.activeChat()).toContainText(UNIQUE_MODEL_VARIATION);
await testPage.reload();
await session.waitForLoad();
await expect(
session.activeChat().getByText("The executor could not use the saved model selection."),
).toBeVisible({ timeout: 15_000 });
await expect(session.activeChat()).toContainText(UNIQUE_MODEL_VARIATION);
await expect
.poll(() => readModelSelectionWarnings(apiClient, task.session_id!))
.toHaveLength(1);
} finally {
await apiClient.deleteAgentProfile(profile.id, true).catch(() => {});
}
});
test("does not infer a model when the executor advertises multiple variations", async ({
apiClient,
seedData,
}) => {
test.setTimeout(120_000);
const profile = await createModelVariationProfile(
apiClient,
"Ambiguous model variation profile",
"ambiguous",
);
try {
const task = await apiClient.createTaskWithAgent(
seedData.workspaceId,
"Ambiguous model variation warning",
profile.id,
{
description: "/e2e:simple-message",
workflow_id: seedData.workflowId,
workflow_step_id: seedData.startStepId,
repository_ids: [seedData.repositoryId],
},
);
expect(task.session_id).toBeTruthy();
await waitForSessionDone(
apiClient,
task.id,
task.session_id!,
"Waiting for ambiguous model variation task continuation",
);
await expect
.poll(() => readModelSelectionWarnings(apiClient, task.session_id!), {
timeout: 15_000,
message: "Waiting for the ambiguous model variation warning",
})
.toHaveLength(1);
const warnings = await readModelSelectionWarnings(apiClient, task.session_id!);
expect(warnings[0]?.metadata).toMatchObject({
kind: "model_selection_warning",
reason: "requested_not_advertised",
requested_model: MODEL_VARIATION_BASE,
effective_model: "mock-fast",
agent_id: "mock-agent",
});
expect((await apiClient.getAgentProfile(profile.id)).model).toBe(MODEL_VARIATION_BASE);
} finally {
await apiClient.deleteAgentProfile(profile.id, true).catch(() => {});
}
});
});
@@ -2,7 +2,10 @@ import { expect, test } from "../../fixtures/test-base";
import { KanbanPage } from "../../pages/kanban-page";
import { assertNoDocumentHorizontalOverflow } from "../../helpers/layout-assertions";
import {
createModelVariationProfile,
createMismatchedProfile,
MODEL_VARIATION_BASE,
UNIQUE_MODEL_VARIATION,
UNADVERTISED_MODEL,
} from "../session/model-mismatch-warning-helpers";
@@ -41,4 +44,44 @@ test.describe("executor-authoritative model selection on mobile", () => {
await apiClient.deleteAgentProfile(profile.id, true).catch(() => {});
}
});
test("opens the unique variation advisory by touch without overflow", async ({
testPage,
apiClient,
}) => {
const profile = await createModelVariationProfile(
apiClient,
"Mobile unique host variation profile",
"unique",
);
try {
const kanban = new KanbanPage(testPage);
await kanban.goto();
await testPage.reload({ waitUntil: "networkidle" });
await testPage.getByRole("button", { name: "Add task" }).tap();
const dialog = testPage.getByTestId("create-task-dialog");
await expect(dialog).toBeVisible();
const selector = dialog.getByTestId("agent-profile-selector");
await selector.tap();
const option = testPage
.getByRole("listbox")
.getByRole("option", { name: profile.name, exact: false });
await expect(option).toBeVisible();
await expect(option).toBeEnabled();
const warning = option.getByTestId("agent-profile-model-probe-warning");
await expect(warning).toBeVisible();
const warningText = `The host probe found one possible variation of ${MODEL_VARIATION_BASE}: ${UNIQUE_MODEL_VARIATION}. The selected executor will decide the model at launch.`;
await warning.tap();
await expect(
testPage
.locator('[data-slot="drawer-content"][data-state="open"]')
.filter({ hasText: warningText }),
).toBeVisible();
await assertNoDocumentHorizontalOverflow(testPage, "mobile unique variation advisory");
} finally {
await apiClient.deleteAgentProfile(profile.id, true).catch(() => {});
}
});
});
@@ -1,7 +1,10 @@
import { expect, test } from "../../fixtures/test-base";
import { KanbanPage } from "../../pages/kanban-page";
import {
createModelVariationProfile,
createMismatchedProfile,
MODEL_VARIATION_BASE,
UNIQUE_MODEL_VARIATION,
UNADVERTISED_MODEL,
} from "../session/model-mismatch-warning-helpers";
@@ -41,4 +44,44 @@ test.describe("executor-authoritative model selection", () => {
await apiClient.deleteAgentProfile(profile.id, true).catch(() => {});
}
});
test("names one host-advertised variation while keeping the profile selectable", async ({
testPage,
apiClient,
}) => {
const profile = await createModelVariationProfile(
apiClient,
"Unique host variation selectable profile",
"unique",
);
try {
const kanban = new KanbanPage(testPage);
await kanban.goto();
await testPage.reload({ waitUntil: "networkidle" });
await kanban.createTaskButton.first().click();
const dialog = testPage.getByTestId("create-task-dialog");
await expect(dialog).toBeVisible();
const selector = dialog.getByTestId("agent-profile-selector");
await selector.click();
const option = testPage
.getByRole("listbox")
.getByRole("option", { name: profile.name, exact: false });
await expect(option).toBeVisible();
await expect(option).toBeEnabled();
const warning = option.getByTestId("agent-profile-model-probe-warning");
await expect(warning).toBeVisible();
const warningText = `The host probe found one possible variation of ${MODEL_VARIATION_BASE}: ${UNIQUE_MODEL_VARIATION}. The selected executor will decide the model at launch.`;
await warning.hover();
await expect(
testPage
.locator('[data-slot="tooltip-content"]:not([data-state="closed"])')
.filter({ hasText: warningText }),
).toBeVisible();
await option.click();
} finally {
await apiClient.deleteAgentProfile(profile.id, true).catch(() => {});
}
});
});
+57
View File
@@ -0,0 +1,57 @@
import { describe, expect, it } from "vitest";
import { findUniqueModelVariation } from "./model-variation";
describe("findUniqueModelVariation", () => {
it.each([
{
name: "finds one bracketed variation",
requested: "opus",
advertised: ["opus[1m]"],
expected: "opus[1m]",
},
{
name: "deduplicates the same variation",
requested: "opus",
advertised: ["opus[1m]", "opus[1m]"],
expected: "opus[1m]",
},
{
name: "keeps opaque variation text",
requested: "opus",
advertised: ["opus[1m, fast]"],
expected: "opus[1m, fast]",
},
])("$name", ({ requested, advertised, expected }) => {
expect(findUniqueModelVariation(requested, advertised)).toBe(expected);
});
it.each([
{
name: "returns no candidate for an empty catalog",
requested: "opus",
advertised: [],
},
{
name: "returns no candidate for ambiguous variations",
requested: "opus",
advertised: ["opus[1m]", "opus[270k]"],
},
{
name: "returns no candidate for case differences",
requested: "opus",
advertised: ["Opus[1m]"],
},
{
name: "returns no candidate for a bracketed request",
requested: "opus[1m]",
advertised: ["opus[2m]"],
},
{
name: "returns no candidate for malformed IDs",
requested: "opus",
advertised: ["opus[]", "opus[1m", "opus[1m][fast]", "opus-pro[1m]"],
},
])("$name", ({ requested, advertised }) => {
expect(findUniqueModelVariation(requested, advertised)).toBeNull();
});
});
+17
View File
@@ -0,0 +1,17 @@
function isModelVariation(requested: string, advertised: string): boolean {
const prefix = `${requested}[`;
if (!advertised.startsWith(prefix) || !advertised.endsWith("]")) return false;
const variant = advertised.slice(prefix.length, -1);
return variant.length > 0 && !variant.includes("[") && !variant.includes("]");
}
export function findUniqueModelVariation(
requested: string,
advertised: readonly string[],
): string | null {
if (!requested || requested.includes("[") || requested.includes("]")) return null;
const candidates = new Set(advertised.filter((model) => isModelVariation(requested, model)));
return candidates.size === 1 ? ([...candidates][0] ?? null) : null;
}
+2
View File
@@ -504,6 +504,8 @@
"profileForTasksCreatedByAgents": "Profile for Tasks Created by Agents",
"programming": "Programming",
"profileStartModelNotAdvertisedOnHost": "The host probe did not advertise {{model}}. The selected executor will decide the model at launch.",
"profileStartModelUniqueVariationOnHost": "The host probe found one possible variation of {{model}}: {{variation}}. The selected executor will decide the model at launch.",
"modelVariationAdvisory": "The host catalog has one possible variation of {{model}}: {{variation}}. The saved model remains unchanged until launch.",
"promptAdd": "Add prompt",
"promptEditorAriaLabel": "Prompt editor",
"promptEditorBothHint": "Type {{placeholder}} for available placeholders or {{mention}} for saved prompts.",
+1
View File
@@ -883,6 +883,7 @@
"modelSelectionReasonCatalogEmpty": "The executor did not advertise a model catalog.",
"modelSelectionReasonFallbackNotAdvertised": "The saved fallback model was not advertised by the executor.",
"modelSelectionReasonRequestedNotAdvertised": "The saved model was not advertised by the executor.",
"modelSelectionReasonUniqueVariation": "The executor applied the only advertised variation of the saved model.",
"modelSelectionReasonUnknown": "{{reason}}",
"modelSelectionReasonUnsupported": "The executor does not support model selection.",
"modelSelectionRemediationAgentVersion": "Check the agent version in the executor.",
@@ -504,6 +504,8 @@
"profileForTasksCreatedByAgents": "Ƥŕōƒĩĺē ƒōŕ Ţàśķś Ćŕēàţēď ƀŷ Àĝēńţś",
"programming": "Ƥŕōĝŕàḿḿĩńĝ",
"profileStartModelNotAdvertisedOnHost": "Ţĥē ĥōśţ ƥŕōƀē ďĩď ńōţ àďvēŕţĩśē {{model}}. Ţĥē śēĺēćţēď ēxēćũţōŕ ŵĩĺĺ ďēćĩďē ţĥē ḿōďēĺ àţ ĺàũńćĥ.",
"profileStartModelUniqueVariationOnHost": "Ţĥē ĥōśţ ƥŕōƀē ƒōũńď ōńē ƥōśśĩƀĺē vàŕĩàţĩōń ōƒ {{model}}: {{variation}}. Ţĥē śēĺēćţēď ēxēćũţōŕ ŵĩĺĺ ďēćĩďē ţĥē ḿōďēĺ àţ ĺàũńćĥ.",
"modelVariationAdvisory": "Ţĥē ĥōśţ ćàţàĺōĝ ĥàś ōńē ƥōśśĩƀĺē vàŕĩàţĩōń ōƒ {{model}}: {{variation}}. Ţĥē śàvēď ḿōďēĺ ŕēḿàĩńś ũńćĥàńĝēď ũńţĩĺ ĺàũńćĥ.",
"promptAdd": "Àďď ƥŕōḿƥţ",
"promptEditorAriaLabel": "Ƥŕōḿƥţ ēďĩţōŕ",
"promptEditorBothHint": "Ţŷƥē {{placeholder}} ƒōŕ àvàĩĺàƀĺē ƥĺàćēĥōĺďēŕś ōŕ {{mention}} ƒōŕ śàvēď ƥŕōḿƥţś.",
+1
View File
@@ -883,6 +883,7 @@
"modelSelectionReasonCatalogEmpty": "Ţĥē ēxēćũţōŕ ďĩď ńōţ àďvēŕţĩśē à ḿōďēĺ ćàţàĺōĝ.",
"modelSelectionReasonFallbackNotAdvertised": "Ţĥē śàvēď ƒàĺĺƀàćķ ḿōďēĺ ŵàś ńōţ àďvēŕţĩśēď ƀŷ ţĥē ēxēćũţōŕ.",
"modelSelectionReasonRequestedNotAdvertised": "Ţĥē śàvēď ḿōďēĺ ŵàś ńōţ àďvēŕţĩśēď ƀŷ ţĥē ēxēćũţōŕ.",
"modelSelectionReasonUniqueVariation": "Ţĥē ēxēćũţōŕ àƥƥĺĩēď ţĥē ōńĺŷ àďvēŕţĩśēď vàŕĩàţĩōń ōƒ ţĥē śàvēď ḿōďēĺ.",
"modelSelectionReasonUnknown": "{{reason}}",
"modelSelectionReasonUnsupported": "Ţĥē ēxēćũţōŕ ďōēś ńōţ śũƥƥōŕţ ḿōďēĺ śēĺēćţĩōń.",
"modelSelectionRemediationAgentVersion": "Ćĥēćķ ţĥē àĝēńţ vēŕśĩōń ĩń ţĥē ēxēćũţōŕ.",
+2
View File
@@ -751,6 +751,8 @@
"terminalAndEditors": "Terminal e editores",
"workspacesAndAccess": "Workspaces e acesso",
"profileStartModelNotAdvertisedOnHost": "A sondagem do anfitrião não anunciou {{model}}. O executor selecionado decidirá o modelo no arranque.",
"profileStartModelUniqueVariationOnHost": "A sondagem do anfitrião encontrou uma única variação possível de {{model}}: {{variation}}. O executor selecionado decidirá o modelo no arranque.",
"modelVariationAdvisory": "O catálogo do anfitrião tem uma única variação possível de {{model}}: {{variation}}. O modelo guardado permanece inalterado até ao arranque.",
"navUnits": "Unidades da organização",
"unitsTitle": "Unidades da organização",
"unitsDescription": "Uma unidade contém equipas, áreas de trabalho e pessoas. Pertencer a uma unidade dá acesso a todas as áreas de trabalho abaixo dela, incluindo as criadas depois.",
+1
View File
@@ -1958,6 +1958,7 @@
"modelSelectionReasonCatalogEmpty": "O executor não anunciou um catálogo de modelos.",
"modelSelectionReasonFallbackNotAdvertised": "O modelo alternativo guardado não foi anunciado pelo executor.",
"modelSelectionReasonRequestedNotAdvertised": "O modelo guardado não foi anunciado pelo executor.",
"modelSelectionReasonUniqueVariation": "O executor aplicou a única variação anunciada do modelo guardado.",
"modelSelectionReasonUnknown": "{{reason}}",
"modelSelectionReasonUnsupported": "O executor não suporta seleção de modelo.",
"modelSelectionRemediationAgentVersion": "Verifique a versão do agente no executor.",
+2
View File
@@ -751,6 +751,8 @@
"keyValueKeyPlaceholder": "键",
"keyValueValuePlaceholder": "值",
"profileStartModelNotAdvertisedOnHost": "主机探测未公布 {{model}}。所选执行器将在启动时决定使用的模型。",
"profileStartModelUniqueVariationOnHost": "主机探测发现 {{model}} 的一个可能变体:{{variation}}。所选执行器将在启动时决定使用的模型。",
"modelVariationAdvisory": "主机目录中有一个可能的 {{model}} 变体:{{variation}}。保存的模型在启动前保持不变。",
"navUnits": "组织单元",
"unitsTitle": "组织单元",
"unitsDescription": "单元包含团队、工作区和成员。加入某个单元即可访问其下的所有工作区,包括之后创建的。",
+1
View File
@@ -1958,6 +1958,7 @@
"modelSelectionReasonCatalogEmpty": "执行器未公布模型目录。",
"modelSelectionReasonFallbackNotAdvertised": "执行器未公布已保存的备选模型。",
"modelSelectionReasonRequestedNotAdvertised": "执行器未公布已保存的模型。",
"modelSelectionReasonUniqueVariation": "执行器应用了已保存模型唯一公布的变体。",
"modelSelectionReasonUnknown": "{{reason}}",
"modelSelectionReasonUnsupported": "该执行器不支持选择模型。",
"modelSelectionRemediationAgentVersion": "请检查执行器中的智能体版本。",
+2
View File
@@ -751,6 +751,8 @@
"keyValueKeyPlaceholder": "鍵",
"keyValueValuePlaceholder": "值",
"profileStartModelNotAdvertisedOnHost": "主機探測未公佈 {{model}}。所選執行器將在啓動時決定使用的模型。",
"profileStartModelUniqueVariationOnHost": "主機探測發現 {{model}} 的一個可能變體:{{variation}}。所選執行器將在啓動時決定使用的模型。",
"modelVariationAdvisory": "主機目錄中有一個可能的 {{model}} 變體:{{variation}}。已儲存的模型在啟動前保持不變。",
"navUnits": "組織單元",
"unitsTitle": "組織單元",
"unitsDescription": "單元包含團隊、工作區和成員。加入某個單元即可存取其下的所有工作區,包括之後建立的。",
+1
View File
@@ -1958,6 +1958,7 @@
"modelSelectionReasonCatalogEmpty": "執行器未公佈模型目錄。",
"modelSelectionReasonFallbackNotAdvertised": "執行器未公佈已儲存的備選模型。",
"modelSelectionReasonRequestedNotAdvertised": "執行器未公佈已儲存的模型。",
"modelSelectionReasonUniqueVariation": "執行器套用了已儲存模型唯一公佈的變體。",
"modelSelectionReasonUnknown": "{{reason}}",
"modelSelectionReasonUnsupported": "該執行器不支援選擇模型。",
"modelSelectionRemediationAgentVersion": "請檢查執行器中的代理程式版本。",
+2
View File
@@ -751,6 +751,8 @@
"keyValueKeyPlaceholder": "鍵",
"keyValueValuePlaceholder": "值",
"profileStartModelNotAdvertisedOnHost": "主機探測未公佈 {{model}}。所選執行器將在啟動時決定使用的模型。",
"profileStartModelUniqueVariationOnHost": "主機探測發現 {{model}} 的一個可能變體:{{variation}}。所選執行器將在啟動時決定使用的模型。",
"modelVariationAdvisory": "主機目錄中有一個可能的 {{model}} 變體:{{variation}}。已儲存的模型在啟動前保持不變。",
"navUnits": "組織單元",
"unitsTitle": "組織單元",
"unitsDescription": "單元包含團隊、工作區和成員。加入某個單元即可存取其下的所有工作區,包括之後建立的。",
+1
View File
@@ -1958,6 +1958,7 @@
"modelSelectionReasonCatalogEmpty": "執行器未公佈模型目錄。",
"modelSelectionReasonFallbackNotAdvertised": "執行器未公佈已儲存的備選模型。",
"modelSelectionReasonRequestedNotAdvertised": "執行器未公佈已儲存的模型。",
"modelSelectionReasonUniqueVariation": "執行器套用了已儲存模型唯一公佈的變體。",
"modelSelectionReasonUnknown": "{{reason}}",
"modelSelectionReasonUnsupported": "該執行器不支援選擇模型。",
"modelSelectionRemediationAgentVersion": "請檢查執行器中的代理程式版本。",
@@ -0,0 +1,84 @@
# ADR-2026-09-07: Resolve Only One Advertised Model Variation
**Status:** proposed
**Date:** 2026-09-07
**Area:** backend, frontend, protocol
**Amends:** `2026-08-15-executor-authoritative-model-selection`
## Context
An agent CLI can replace a bare model ID with bracketed variations. A profile
can keep `opus` while the executor later advertises only `opus[1m]`.
The current executor-authoritative policy treats the saved ID as absent. It
uses an explicit fallback when available. Otherwise, it continues with the
provider current or default model and records a warning. This result can move a
session to an unrelated model even when the catalog has one clear variation of
the requested model.
Variation text is provider-owned. Kandev cannot safely rank `270k`, `1m`,
`fast`, or future labels. Catalog order and the current-model marker do not
express user intent.
## Decision
Kandev recognizes a model variation only by a narrow bracketed ID shape. A bare
requested ID `x` matches an advertised ID `x[v]` when `v` is non-empty and
contains no bracket. Matching is exact and case-sensitive. Kandev treats `v` as
opaque text.
For profiles with `auto_fallback = false`, the executor-authoritative resolution
order is:
1. Exact requested model.
2. Advertised explicit fallback from the profile.
3. One distinct advertised variation of the requested bare model.
4. Provider current or default model, with no speculative model-selection call.
Kandev applies step 3 only when exactly one distinct candidate exists. Zero or
multiple candidates produce no inferred selection. An unadvertised explicit
fallback does not block step 3.
A requested ID that contains a bracket is an exact ID only. Kandev does not
infer another variation from it.
Profiles with `auto_fallback = true` retain the legacy path when the requested
model is absent. Kandev ignores the configured explicit fallback, does not
infer a variation, and continues with the provider current or default model.
Apply errors for an advertised model remain best-effort in that mode.
The lifecycle stores `unique_variation` as the decision outcome and
`unique_variation_applied` as the warning reason. The effective model contains
the advertised variation. The explicit fallback field keeps its existing
meaning and stays empty when Kandev infers a variation.
The saved profile model does not change. The rule runs for initial launch,
context reset, and workspace rebind. The executor catalog remains authoritative.
Any host-catalog result remains an advisory.
## Consequences
- A profile with `opus` follows `opus[1m]` when it is the only advertised
variation.
- A catalog with `opus[270k]` and `opus[1m, fast]` remains ambiguous and uses
the existing provider-default path.
- An explicit advertised fallback remains stronger than inference.
- Kandev never sends a model ID that the executor did not advertise.
- Users receive a durable warning because the effective model differs from the
saved request.
- Provider-specific ranking logic does not enter the lifecycle policy.
- Backend and frontend helpers need mirrored contract tests. The backend result
remains authoritative when their input catalogs differ.
## Alternatives Considered
1. **Always choose the first variation.** Rejected because catalog order is not
a user preference or a stable provider contract.
2. **Prefer the provider current model.** Rejected because current state can
reflect a default or an earlier session, not the saved profile intent.
3. **Rank known labels such as context size or fast mode.** Rejected because
labels are provider-owned and can change without a Kandev release.
4. **Rewrite the saved profile to the inferred model.** Rejected because one
executor observation must not change behavior for every executor.
5. **Infer from an already bracketed request.** Rejected because it can replace
one explicit variation with another without user intent.
+1
View File
@@ -95,6 +95,7 @@ Read individual ADRs for full context. Create new ones via `/record decision` or
| 2026-08-15-subagent-context-schema-boundary | [Start with the execution-aware subagent schema](2026-08-15-subagent-context-schema-boundary.md) | accepted | backend | 2026-08-15 |
| 2026-08-15-portable-agent-configuration-bundles | [Use Explicit Portable Agent Configuration Bundles](2026-08-15-portable-agent-configuration-bundles.md) | accepted | backend, frontend, protocol, security | 2026-08-15 |
| 2026-08-15-executor-authoritative-model-selection | [Let the Executor Own Model Selection](2026-08-15-executor-authoritative-model-selection.md) | accepted | backend, frontend, protocol, persistence | 2026-08-15 |
| 2026-09-07-unique-model-variation-resolution | [Resolve Only One Advertised Model Variation](2026-09-07-unique-model-variation-resolution.md) | proposed | backend, frontend, protocol | 2026-09-07 |
| 2026-08-18-context-reset-preserves-runtime-configuration | [Preserve ACP Runtime Configuration Across Context Reset](2026-08-18-context-reset-preserves-runtime-configuration.md) | accepted | backend, protocol, workflow | 2026-08-18 |
| 2026-08-19-parent-authorized-child-workspace-sources | [Parent-authorized child workspace sources](2026-08-19-parent-authorized-child-workspace-sources.md) | accepted | backend, protocol, security | 2026-08-19 |
| 2026-08-24-task-snapshotted-branch-policies | [Snapshot Repository Branch Policies on Tasks](2026-08-24-task-snapshotted-branch-policies.md) | proposed | backend, frontend, persistence | 2026-08-24 |
@@ -0,0 +1,129 @@
---
created: 2026-09-07
status: complete
requirements:
- REQ-AGENTS-NO-SILENT-MODEL-FALLBACK-001
- REQ-AGENTS-NO-SILENT-MODEL-FALLBACK-002
system_design:
- ../../specs/agents/system-design/no-silent-model-fallback-01.md
- ../../specs/agents/system-design/no-silent-model-fallback-02.md
legacy_specs: []
---
# Implementation Plan: Unique Model Variation Fallback
## Overview
Extend the central executor-authoritative model policy with one deterministic
inference step. Implement the runtime contract first, expose its stable decision
in the existing frontend surfaces, then prove and document the user outcome.
## Scope
### In scope
- Resolve a missing bare model to its only distinct advertised bracketed
variation.
- Keep exact models and advertised explicit fallbacks at higher precedence.
- Persist a distinct warning outcome and reason without rewriting the profile.
- Present host results as advisory on desktop and touch interfaces.
- Test and document unique and ambiguous catalogs.
### Out of scope
- Provider-specific ranking of variation labels.
- Selection among multiple variations.
- Profile schema changes or saved-model migration.
- Office post-start route changes or mid-turn model switching.
## Technical approach
### Runtime policy
Add a pure unique-candidate helper in
`apps/backend/internal/agent/runtime/lifecycle/start_model.go`. It accepts the
requested ID and executor model IDs. It returns one distinct advertised ID only
when the structural contract matches.
For profiles with automatic fallback disabled, extend `applyStartModelPolicy`
with this order: exact request, advertised explicit fallback, unique variation,
provider current or default. Automatic-fallback profiles retain their legacy
no-selection behavior when the requested model is absent. Add
`unique_variation` to `ModelSelectionOutcome` and
`unique_variation_applied` to stable warning reasons.
Keep `EffectiveModel` as the inferred ID. Do not set `FallbackModel` for this
outcome. `publishModelSelectionWarningEvent` must therefore emit the new reason
without the legacy `session_model_fallback` compatibility event.
Initial launch, context reset, and workspace rebind already use this policy.
Targeted tests must prove that no separate path bypasses the new order.
### Frontend advisory
Add the mirrored pure matcher under `apps/web/lib/`. Its table tests use the
same contract cases as the Go helper. Backend lifecycle data remains the launch
authority.
Use the helper in `task-create-dialog-options.tsx` and
`settings/profile-model-fields.tsx`. One host variation names the possible
target while keeping the saved value and selectable profile. Zero or multiple
variations keep the current missing-model presentation.
Map `unique_variation_applied` in `status-message.tsx`. Add localized copy to
all required `settings` and `task` catalogs. Reuse the existing desktop tooltip,
touch drawer, responsive model selector, and task-chat status layout.
### Documentation
Update `docs/public/agents-and-profiles.md` and `docs/public/executors.md` with
the four-step order, one unique example, one ambiguous example, and the
unchanged saved-profile rule.
## Tests
- AC-002.1 and AC-002.2: Go policy tests prove exact and explicit-fallback
precedence before one inferred variation, including the legacy
auto-fallback path.
- AC-002.3 through AC-002.5: mirrored Go and TypeScript table tests prove zero,
duplicate, multiple, case-different, bracketed, prefix, and malformed cases.
- AC-002.6: lifecycle event and React status-message tests prove the distinct
outcome, effective model, absent explicit-fallback field, and durable copy.
- AC-002.7: task picker and profile editor tests prove advisory host behavior.
- AC-002.8: lifecycle integration tests prove launch, reset, and rebind parity.
## E2E tests
- AC-002.2 and AC-002.6: extend
`apps/web/e2e/tests/session/model-mismatch-warning.spec.ts` to launch `opus`
against one `opus[1m]`, show the effective model, and retain one warning after
reload.
- AC-002.3: add an ambiguous catalog with `opus[270k]` and
`opus[1m, fast]`; prove that Kandev does not select either variation.
- AC-002.7: extend the desktop and mobile settings mismatch specs. Prove that
the profile remains selectable, the advisory is keyboard or tap accessible,
and mobile has no horizontal document overflow.
## Work orders
- [x] [Task 01: Resolve one executor variation](task-01-runtime-resolution.md)
- [x] [Task 02: Present variation decisions](task-02-frontend-advisory.md)
- [x] [Task 03: Prove variation workflows](task-03-e2e-verification.md)
- [x] [Task 04: Document variation fallback](task-04-public-docs.md)
## Verification results
Backend verification passed: 2,364 lifecycle tests, 29 focused model-policy
tests, and 212 mock-agent tests. Frontend verification passed: 51 focused
unit tests, typecheck, lint, `i18n:check`, and `i18n:ratchet`. Desktop E2E
passed 5 tests and mobile E2E passed 3 tests. Public-document validation
passed 61 tests and validated 46 published pages. Specification lint passed.
## Risks
- Provider variation labels can change. Kandev must keep their content opaque.
- Host and executor catalogs can differ. Host preview logic cannot become a
launch gate.
- Older frontends can render the new reason as unknown during mixed-version
use. Requested and effective fields must remain useful.
- Duplicate or malformed catalog rows can make a naive candidate count unstable.
@@ -0,0 +1,100 @@
---
id: "01-runtime-resolution"
title: "Resolve one executor variation"
status: done
wave: 1
depends_on: []
plan: "plan.md"
requirements:
- REQ-AGENTS-NO-SILENT-MODEL-FALLBACK-001
- REQ-AGENTS-NO-SILENT-MODEL-FALLBACK-002
acceptance_criteria:
- AC-AGENTS-NO-SILENT-MODEL-FALLBACK-002.1
- AC-AGENTS-NO-SILENT-MODEL-FALLBACK-002.2
- AC-AGENTS-NO-SILENT-MODEL-FALLBACK-002.3
- AC-AGENTS-NO-SILENT-MODEL-FALLBACK-002.4
- AC-AGENTS-NO-SILENT-MODEL-FALLBACK-002.5
- AC-AGENTS-NO-SILENT-MODEL-FALLBACK-002.6
- AC-AGENTS-NO-SILENT-MODEL-FALLBACK-002.8
system_design:
- ../../specs/agents/system-design/no-silent-model-fallback-01.md
- ../../specs/agents/system-design/no-silent-model-fallback-02.md
---
# Task 01: Resolve One Executor Variation
## Summary
Extend the single lifecycle model policy with safe unique-variation inference.
Expose a distinct decision without changing explicit fallback semantics.
## In scope
- Add the pure backend candidate matcher.
- Apply the defined four-step precedence order.
- Extend decision, logging, and warning metadata behavior.
- Cover initial launch, context reset, and workspace rebind.
## Out of scope
- Host advisory UI and translations.
- Provider-specific variation parsing or ranking.
- Profile persistence changes.
## Acceptance
- One valid distinct variation is the only inferred model-selection call.
- Exact and advertised explicit fallback models remain higher priority; zero or
multiple variations cause no inference.
- Automatic-fallback profiles keep the legacy no-selection behavior when the
requested model is absent.
- The warning records the unique outcome and effective model without setting
the explicit fallback field or compatibility fallback event.
## Verification
```bash
(
cd apps/backend
go test -tags fts5 -run 'TestApplyStartModelPolicy|TestInitializeAndPrompt.*Model|TestReapplySessionModel|TestWorkspaceRebind.*Model|Test.*ModelSelectionWarning' ./internal/agent/runtime/lifecycle
go test -tags fts5 ./internal/agent/runtime/lifecycle
)
git diff --check
```
## Files likely touched
- `apps/backend/internal/agent/runtime/lifecycle/start_model.go`
- `apps/backend/internal/agent/runtime/lifecycle/start_model_executor_authority_test.go`
- `apps/backend/internal/agent/runtime/lifecycle/session.go`
- `apps/backend/internal/agent/runtime/lifecycle/session_model_selection_warning_test.go`
- `apps/backend/internal/agentctl/types/streams/agent.go`
## Dependencies
None.
## Risks
- The Cursor-specific configuration parser rejects labels such as `1m`. This
matcher must remain a separate model-ID helper.
- A special inferred-model apply path can weaken existing error handling.
## Parallelism
`sequential`
## Inputs
- Requirement 002 and its matching examples.
- The unique-variation ADR.
- Existing `applyStartModelPolicy` and warning publication tests.
## Results
Implemented the executor-authoritative unique-variation policy in
`start_model.go`, including explicit-fallback precedence, opaque and
case-sensitive matching, the legacy automatic-fallback path, and distinct
warning metadata without a legacy fallback event. Initial launch, context
reset, and workspace rebind all use the same policy. Focused model-policy
tests passed (29), and the full lifecycle suite passed (2,364).
@@ -0,0 +1,102 @@
---
id: "02-frontend-advisory"
title: "Present variation decisions"
status: done
wave: 2
depends_on: ["01-runtime-resolution"]
plan: "plan.md"
requirements:
- REQ-AGENTS-NO-SILENT-MODEL-FALLBACK-002
acceptance_criteria:
- AC-AGENTS-NO-SILENT-MODEL-FALLBACK-002.3
- AC-AGENTS-NO-SILENT-MODEL-FALLBACK-002.4
- AC-AGENTS-NO-SILENT-MODEL-FALLBACK-002.5
- AC-AGENTS-NO-SILENT-MODEL-FALLBACK-002.6
- AC-AGENTS-NO-SILENT-MODEL-FALLBACK-002.7
system_design:
- ../../specs/agents/system-design/no-silent-model-fallback-01.md
---
# Task 02: Present Variation Decisions
## Summary
Present one host variation as an advisory and render the runtime decision in
task chat. Keep the host probe non-authoritative on every viewport.
## In scope
- Add the mirrored frontend matcher and table tests.
- Update task-create and profile-editor missing-model states.
- Render the new warning reason with requested and effective models.
- Add all required localized copy.
## Out of scope
- Runtime model selection.
- New dialog, drawer, or navigation patterns.
- Changes to the saved profile model.
## Acceptance
- One host variation names a possible target and keeps the profile selectable.
- Zero or multiple host variations keep the existing missing-model state.
- Desktop, keyboard, and touch users can reach the advisory and durable warning.
## Verification
```bash
(
cd apps
pnpm --filter @kandev/web test -- --run components/task-create-dialog-options components/settings/profile-form-fields components/task/chat/messages/status-message lib/model-variation
pnpm --filter @kandev/web lint
)
(
cd apps/web
pnpm run typecheck
pnpm run i18n:check
pnpm run i18n:ratchet
)
git diff --check
```
## Files likely touched
- `apps/web/lib/model-variation.ts`
- `apps/web/lib/model-variation.test.ts`
- `apps/web/components/task-create-dialog-options.tsx`
- `apps/web/components/task-create-dialog-options.test.tsx`
- `apps/web/components/settings/profile-model-fields.tsx`
- `apps/web/components/settings/profile-form-fields.test.tsx`
- `apps/web/components/task/chat/messages/status-message.tsx`
- `apps/web/components/task/chat/messages/status-message.test.tsx`
- `apps/web/src/locales/*/settings.json`
- `apps/web/src/locales/*/task.json`
## Dependencies
Task 01.
## Risks
- Host preview text can sound authoritative when another executor can advertise
a different catalog.
- Missing and resolvable states must be mutually exclusive in the profile row.
## Parallelism
`sequential`
## Inputs
- Task 01 warning reason and payload fields.
- Existing host warning tooltip, touch drawer, profile selector, and status
message patterns.
## Results
Added the mirrored structural matcher, host advisory states, and the task-chat
warning reason while preserving the saved profile model. Added localized copy
for all required catalogs and retained the existing desktop tooltip and mobile
touch-drawer patterns. Focused frontend tests passed (51), typecheck and lint
passed, and both i18n checks passed.
@@ -0,0 +1,84 @@
---
id: "03-e2e-verification"
title: "Prove variation workflows"
status: done
wave: 3
depends_on: ["01-runtime-resolution", "02-frontend-advisory"]
plan: "plan.md"
requirements:
- REQ-AGENTS-NO-SILENT-MODEL-FALLBACK-002
acceptance_criteria:
- AC-AGENTS-NO-SILENT-MODEL-FALLBACK-002.2
- AC-AGENTS-NO-SILENT-MODEL-FALLBACK-002.3
- AC-AGENTS-NO-SILENT-MODEL-FALLBACK-002.6
- AC-AGENTS-NO-SILENT-MODEL-FALLBACK-002.7
system_design:
- ../../specs/agents/system-design/no-silent-model-fallback-01.md
---
# Task 03: Prove Variation Workflows
## Summary
Extend the existing model-mismatch browser flows with unique and ambiguous
catalogs. Prove the launch result, durable warning, and touch interaction.
## In scope
- Desktop launch and reload coverage for one variation.
- Desktop no-inference coverage for multiple variations.
- Mobile advisory and warning access through tap.
- Horizontal overflow checks for the changed mobile states.
## Out of scope
- New browser fixtures unrelated to model catalogs.
- Full E2E suite execution.
- Public documentation.
## Acceptance
- `opus` launches as `opus[1m]` only when that is the sole distinct variation.
- Two variations do not select either candidate.
- The warning survives reload and the mobile flow needs no hover interaction.
## Verification
```bash
cd apps/web && pnpm e2e:run tests/settings/no-silent-model-fallback.spec.ts tests/session/model-mismatch-warning.spec.ts
cd apps/web && pnpm e2e:run --project mobile-chrome --no-build tests/settings/mobile-no-silent-model-fallback.spec.ts tests/session/mobile-model-mismatch-warning.spec.ts
git diff --check
```
## Files likely touched
- `apps/web/e2e/tests/settings/no-silent-model-fallback.spec.ts`
- `apps/web/e2e/tests/settings/mobile-no-silent-model-fallback.spec.ts`
- `apps/web/e2e/tests/session/model-mismatch-warning.spec.ts`
- `apps/web/e2e/tests/session/mobile-model-mismatch-warning.spec.ts`
- `apps/web/e2e/tests/session/model-mismatch-warning-helpers.ts`
## Dependencies
Tasks 01 and 02.
## Risks
- Mock catalogs must prove the executor decision, not only the host advisory.
- Each scenario needs isolated profile data so another catalog does not change
the distinct candidate count.
## Parallelism
`parallel-safe` with Task 04 after Tasks 01 and 02.
## Inputs
- Completed runtime and frontend work.
- Existing desktop and mobile model-mismatch fixtures.
## Results
Added unique and ambiguous mock-agent catalogs and covered launch, reload,
profile persistence, host advisory access, and mobile overflow. The focused
desktop E2E run passed 5 tests; the focused mobile E2E run passed 3 tests.
@@ -0,0 +1,83 @@
---
id: "04-public-docs"
title: "Document variation fallback"
status: done
wave: 3
depends_on: ["01-runtime-resolution", "02-frontend-advisory"]
plan: "plan.md"
requirements:
- REQ-AGENTS-NO-SILENT-MODEL-FALLBACK-002
acceptance_criteria:
- AC-AGENTS-NO-SILENT-MODEL-FALLBACK-002.1
- AC-AGENTS-NO-SILENT-MODEL-FALLBACK-002.2
- AC-AGENTS-NO-SILENT-MODEL-FALLBACK-002.3
- AC-AGENTS-NO-SILENT-MODEL-FALLBACK-002.5
- AC-AGENTS-NO-SILENT-MODEL-FALLBACK-002.6
system_design:
- ../../specs/agents/system-design/no-silent-model-fallback-01.md
---
# Task 04: Document Variation Fallback
## Summary
Update the public profile and executor guides with the unique-variation rule.
Give users enough detail to predict unique and ambiguous results.
## In scope
- Document the four-step launch order.
- Add one unique and one ambiguous example.
- State that labels remain opaque and saved profiles do not change.
- Run the public-doc validation commands.
## Out of scope
- Release notes and changelog generation.
- Internal implementation detail beyond user-visible behavior.
- New screenshots or media.
## Acceptance
- Both guides describe the same precedence and ambiguity rule.
- The docs state that the executor is authoritative and the profile stays
unchanged.
- Public-doc validators pass.
## Verification
```bash
node --test scripts/validate-public-docs.test.mjs
node scripts/validate-public-docs.mjs
git diff --check
```
## Files likely touched
- `docs/public/agents-and-profiles.md`
- `docs/public/executors.md`
## Dependencies
Tasks 01 and 02.
## Risks
- Examples can imply that Kandev understands `1m` or `fast`. The docs must state
that it compares only the ID shape and candidate count.
## Parallelism
`parallel-safe` with Task 03 after Tasks 01 and 02.
## Inputs
- Final runtime precedence and warning behavior.
- Existing host-probe and remote-executor guide sections.
## Results
Updated the public agent-profile and executor guides with the four-step
resolution order, unique and ambiguous examples, opaque variation matching,
and the unchanged saved-profile rule. Public-document tests passed (61) and
the validator checked 46 published pages.
+32 -7
View File
@@ -171,6 +171,31 @@ change makes a saved option value unsupported, Kandev removes that value after
a successful resolution; a failed resolution keeps the draft unchanged so you
can retry it.
### Model IDs and executor catalogs
The host model probe is an editing hint. The selected executor owns the model
catalog at launch. Kandev resolves a saved model in this order:
1. Use the exact requested model when the executor advertises it.
2. Use the advertised explicit fallback when the requested model is absent.
3. Use one unique bracketed variation when the requested model is absent and
the executor advertises exactly one matching ID.
4. Use the agent's current or default model when no earlier choice applies.
Profiles with automatic fallback enabled keep the legacy behavior when the
saved model is absent: Kandev ignores the explicit fallback and does not infer
a variation. It uses the agent's current or default model instead.
For example, a saved `opus` model can launch as `opus[1m]` when that is the
only advertised `opus[...]` ID. If the executor advertises both
`opus[270k]` and `opus[1m, fast]`, Kandev does not choose either variation.
Variation text is opaque and model IDs remain case-sensitive.
Kandev shows a warning when the launch result differs from the saved model.
The saved profile remains `opus`; Kandev does not rewrite it after applying a
unique variation or using the agent default. Recheck the executor catalog when
the warning repeats after credentials, copied configuration, or agent updates.
### Use a dynamic profile
> [!EXPERIMENTAL]
@@ -228,13 +253,13 @@ The model list shown while editing a profile comes from a host probe. It is an
editing hint, not a launch gate. A profile remains selectable when its saved
model is missing from that host list.
At task launch, the selected executor's ACP catalog is authoritative. Kandev
sends the requested model only when the executor advertises it. If it does
not, Kandev uses an advertised fallback when available, or sends no model
request and continues with the agent's current or default model. Kandev stores
one warning in task chat with the requested model and the effective model when
known. The warning also identifies the agent and executor and asks you to
check credentials, copied configuration, and the agent version.
At task launch, the selected executor's ACP catalog is authoritative. For
profiles without automatic fallback, Kandev follows the four-step order above.
For profiles with automatic fallback enabled, an absent saved model causes no
model request, and Kandev ignores the explicit fallback and any variation.
Kandev stores one warning in task chat with the requested model and the
effective model when known. The warning also identifies the agent and executor
and asks you to check credentials, copied configuration, and the agent version.
The saved profile model is not changed. Optional portable configuration can
copy selected allowlisted files into a remote executor, but it cannot guarantee
+23 -8
View File
@@ -101,15 +101,30 @@ Review the selected bundles before saving the profile.
### Model selection in remote executors
The host model probe helps edit a profile, but it is not the launch authority.
At launch, the selected executor's advertised ACP catalog decides whether
Kandev sends the saved model. If the executor does not advertise that model,
Kandev sends no request for it. It uses an advertised fallback only when one
exists; otherwise the agent uses its current or default model.
At launch, the selected executor's advertised ACP catalog decides the model.
For profiles without automatic fallback, Kandev applies this deterministic
order:
Kandev writes one warning to task chat when this happens. The warning can list
the requested model, effective model, agent, executor, and executor profile.
It also tells you to check executor credentials, copied agent configuration,
and the agent version. Kandev does not rewrite the saved profile model.
1. An exact advertised model ID.
2. An advertised explicit fallback.
3. One unique advertised bracketed variation of the saved model.
4. The agent's current or default model.
For profiles with automatic fallback enabled, an absent saved model keeps the
legacy behavior: Kandev ignores the explicit fallback and does not infer a
variation. It uses the agent's current or default model instead.
For example, a saved `opus` model uses `opus[1m]` when that is the only
advertised variation. With both `opus[270k]` and `opus[1m, fast]`, Kandev
does not infer a choice and uses the agent's current or default model instead.
It treats model IDs as case-sensitive and variation text as opaque.
Kandev writes one warning to task chat when the effective model differs from
the saved model. The warning can list the requested model, effective model,
agent, executor, and executor profile. It also tells you to check executor
credentials, copied agent configuration, and the agent version. Kandev does
not rewrite the saved profile model, including after applying a unique
variation.
Portable configuration can improve parity, but it does not guarantee equal
host and executor model catalogs.
@@ -1,26 +1,101 @@
---
status: draft
status: active
system: agents
created: 2026-08-23
owners:
- kandev
---
# No Silent Model Fallback Requirements
## Overview
**Status**: implemented (executor-authority amendment 2026-08-15) **Date**: 2026-08-04 **Slug**: `no-silent-model-fallback`
Kandev applies an agent profile model against the model catalog from the
selected executor. A different effective model must be deterministic and
visible to the user. The agent system owns this contract because it owns
profiles, ACP session startup, and model-selection warnings.
## Terminology
- **Requested model:** The model ID stored in the agent profile.
- **Explicit fallback:** The optional fallback model ID stored in the profile.
- **Bare model ID:** A model ID that contains no `[` or `]` character.
- **Model variation:** An advertised model ID with the exact form
`<bare-model-id>[<non-empty-variant>]`. The variant contains no bracket
character. Its content is otherwise opaque.
- **Unique variation:** The only distinct advertised variation for a requested
bare model ID.
## Requirements
### REQ-AGENTS-NO-SILENT-MODEL-FALLBACK-001: No Silent Model Fallback
### REQ-AGENTS-NO-SILENT-MODEL-FALLBACK-001: Use executor-authoritative model selection
**Intent:** **Status**: implemented (executor-authority amendment 2026-08-15) **Date**: 2026-08-04 **Slug**: `no-silent-model-fallback`
**Intent:** Keep task launch operational without hiding a difference between
the saved model and the effective executor model.
**User story:** As a user, I want Kandev to use the selected executor's model
catalog and report any fallback, so that I can understand which model runs.
#### Acceptance criteria
- **AC-AGENTS-NO-SILENT-MODEL-FALLBACK-001.1:** When a consumer uses **Status**: implemented (executor-authority amendment 2026-08-15), the system shall provide the documented behavior.
- **AC-AGENTS-NO-SILENT-MODEL-FALLBACK-001.1:** When the executor advertises
the requested model, the system shall apply that exact model.
- **AC-AGENTS-NO-SILENT-MODEL-FALLBACK-001.2:** When the requested model is
absent and the executor advertises the configured explicit fallback, the
system shall apply the explicit fallback.
- **AC-AGENTS-NO-SILENT-MODEL-FALLBACK-001.3:** When no permitted advertised
model applies, the system shall make no speculative model-selection call and
shall continue with the provider current or default model.
- **AC-AGENTS-NO-SILENT-MODEL-FALLBACK-001.4:** When the effective model differs
from the requested model, the system shall persist one structured warning
that identifies the requested and effective models when known.
- **AC-AGENTS-NO-SILENT-MODEL-FALLBACK-001.5:** The system shall not rewrite the
saved profile model from an executor model-selection decision.
### REQ-AGENTS-NO-SILENT-MODEL-FALLBACK-002: Resolve one advertised model variation
**Intent:** Preserve a user's bare model choice when a CLI replaces that model
ID with one unambiguous bracketed variation.
**User story:** As a user, I want `opus` to resolve to `opus[1m]` when that is
the only advertised variation, so that a provider catalog change does not move
my session to an unrelated default model.
#### Acceptance criteria
- **AC-AGENTS-NO-SILENT-MODEL-FALLBACK-002.1:** When automatic fallback is
disabled, the exact requested model is absent, and an advertised explicit
fallback exists, the system shall apply the explicit fallback before it
considers an inferred variation. Automatic-fallback profiles shall retain
their legacy no-selection behavior for an absent requested model.
- **AC-AGENTS-NO-SILENT-MODEL-FALLBACK-002.2:** When no advertised explicit
fallback exists and the catalog contains exactly one distinct variation of a
requested bare model ID, the system shall apply that advertised variation.
- **AC-AGENTS-NO-SILENT-MODEL-FALLBACK-002.3:** When the catalog contains zero
or more than one distinct variation, the system shall not infer a model.
- **AC-AGENTS-NO-SILENT-MODEL-FALLBACK-002.4:** When the requested ID contains a
bracket, the system shall treat it as an exact ID and shall not infer another
variation.
- **AC-AGENTS-NO-SILENT-MODEL-FALLBACK-002.5:** Matching shall be case-sensitive
and shall not parse, rank, or assign meaning to variation text.
- **AC-AGENTS-NO-SILENT-MODEL-FALLBACK-002.6:** When Kandev applies a unique
variation, it shall keep the saved model unchanged and shall persist a
warning that identifies the inferred effective model.
- **AC-AGENTS-NO-SILENT-MODEL-FALLBACK-002.7:** The host profile probe shall
present unique-variation resolution as an advisory. It shall not become the
launch authority or disable the profile.
- **AC-AGENTS-NO-SILENT-MODEL-FALLBACK-002.8:** The same resolution order shall
apply at initial launch, context reset, and workspace rebind.
## Out of scope
- Ranking multiple variations by context size, speed, price, or display order.
- Parsing provider-specific variation syntax beyond the bracketed ID shape.
- Inferring a base model from an already bracketed request.
- Rewriting the profile model to the current executor's advertised variation.
- Changing Office post-start provider routing or mid-turn model switching.
## System design
The migrated technical source is split into [part 1](../system-design/no-silent-model-fallback-01.md), [part 2](../system-design/no-silent-model-fallback-02.md).
The technical source is split into [part 1](../system-design/no-silent-model-fallback-01.md)
and [part 2](../system-design/no-silent-model-fallback-02.md).
@@ -1,8 +1,9 @@
---
status: draft
status: current
system: agents
requirements:
- REQ-AGENTS-NO-SILENT-MODEL-FALLBACK-001
- REQ-AGENTS-NO-SILENT-MODEL-FALLBACK-002
created: 2026-08-23
owners:
- kandev
@@ -11,13 +12,15 @@ owners:
## Purpose and boundaries
This design preserves the technical source detail for `REQ-AGENTS-NO-SILENT-MODEL-FALLBACK-001` during migration.
This design defines executor-authoritative model fallback, including safe
resolution from a bare requested model to one advertised bracketed variation.
## Requirement mapping
| Requirement | Design section |
| --- | --- |
| `REQ-AGENTS-NO-SILENT-MODEL-FALLBACK-001` | [Migrated source detail](#migrated-source-detail) |
| `REQ-AGENTS-NO-SILENT-MODEL-FALLBACK-002` | [Unique model-variation resolution](#unique-model-variation-resolution) |
## Migrated source detail
@@ -146,6 +149,121 @@ the launch. The agent uses its default and Kandev persists a warning.
The executor ACP catalog is authoritative for launch.
The host probe remains an editing hint and does not block profile selection.
## Unique model-variation resolution
### Matching contract
ACP model IDs remain opaque and case-sensitive. Kandev recognizes only this
structural relationship:
```text
<requested>[<variant>]
```
The requested ID must contain no `[` or `]` character. The advertised ID must
start with the complete requested ID followed by `[`, end with `]`, and contain
a non-empty variant with no bracket character. Kandev does not trim, parse, or
rank the variant.
Examples:
| Requested | Advertised candidates | Result |
| --- | --- | --- |
| `opus` | `opus[1m]` | Apply `opus[1m]`. |
| `opus` | `opus[270k]`, `opus[1m, fast]` | Do not infer. |
| `opus[1m]` | `opus[270k]` | Do not infer. |
| `opus` | `myopus[1m]`, `opus-extra[1m]` | Do not infer. |
| `opus` | `opus[]`, `opus[1m][fast]` | Do not infer. |
Duplicate catalog rows with the same model ID count as one distinct candidate.
This rule keeps the outcome stable when an agent repeats a catalog entry.
### Resolution order
For profiles with `auto_fallback = false`, the lifecycle applies this order
after ACP initialization:
1. Apply the exact requested model when advertised.
2. Apply the configured explicit fallback when advertised.
3. Apply the requested model's unique advertised variation when one exists.
4. Otherwise, make no model-selection call and keep the provider current or
default model.
An unadvertised explicit fallback does not block step 3. More than one distinct
variation blocks inference even when one candidate appears first or is the
provider current model.
When `auto_fallback = true`, the legacy path takes precedence when the
requested model is absent: Kandev ignores the explicit fallback and does not
infer a variation. It makes no model-selection call and continues on the
provider current or default model with a warning. Apply errors for an
advertised model remain best-effort in that mode.
`applyStartModelPolicy` remains the single runtime owner. A pure helper returns
the unique candidate or no candidate. Initial launch, context reset, and
workspace rebind continue to call the same policy.
### Decision and warning contract
The typed decision adds the outcome `unique_variation` and the stable reason
`unique_variation_applied`. `EffectiveModel` contains the advertised variation.
`FallbackModel` remains reserved for a configured explicit fallback and is not
overloaded with an inferred value.
Kandev calls `SetModel` only with the candidate's exact advertised ID. A failed
call follows the existing advertised-model rule: fail the operation unless
legacy `auto_fallback` permits provider-default continuation. A
method-not-supported response also follows the current provider-default path.
The persisted `model_selection_warning` event uses the new reason and existing
requested and effective model fields. The profile remains unchanged. The
decision ID continues to include the requested model, reason, and effective
model, so the warning stays idempotent for one session decision.
### Host advisory and mobile behavior
The frontend uses a small pure helper with the same case-sensitive matching
table. This calculation is advisory because the host catalog can differ from
the selected executor catalog.
When the host advertises one variation, the task profile option remains
selectable and identifies the possible target in its existing amber advisory.
The profile editor keeps the saved bare ID visible and describes the unique
host variation instead of reporting that the model is simply gone. Selecting
the advertised variation remains an explicit way to update the profile.
When the host advertises zero or multiple variations, the current missing-model
presentation remains. The runtime can still reach a different decision from
the selected executor catalog.
The task-create advisory keeps its existing desktop tooltip and coarse-pointer
drawer. The profile editor uses its existing responsive selector. The durable
chat warning uses the existing status-message layout on desktop and mobile.
This change adds no nested scroll area or hover-only information.
### Tests and operational evidence
Backend table tests cover exact precedence, explicit-fallback precedence,
one variation, duplicate rows, multiple variations, bracketed requests,
case differences, malformed IDs, method-not-supported, and apply errors.
Existing launch, reset, rebind, warning persistence, and reload tests cover the
new decision outcome.
Frontend unit tests mirror the matching table and cover the task picker,
profile editor, and localized status warning. Existing desktop and mobile model
mismatch E2E flows add the unique and ambiguous catalog cases.
Structured lifecycle logs record the requested model, effective model, outcome,
and distinct candidate count. They do not record credentials or configuration
file contents.
No database migration is required.
## Related decisions
- [Let the Executor Own Model Selection](../../../decisions/2026-08-15-executor-authoritative-model-selection.md)
- [Resolve Only One Advertised Model Variation](../../../decisions/2026-09-07-unique-model-variation-resolution.md)
## Backend Changes
### 1. Agent profile schema (new columns)
@@ -1,8 +1,9 @@
---
status: draft
status: current
system: agents
requirements:
- REQ-AGENTS-NO-SILENT-MODEL-FALLBACK-001
- REQ-AGENTS-NO-SILENT-MODEL-FALLBACK-002
created: 2026-08-23
owners:
- kandev
@@ -11,13 +12,15 @@ owners:
## Purpose and boundaries
This design preserves the technical source detail for `REQ-AGENTS-NO-SILENT-MODEL-FALLBACK-001` during migration.
This design records risks for executor-authoritative model fallback and unique
variation resolution.
## Requirement mapping
| Requirement | Design section |
| --- | --- |
| `REQ-AGENTS-NO-SILENT-MODEL-FALLBACK-001` | [Migrated source detail](#migrated-source-detail) |
| `REQ-AGENTS-NO-SILENT-MODEL-FALLBACK-002` | [Risks & Open Questions](#risks--open-questions) |
## Migrated source detail
@@ -51,3 +54,14 @@ This design preserves the technical source detail for `REQ-AGENTS-NO-SILENT-MODE
- **Hover is supplementary**: every info icon is focusable, and coarse-pointer
devices receive the same content in a drawer. The visible option helper copy
remains the baseline explanation.
- **Provider-specific labels remain opaque**: `1m`, `270k`, and `fast` have no
Kandev-defined order. More than one variation therefore fails closed to the
existing provider-default path.
- **Host and executor decisions can differ**: a unique host variation is only
an advisory. The selected executor can advertise zero, one, or multiple
variations at launch.
- **Bracketed requests do not drift sideways**: a request such as `opus[1m]`
does not infer `opus[270k]`. The user must select another explicit model.
- **Legacy automatic fallback remains unchanged**: when `auto_fallback` is
enabled and the requested model is absent, Kandev does not apply an explicit
fallback or infer a variation. It continues with the provider default.