feat(agents): resolve unique advertised model variations (#3472)
This commit is contained in:
@@ -129,7 +129,7 @@ class PRWalkthroughWorkflowContractTest(unittest.TestCase):
|
||||
'PUBLIC_URL="${WALKTHROUGH_BASE_URL%/}/pr/${PR_NUMBER}/${SHORT_HEAD_SHA}.html"',
|
||||
"RESPONSE_META_PATH=pr-walkthrough-public.response",
|
||||
"curl --fail",
|
||||
"--write-out '%{http_code}\\t%{content_type}\\t%{url_effective}'",
|
||||
"--write-out '%{http_code}\\t%{content_type}\\t%{url_effective}\\n'",
|
||||
"response_status",
|
||||
'response_effective_url" != "$PUBLIC_URL"',
|
||||
"response_content_type",
|
||||
|
||||
@@ -69,7 +69,7 @@ jobs:
|
||||
if ! curl --fail --silent --show-error --location --max-time 30 \
|
||||
--retry 3 --retry-all-errors --retry-delay 2 --retry-max-time 30 \
|
||||
--output "$PUBLIC_HTML_PATH" \
|
||||
--write-out '%{http_code}\t%{content_type}\t%{url_effective}' \
|
||||
--write-out '%{http_code}\t%{content_type}\t%{url_effective}\n' \
|
||||
> "$RESPONSE_META_PATH" \
|
||||
"$PUBLIC_URL"; then
|
||||
echo "Canonical walkthrough is not publicly available; skipping repair." >> "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
@@ -24,6 +24,9 @@ const (
|
||||
modelFast = "mock-fast"
|
||||
modelSmart = "mock-smart"
|
||||
modelSlow = "mock-slow"
|
||||
modelUnique = "opus[1m]"
|
||||
modelAmbiguousFirst = "opus[270k]"
|
||||
modelAmbiguousLast = "opus[1m, fast]"
|
||||
reasoningEffortLow = "low"
|
||||
reasoningEffortMed = "medium"
|
||||
reasoningEffortHigh = "high"
|
||||
@@ -207,22 +210,24 @@ func mockSessionConfigOptionsForModel(model string) []acp.SessionConfigOption {
|
||||
{Value: "max", Name: "Max", Description: ptr("Use maximum reasoning")},
|
||||
}
|
||||
}
|
||||
modelOptions := acp.SessionConfigSelectOptionsUngrouped{
|
||||
{Value: modelFast, Name: "Mock Fast", Description: ptr("Fast mock model for testing")},
|
||||
{Value: modelSmart, Name: "Mock Smart", Description: ptr("Smart mock model for testing")},
|
||||
// E2E fixtures use "mock-slow" for the slow-response delay tier.
|
||||
// It must be advertised so the no-silent-model-fallback strict
|
||||
// policy (which fails session start when the profile model is
|
||||
// absent from the advertised list) does not reject it.
|
||||
{Value: modelSlow, Name: "Mock Slow", Description: ptr("Slow mock model for testing")},
|
||||
}
|
||||
modelOptions = append(modelOptions, mockModelVariationOptions()...)
|
||||
return []acp.SessionConfigOption{
|
||||
{Select: &acp.SessionConfigOptionSelect{
|
||||
Category: &modelCat,
|
||||
CurrentValue: acp.SessionConfigValueId(model),
|
||||
Id: "model",
|
||||
Name: "Model",
|
||||
Options: acp.SessionConfigSelectOptions{Ungrouped: &acp.SessionConfigSelectOptionsUngrouped{
|
||||
{Value: modelFast, Name: "Mock Fast", Description: ptr("Fast mock model for testing")},
|
||||
{Value: modelSmart, Name: "Mock Smart", Description: ptr("Smart mock model for testing")},
|
||||
// E2E fixtures use "mock-slow" for the slow-response delay tier.
|
||||
// It must be advertised so the no-silent-model-fallback strict
|
||||
// policy (which fails session start when the profile model is
|
||||
// absent from the advertised list) does not reject it.
|
||||
{Value: modelSlow, Name: "Mock Slow", Description: ptr("Slow mock model for testing")},
|
||||
}},
|
||||
Type: "select",
|
||||
Options: acp.SessionConfigSelectOptions{Ungrouped: &modelOptions},
|
||||
Type: "select",
|
||||
}},
|
||||
{Select: &acp.SessionConfigOptionSelect{
|
||||
Category: &modeCat,
|
||||
@@ -247,6 +252,22 @@ func mockSessionConfigOptionsForModel(model string) []acp.SessionConfigOption {
|
||||
}
|
||||
}
|
||||
|
||||
func mockModelVariationOptions() []acp.SessionConfigSelectOption {
|
||||
catalog := strings.ToLower(strings.TrimSpace(os.Getenv("MOCK_AGENT_MODEL_CATALOG")))
|
||||
if catalog == "ambiguous" {
|
||||
return []acp.SessionConfigSelectOption{
|
||||
{Value: modelAmbiguousFirst, Name: "Opus (270k)", Description: ptr("Ambiguous variation fixture")},
|
||||
{Value: modelAmbiguousLast, Name: "Opus (1m, fast)", Description: ptr("Ambiguous variation fixture")},
|
||||
}
|
||||
}
|
||||
if catalog == "unique" || (catalog == "" && strings.EqualFold(os.Getenv("KANDEV_E2E_MOCK"), "true")) {
|
||||
return []acp.SessionConfigSelectOption{
|
||||
{Value: modelUnique, Name: "Opus (1m)", Description: ptr("Unique variation fixture")},
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func cloneSessionConfigOptions(options []acp.SessionConfigOption) []acp.SessionConfigOption {
|
||||
cloned := make([]acp.SessionConfigOption, len(options))
|
||||
copy(cloned, options)
|
||||
|
||||
@@ -224,6 +224,26 @@ func TestMockSessionConfigOptionsForModelAdvertisesSlowModel(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestMockSessionConfigOptionsVariationCatalogs(t *testing.T) {
|
||||
t.Setenv("KANDEV_E2E_MOCK", "true")
|
||||
t.Setenv("MOCK_AGENT_MODEL_CATALOG", "")
|
||||
unique := modelIDs(findModelOption(t))
|
||||
if !containsString(unique, modelUnique) {
|
||||
t.Fatalf("unique catalog = %v, missing %s", unique, modelUnique)
|
||||
}
|
||||
if containsString(unique, modelAmbiguousFirst) || containsString(unique, modelAmbiguousLast) {
|
||||
t.Fatalf("unique catalog = %v, must not include ambiguous variations", unique)
|
||||
}
|
||||
|
||||
t.Setenv("MOCK_AGENT_MODEL_CATALOG", "ambiguous")
|
||||
ambiguous := modelIDs(findModelOption(t))
|
||||
for _, want := range []string{modelAmbiguousFirst, modelAmbiguousLast} {
|
||||
if !containsString(ambiguous, want) {
|
||||
t.Errorf("ambiguous catalog = %v, missing %s", ambiguous, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func findModelOption(t *testing.T) acp.SessionConfigOption {
|
||||
t.Helper()
|
||||
for _, option := range mockSessionConfigOptionsForModel(modelFast) {
|
||||
@@ -235,6 +255,17 @@ func findModelOption(t *testing.T) acp.SessionConfigOption {
|
||||
return acp.SessionConfigOption{}
|
||||
}
|
||||
|
||||
func modelIDs(option acp.SessionConfigOption) []string {
|
||||
if option.Select == nil || option.Select.Options.Ungrouped == nil {
|
||||
return nil
|
||||
}
|
||||
ids := make([]string, 0, len(*option.Select.Options.Ungrouped))
|
||||
for _, model := range *option.Select.Options.Ungrouped {
|
||||
ids = append(ids, string(model.Value))
|
||||
}
|
||||
return ids
|
||||
}
|
||||
|
||||
func containsString(values []string, want string) bool {
|
||||
for _, v := range values {
|
||||
if v == want {
|
||||
|
||||
@@ -543,7 +543,9 @@ func (m *Manager) reapplySessionModelAfterReset(
|
||||
m.sessionManager.publishModelSelectionWarningEvent(execution, newSessionID, decision)
|
||||
}
|
||||
if decision.EffectiveModel != "" &&
|
||||
(decision.Outcome == ModelSelectionOutcomeApplied || decision.Outcome == ModelSelectionOutcomeExplicitFallback) {
|
||||
(decision.Outcome == ModelSelectionOutcomeApplied ||
|
||||
decision.Outcome == ModelSelectionOutcomeExplicitFallback ||
|
||||
decision.Outcome == ModelSelectionOutcomeUniqueVariation) {
|
||||
m.logger.Info("re-applied session model after context reset",
|
||||
zap.String("execution_id", execution.ID),
|
||||
zap.String("session_id", execution.SessionID),
|
||||
|
||||
+45
@@ -99,6 +99,51 @@ func TestManager_ResetAgentContext_ReappliesSessionRuntimeConfig(t *testing.T) {
|
||||
require.Equal(t, v1.AgentStatusReady, exec.Status)
|
||||
}
|
||||
|
||||
func TestReapplySessionModel_UsesUniqueAdvertisedVariation(t *testing.T) {
|
||||
mgr := newTestManager(t)
|
||||
mock := newRestartMockAgentctlServer(t, false, false)
|
||||
client := createTestClient(t, mock.server.URL)
|
||||
t.Cleanup(client.Close)
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
|
||||
t.Cleanup(cancel)
|
||||
require.NoError(t, client.StreamUpdates(ctx, func(agentctl.AgentEvent) {}, nil, nil))
|
||||
|
||||
exec := runtimeConfigResetExecution(client, true)
|
||||
exec.SetModelState(&CachedModelState{
|
||||
CurrentModelID: "provider-default",
|
||||
Models: []streams.SessionModelInfo{{ModelID: "opus[1m]"}},
|
||||
})
|
||||
|
||||
require.NoError(t, mgr.reapplySessionModelAfterReset(ctx, exec, "reset-session", "opus"))
|
||||
require.Equal(t, []string{"opus[1m]"}, mock.getSetModelIDs())
|
||||
}
|
||||
|
||||
func TestWorkspaceRebindModel_UsesUniqueAdvertisedVariation(t *testing.T) {
|
||||
mgr := newTestManager(t)
|
||||
mock := newRestartMockAgentctlServer(t, false, false)
|
||||
client := createTestClient(t, mock.server.URL)
|
||||
t.Cleanup(client.Close)
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
|
||||
t.Cleanup(cancel)
|
||||
require.NoError(t, client.StreamUpdates(ctx, func(agentctl.AgentEvent) {}, nil, nil))
|
||||
|
||||
exec := runtimeConfigResetExecution(client, true)
|
||||
exec.SetModelState(&CachedModelState{
|
||||
CurrentModelID: "provider-default",
|
||||
Models: []streams.SessionModelInfo{{ModelID: "opus[1m]"}},
|
||||
})
|
||||
|
||||
require.NoError(t, mgr.reapplyReboundSessionConfig(
|
||||
ctx,
|
||||
exec,
|
||||
"rebind-session",
|
||||
&CachedModelState{},
|
||||
"opus",
|
||||
nil,
|
||||
))
|
||||
require.Equal(t, []string{"opus[1m]"}, mock.getSetModelIDs())
|
||||
}
|
||||
|
||||
func TestManager_RestartAgentProcess_ReappliesSessionRuntimeConfig(t *testing.T) {
|
||||
mgr := newTestManager(t)
|
||||
provider := runtimeConfigResetWorkspaceInfo()
|
||||
|
||||
@@ -513,10 +513,12 @@ func (sm *SessionManager) applyStartModelPolicyToEffectiveModel(
|
||||
}
|
||||
effective.decision = decision
|
||||
if decision.Outcome == ModelSelectionOutcomeApplied ||
|
||||
decision.Outcome == ModelSelectionOutcomeExplicitFallback {
|
||||
decision.Outcome == ModelSelectionOutcomeExplicitFallback ||
|
||||
decision.Outcome == ModelSelectionOutcomeUniqueVariation {
|
||||
effective.appliedModel = decision.EffectiveModel
|
||||
}
|
||||
if decision.Outcome == ModelSelectionOutcomeExplicitFallback {
|
||||
if decision.Outcome == ModelSelectionOutcomeExplicitFallback ||
|
||||
decision.Outcome == ModelSelectionOutcomeUniqueVariation {
|
||||
effective.model = decision.EffectiveModel
|
||||
}
|
||||
return effective
|
||||
|
||||
+100
@@ -0,0 +1,100 @@
|
||||
package lifecycle
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/kandev/kandev/internal/agent/agents"
|
||||
agentctl "github.com/kandev/kandev/internal/agent/runtime/agentctl"
|
||||
"github.com/kandev/kandev/internal/agentctl/types/streams"
|
||||
"github.com/kandev/kandev/pkg/agent"
|
||||
)
|
||||
|
||||
func TestInitializeAndPrompt_UniqueVariationPublishesProviderNeutralWarning(t *testing.T) {
|
||||
mock := newMockAgentServer(t)
|
||||
defer mock.Close()
|
||||
|
||||
log := newSessionTestLogger()
|
||||
stopCh := newTestStopCh(t)
|
||||
sm := NewSessionManager(log, stopCh)
|
||||
streamMgr := NewStreamManager(log, StreamCallbacks{
|
||||
OnAgentEvent: func(execution *AgentExecution, event agentctl.AgentEvent) {},
|
||||
}, nil, stopCh)
|
||||
cleanupStreamManager(t, stopCh, streamMgr)
|
||||
eventBus := &MockEventBusWithTracking{}
|
||||
sm.SetDependencies(NewEventPublisher(eventBus, log), streamMgr, nil, nil)
|
||||
|
||||
client := createTestClient(t, mock.server.URL)
|
||||
defer client.Close()
|
||||
execution := &AgentExecution{
|
||||
ID: "exec-unique-variation",
|
||||
TaskID: "task-1",
|
||||
SessionID: "session-1",
|
||||
WorkspacePath: "/workspace",
|
||||
agentctl: client,
|
||||
promptDoneCh: make(chan PromptCompletionSignal, 1),
|
||||
}
|
||||
execution.SetModelState(modelState("opus[1m]"))
|
||||
agentConfig := &testAgent{
|
||||
id: "test-agent",
|
||||
enabled: true,
|
||||
runtimeConfig: &agents.RuntimeConfig{
|
||||
Cmd: agents.NewCommand("test-agent"),
|
||||
Protocol: agent.ProtocolACP,
|
||||
SessionConfig: agents.SessionConfig{},
|
||||
ResourceLimits: agents.ResourceLimits{MemoryMB: 512, CPUCores: 0.5, Timeout: time.Hour},
|
||||
},
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
|
||||
defer cancel()
|
||||
err := sm.InitializeAndPrompt(ctx, execution, agentConfig, "", nil, nil,
|
||||
func(executionID string) error { return nil },
|
||||
StartModelPolicy{Model: "opus"}, "plan", nil)
|
||||
if err != nil {
|
||||
t.Fatalf("InitializeAndPrompt failed: %v", err)
|
||||
}
|
||||
|
||||
setModelCalls := 0
|
||||
for _, action := range mock.getActionLog() {
|
||||
if action == "agent.session.set_model" {
|
||||
setModelCalls++
|
||||
}
|
||||
}
|
||||
if setModelCalls != 1 {
|
||||
t.Fatalf("set_model calls = %d, want exactly 1", setModelCalls)
|
||||
}
|
||||
|
||||
var warning *streams.ModelSelectionWarning
|
||||
legacyFallbackEvents := 0
|
||||
for _, event := range eventBus.getStreamEvents() {
|
||||
if event.Data == nil {
|
||||
continue
|
||||
}
|
||||
if event.Data.ModelSelectionWarning != nil {
|
||||
warning = event.Data.ModelSelectionWarning
|
||||
}
|
||||
if event.Data.Type == streams.EventTypeSessionModelFallback {
|
||||
legacyFallbackEvents++
|
||||
}
|
||||
}
|
||||
if warning == nil {
|
||||
t.Fatal("expected a model-selection warning event")
|
||||
}
|
||||
if warning.RequestedModel != "opus" {
|
||||
t.Errorf("warning requested model = %q, want opus", warning.RequestedModel)
|
||||
}
|
||||
if warning.EffectiveModel != "opus[1m]" {
|
||||
t.Errorf("warning effective model = %q, want opus[1m]", warning.EffectiveModel)
|
||||
}
|
||||
if warning.FallbackModel != "" {
|
||||
t.Errorf("warning fallback model = %q, want empty", warning.FallbackModel)
|
||||
}
|
||||
if warning.Reason != ModelSelectionReasonUniqueVariationApplied {
|
||||
t.Errorf("warning reason = %q, want %q", warning.Reason, ModelSelectionReasonUniqueVariationApplied)
|
||||
}
|
||||
if legacyFallbackEvents != 0 {
|
||||
t.Fatalf("legacy fallback events = %d, want zero for unique variation", legacyFallbackEvents)
|
||||
}
|
||||
}
|
||||
@@ -3,6 +3,7 @@ package lifecycle
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/kandev/kandev/internal/agentctl/sessionmodel"
|
||||
"github.com/kandev/kandev/internal/common/logger"
|
||||
@@ -31,6 +32,7 @@ const (
|
||||
ModelSelectionOutcomeNone ModelSelectionOutcome = ""
|
||||
ModelSelectionOutcomeApplied ModelSelectionOutcome = "applied"
|
||||
ModelSelectionOutcomeExplicitFallback ModelSelectionOutcome = "explicit_fallback"
|
||||
ModelSelectionOutcomeUniqueVariation ModelSelectionOutcome = "unique_variation"
|
||||
ModelSelectionOutcomeProviderDefault ModelSelectionOutcome = "provider_default"
|
||||
)
|
||||
|
||||
@@ -42,6 +44,7 @@ const (
|
||||
ModelSelectionReasonCatalogEmpty = "catalog_empty"
|
||||
ModelSelectionReasonSelectionUnsupported = "selection_unsupported"
|
||||
ModelSelectionReasonSelectionFailedAutoFallback = "selection_failed_auto_fallback"
|
||||
ModelSelectionReasonUniqueVariationApplied = "unique_variation_applied"
|
||||
)
|
||||
|
||||
// ModelSelectionDecision is the single model-selection result shared by
|
||||
@@ -84,6 +87,32 @@ func containsModel(ids []string, id string) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
func uniqueAdvertisedModelVariation(requested string, advertised []string) string {
|
||||
if requested == "" || strings.ContainsAny(requested, "[]") {
|
||||
return ""
|
||||
}
|
||||
|
||||
prefix := requested + "["
|
||||
candidates := make(map[string]struct{})
|
||||
for _, id := range advertised {
|
||||
if !strings.HasPrefix(id, prefix) || !strings.HasSuffix(id, "]") {
|
||||
continue
|
||||
}
|
||||
variant := strings.TrimSuffix(strings.TrimPrefix(id, prefix), "]")
|
||||
if variant == "" || strings.ContainsAny(variant, "[]") {
|
||||
continue
|
||||
}
|
||||
candidates[id] = struct{}{}
|
||||
if len(candidates) > 1 {
|
||||
return ""
|
||||
}
|
||||
}
|
||||
for candidate := range candidates {
|
||||
return candidate
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func providerDefaultDecision(state *CachedModelState, policy StartModelPolicy, reason string) ModelSelectionDecision {
|
||||
decision := ModelSelectionDecision{
|
||||
RequestedModel: policy.Model,
|
||||
@@ -114,6 +143,9 @@ func applyStartModelPolicy(
|
||||
if policy.Model == "" {
|
||||
return ModelSelectionDecision{Outcome: ModelSelectionOutcomeNone}, nil
|
||||
}
|
||||
if policy.AutoFallback {
|
||||
policy.FallbackModel = ""
|
||||
}
|
||||
|
||||
decision := ModelSelectionDecision{
|
||||
RequestedModel: policy.Model,
|
||||
@@ -125,9 +157,15 @@ func applyStartModelPolicy(
|
||||
}
|
||||
|
||||
if !containsModel(advertised, policy.Model) {
|
||||
if policy.AutoFallback {
|
||||
return providerDefaultDecision(state, policy, ModelSelectionReasonRequestedNotAdvertised), nil
|
||||
}
|
||||
if policy.FallbackModel != "" && containsModel(advertised, policy.FallbackModel) {
|
||||
return applyAdvertisedFallback(ctx, log, applier, state, policy, decision)
|
||||
}
|
||||
if variation := uniqueAdvertisedModelVariation(policy.Model, advertised); variation != "" {
|
||||
return applyUniqueAdvertisedVariation(ctx, log, applier, state, policy, decision, variation)
|
||||
}
|
||||
reason := ModelSelectionReasonRequestedNotAdvertised
|
||||
if policy.FallbackModel != "" {
|
||||
reason = ModelSelectionReasonFallbackNotAdvertised
|
||||
@@ -159,6 +197,41 @@ func applyStartModelPolicy(
|
||||
return decision, nil
|
||||
}
|
||||
|
||||
func applyUniqueAdvertisedVariation(
|
||||
ctx context.Context,
|
||||
log *logger.Logger,
|
||||
applier modelApplier,
|
||||
state *CachedModelState,
|
||||
policy StartModelPolicy,
|
||||
decision ModelSelectionDecision,
|
||||
variation string,
|
||||
) (ModelSelectionDecision, error) {
|
||||
decision.FallbackModel = ""
|
||||
policy.FallbackModel = ""
|
||||
decision.SetModelCalled = true
|
||||
if err := applier.SetModel(ctx, variation); err != nil {
|
||||
if sessionmodel.IsMethodNotFound(err) {
|
||||
decision = providerDefaultDecision(state, policy, ModelSelectionReasonSelectionUnsupported)
|
||||
decision.SetModelCalled = true
|
||||
return decision, nil
|
||||
}
|
||||
if policy.AutoFallback {
|
||||
decision = providerDefaultDecision(state, policy, ModelSelectionReasonSelectionFailedAutoFallback)
|
||||
decision.SetModelCalled = true
|
||||
return decision, nil
|
||||
}
|
||||
return decision, fmt.Errorf("failed to set unique model variation %q: %w", variation, err)
|
||||
}
|
||||
decision.EffectiveModel = variation
|
||||
decision.Outcome = ModelSelectionOutcomeUniqueVariation
|
||||
decision.Reason = ModelSelectionReasonUniqueVariationApplied
|
||||
decision.Warning = true
|
||||
log.Info("start model unavailable, using unique advertised variation",
|
||||
zap.String("start_model", policy.Model),
|
||||
zap.String("variation_model", variation))
|
||||
return decision, nil
|
||||
}
|
||||
|
||||
func applyAdvertisedFallback(
|
||||
ctx context.Context,
|
||||
log *logger.Logger,
|
||||
|
||||
@@ -47,6 +47,104 @@ func TestApplyStartModelPolicyExecutorAuthority(t *testing.T) {
|
||||
wantEffective: "fallback",
|
||||
wantWarning: true,
|
||||
},
|
||||
{
|
||||
name: "unique variation is applied when fallback is absent",
|
||||
state: modelState("opus[1m]"),
|
||||
policy: StartModelPolicy{Model: "opus"},
|
||||
wantCalls: []string{"opus[1m]"},
|
||||
wantOutcome: ModelSelectionOutcomeUniqueVariation,
|
||||
wantReason: ModelSelectionReasonUniqueVariationApplied,
|
||||
wantEffective: "opus[1m]",
|
||||
wantWarning: true,
|
||||
},
|
||||
{
|
||||
name: "advertised fallback takes precedence over unique variation",
|
||||
state: modelState("fallback", "opus[1m]"),
|
||||
policy: StartModelPolicy{Model: "opus", FallbackModel: "fallback"},
|
||||
wantCalls: []string{"fallback"},
|
||||
wantOutcome: ModelSelectionOutcomeExplicitFallback,
|
||||
wantReason: ModelSelectionReasonRequestedNotAdvertised,
|
||||
wantEffective: "fallback",
|
||||
wantWarning: true,
|
||||
},
|
||||
{
|
||||
name: "unadvertised fallback does not block unique variation",
|
||||
state: modelState("opus[1m]"),
|
||||
policy: StartModelPolicy{Model: "opus", FallbackModel: "fallback"},
|
||||
wantCalls: []string{"opus[1m]"},
|
||||
wantOutcome: ModelSelectionOutcomeUniqueVariation,
|
||||
wantReason: ModelSelectionReasonUniqueVariationApplied,
|
||||
wantEffective: "opus[1m]",
|
||||
wantWarning: true,
|
||||
},
|
||||
{
|
||||
name: "auto fallback keeps unavailable model on provider default",
|
||||
state: modelState("fallback", "opus[1m]"),
|
||||
policy: StartModelPolicy{Model: "opus", FallbackModel: "fallback", AutoFallback: true},
|
||||
wantOutcome: ModelSelectionOutcomeProviderDefault,
|
||||
wantReason: ModelSelectionReasonRequestedNotAdvertised,
|
||||
wantWarning: true,
|
||||
},
|
||||
{
|
||||
name: "duplicate unique variation is one candidate",
|
||||
state: modelState("opus[1m]", "opus[1m]"),
|
||||
policy: StartModelPolicy{Model: "opus"},
|
||||
wantCalls: []string{"opus[1m]"},
|
||||
wantOutcome: ModelSelectionOutcomeUniqueVariation,
|
||||
wantReason: ModelSelectionReasonUniqueVariationApplied,
|
||||
wantEffective: "opus[1m]",
|
||||
wantWarning: true,
|
||||
},
|
||||
{
|
||||
name: "multiple variations keep provider default",
|
||||
state: modelState("opus[1m]", "opus[270k]"),
|
||||
policy: StartModelPolicy{Model: "opus"},
|
||||
wantOutcome: ModelSelectionOutcomeProviderDefault,
|
||||
wantReason: ModelSelectionReasonRequestedNotAdvertised,
|
||||
wantWarning: true,
|
||||
},
|
||||
{
|
||||
name: "malformed variations keep provider default",
|
||||
state: modelState("opus[]", "opus[1m", "opus[1m][fast]", "opus-pro[1m]"),
|
||||
policy: StartModelPolicy{Model: "opus"},
|
||||
wantOutcome: ModelSelectionOutcomeProviderDefault,
|
||||
wantReason: ModelSelectionReasonRequestedNotAdvertised,
|
||||
wantWarning: true,
|
||||
},
|
||||
{
|
||||
name: "variation text remains opaque",
|
||||
state: modelState("opus[1m, fast]"),
|
||||
policy: StartModelPolicy{Model: "opus"},
|
||||
wantCalls: []string{"opus[1m, fast]"},
|
||||
wantOutcome: ModelSelectionOutcomeUniqueVariation,
|
||||
wantReason: ModelSelectionReasonUniqueVariationApplied,
|
||||
wantEffective: "opus[1m, fast]",
|
||||
wantWarning: true,
|
||||
},
|
||||
{
|
||||
name: "case-sensitive variation matching keeps provider default",
|
||||
state: modelState("Opus[1m]"),
|
||||
policy: StartModelPolicy{Model: "opus"},
|
||||
wantOutcome: ModelSelectionOutcomeProviderDefault,
|
||||
wantReason: ModelSelectionReasonRequestedNotAdvertised,
|
||||
wantWarning: true,
|
||||
},
|
||||
{
|
||||
name: "bracketed request does not infer another variation",
|
||||
state: modelState("opus[2m]"),
|
||||
policy: StartModelPolicy{Model: "opus[1m]"},
|
||||
wantOutcome: ModelSelectionOutcomeProviderDefault,
|
||||
wantReason: ModelSelectionReasonRequestedNotAdvertised,
|
||||
wantWarning: true,
|
||||
},
|
||||
{
|
||||
name: "exact bracketed request remains exact",
|
||||
state: modelState("opus[1m]", "opus[2m]"),
|
||||
policy: StartModelPolicy{Model: "opus[1m]"},
|
||||
wantCalls: []string{"opus[1m]"},
|
||||
wantOutcome: ModelSelectionOutcomeApplied,
|
||||
wantEffective: "opus[1m]",
|
||||
},
|
||||
{
|
||||
name: "advertised fallback method not supported keeps provider default",
|
||||
state: modelState("fallback"),
|
||||
@@ -57,6 +155,16 @@ func TestApplyStartModelPolicyExecutorAuthority(t *testing.T) {
|
||||
wantReason: ModelSelectionReasonSelectionUnsupported,
|
||||
wantWarning: true,
|
||||
},
|
||||
{
|
||||
name: "unique variation method not supported keeps provider default",
|
||||
state: modelState("opus[1m]"),
|
||||
policy: StartModelPolicy{Model: "opus", FallbackModel: "fallback"},
|
||||
applierErrors: []error{methodNotFoundErr()},
|
||||
wantCalls: []string{"opus[1m]"},
|
||||
wantOutcome: ModelSelectionOutcomeProviderDefault,
|
||||
wantReason: ModelSelectionReasonSelectionUnsupported,
|
||||
wantWarning: true,
|
||||
},
|
||||
{
|
||||
name: "unadvertised fallback keeps provider default",
|
||||
state: modelState("executor-default"),
|
||||
@@ -132,6 +240,12 @@ func TestApplyStartModelPolicyExecutorAuthority(t *testing.T) {
|
||||
if decision.Warning != tt.wantWarning {
|
||||
t.Errorf("warning = %v, want %v", decision.Warning, tt.wantWarning)
|
||||
}
|
||||
if decision.Outcome == ModelSelectionOutcomeUniqueVariation && decision.FallbackModel != "" {
|
||||
t.Errorf("unique variation fallback model = %q, want empty", decision.FallbackModel)
|
||||
}
|
||||
if tt.policy.AutoFallback && decision.FallbackModel != "" {
|
||||
t.Errorf("auto-fallback fallback model = %q, want empty", decision.FallbackModel)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -150,6 +150,20 @@ describe("ProfileFormFields no-silent-model-fallback rows", () => {
|
||||
expect(trigger.textContent).toContain("claude-gone");
|
||||
});
|
||||
|
||||
it("names a unique advertised variation while preserving the saved model", () => {
|
||||
renderForm(formData({ model: "opus" }), {
|
||||
...modelConfig,
|
||||
available_models: [{ id: "opus[1m]", name: "Opus (1m)" }],
|
||||
});
|
||||
|
||||
const advisory = screen.getByTestId("profile-model-variation-advisory");
|
||||
expect(advisory.textContent).toContain("opus[1m]");
|
||||
expect(advisory.textContent).toContain("opus");
|
||||
expect(
|
||||
screen.getByRole("button", { name: profileStartModelSettingsLabel }).textContent,
|
||||
).toContain("opus");
|
||||
});
|
||||
|
||||
it("shows the agent fallback row when auto-fallback is off", () => {
|
||||
renderForm(formData({ auto_fallback: false }));
|
||||
expandFallbackSettings();
|
||||
|
||||
@@ -23,6 +23,7 @@ import {
|
||||
import type { ModelConfig, ModeEntry, ModelEntry } from "@/lib/types/http";
|
||||
import type { PermissionKey } from "@/lib/agent-permissions";
|
||||
import type { CLIFlag } from "@/lib/types/http";
|
||||
import { findUniqueModelVariation } from "@/lib/model-variation";
|
||||
import {
|
||||
SettingsFieldDescription,
|
||||
SettingsFieldLabel,
|
||||
@@ -83,6 +84,12 @@ export function ModelPicker({
|
||||
// in the list, greyed out and unselectable, so the user sees what was
|
||||
// configured instead of it silently vanishing.
|
||||
const modelIsGone = Boolean(profile.model && !modelOptions.some((m) => m.id === profile.model));
|
||||
const uniqueVariation = modelIsGone
|
||||
? findUniqueModelVariation(
|
||||
profile.model,
|
||||
modelOptions.map((model) => model.id),
|
||||
)
|
||||
: null;
|
||||
if (modelIsGone) {
|
||||
modelOptions.unshift({
|
||||
id: profile.model!,
|
||||
@@ -99,22 +106,32 @@ export function ModelPicker({
|
||||
}));
|
||||
|
||||
return (
|
||||
<ModelConfigSelector
|
||||
modelOptions={modelOptions}
|
||||
currentModel={currentModel}
|
||||
configOptions={selectedConfigOptions}
|
||||
onModelChange={(value) => onChange({ model: value })}
|
||||
onConfigChange={(configId, value) =>
|
||||
onChange({ config_options: { ...(profile.config_options ?? {}), [configId]: value } })
|
||||
}
|
||||
placeholder={placeholder ?? t("settings:selectAModel")}
|
||||
ariaLabel={ariaLabel}
|
||||
popoverAlign="start"
|
||||
disabled={disabled}
|
||||
configOptionsLoading={configOptionsLoading}
|
||||
keepOpenOnModelChange={keepOpenOnModelChange}
|
||||
triggerClassName={modelIsGone ? "text-destructive" : undefined}
|
||||
/>
|
||||
<div className="space-y-1.5">
|
||||
<ModelConfigSelector
|
||||
modelOptions={modelOptions}
|
||||
currentModel={currentModel}
|
||||
configOptions={selectedConfigOptions}
|
||||
onModelChange={(value) => onChange({ model: value })}
|
||||
onConfigChange={(configId, value) =>
|
||||
onChange({ config_options: { ...(profile.config_options ?? {}), [configId]: value } })
|
||||
}
|
||||
placeholder={placeholder ?? t("settings:selectAModel")}
|
||||
ariaLabel={ariaLabel}
|
||||
popoverAlign="start"
|
||||
disabled={disabled}
|
||||
configOptionsLoading={configOptionsLoading}
|
||||
keepOpenOnModelChange={keepOpenOnModelChange}
|
||||
triggerClassName={modelIsGone ? "text-destructive" : undefined}
|
||||
/>
|
||||
{uniqueVariation && (
|
||||
<p className="text-xs text-muted-foreground" data-testid="profile-model-variation-advisory">
|
||||
{t("settings:modelVariationAdvisory", {
|
||||
model: profile.model,
|
||||
variation: uniqueVariation,
|
||||
})}
|
||||
</p>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
@@ -58,6 +58,19 @@ const AGENT_WITH_GPT: AvailableAgent = {
|
||||
},
|
||||
} as unknown as AvailableAgent;
|
||||
|
||||
const AGENT_WITH_OPUS_VARIATION: AvailableAgent = {
|
||||
name: "omp-acp",
|
||||
available: true,
|
||||
model_config: {
|
||||
default_model: "opus[1m]",
|
||||
available_models: [{ id: "opus[1m]", name: "Opus (1m)" }],
|
||||
current_model_id: "opus[1m]",
|
||||
available_modes: [],
|
||||
supports_dynamic_models: false,
|
||||
status: "ok",
|
||||
},
|
||||
} as unknown as AvailableAgent;
|
||||
|
||||
const GONE_MODEL = "claude-gone";
|
||||
const DATA_DISABLED = "data-disabled";
|
||||
const MODEL_PROBE_WARNING_TEST_ID = "agent-profile-model-probe-warning";
|
||||
@@ -225,6 +238,14 @@ describe("useAgentProfileOptions executor-authoritative model hint", () => {
|
||||
expect(option.getAttribute(DATA_DISABLED)).toBeNull();
|
||||
});
|
||||
|
||||
it("names a unique advertised variation without disabling the profile", () => {
|
||||
setAvailableAgents([AGENT_WITH_OPUS_VARIATION]);
|
||||
const option = renderOptions([profileOption({ model: "opus" })]);
|
||||
|
||||
expect(option.getAttribute(DATA_DISABLED)).toBeNull();
|
||||
expect(getModelProbeWarningLabel()).toContain("opus[1m]");
|
||||
});
|
||||
|
||||
it("keeps a profile with an empty (agent default) model selectable", () => {
|
||||
const option = renderOptions([profileOption({ model: "" })]);
|
||||
expect(option.getAttribute(DATA_DISABLED)).toBeNull();
|
||||
|
||||
@@ -32,6 +32,7 @@ import { formatUserHomePath, truncateRepoPath } from "@/lib/utils";
|
||||
import { getExecutorIcon } from "@/lib/executor-icons";
|
||||
import { AgentLogo } from "@/components/agent-logo";
|
||||
import { getCapabilityWarning } from "@/lib/capability-warning";
|
||||
import { findUniqueModelVariation } from "@/lib/model-variation";
|
||||
import { useTouchDrawer } from "@/hooks/use-compact-task-chrome";
|
||||
import { buildBranchKeywords } from "./branch-picker-options";
|
||||
import {
|
||||
@@ -229,9 +230,18 @@ export function useAgentProfileOptions(
|
||||
const startModelGone = Boolean(
|
||||
profile.model && advertised.length > 0 && !advertised.includes(profile.model),
|
||||
);
|
||||
const modelProbeNote = startModelGone
|
||||
? t("settings:profileStartModelNotAdvertisedOnHost", { model: profile.model })
|
||||
: undefined;
|
||||
const uniqueVariation = startModelGone
|
||||
? findUniqueModelVariation(profile.model ?? "", advertised)
|
||||
: null;
|
||||
let modelProbeNote: string | undefined;
|
||||
if (startModelGone) {
|
||||
modelProbeNote = uniqueVariation
|
||||
? t("settings:profileStartModelUniqueVariationOnHost", {
|
||||
model: profile.model,
|
||||
variation: uniqueVariation,
|
||||
})
|
||||
: t("settings:profileStartModelNotAdvertisedOnHost", { model: profile.model });
|
||||
}
|
||||
const renderProfileLabel = (modelProbeWarning: React.ReactNode) => (
|
||||
<span className="flex min-w-0 flex-1 flex-col gap-1">
|
||||
<span className="flex shrink-0 items-center justify-between gap-2">
|
||||
|
||||
@@ -42,6 +42,23 @@ describe("StatusMessage model selection warnings", () => {
|
||||
expect(screen.getByText("Check the agent version in the executor.")).toBeTruthy();
|
||||
expect(screen.getByText("The saved model was not advertised by the executor.")).toBeTruthy();
|
||||
});
|
||||
|
||||
it("explains when the executor applied the only advertised variation", () => {
|
||||
const comment = modelSelectionWarningMessage();
|
||||
comment.metadata = {
|
||||
...comment.metadata,
|
||||
reason: "unique_variation_applied",
|
||||
effective_model: "opus[1m]",
|
||||
fallback_model: undefined,
|
||||
};
|
||||
|
||||
render(<StatusMessage comment={comment} />);
|
||||
|
||||
expect(
|
||||
screen.getByText("The executor applied the only advertised variation of the saved model."),
|
||||
).toBeTruthy();
|
||||
expect(screen.queryByText(/fallback model/i)).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe("StatusMessage branch replacement warnings", () => {
|
||||
|
||||
@@ -180,6 +180,7 @@ function modelSelectionReasonLabel(reason: string | undefined): string {
|
||||
const keyByReason: Record<string, string> = {
|
||||
requested_not_advertised: "task:modelSelectionReasonRequestedNotAdvertised",
|
||||
fallback_not_advertised: "task:modelSelectionReasonFallbackNotAdvertised",
|
||||
unique_variation_applied: "task:modelSelectionReasonUniqueVariation",
|
||||
catalog_empty: "task:modelSelectionReasonCatalogEmpty",
|
||||
selection_unsupported: "task:modelSelectionReasonUnsupported",
|
||||
selection_failed_auto_fallback: "task:modelSelectionReasonAutoFallback",
|
||||
|
||||
@@ -2,6 +2,10 @@ import type { AgentProfile } from "../../../lib/types/http-agents";
|
||||
import type { ApiClient } from "../../helpers/api-client";
|
||||
|
||||
export const UNADVERTISED_MODEL = "host-only-model";
|
||||
export const MODEL_VARIATION_BASE = "opus";
|
||||
export const UNIQUE_MODEL_VARIATION = "opus[1m]";
|
||||
export const AMBIGUOUS_MODEL_VARIATIONS = ["opus[270k]", "opus[1m, fast]"] as const;
|
||||
const MODEL_CATALOG_ENV = "MOCK_AGENT_MODEL_CATALOG";
|
||||
|
||||
export async function createMismatchedProfile(
|
||||
apiClient: ApiClient,
|
||||
@@ -13,6 +17,20 @@ export async function createMismatchedProfile(
|
||||
return apiClient.createAgentProfile(agent.id, name, { model: UNADVERTISED_MODEL });
|
||||
}
|
||||
|
||||
export async function createModelVariationProfile(
|
||||
apiClient: ApiClient,
|
||||
name: string,
|
||||
catalog: "unique" | "ambiguous",
|
||||
): Promise<AgentProfile> {
|
||||
const { agents } = await apiClient.listAgents();
|
||||
const agent = agents.find((item) => item.name === "mock-agent") ?? agents[0];
|
||||
if (!agent) throw new Error("The E2E fixture must provide a mock agent");
|
||||
return apiClient.createAgentProfile(agent.id, name, {
|
||||
model: MODEL_VARIATION_BASE,
|
||||
env_vars: [{ key: MODEL_CATALOG_ENV, value: catalog }],
|
||||
});
|
||||
}
|
||||
|
||||
export async function readModelSelectionWarnings(
|
||||
apiClient: ApiClient,
|
||||
sessionId: string,
|
||||
|
||||
@@ -2,7 +2,10 @@ import { expect, test } from "../../fixtures/test-base";
|
||||
import { waitForSessionDone } from "../../helpers/session";
|
||||
import { SessionPage } from "../../pages/session-page";
|
||||
import {
|
||||
createModelVariationProfile,
|
||||
createMismatchedProfile,
|
||||
MODEL_VARIATION_BASE,
|
||||
UNIQUE_MODEL_VARIATION,
|
||||
readModelSelectionWarnings,
|
||||
UNADVERTISED_MODEL,
|
||||
} from "./model-mismatch-warning-helpers";
|
||||
@@ -73,4 +76,124 @@ test.describe("model mismatch warning", () => {
|
||||
await apiClient.deleteAgentProfile(profile.id, true).catch(() => {});
|
||||
}
|
||||
});
|
||||
|
||||
test("applies one unique advertised variation and persists its warning", async ({
|
||||
testPage,
|
||||
apiClient,
|
||||
seedData,
|
||||
}) => {
|
||||
test.setTimeout(120_000);
|
||||
const profile = await createModelVariationProfile(
|
||||
apiClient,
|
||||
"Unique model variation profile",
|
||||
"unique",
|
||||
);
|
||||
try {
|
||||
const task = await apiClient.createTaskWithAgent(
|
||||
seedData.workspaceId,
|
||||
"Unique model variation warning",
|
||||
profile.id,
|
||||
{
|
||||
description: "/e2e:simple-message",
|
||||
workflow_id: seedData.workflowId,
|
||||
workflow_step_id: seedData.startStepId,
|
||||
repository_ids: [seedData.repositoryId],
|
||||
},
|
||||
);
|
||||
expect(task.session_id).toBeTruthy();
|
||||
await waitForSessionDone(
|
||||
apiClient,
|
||||
task.id,
|
||||
task.session_id!,
|
||||
"Waiting for unique model variation task continuation",
|
||||
);
|
||||
|
||||
await expect
|
||||
.poll(() => readModelSelectionWarnings(apiClient, task.session_id!), {
|
||||
timeout: 15_000,
|
||||
message: "Waiting for the unique model variation warning",
|
||||
})
|
||||
.toHaveLength(1);
|
||||
const warnings = await readModelSelectionWarnings(apiClient, task.session_id!);
|
||||
expect(warnings[0]?.metadata).toMatchObject({
|
||||
kind: "model_selection_warning",
|
||||
reason: "unique_variation_applied",
|
||||
requested_model: MODEL_VARIATION_BASE,
|
||||
effective_model: UNIQUE_MODEL_VARIATION,
|
||||
agent_id: "mock-agent",
|
||||
});
|
||||
expect(warnings[0]?.metadata?.fallback_model).toBeUndefined();
|
||||
expect((await apiClient.getAgentProfile(profile.id)).model).toBe(MODEL_VARIATION_BASE);
|
||||
|
||||
await testPage.goto(`/t/${task.id}`);
|
||||
const session = new SessionPage(testPage);
|
||||
await session.waitForLoad();
|
||||
await expect(
|
||||
session.activeChat().getByText("The executor could not use the saved model selection."),
|
||||
).toBeVisible({ timeout: 15_000 });
|
||||
await expect(session.activeChat()).toContainText(UNIQUE_MODEL_VARIATION);
|
||||
|
||||
await testPage.reload();
|
||||
await session.waitForLoad();
|
||||
await expect(
|
||||
session.activeChat().getByText("The executor could not use the saved model selection."),
|
||||
).toBeVisible({ timeout: 15_000 });
|
||||
await expect(session.activeChat()).toContainText(UNIQUE_MODEL_VARIATION);
|
||||
await expect
|
||||
.poll(() => readModelSelectionWarnings(apiClient, task.session_id!))
|
||||
.toHaveLength(1);
|
||||
} finally {
|
||||
await apiClient.deleteAgentProfile(profile.id, true).catch(() => {});
|
||||
}
|
||||
});
|
||||
|
||||
test("does not infer a model when the executor advertises multiple variations", async ({
|
||||
apiClient,
|
||||
seedData,
|
||||
}) => {
|
||||
test.setTimeout(120_000);
|
||||
const profile = await createModelVariationProfile(
|
||||
apiClient,
|
||||
"Ambiguous model variation profile",
|
||||
"ambiguous",
|
||||
);
|
||||
try {
|
||||
const task = await apiClient.createTaskWithAgent(
|
||||
seedData.workspaceId,
|
||||
"Ambiguous model variation warning",
|
||||
profile.id,
|
||||
{
|
||||
description: "/e2e:simple-message",
|
||||
workflow_id: seedData.workflowId,
|
||||
workflow_step_id: seedData.startStepId,
|
||||
repository_ids: [seedData.repositoryId],
|
||||
},
|
||||
);
|
||||
expect(task.session_id).toBeTruthy();
|
||||
await waitForSessionDone(
|
||||
apiClient,
|
||||
task.id,
|
||||
task.session_id!,
|
||||
"Waiting for ambiguous model variation task continuation",
|
||||
);
|
||||
|
||||
await expect
|
||||
.poll(() => readModelSelectionWarnings(apiClient, task.session_id!), {
|
||||
timeout: 15_000,
|
||||
message: "Waiting for the ambiguous model variation warning",
|
||||
})
|
||||
.toHaveLength(1);
|
||||
const warnings = await readModelSelectionWarnings(apiClient, task.session_id!);
|
||||
expect(warnings[0]?.metadata).toMatchObject({
|
||||
kind: "model_selection_warning",
|
||||
reason: "requested_not_advertised",
|
||||
requested_model: MODEL_VARIATION_BASE,
|
||||
effective_model: "mock-fast",
|
||||
agent_id: "mock-agent",
|
||||
});
|
||||
expect((await apiClient.getAgentProfile(profile.id)).model).toBe(MODEL_VARIATION_BASE);
|
||||
} finally {
|
||||
await apiClient.deleteAgentProfile(profile.id, true).catch(() => {});
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
@@ -2,7 +2,10 @@ import { expect, test } from "../../fixtures/test-base";
|
||||
import { KanbanPage } from "../../pages/kanban-page";
|
||||
import { assertNoDocumentHorizontalOverflow } from "../../helpers/layout-assertions";
|
||||
import {
|
||||
createModelVariationProfile,
|
||||
createMismatchedProfile,
|
||||
MODEL_VARIATION_BASE,
|
||||
UNIQUE_MODEL_VARIATION,
|
||||
UNADVERTISED_MODEL,
|
||||
} from "../session/model-mismatch-warning-helpers";
|
||||
|
||||
@@ -41,4 +44,44 @@ test.describe("executor-authoritative model selection on mobile", () => {
|
||||
await apiClient.deleteAgentProfile(profile.id, true).catch(() => {});
|
||||
}
|
||||
});
|
||||
|
||||
test("opens the unique variation advisory by touch without overflow", async ({
|
||||
testPage,
|
||||
apiClient,
|
||||
}) => {
|
||||
const profile = await createModelVariationProfile(
|
||||
apiClient,
|
||||
"Mobile unique host variation profile",
|
||||
"unique",
|
||||
);
|
||||
try {
|
||||
const kanban = new KanbanPage(testPage);
|
||||
await kanban.goto();
|
||||
await testPage.reload({ waitUntil: "networkidle" });
|
||||
await testPage.getByRole("button", { name: "Add task" }).tap();
|
||||
|
||||
const dialog = testPage.getByTestId("create-task-dialog");
|
||||
await expect(dialog).toBeVisible();
|
||||
const selector = dialog.getByTestId("agent-profile-selector");
|
||||
await selector.tap();
|
||||
|
||||
const option = testPage
|
||||
.getByRole("listbox")
|
||||
.getByRole("option", { name: profile.name, exact: false });
|
||||
await expect(option).toBeVisible();
|
||||
await expect(option).toBeEnabled();
|
||||
const warning = option.getByTestId("agent-profile-model-probe-warning");
|
||||
await expect(warning).toBeVisible();
|
||||
const warningText = `The host probe found one possible variation of ${MODEL_VARIATION_BASE}: ${UNIQUE_MODEL_VARIATION}. The selected executor will decide the model at launch.`;
|
||||
await warning.tap();
|
||||
await expect(
|
||||
testPage
|
||||
.locator('[data-slot="drawer-content"][data-state="open"]')
|
||||
.filter({ hasText: warningText }),
|
||||
).toBeVisible();
|
||||
await assertNoDocumentHorizontalOverflow(testPage, "mobile unique variation advisory");
|
||||
} finally {
|
||||
await apiClient.deleteAgentProfile(profile.id, true).catch(() => {});
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,7 +1,10 @@
|
||||
import { expect, test } from "../../fixtures/test-base";
|
||||
import { KanbanPage } from "../../pages/kanban-page";
|
||||
import {
|
||||
createModelVariationProfile,
|
||||
createMismatchedProfile,
|
||||
MODEL_VARIATION_BASE,
|
||||
UNIQUE_MODEL_VARIATION,
|
||||
UNADVERTISED_MODEL,
|
||||
} from "../session/model-mismatch-warning-helpers";
|
||||
|
||||
@@ -41,4 +44,44 @@ test.describe("executor-authoritative model selection", () => {
|
||||
await apiClient.deleteAgentProfile(profile.id, true).catch(() => {});
|
||||
}
|
||||
});
|
||||
|
||||
test("names one host-advertised variation while keeping the profile selectable", async ({
|
||||
testPage,
|
||||
apiClient,
|
||||
}) => {
|
||||
const profile = await createModelVariationProfile(
|
||||
apiClient,
|
||||
"Unique host variation selectable profile",
|
||||
"unique",
|
||||
);
|
||||
try {
|
||||
const kanban = new KanbanPage(testPage);
|
||||
await kanban.goto();
|
||||
await testPage.reload({ waitUntil: "networkidle" });
|
||||
await kanban.createTaskButton.first().click();
|
||||
|
||||
const dialog = testPage.getByTestId("create-task-dialog");
|
||||
await expect(dialog).toBeVisible();
|
||||
const selector = dialog.getByTestId("agent-profile-selector");
|
||||
await selector.click();
|
||||
|
||||
const option = testPage
|
||||
.getByRole("listbox")
|
||||
.getByRole("option", { name: profile.name, exact: false });
|
||||
await expect(option).toBeVisible();
|
||||
await expect(option).toBeEnabled();
|
||||
const warning = option.getByTestId("agent-profile-model-probe-warning");
|
||||
await expect(warning).toBeVisible();
|
||||
const warningText = `The host probe found one possible variation of ${MODEL_VARIATION_BASE}: ${UNIQUE_MODEL_VARIATION}. The selected executor will decide the model at launch.`;
|
||||
await warning.hover();
|
||||
await expect(
|
||||
testPage
|
||||
.locator('[data-slot="tooltip-content"]:not([data-state="closed"])')
|
||||
.filter({ hasText: warningText }),
|
||||
).toBeVisible();
|
||||
await option.click();
|
||||
} finally {
|
||||
await apiClient.deleteAgentProfile(profile.id, true).catch(() => {});
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
@@ -0,0 +1,57 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { findUniqueModelVariation } from "./model-variation";
|
||||
|
||||
describe("findUniqueModelVariation", () => {
|
||||
it.each([
|
||||
{
|
||||
name: "finds one bracketed variation",
|
||||
requested: "opus",
|
||||
advertised: ["opus[1m]"],
|
||||
expected: "opus[1m]",
|
||||
},
|
||||
{
|
||||
name: "deduplicates the same variation",
|
||||
requested: "opus",
|
||||
advertised: ["opus[1m]", "opus[1m]"],
|
||||
expected: "opus[1m]",
|
||||
},
|
||||
{
|
||||
name: "keeps opaque variation text",
|
||||
requested: "opus",
|
||||
advertised: ["opus[1m, fast]"],
|
||||
expected: "opus[1m, fast]",
|
||||
},
|
||||
])("$name", ({ requested, advertised, expected }) => {
|
||||
expect(findUniqueModelVariation(requested, advertised)).toBe(expected);
|
||||
});
|
||||
|
||||
it.each([
|
||||
{
|
||||
name: "returns no candidate for an empty catalog",
|
||||
requested: "opus",
|
||||
advertised: [],
|
||||
},
|
||||
{
|
||||
name: "returns no candidate for ambiguous variations",
|
||||
requested: "opus",
|
||||
advertised: ["opus[1m]", "opus[270k]"],
|
||||
},
|
||||
{
|
||||
name: "returns no candidate for case differences",
|
||||
requested: "opus",
|
||||
advertised: ["Opus[1m]"],
|
||||
},
|
||||
{
|
||||
name: "returns no candidate for a bracketed request",
|
||||
requested: "opus[1m]",
|
||||
advertised: ["opus[2m]"],
|
||||
},
|
||||
{
|
||||
name: "returns no candidate for malformed IDs",
|
||||
requested: "opus",
|
||||
advertised: ["opus[]", "opus[1m", "opus[1m][fast]", "opus-pro[1m]"],
|
||||
},
|
||||
])("$name", ({ requested, advertised }) => {
|
||||
expect(findUniqueModelVariation(requested, advertised)).toBeNull();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,17 @@
|
||||
function isModelVariation(requested: string, advertised: string): boolean {
|
||||
const prefix = `${requested}[`;
|
||||
if (!advertised.startsWith(prefix) || !advertised.endsWith("]")) return false;
|
||||
|
||||
const variant = advertised.slice(prefix.length, -1);
|
||||
return variant.length > 0 && !variant.includes("[") && !variant.includes("]");
|
||||
}
|
||||
|
||||
export function findUniqueModelVariation(
|
||||
requested: string,
|
||||
advertised: readonly string[],
|
||||
): string | null {
|
||||
if (!requested || requested.includes("[") || requested.includes("]")) return null;
|
||||
|
||||
const candidates = new Set(advertised.filter((model) => isModelVariation(requested, model)));
|
||||
return candidates.size === 1 ? ([...candidates][0] ?? null) : null;
|
||||
}
|
||||
@@ -504,6 +504,8 @@
|
||||
"profileForTasksCreatedByAgents": "Profile for Tasks Created by Agents",
|
||||
"programming": "Programming",
|
||||
"profileStartModelNotAdvertisedOnHost": "The host probe did not advertise {{model}}. The selected executor will decide the model at launch.",
|
||||
"profileStartModelUniqueVariationOnHost": "The host probe found one possible variation of {{model}}: {{variation}}. The selected executor will decide the model at launch.",
|
||||
"modelVariationAdvisory": "The host catalog has one possible variation of {{model}}: {{variation}}. The saved model remains unchanged until launch.",
|
||||
"promptAdd": "Add prompt",
|
||||
"promptEditorAriaLabel": "Prompt editor",
|
||||
"promptEditorBothHint": "Type {{placeholder}} for available placeholders or {{mention}} for saved prompts.",
|
||||
|
||||
@@ -883,6 +883,7 @@
|
||||
"modelSelectionReasonCatalogEmpty": "The executor did not advertise a model catalog.",
|
||||
"modelSelectionReasonFallbackNotAdvertised": "The saved fallback model was not advertised by the executor.",
|
||||
"modelSelectionReasonRequestedNotAdvertised": "The saved model was not advertised by the executor.",
|
||||
"modelSelectionReasonUniqueVariation": "The executor applied the only advertised variation of the saved model.",
|
||||
"modelSelectionReasonUnknown": "{{reason}}",
|
||||
"modelSelectionReasonUnsupported": "The executor does not support model selection.",
|
||||
"modelSelectionRemediationAgentVersion": "Check the agent version in the executor.",
|
||||
|
||||
@@ -504,6 +504,8 @@
|
||||
"profileForTasksCreatedByAgents": "Ƥŕōƒĩĺē ƒōŕ Ţàśķś Ćŕēàţēď ƀŷ Àĝēńţś",
|
||||
"programming": "Ƥŕōĝŕàḿḿĩńĝ",
|
||||
"profileStartModelNotAdvertisedOnHost": "Ţĥē ĥōśţ ƥŕōƀē ďĩď ńōţ àďvēŕţĩśē {{model}}. Ţĥē śēĺēćţēď ēxēćũţōŕ ŵĩĺĺ ďēćĩďē ţĥē ḿōďēĺ àţ ĺàũńćĥ.",
|
||||
"profileStartModelUniqueVariationOnHost": "Ţĥē ĥōśţ ƥŕōƀē ƒōũńď ōńē ƥōśśĩƀĺē vàŕĩàţĩōń ōƒ {{model}}: {{variation}}. Ţĥē śēĺēćţēď ēxēćũţōŕ ŵĩĺĺ ďēćĩďē ţĥē ḿōďēĺ àţ ĺàũńćĥ.",
|
||||
"modelVariationAdvisory": "Ţĥē ĥōśţ ćàţàĺōĝ ĥàś ōńē ƥōśśĩƀĺē vàŕĩàţĩōń ōƒ {{model}}: {{variation}}. Ţĥē śàvēď ḿōďēĺ ŕēḿàĩńś ũńćĥàńĝēď ũńţĩĺ ĺàũńćĥ.",
|
||||
"promptAdd": "Àďď ƥŕōḿƥţ",
|
||||
"promptEditorAriaLabel": "Ƥŕōḿƥţ ēďĩţōŕ",
|
||||
"promptEditorBothHint": "Ţŷƥē {{placeholder}} ƒōŕ àvàĩĺàƀĺē ƥĺàćēĥōĺďēŕś ōŕ {{mention}} ƒōŕ śàvēď ƥŕōḿƥţś.",
|
||||
|
||||
@@ -883,6 +883,7 @@
|
||||
"modelSelectionReasonCatalogEmpty": "Ţĥē ēxēćũţōŕ ďĩď ńōţ àďvēŕţĩśē à ḿōďēĺ ćàţàĺōĝ.",
|
||||
"modelSelectionReasonFallbackNotAdvertised": "Ţĥē śàvēď ƒàĺĺƀàćķ ḿōďēĺ ŵàś ńōţ àďvēŕţĩśēď ƀŷ ţĥē ēxēćũţōŕ.",
|
||||
"modelSelectionReasonRequestedNotAdvertised": "Ţĥē śàvēď ḿōďēĺ ŵàś ńōţ àďvēŕţĩśēď ƀŷ ţĥē ēxēćũţōŕ.",
|
||||
"modelSelectionReasonUniqueVariation": "Ţĥē ēxēćũţōŕ àƥƥĺĩēď ţĥē ōńĺŷ àďvēŕţĩśēď vàŕĩàţĩōń ōƒ ţĥē śàvēď ḿōďēĺ.",
|
||||
"modelSelectionReasonUnknown": "{{reason}}",
|
||||
"modelSelectionReasonUnsupported": "Ţĥē ēxēćũţōŕ ďōēś ńōţ śũƥƥōŕţ ḿōďēĺ śēĺēćţĩōń.",
|
||||
"modelSelectionRemediationAgentVersion": "Ćĥēćķ ţĥē àĝēńţ vēŕśĩōń ĩń ţĥē ēxēćũţōŕ.",
|
||||
|
||||
@@ -751,6 +751,8 @@
|
||||
"terminalAndEditors": "Terminal e editores",
|
||||
"workspacesAndAccess": "Workspaces e acesso",
|
||||
"profileStartModelNotAdvertisedOnHost": "A sondagem do anfitrião não anunciou {{model}}. O executor selecionado decidirá o modelo no arranque.",
|
||||
"profileStartModelUniqueVariationOnHost": "A sondagem do anfitrião encontrou uma única variação possível de {{model}}: {{variation}}. O executor selecionado decidirá o modelo no arranque.",
|
||||
"modelVariationAdvisory": "O catálogo do anfitrião tem uma única variação possível de {{model}}: {{variation}}. O modelo guardado permanece inalterado até ao arranque.",
|
||||
"navUnits": "Unidades da organização",
|
||||
"unitsTitle": "Unidades da organização",
|
||||
"unitsDescription": "Uma unidade contém equipas, áreas de trabalho e pessoas. Pertencer a uma unidade dá acesso a todas as áreas de trabalho abaixo dela, incluindo as criadas depois.",
|
||||
|
||||
@@ -1958,6 +1958,7 @@
|
||||
"modelSelectionReasonCatalogEmpty": "O executor não anunciou um catálogo de modelos.",
|
||||
"modelSelectionReasonFallbackNotAdvertised": "O modelo alternativo guardado não foi anunciado pelo executor.",
|
||||
"modelSelectionReasonRequestedNotAdvertised": "O modelo guardado não foi anunciado pelo executor.",
|
||||
"modelSelectionReasonUniqueVariation": "O executor aplicou a única variação anunciada do modelo guardado.",
|
||||
"modelSelectionReasonUnknown": "{{reason}}",
|
||||
"modelSelectionReasonUnsupported": "O executor não suporta seleção de modelo.",
|
||||
"modelSelectionRemediationAgentVersion": "Verifique a versão do agente no executor.",
|
||||
|
||||
@@ -751,6 +751,8 @@
|
||||
"keyValueKeyPlaceholder": "键",
|
||||
"keyValueValuePlaceholder": "值",
|
||||
"profileStartModelNotAdvertisedOnHost": "主机探测未公布 {{model}}。所选执行器将在启动时决定使用的模型。",
|
||||
"profileStartModelUniqueVariationOnHost": "主机探测发现 {{model}} 的一个可能变体:{{variation}}。所选执行器将在启动时决定使用的模型。",
|
||||
"modelVariationAdvisory": "主机目录中有一个可能的 {{model}} 变体:{{variation}}。保存的模型在启动前保持不变。",
|
||||
"navUnits": "组织单元",
|
||||
"unitsTitle": "组织单元",
|
||||
"unitsDescription": "单元包含团队、工作区和成员。加入某个单元即可访问其下的所有工作区,包括之后创建的。",
|
||||
|
||||
@@ -1958,6 +1958,7 @@
|
||||
"modelSelectionReasonCatalogEmpty": "执行器未公布模型目录。",
|
||||
"modelSelectionReasonFallbackNotAdvertised": "执行器未公布已保存的备选模型。",
|
||||
"modelSelectionReasonRequestedNotAdvertised": "执行器未公布已保存的模型。",
|
||||
"modelSelectionReasonUniqueVariation": "执行器应用了已保存模型唯一公布的变体。",
|
||||
"modelSelectionReasonUnknown": "{{reason}}",
|
||||
"modelSelectionReasonUnsupported": "该执行器不支持选择模型。",
|
||||
"modelSelectionRemediationAgentVersion": "请检查执行器中的智能体版本。",
|
||||
|
||||
@@ -751,6 +751,8 @@
|
||||
"keyValueKeyPlaceholder": "鍵",
|
||||
"keyValueValuePlaceholder": "值",
|
||||
"profileStartModelNotAdvertisedOnHost": "主機探測未公佈 {{model}}。所選執行器將在啓動時決定使用的模型。",
|
||||
"profileStartModelUniqueVariationOnHost": "主機探測發現 {{model}} 的一個可能變體:{{variation}}。所選執行器將在啓動時決定使用的模型。",
|
||||
"modelVariationAdvisory": "主機目錄中有一個可能的 {{model}} 變體:{{variation}}。已儲存的模型在啟動前保持不變。",
|
||||
"navUnits": "組織單元",
|
||||
"unitsTitle": "組織單元",
|
||||
"unitsDescription": "單元包含團隊、工作區和成員。加入某個單元即可存取其下的所有工作區,包括之後建立的。",
|
||||
|
||||
@@ -1958,6 +1958,7 @@
|
||||
"modelSelectionReasonCatalogEmpty": "執行器未公佈模型目錄。",
|
||||
"modelSelectionReasonFallbackNotAdvertised": "執行器未公佈已儲存的備選模型。",
|
||||
"modelSelectionReasonRequestedNotAdvertised": "執行器未公佈已儲存的模型。",
|
||||
"modelSelectionReasonUniqueVariation": "執行器套用了已儲存模型唯一公佈的變體。",
|
||||
"modelSelectionReasonUnknown": "{{reason}}",
|
||||
"modelSelectionReasonUnsupported": "該執行器不支援選擇模型。",
|
||||
"modelSelectionRemediationAgentVersion": "請檢查執行器中的代理程式版本。",
|
||||
|
||||
@@ -751,6 +751,8 @@
|
||||
"keyValueKeyPlaceholder": "鍵",
|
||||
"keyValueValuePlaceholder": "值",
|
||||
"profileStartModelNotAdvertisedOnHost": "主機探測未公佈 {{model}}。所選執行器將在啟動時決定使用的模型。",
|
||||
"profileStartModelUniqueVariationOnHost": "主機探測發現 {{model}} 的一個可能變體:{{variation}}。所選執行器將在啟動時決定使用的模型。",
|
||||
"modelVariationAdvisory": "主機目錄中有一個可能的 {{model}} 變體:{{variation}}。已儲存的模型在啟動前保持不變。",
|
||||
"navUnits": "組織單元",
|
||||
"unitsTitle": "組織單元",
|
||||
"unitsDescription": "單元包含團隊、工作區和成員。加入某個單元即可存取其下的所有工作區,包括之後建立的。",
|
||||
|
||||
@@ -1958,6 +1958,7 @@
|
||||
"modelSelectionReasonCatalogEmpty": "執行器未公佈模型目錄。",
|
||||
"modelSelectionReasonFallbackNotAdvertised": "執行器未公佈已儲存的備選模型。",
|
||||
"modelSelectionReasonRequestedNotAdvertised": "執行器未公佈已儲存的模型。",
|
||||
"modelSelectionReasonUniqueVariation": "執行器套用了已儲存模型唯一公佈的變體。",
|
||||
"modelSelectionReasonUnknown": "{{reason}}",
|
||||
"modelSelectionReasonUnsupported": "該執行器不支援選擇模型。",
|
||||
"modelSelectionRemediationAgentVersion": "請檢查執行器中的代理程式版本。",
|
||||
|
||||
@@ -0,0 +1,84 @@
|
||||
# ADR-2026-09-07: Resolve Only One Advertised Model Variation
|
||||
|
||||
**Status:** proposed
|
||||
**Date:** 2026-09-07
|
||||
**Area:** backend, frontend, protocol
|
||||
**Amends:** `2026-08-15-executor-authoritative-model-selection`
|
||||
|
||||
## Context
|
||||
|
||||
An agent CLI can replace a bare model ID with bracketed variations. A profile
|
||||
can keep `opus` while the executor later advertises only `opus[1m]`.
|
||||
|
||||
The current executor-authoritative policy treats the saved ID as absent. It
|
||||
uses an explicit fallback when available. Otherwise, it continues with the
|
||||
provider current or default model and records a warning. This result can move a
|
||||
session to an unrelated model even when the catalog has one clear variation of
|
||||
the requested model.
|
||||
|
||||
Variation text is provider-owned. Kandev cannot safely rank `270k`, `1m`,
|
||||
`fast`, or future labels. Catalog order and the current-model marker do not
|
||||
express user intent.
|
||||
|
||||
## Decision
|
||||
|
||||
Kandev recognizes a model variation only by a narrow bracketed ID shape. A bare
|
||||
requested ID `x` matches an advertised ID `x[v]` when `v` is non-empty and
|
||||
contains no bracket. Matching is exact and case-sensitive. Kandev treats `v` as
|
||||
opaque text.
|
||||
|
||||
For profiles with `auto_fallback = false`, the executor-authoritative resolution
|
||||
order is:
|
||||
|
||||
1. Exact requested model.
|
||||
2. Advertised explicit fallback from the profile.
|
||||
3. One distinct advertised variation of the requested bare model.
|
||||
4. Provider current or default model, with no speculative model-selection call.
|
||||
|
||||
Kandev applies step 3 only when exactly one distinct candidate exists. Zero or
|
||||
multiple candidates produce no inferred selection. An unadvertised explicit
|
||||
fallback does not block step 3.
|
||||
|
||||
A requested ID that contains a bracket is an exact ID only. Kandev does not
|
||||
infer another variation from it.
|
||||
|
||||
Profiles with `auto_fallback = true` retain the legacy path when the requested
|
||||
model is absent. Kandev ignores the configured explicit fallback, does not
|
||||
infer a variation, and continues with the provider current or default model.
|
||||
Apply errors for an advertised model remain best-effort in that mode.
|
||||
|
||||
The lifecycle stores `unique_variation` as the decision outcome and
|
||||
`unique_variation_applied` as the warning reason. The effective model contains
|
||||
the advertised variation. The explicit fallback field keeps its existing
|
||||
meaning and stays empty when Kandev infers a variation.
|
||||
|
||||
The saved profile model does not change. The rule runs for initial launch,
|
||||
context reset, and workspace rebind. The executor catalog remains authoritative.
|
||||
Any host-catalog result remains an advisory.
|
||||
|
||||
## Consequences
|
||||
|
||||
- A profile with `opus` follows `opus[1m]` when it is the only advertised
|
||||
variation.
|
||||
- A catalog with `opus[270k]` and `opus[1m, fast]` remains ambiguous and uses
|
||||
the existing provider-default path.
|
||||
- An explicit advertised fallback remains stronger than inference.
|
||||
- Kandev never sends a model ID that the executor did not advertise.
|
||||
- Users receive a durable warning because the effective model differs from the
|
||||
saved request.
|
||||
- Provider-specific ranking logic does not enter the lifecycle policy.
|
||||
- Backend and frontend helpers need mirrored contract tests. The backend result
|
||||
remains authoritative when their input catalogs differ.
|
||||
|
||||
## Alternatives Considered
|
||||
|
||||
1. **Always choose the first variation.** Rejected because catalog order is not
|
||||
a user preference or a stable provider contract.
|
||||
2. **Prefer the provider current model.** Rejected because current state can
|
||||
reflect a default or an earlier session, not the saved profile intent.
|
||||
3. **Rank known labels such as context size or fast mode.** Rejected because
|
||||
labels are provider-owned and can change without a Kandev release.
|
||||
4. **Rewrite the saved profile to the inferred model.** Rejected because one
|
||||
executor observation must not change behavior for every executor.
|
||||
5. **Infer from an already bracketed request.** Rejected because it can replace
|
||||
one explicit variation with another without user intent.
|
||||
@@ -95,6 +95,7 @@ Read individual ADRs for full context. Create new ones via `/record decision` or
|
||||
| 2026-08-15-subagent-context-schema-boundary | [Start with the execution-aware subagent schema](2026-08-15-subagent-context-schema-boundary.md) | accepted | backend | 2026-08-15 |
|
||||
| 2026-08-15-portable-agent-configuration-bundles | [Use Explicit Portable Agent Configuration Bundles](2026-08-15-portable-agent-configuration-bundles.md) | accepted | backend, frontend, protocol, security | 2026-08-15 |
|
||||
| 2026-08-15-executor-authoritative-model-selection | [Let the Executor Own Model Selection](2026-08-15-executor-authoritative-model-selection.md) | accepted | backend, frontend, protocol, persistence | 2026-08-15 |
|
||||
| 2026-09-07-unique-model-variation-resolution | [Resolve Only One Advertised Model Variation](2026-09-07-unique-model-variation-resolution.md) | proposed | backend, frontend, protocol | 2026-09-07 |
|
||||
| 2026-08-18-context-reset-preserves-runtime-configuration | [Preserve ACP Runtime Configuration Across Context Reset](2026-08-18-context-reset-preserves-runtime-configuration.md) | accepted | backend, protocol, workflow | 2026-08-18 |
|
||||
| 2026-08-19-parent-authorized-child-workspace-sources | [Parent-authorized child workspace sources](2026-08-19-parent-authorized-child-workspace-sources.md) | accepted | backend, protocol, security | 2026-08-19 |
|
||||
| 2026-08-24-task-snapshotted-branch-policies | [Snapshot Repository Branch Policies on Tasks](2026-08-24-task-snapshotted-branch-policies.md) | proposed | backend, frontend, persistence | 2026-08-24 |
|
||||
|
||||
@@ -0,0 +1,129 @@
|
||||
---
|
||||
created: 2026-09-07
|
||||
status: complete
|
||||
requirements:
|
||||
- REQ-AGENTS-NO-SILENT-MODEL-FALLBACK-001
|
||||
- REQ-AGENTS-NO-SILENT-MODEL-FALLBACK-002
|
||||
system_design:
|
||||
- ../../specs/agents/system-design/no-silent-model-fallback-01.md
|
||||
- ../../specs/agents/system-design/no-silent-model-fallback-02.md
|
||||
legacy_specs: []
|
||||
---
|
||||
|
||||
# Implementation Plan: Unique Model Variation Fallback
|
||||
|
||||
## Overview
|
||||
|
||||
Extend the central executor-authoritative model policy with one deterministic
|
||||
inference step. Implement the runtime contract first, expose its stable decision
|
||||
in the existing frontend surfaces, then prove and document the user outcome.
|
||||
|
||||
## Scope
|
||||
|
||||
### In scope
|
||||
|
||||
- Resolve a missing bare model to its only distinct advertised bracketed
|
||||
variation.
|
||||
- Keep exact models and advertised explicit fallbacks at higher precedence.
|
||||
- Persist a distinct warning outcome and reason without rewriting the profile.
|
||||
- Present host results as advisory on desktop and touch interfaces.
|
||||
- Test and document unique and ambiguous catalogs.
|
||||
|
||||
### Out of scope
|
||||
|
||||
- Provider-specific ranking of variation labels.
|
||||
- Selection among multiple variations.
|
||||
- Profile schema changes or saved-model migration.
|
||||
- Office post-start route changes or mid-turn model switching.
|
||||
|
||||
## Technical approach
|
||||
|
||||
### Runtime policy
|
||||
|
||||
Add a pure unique-candidate helper in
|
||||
`apps/backend/internal/agent/runtime/lifecycle/start_model.go`. It accepts the
|
||||
requested ID and executor model IDs. It returns one distinct advertised ID only
|
||||
when the structural contract matches.
|
||||
|
||||
For profiles with automatic fallback disabled, extend `applyStartModelPolicy`
|
||||
with this order: exact request, advertised explicit fallback, unique variation,
|
||||
provider current or default. Automatic-fallback profiles retain their legacy
|
||||
no-selection behavior when the requested model is absent. Add
|
||||
`unique_variation` to `ModelSelectionOutcome` and
|
||||
`unique_variation_applied` to stable warning reasons.
|
||||
|
||||
Keep `EffectiveModel` as the inferred ID. Do not set `FallbackModel` for this
|
||||
outcome. `publishModelSelectionWarningEvent` must therefore emit the new reason
|
||||
without the legacy `session_model_fallback` compatibility event.
|
||||
|
||||
Initial launch, context reset, and workspace rebind already use this policy.
|
||||
Targeted tests must prove that no separate path bypasses the new order.
|
||||
|
||||
### Frontend advisory
|
||||
|
||||
Add the mirrored pure matcher under `apps/web/lib/`. Its table tests use the
|
||||
same contract cases as the Go helper. Backend lifecycle data remains the launch
|
||||
authority.
|
||||
|
||||
Use the helper in `task-create-dialog-options.tsx` and
|
||||
`settings/profile-model-fields.tsx`. One host variation names the possible
|
||||
target while keeping the saved value and selectable profile. Zero or multiple
|
||||
variations keep the current missing-model presentation.
|
||||
|
||||
Map `unique_variation_applied` in `status-message.tsx`. Add localized copy to
|
||||
all required `settings` and `task` catalogs. Reuse the existing desktop tooltip,
|
||||
touch drawer, responsive model selector, and task-chat status layout.
|
||||
|
||||
### Documentation
|
||||
|
||||
Update `docs/public/agents-and-profiles.md` and `docs/public/executors.md` with
|
||||
the four-step order, one unique example, one ambiguous example, and the
|
||||
unchanged saved-profile rule.
|
||||
|
||||
## Tests
|
||||
|
||||
- AC-002.1 and AC-002.2: Go policy tests prove exact and explicit-fallback
|
||||
precedence before one inferred variation, including the legacy
|
||||
auto-fallback path.
|
||||
- AC-002.3 through AC-002.5: mirrored Go and TypeScript table tests prove zero,
|
||||
duplicate, multiple, case-different, bracketed, prefix, and malformed cases.
|
||||
- AC-002.6: lifecycle event and React status-message tests prove the distinct
|
||||
outcome, effective model, absent explicit-fallback field, and durable copy.
|
||||
- AC-002.7: task picker and profile editor tests prove advisory host behavior.
|
||||
- AC-002.8: lifecycle integration tests prove launch, reset, and rebind parity.
|
||||
|
||||
## E2E tests
|
||||
|
||||
- AC-002.2 and AC-002.6: extend
|
||||
`apps/web/e2e/tests/session/model-mismatch-warning.spec.ts` to launch `opus`
|
||||
against one `opus[1m]`, show the effective model, and retain one warning after
|
||||
reload.
|
||||
- AC-002.3: add an ambiguous catalog with `opus[270k]` and
|
||||
`opus[1m, fast]`; prove that Kandev does not select either variation.
|
||||
- AC-002.7: extend the desktop and mobile settings mismatch specs. Prove that
|
||||
the profile remains selectable, the advisory is keyboard or tap accessible,
|
||||
and mobile has no horizontal document overflow.
|
||||
|
||||
## Work orders
|
||||
|
||||
- [x] [Task 01: Resolve one executor variation](task-01-runtime-resolution.md)
|
||||
- [x] [Task 02: Present variation decisions](task-02-frontend-advisory.md)
|
||||
- [x] [Task 03: Prove variation workflows](task-03-e2e-verification.md)
|
||||
- [x] [Task 04: Document variation fallback](task-04-public-docs.md)
|
||||
|
||||
## Verification results
|
||||
|
||||
Backend verification passed: 2,364 lifecycle tests, 29 focused model-policy
|
||||
tests, and 212 mock-agent tests. Frontend verification passed: 51 focused
|
||||
unit tests, typecheck, lint, `i18n:check`, and `i18n:ratchet`. Desktop E2E
|
||||
passed 5 tests and mobile E2E passed 3 tests. Public-document validation
|
||||
passed 61 tests and validated 46 published pages. Specification lint passed.
|
||||
|
||||
## Risks
|
||||
|
||||
- Provider variation labels can change. Kandev must keep their content opaque.
|
||||
- Host and executor catalogs can differ. Host preview logic cannot become a
|
||||
launch gate.
|
||||
- Older frontends can render the new reason as unknown during mixed-version
|
||||
use. Requested and effective fields must remain useful.
|
||||
- Duplicate or malformed catalog rows can make a naive candidate count unstable.
|
||||
@@ -0,0 +1,100 @@
|
||||
---
|
||||
id: "01-runtime-resolution"
|
||||
title: "Resolve one executor variation"
|
||||
status: done
|
||||
wave: 1
|
||||
depends_on: []
|
||||
plan: "plan.md"
|
||||
requirements:
|
||||
- REQ-AGENTS-NO-SILENT-MODEL-FALLBACK-001
|
||||
- REQ-AGENTS-NO-SILENT-MODEL-FALLBACK-002
|
||||
acceptance_criteria:
|
||||
- AC-AGENTS-NO-SILENT-MODEL-FALLBACK-002.1
|
||||
- AC-AGENTS-NO-SILENT-MODEL-FALLBACK-002.2
|
||||
- AC-AGENTS-NO-SILENT-MODEL-FALLBACK-002.3
|
||||
- AC-AGENTS-NO-SILENT-MODEL-FALLBACK-002.4
|
||||
- AC-AGENTS-NO-SILENT-MODEL-FALLBACK-002.5
|
||||
- AC-AGENTS-NO-SILENT-MODEL-FALLBACK-002.6
|
||||
- AC-AGENTS-NO-SILENT-MODEL-FALLBACK-002.8
|
||||
system_design:
|
||||
- ../../specs/agents/system-design/no-silent-model-fallback-01.md
|
||||
- ../../specs/agents/system-design/no-silent-model-fallback-02.md
|
||||
---
|
||||
|
||||
# Task 01: Resolve One Executor Variation
|
||||
|
||||
## Summary
|
||||
|
||||
Extend the single lifecycle model policy with safe unique-variation inference.
|
||||
Expose a distinct decision without changing explicit fallback semantics.
|
||||
|
||||
## In scope
|
||||
|
||||
- Add the pure backend candidate matcher.
|
||||
- Apply the defined four-step precedence order.
|
||||
- Extend decision, logging, and warning metadata behavior.
|
||||
- Cover initial launch, context reset, and workspace rebind.
|
||||
|
||||
## Out of scope
|
||||
|
||||
- Host advisory UI and translations.
|
||||
- Provider-specific variation parsing or ranking.
|
||||
- Profile persistence changes.
|
||||
|
||||
## Acceptance
|
||||
|
||||
- One valid distinct variation is the only inferred model-selection call.
|
||||
- Exact and advertised explicit fallback models remain higher priority; zero or
|
||||
multiple variations cause no inference.
|
||||
- Automatic-fallback profiles keep the legacy no-selection behavior when the
|
||||
requested model is absent.
|
||||
- The warning records the unique outcome and effective model without setting
|
||||
the explicit fallback field or compatibility fallback event.
|
||||
|
||||
## Verification
|
||||
|
||||
```bash
|
||||
(
|
||||
cd apps/backend
|
||||
go test -tags fts5 -run 'TestApplyStartModelPolicy|TestInitializeAndPrompt.*Model|TestReapplySessionModel|TestWorkspaceRebind.*Model|Test.*ModelSelectionWarning' ./internal/agent/runtime/lifecycle
|
||||
go test -tags fts5 ./internal/agent/runtime/lifecycle
|
||||
)
|
||||
git diff --check
|
||||
```
|
||||
|
||||
## Files likely touched
|
||||
|
||||
- `apps/backend/internal/agent/runtime/lifecycle/start_model.go`
|
||||
- `apps/backend/internal/agent/runtime/lifecycle/start_model_executor_authority_test.go`
|
||||
- `apps/backend/internal/agent/runtime/lifecycle/session.go`
|
||||
- `apps/backend/internal/agent/runtime/lifecycle/session_model_selection_warning_test.go`
|
||||
- `apps/backend/internal/agentctl/types/streams/agent.go`
|
||||
|
||||
## Dependencies
|
||||
|
||||
None.
|
||||
|
||||
## Risks
|
||||
|
||||
- The Cursor-specific configuration parser rejects labels such as `1m`. This
|
||||
matcher must remain a separate model-ID helper.
|
||||
- A special inferred-model apply path can weaken existing error handling.
|
||||
|
||||
## Parallelism
|
||||
|
||||
`sequential`
|
||||
|
||||
## Inputs
|
||||
|
||||
- Requirement 002 and its matching examples.
|
||||
- The unique-variation ADR.
|
||||
- Existing `applyStartModelPolicy` and warning publication tests.
|
||||
|
||||
## Results
|
||||
|
||||
Implemented the executor-authoritative unique-variation policy in
|
||||
`start_model.go`, including explicit-fallback precedence, opaque and
|
||||
case-sensitive matching, the legacy automatic-fallback path, and distinct
|
||||
warning metadata without a legacy fallback event. Initial launch, context
|
||||
reset, and workspace rebind all use the same policy. Focused model-policy
|
||||
tests passed (29), and the full lifecycle suite passed (2,364).
|
||||
@@ -0,0 +1,102 @@
|
||||
---
|
||||
id: "02-frontend-advisory"
|
||||
title: "Present variation decisions"
|
||||
status: done
|
||||
wave: 2
|
||||
depends_on: ["01-runtime-resolution"]
|
||||
plan: "plan.md"
|
||||
requirements:
|
||||
- REQ-AGENTS-NO-SILENT-MODEL-FALLBACK-002
|
||||
acceptance_criteria:
|
||||
- AC-AGENTS-NO-SILENT-MODEL-FALLBACK-002.3
|
||||
- AC-AGENTS-NO-SILENT-MODEL-FALLBACK-002.4
|
||||
- AC-AGENTS-NO-SILENT-MODEL-FALLBACK-002.5
|
||||
- AC-AGENTS-NO-SILENT-MODEL-FALLBACK-002.6
|
||||
- AC-AGENTS-NO-SILENT-MODEL-FALLBACK-002.7
|
||||
system_design:
|
||||
- ../../specs/agents/system-design/no-silent-model-fallback-01.md
|
||||
---
|
||||
|
||||
# Task 02: Present Variation Decisions
|
||||
|
||||
## Summary
|
||||
|
||||
Present one host variation as an advisory and render the runtime decision in
|
||||
task chat. Keep the host probe non-authoritative on every viewport.
|
||||
|
||||
## In scope
|
||||
|
||||
- Add the mirrored frontend matcher and table tests.
|
||||
- Update task-create and profile-editor missing-model states.
|
||||
- Render the new warning reason with requested and effective models.
|
||||
- Add all required localized copy.
|
||||
|
||||
## Out of scope
|
||||
|
||||
- Runtime model selection.
|
||||
- New dialog, drawer, or navigation patterns.
|
||||
- Changes to the saved profile model.
|
||||
|
||||
## Acceptance
|
||||
|
||||
- One host variation names a possible target and keeps the profile selectable.
|
||||
- Zero or multiple host variations keep the existing missing-model state.
|
||||
- Desktop, keyboard, and touch users can reach the advisory and durable warning.
|
||||
|
||||
## Verification
|
||||
|
||||
```bash
|
||||
(
|
||||
cd apps
|
||||
pnpm --filter @kandev/web test -- --run components/task-create-dialog-options components/settings/profile-form-fields components/task/chat/messages/status-message lib/model-variation
|
||||
pnpm --filter @kandev/web lint
|
||||
)
|
||||
(
|
||||
cd apps/web
|
||||
pnpm run typecheck
|
||||
pnpm run i18n:check
|
||||
pnpm run i18n:ratchet
|
||||
)
|
||||
git diff --check
|
||||
```
|
||||
|
||||
## Files likely touched
|
||||
|
||||
- `apps/web/lib/model-variation.ts`
|
||||
- `apps/web/lib/model-variation.test.ts`
|
||||
- `apps/web/components/task-create-dialog-options.tsx`
|
||||
- `apps/web/components/task-create-dialog-options.test.tsx`
|
||||
- `apps/web/components/settings/profile-model-fields.tsx`
|
||||
- `apps/web/components/settings/profile-form-fields.test.tsx`
|
||||
- `apps/web/components/task/chat/messages/status-message.tsx`
|
||||
- `apps/web/components/task/chat/messages/status-message.test.tsx`
|
||||
- `apps/web/src/locales/*/settings.json`
|
||||
- `apps/web/src/locales/*/task.json`
|
||||
|
||||
## Dependencies
|
||||
|
||||
Task 01.
|
||||
|
||||
## Risks
|
||||
|
||||
- Host preview text can sound authoritative when another executor can advertise
|
||||
a different catalog.
|
||||
- Missing and resolvable states must be mutually exclusive in the profile row.
|
||||
|
||||
## Parallelism
|
||||
|
||||
`sequential`
|
||||
|
||||
## Inputs
|
||||
|
||||
- Task 01 warning reason and payload fields.
|
||||
- Existing host warning tooltip, touch drawer, profile selector, and status
|
||||
message patterns.
|
||||
|
||||
## Results
|
||||
|
||||
Added the mirrored structural matcher, host advisory states, and the task-chat
|
||||
warning reason while preserving the saved profile model. Added localized copy
|
||||
for all required catalogs and retained the existing desktop tooltip and mobile
|
||||
touch-drawer patterns. Focused frontend tests passed (51), typecheck and lint
|
||||
passed, and both i18n checks passed.
|
||||
@@ -0,0 +1,84 @@
|
||||
---
|
||||
id: "03-e2e-verification"
|
||||
title: "Prove variation workflows"
|
||||
status: done
|
||||
wave: 3
|
||||
depends_on: ["01-runtime-resolution", "02-frontend-advisory"]
|
||||
plan: "plan.md"
|
||||
requirements:
|
||||
- REQ-AGENTS-NO-SILENT-MODEL-FALLBACK-002
|
||||
acceptance_criteria:
|
||||
- AC-AGENTS-NO-SILENT-MODEL-FALLBACK-002.2
|
||||
- AC-AGENTS-NO-SILENT-MODEL-FALLBACK-002.3
|
||||
- AC-AGENTS-NO-SILENT-MODEL-FALLBACK-002.6
|
||||
- AC-AGENTS-NO-SILENT-MODEL-FALLBACK-002.7
|
||||
system_design:
|
||||
- ../../specs/agents/system-design/no-silent-model-fallback-01.md
|
||||
---
|
||||
|
||||
# Task 03: Prove Variation Workflows
|
||||
|
||||
## Summary
|
||||
|
||||
Extend the existing model-mismatch browser flows with unique and ambiguous
|
||||
catalogs. Prove the launch result, durable warning, and touch interaction.
|
||||
|
||||
## In scope
|
||||
|
||||
- Desktop launch and reload coverage for one variation.
|
||||
- Desktop no-inference coverage for multiple variations.
|
||||
- Mobile advisory and warning access through tap.
|
||||
- Horizontal overflow checks for the changed mobile states.
|
||||
|
||||
## Out of scope
|
||||
|
||||
- New browser fixtures unrelated to model catalogs.
|
||||
- Full E2E suite execution.
|
||||
- Public documentation.
|
||||
|
||||
## Acceptance
|
||||
|
||||
- `opus` launches as `opus[1m]` only when that is the sole distinct variation.
|
||||
- Two variations do not select either candidate.
|
||||
- The warning survives reload and the mobile flow needs no hover interaction.
|
||||
|
||||
## Verification
|
||||
|
||||
```bash
|
||||
cd apps/web && pnpm e2e:run tests/settings/no-silent-model-fallback.spec.ts tests/session/model-mismatch-warning.spec.ts
|
||||
cd apps/web && pnpm e2e:run --project mobile-chrome --no-build tests/settings/mobile-no-silent-model-fallback.spec.ts tests/session/mobile-model-mismatch-warning.spec.ts
|
||||
git diff --check
|
||||
```
|
||||
|
||||
## Files likely touched
|
||||
|
||||
- `apps/web/e2e/tests/settings/no-silent-model-fallback.spec.ts`
|
||||
- `apps/web/e2e/tests/settings/mobile-no-silent-model-fallback.spec.ts`
|
||||
- `apps/web/e2e/tests/session/model-mismatch-warning.spec.ts`
|
||||
- `apps/web/e2e/tests/session/mobile-model-mismatch-warning.spec.ts`
|
||||
- `apps/web/e2e/tests/session/model-mismatch-warning-helpers.ts`
|
||||
|
||||
## Dependencies
|
||||
|
||||
Tasks 01 and 02.
|
||||
|
||||
## Risks
|
||||
|
||||
- Mock catalogs must prove the executor decision, not only the host advisory.
|
||||
- Each scenario needs isolated profile data so another catalog does not change
|
||||
the distinct candidate count.
|
||||
|
||||
## Parallelism
|
||||
|
||||
`parallel-safe` with Task 04 after Tasks 01 and 02.
|
||||
|
||||
## Inputs
|
||||
|
||||
- Completed runtime and frontend work.
|
||||
- Existing desktop and mobile model-mismatch fixtures.
|
||||
|
||||
## Results
|
||||
|
||||
Added unique and ambiguous mock-agent catalogs and covered launch, reload,
|
||||
profile persistence, host advisory access, and mobile overflow. The focused
|
||||
desktop E2E run passed 5 tests; the focused mobile E2E run passed 3 tests.
|
||||
@@ -0,0 +1,83 @@
|
||||
---
|
||||
id: "04-public-docs"
|
||||
title: "Document variation fallback"
|
||||
status: done
|
||||
wave: 3
|
||||
depends_on: ["01-runtime-resolution", "02-frontend-advisory"]
|
||||
plan: "plan.md"
|
||||
requirements:
|
||||
- REQ-AGENTS-NO-SILENT-MODEL-FALLBACK-002
|
||||
acceptance_criteria:
|
||||
- AC-AGENTS-NO-SILENT-MODEL-FALLBACK-002.1
|
||||
- AC-AGENTS-NO-SILENT-MODEL-FALLBACK-002.2
|
||||
- AC-AGENTS-NO-SILENT-MODEL-FALLBACK-002.3
|
||||
- AC-AGENTS-NO-SILENT-MODEL-FALLBACK-002.5
|
||||
- AC-AGENTS-NO-SILENT-MODEL-FALLBACK-002.6
|
||||
system_design:
|
||||
- ../../specs/agents/system-design/no-silent-model-fallback-01.md
|
||||
---
|
||||
|
||||
# Task 04: Document Variation Fallback
|
||||
|
||||
## Summary
|
||||
|
||||
Update the public profile and executor guides with the unique-variation rule.
|
||||
Give users enough detail to predict unique and ambiguous results.
|
||||
|
||||
## In scope
|
||||
|
||||
- Document the four-step launch order.
|
||||
- Add one unique and one ambiguous example.
|
||||
- State that labels remain opaque and saved profiles do not change.
|
||||
- Run the public-doc validation commands.
|
||||
|
||||
## Out of scope
|
||||
|
||||
- Release notes and changelog generation.
|
||||
- Internal implementation detail beyond user-visible behavior.
|
||||
- New screenshots or media.
|
||||
|
||||
## Acceptance
|
||||
|
||||
- Both guides describe the same precedence and ambiguity rule.
|
||||
- The docs state that the executor is authoritative and the profile stays
|
||||
unchanged.
|
||||
- Public-doc validators pass.
|
||||
|
||||
## Verification
|
||||
|
||||
```bash
|
||||
node --test scripts/validate-public-docs.test.mjs
|
||||
node scripts/validate-public-docs.mjs
|
||||
git diff --check
|
||||
```
|
||||
|
||||
## Files likely touched
|
||||
|
||||
- `docs/public/agents-and-profiles.md`
|
||||
- `docs/public/executors.md`
|
||||
|
||||
## Dependencies
|
||||
|
||||
Tasks 01 and 02.
|
||||
|
||||
## Risks
|
||||
|
||||
- Examples can imply that Kandev understands `1m` or `fast`. The docs must state
|
||||
that it compares only the ID shape and candidate count.
|
||||
|
||||
## Parallelism
|
||||
|
||||
`parallel-safe` with Task 03 after Tasks 01 and 02.
|
||||
|
||||
## Inputs
|
||||
|
||||
- Final runtime precedence and warning behavior.
|
||||
- Existing host-probe and remote-executor guide sections.
|
||||
|
||||
## Results
|
||||
|
||||
Updated the public agent-profile and executor guides with the four-step
|
||||
resolution order, unique and ambiguous examples, opaque variation matching,
|
||||
and the unchanged saved-profile rule. Public-document tests passed (61) and
|
||||
the validator checked 46 published pages.
|
||||
@@ -171,6 +171,31 @@ change makes a saved option value unsupported, Kandev removes that value after
|
||||
a successful resolution; a failed resolution keeps the draft unchanged so you
|
||||
can retry it.
|
||||
|
||||
### Model IDs and executor catalogs
|
||||
|
||||
The host model probe is an editing hint. The selected executor owns the model
|
||||
catalog at launch. Kandev resolves a saved model in this order:
|
||||
|
||||
1. Use the exact requested model when the executor advertises it.
|
||||
2. Use the advertised explicit fallback when the requested model is absent.
|
||||
3. Use one unique bracketed variation when the requested model is absent and
|
||||
the executor advertises exactly one matching ID.
|
||||
4. Use the agent's current or default model when no earlier choice applies.
|
||||
|
||||
Profiles with automatic fallback enabled keep the legacy behavior when the
|
||||
saved model is absent: Kandev ignores the explicit fallback and does not infer
|
||||
a variation. It uses the agent's current or default model instead.
|
||||
|
||||
For example, a saved `opus` model can launch as `opus[1m]` when that is the
|
||||
only advertised `opus[...]` ID. If the executor advertises both
|
||||
`opus[270k]` and `opus[1m, fast]`, Kandev does not choose either variation.
|
||||
Variation text is opaque and model IDs remain case-sensitive.
|
||||
|
||||
Kandev shows a warning when the launch result differs from the saved model.
|
||||
The saved profile remains `opus`; Kandev does not rewrite it after applying a
|
||||
unique variation or using the agent default. Recheck the executor catalog when
|
||||
the warning repeats after credentials, copied configuration, or agent updates.
|
||||
|
||||
### Use a dynamic profile
|
||||
|
||||
> [!EXPERIMENTAL]
|
||||
@@ -228,13 +253,13 @@ The model list shown while editing a profile comes from a host probe. It is an
|
||||
editing hint, not a launch gate. A profile remains selectable when its saved
|
||||
model is missing from that host list.
|
||||
|
||||
At task launch, the selected executor's ACP catalog is authoritative. Kandev
|
||||
sends the requested model only when the executor advertises it. If it does
|
||||
not, Kandev uses an advertised fallback when available, or sends no model
|
||||
request and continues with the agent's current or default model. Kandev stores
|
||||
one warning in task chat with the requested model and the effective model when
|
||||
known. The warning also identifies the agent and executor and asks you to
|
||||
check credentials, copied configuration, and the agent version.
|
||||
At task launch, the selected executor's ACP catalog is authoritative. For
|
||||
profiles without automatic fallback, Kandev follows the four-step order above.
|
||||
For profiles with automatic fallback enabled, an absent saved model causes no
|
||||
model request, and Kandev ignores the explicit fallback and any variation.
|
||||
Kandev stores one warning in task chat with the requested model and the
|
||||
effective model when known. The warning also identifies the agent and executor
|
||||
and asks you to check credentials, copied configuration, and the agent version.
|
||||
|
||||
The saved profile model is not changed. Optional portable configuration can
|
||||
copy selected allowlisted files into a remote executor, but it cannot guarantee
|
||||
|
||||
@@ -101,15 +101,30 @@ Review the selected bundles before saving the profile.
|
||||
### Model selection in remote executors
|
||||
|
||||
The host model probe helps edit a profile, but it is not the launch authority.
|
||||
At launch, the selected executor's advertised ACP catalog decides whether
|
||||
Kandev sends the saved model. If the executor does not advertise that model,
|
||||
Kandev sends no request for it. It uses an advertised fallback only when one
|
||||
exists; otherwise the agent uses its current or default model.
|
||||
At launch, the selected executor's advertised ACP catalog decides the model.
|
||||
For profiles without automatic fallback, Kandev applies this deterministic
|
||||
order:
|
||||
|
||||
Kandev writes one warning to task chat when this happens. The warning can list
|
||||
the requested model, effective model, agent, executor, and executor profile.
|
||||
It also tells you to check executor credentials, copied agent configuration,
|
||||
and the agent version. Kandev does not rewrite the saved profile model.
|
||||
1. An exact advertised model ID.
|
||||
2. An advertised explicit fallback.
|
||||
3. One unique advertised bracketed variation of the saved model.
|
||||
4. The agent's current or default model.
|
||||
|
||||
For profiles with automatic fallback enabled, an absent saved model keeps the
|
||||
legacy behavior: Kandev ignores the explicit fallback and does not infer a
|
||||
variation. It uses the agent's current or default model instead.
|
||||
|
||||
For example, a saved `opus` model uses `opus[1m]` when that is the only
|
||||
advertised variation. With both `opus[270k]` and `opus[1m, fast]`, Kandev
|
||||
does not infer a choice and uses the agent's current or default model instead.
|
||||
It treats model IDs as case-sensitive and variation text as opaque.
|
||||
|
||||
Kandev writes one warning to task chat when the effective model differs from
|
||||
the saved model. The warning can list the requested model, effective model,
|
||||
agent, executor, and executor profile. It also tells you to check executor
|
||||
credentials, copied agent configuration, and the agent version. Kandev does
|
||||
not rewrite the saved profile model, including after applying a unique
|
||||
variation.
|
||||
Portable configuration can improve parity, but it does not guarantee equal
|
||||
host and executor model catalogs.
|
||||
|
||||
|
||||
@@ -1,26 +1,101 @@
|
||||
---
|
||||
status: draft
|
||||
status: active
|
||||
system: agents
|
||||
created: 2026-08-23
|
||||
owners:
|
||||
- kandev
|
||||
---
|
||||
|
||||
# No Silent Model Fallback Requirements
|
||||
|
||||
## Overview
|
||||
|
||||
**Status**: implemented (executor-authority amendment 2026-08-15) **Date**: 2026-08-04 **Slug**: `no-silent-model-fallback`
|
||||
Kandev applies an agent profile model against the model catalog from the
|
||||
selected executor. A different effective model must be deterministic and
|
||||
visible to the user. The agent system owns this contract because it owns
|
||||
profiles, ACP session startup, and model-selection warnings.
|
||||
|
||||
## Terminology
|
||||
|
||||
- **Requested model:** The model ID stored in the agent profile.
|
||||
- **Explicit fallback:** The optional fallback model ID stored in the profile.
|
||||
- **Bare model ID:** A model ID that contains no `[` or `]` character.
|
||||
- **Model variation:** An advertised model ID with the exact form
|
||||
`<bare-model-id>[<non-empty-variant>]`. The variant contains no bracket
|
||||
character. Its content is otherwise opaque.
|
||||
- **Unique variation:** The only distinct advertised variation for a requested
|
||||
bare model ID.
|
||||
|
||||
## Requirements
|
||||
|
||||
### REQ-AGENTS-NO-SILENT-MODEL-FALLBACK-001: No Silent Model Fallback
|
||||
### REQ-AGENTS-NO-SILENT-MODEL-FALLBACK-001: Use executor-authoritative model selection
|
||||
|
||||
**Intent:** **Status**: implemented (executor-authority amendment 2026-08-15) **Date**: 2026-08-04 **Slug**: `no-silent-model-fallback`
|
||||
**Intent:** Keep task launch operational without hiding a difference between
|
||||
the saved model and the effective executor model.
|
||||
|
||||
**User story:** As a user, I want Kandev to use the selected executor's model
|
||||
catalog and report any fallback, so that I can understand which model runs.
|
||||
|
||||
#### Acceptance criteria
|
||||
|
||||
- **AC-AGENTS-NO-SILENT-MODEL-FALLBACK-001.1:** When a consumer uses **Status**: implemented (executor-authority amendment 2026-08-15), the system shall provide the documented behavior.
|
||||
- **AC-AGENTS-NO-SILENT-MODEL-FALLBACK-001.1:** When the executor advertises
|
||||
the requested model, the system shall apply that exact model.
|
||||
- **AC-AGENTS-NO-SILENT-MODEL-FALLBACK-001.2:** When the requested model is
|
||||
absent and the executor advertises the configured explicit fallback, the
|
||||
system shall apply the explicit fallback.
|
||||
- **AC-AGENTS-NO-SILENT-MODEL-FALLBACK-001.3:** When no permitted advertised
|
||||
model applies, the system shall make no speculative model-selection call and
|
||||
shall continue with the provider current or default model.
|
||||
- **AC-AGENTS-NO-SILENT-MODEL-FALLBACK-001.4:** When the effective model differs
|
||||
from the requested model, the system shall persist one structured warning
|
||||
that identifies the requested and effective models when known.
|
||||
- **AC-AGENTS-NO-SILENT-MODEL-FALLBACK-001.5:** The system shall not rewrite the
|
||||
saved profile model from an executor model-selection decision.
|
||||
|
||||
### REQ-AGENTS-NO-SILENT-MODEL-FALLBACK-002: Resolve one advertised model variation
|
||||
|
||||
**Intent:** Preserve a user's bare model choice when a CLI replaces that model
|
||||
ID with one unambiguous bracketed variation.
|
||||
|
||||
**User story:** As a user, I want `opus` to resolve to `opus[1m]` when that is
|
||||
the only advertised variation, so that a provider catalog change does not move
|
||||
my session to an unrelated default model.
|
||||
|
||||
#### Acceptance criteria
|
||||
|
||||
- **AC-AGENTS-NO-SILENT-MODEL-FALLBACK-002.1:** When automatic fallback is
|
||||
disabled, the exact requested model is absent, and an advertised explicit
|
||||
fallback exists, the system shall apply the explicit fallback before it
|
||||
considers an inferred variation. Automatic-fallback profiles shall retain
|
||||
their legacy no-selection behavior for an absent requested model.
|
||||
- **AC-AGENTS-NO-SILENT-MODEL-FALLBACK-002.2:** When no advertised explicit
|
||||
fallback exists and the catalog contains exactly one distinct variation of a
|
||||
requested bare model ID, the system shall apply that advertised variation.
|
||||
- **AC-AGENTS-NO-SILENT-MODEL-FALLBACK-002.3:** When the catalog contains zero
|
||||
or more than one distinct variation, the system shall not infer a model.
|
||||
- **AC-AGENTS-NO-SILENT-MODEL-FALLBACK-002.4:** When the requested ID contains a
|
||||
bracket, the system shall treat it as an exact ID and shall not infer another
|
||||
variation.
|
||||
- **AC-AGENTS-NO-SILENT-MODEL-FALLBACK-002.5:** Matching shall be case-sensitive
|
||||
and shall not parse, rank, or assign meaning to variation text.
|
||||
- **AC-AGENTS-NO-SILENT-MODEL-FALLBACK-002.6:** When Kandev applies a unique
|
||||
variation, it shall keep the saved model unchanged and shall persist a
|
||||
warning that identifies the inferred effective model.
|
||||
- **AC-AGENTS-NO-SILENT-MODEL-FALLBACK-002.7:** The host profile probe shall
|
||||
present unique-variation resolution as an advisory. It shall not become the
|
||||
launch authority or disable the profile.
|
||||
- **AC-AGENTS-NO-SILENT-MODEL-FALLBACK-002.8:** The same resolution order shall
|
||||
apply at initial launch, context reset, and workspace rebind.
|
||||
|
||||
## Out of scope
|
||||
|
||||
- Ranking multiple variations by context size, speed, price, or display order.
|
||||
- Parsing provider-specific variation syntax beyond the bracketed ID shape.
|
||||
- Inferring a base model from an already bracketed request.
|
||||
- Rewriting the profile model to the current executor's advertised variation.
|
||||
- Changing Office post-start provider routing or mid-turn model switching.
|
||||
|
||||
## System design
|
||||
|
||||
The migrated technical source is split into [part 1](../system-design/no-silent-model-fallback-01.md), [part 2](../system-design/no-silent-model-fallback-02.md).
|
||||
The technical source is split into [part 1](../system-design/no-silent-model-fallback-01.md)
|
||||
and [part 2](../system-design/no-silent-model-fallback-02.md).
|
||||
|
||||
@@ -1,8 +1,9 @@
|
||||
---
|
||||
status: draft
|
||||
status: current
|
||||
system: agents
|
||||
requirements:
|
||||
- REQ-AGENTS-NO-SILENT-MODEL-FALLBACK-001
|
||||
- REQ-AGENTS-NO-SILENT-MODEL-FALLBACK-002
|
||||
created: 2026-08-23
|
||||
owners:
|
||||
- kandev
|
||||
@@ -11,13 +12,15 @@ owners:
|
||||
|
||||
## Purpose and boundaries
|
||||
|
||||
This design preserves the technical source detail for `REQ-AGENTS-NO-SILENT-MODEL-FALLBACK-001` during migration.
|
||||
This design defines executor-authoritative model fallback, including safe
|
||||
resolution from a bare requested model to one advertised bracketed variation.
|
||||
|
||||
## Requirement mapping
|
||||
|
||||
| Requirement | Design section |
|
||||
| --- | --- |
|
||||
| `REQ-AGENTS-NO-SILENT-MODEL-FALLBACK-001` | [Migrated source detail](#migrated-source-detail) |
|
||||
| `REQ-AGENTS-NO-SILENT-MODEL-FALLBACK-002` | [Unique model-variation resolution](#unique-model-variation-resolution) |
|
||||
|
||||
## Migrated source detail
|
||||
|
||||
@@ -146,6 +149,121 @@ the launch. The agent uses its default and Kandev persists a warning.
|
||||
The executor ACP catalog is authoritative for launch.
|
||||
The host probe remains an editing hint and does not block profile selection.
|
||||
|
||||
## Unique model-variation resolution
|
||||
|
||||
### Matching contract
|
||||
|
||||
ACP model IDs remain opaque and case-sensitive. Kandev recognizes only this
|
||||
structural relationship:
|
||||
|
||||
```text
|
||||
<requested>[<variant>]
|
||||
```
|
||||
|
||||
The requested ID must contain no `[` or `]` character. The advertised ID must
|
||||
start with the complete requested ID followed by `[`, end with `]`, and contain
|
||||
a non-empty variant with no bracket character. Kandev does not trim, parse, or
|
||||
rank the variant.
|
||||
|
||||
Examples:
|
||||
|
||||
| Requested | Advertised candidates | Result |
|
||||
| --- | --- | --- |
|
||||
| `opus` | `opus[1m]` | Apply `opus[1m]`. |
|
||||
| `opus` | `opus[270k]`, `opus[1m, fast]` | Do not infer. |
|
||||
| `opus[1m]` | `opus[270k]` | Do not infer. |
|
||||
| `opus` | `myopus[1m]`, `opus-extra[1m]` | Do not infer. |
|
||||
| `opus` | `opus[]`, `opus[1m][fast]` | Do not infer. |
|
||||
|
||||
Duplicate catalog rows with the same model ID count as one distinct candidate.
|
||||
This rule keeps the outcome stable when an agent repeats a catalog entry.
|
||||
|
||||
### Resolution order
|
||||
|
||||
For profiles with `auto_fallback = false`, the lifecycle applies this order
|
||||
after ACP initialization:
|
||||
|
||||
1. Apply the exact requested model when advertised.
|
||||
2. Apply the configured explicit fallback when advertised.
|
||||
3. Apply the requested model's unique advertised variation when one exists.
|
||||
4. Otherwise, make no model-selection call and keep the provider current or
|
||||
default model.
|
||||
|
||||
An unadvertised explicit fallback does not block step 3. More than one distinct
|
||||
variation blocks inference even when one candidate appears first or is the
|
||||
provider current model.
|
||||
|
||||
When `auto_fallback = true`, the legacy path takes precedence when the
|
||||
requested model is absent: Kandev ignores the explicit fallback and does not
|
||||
infer a variation. It makes no model-selection call and continues on the
|
||||
provider current or default model with a warning. Apply errors for an
|
||||
advertised model remain best-effort in that mode.
|
||||
|
||||
`applyStartModelPolicy` remains the single runtime owner. A pure helper returns
|
||||
the unique candidate or no candidate. Initial launch, context reset, and
|
||||
workspace rebind continue to call the same policy.
|
||||
|
||||
### Decision and warning contract
|
||||
|
||||
The typed decision adds the outcome `unique_variation` and the stable reason
|
||||
`unique_variation_applied`. `EffectiveModel` contains the advertised variation.
|
||||
`FallbackModel` remains reserved for a configured explicit fallback and is not
|
||||
overloaded with an inferred value.
|
||||
|
||||
Kandev calls `SetModel` only with the candidate's exact advertised ID. A failed
|
||||
call follows the existing advertised-model rule: fail the operation unless
|
||||
legacy `auto_fallback` permits provider-default continuation. A
|
||||
method-not-supported response also follows the current provider-default path.
|
||||
|
||||
The persisted `model_selection_warning` event uses the new reason and existing
|
||||
requested and effective model fields. The profile remains unchanged. The
|
||||
decision ID continues to include the requested model, reason, and effective
|
||||
model, so the warning stays idempotent for one session decision.
|
||||
|
||||
### Host advisory and mobile behavior
|
||||
|
||||
The frontend uses a small pure helper with the same case-sensitive matching
|
||||
table. This calculation is advisory because the host catalog can differ from
|
||||
the selected executor catalog.
|
||||
|
||||
When the host advertises one variation, the task profile option remains
|
||||
selectable and identifies the possible target in its existing amber advisory.
|
||||
The profile editor keeps the saved bare ID visible and describes the unique
|
||||
host variation instead of reporting that the model is simply gone. Selecting
|
||||
the advertised variation remains an explicit way to update the profile.
|
||||
|
||||
When the host advertises zero or multiple variations, the current missing-model
|
||||
presentation remains. The runtime can still reach a different decision from
|
||||
the selected executor catalog.
|
||||
|
||||
The task-create advisory keeps its existing desktop tooltip and coarse-pointer
|
||||
drawer. The profile editor uses its existing responsive selector. The durable
|
||||
chat warning uses the existing status-message layout on desktop and mobile.
|
||||
This change adds no nested scroll area or hover-only information.
|
||||
|
||||
### Tests and operational evidence
|
||||
|
||||
Backend table tests cover exact precedence, explicit-fallback precedence,
|
||||
one variation, duplicate rows, multiple variations, bracketed requests,
|
||||
case differences, malformed IDs, method-not-supported, and apply errors.
|
||||
Existing launch, reset, rebind, warning persistence, and reload tests cover the
|
||||
new decision outcome.
|
||||
|
||||
Frontend unit tests mirror the matching table and cover the task picker,
|
||||
profile editor, and localized status warning. Existing desktop and mobile model
|
||||
mismatch E2E flows add the unique and ambiguous catalog cases.
|
||||
|
||||
Structured lifecycle logs record the requested model, effective model, outcome,
|
||||
and distinct candidate count. They do not record credentials or configuration
|
||||
file contents.
|
||||
|
||||
No database migration is required.
|
||||
|
||||
## Related decisions
|
||||
|
||||
- [Let the Executor Own Model Selection](../../../decisions/2026-08-15-executor-authoritative-model-selection.md)
|
||||
- [Resolve Only One Advertised Model Variation](../../../decisions/2026-09-07-unique-model-variation-resolution.md)
|
||||
|
||||
## Backend Changes
|
||||
|
||||
### 1. Agent profile schema (new columns)
|
||||
|
||||
@@ -1,8 +1,9 @@
|
||||
---
|
||||
status: draft
|
||||
status: current
|
||||
system: agents
|
||||
requirements:
|
||||
- REQ-AGENTS-NO-SILENT-MODEL-FALLBACK-001
|
||||
- REQ-AGENTS-NO-SILENT-MODEL-FALLBACK-002
|
||||
created: 2026-08-23
|
||||
owners:
|
||||
- kandev
|
||||
@@ -11,13 +12,15 @@ owners:
|
||||
|
||||
## Purpose and boundaries
|
||||
|
||||
This design preserves the technical source detail for `REQ-AGENTS-NO-SILENT-MODEL-FALLBACK-001` during migration.
|
||||
This design records risks for executor-authoritative model fallback and unique
|
||||
variation resolution.
|
||||
|
||||
## Requirement mapping
|
||||
|
||||
| Requirement | Design section |
|
||||
| --- | --- |
|
||||
| `REQ-AGENTS-NO-SILENT-MODEL-FALLBACK-001` | [Migrated source detail](#migrated-source-detail) |
|
||||
| `REQ-AGENTS-NO-SILENT-MODEL-FALLBACK-002` | [Risks & Open Questions](#risks--open-questions) |
|
||||
|
||||
## Migrated source detail
|
||||
|
||||
@@ -51,3 +54,14 @@ This design preserves the technical source detail for `REQ-AGENTS-NO-SILENT-MODE
|
||||
- **Hover is supplementary**: every info icon is focusable, and coarse-pointer
|
||||
devices receive the same content in a drawer. The visible option helper copy
|
||||
remains the baseline explanation.
|
||||
- **Provider-specific labels remain opaque**: `1m`, `270k`, and `fast` have no
|
||||
Kandev-defined order. More than one variation therefore fails closed to the
|
||||
existing provider-default path.
|
||||
- **Host and executor decisions can differ**: a unique host variation is only
|
||||
an advisory. The selected executor can advertise zero, one, or multiple
|
||||
variations at launch.
|
||||
- **Bracketed requests do not drift sideways**: a request such as `opus[1m]`
|
||||
does not infer `opus[270k]`. The user must select another explicit model.
|
||||
- **Legacy automatic fallback remains unchanged**: when `auto_fallback` is
|
||||
enabled and the requested model is absent, Kandev does not apply an explicit
|
||||
fallback or infer a variation. It continues with the provider default.
|
||||
|
||||
Reference in New Issue
Block a user