package dynamic import ( "context" "errors" "strings" "testing" "time" "github.com/kandev/kandev/internal/agent/runtime/routingerr" "github.com/kandev/kandev/internal/agent/runtime/routingpolicy" ) func TestEngineSelectsFixedOrderAndFencesGenerations(t *testing.T) { now := time.Unix(100, 0) engine := NewEngine(WithClock(func() time.Time { return now })) profile := Profile{ ID: "dynamic-1", Version: 4, Candidates: []Candidate{ {ID: "disabled", Enabled: false, BindingKey: "disabled"}, {ID: "open", Enabled: true, BindingKey: "open"}, {ID: "fallback", Enabled: true, BindingKey: "fallback"}, }, } engine.Circuits().Open("open", now.Add(time.Hour), routingerr.CodeProviderUnavailable) decision, err := engine.Select("session-1", profile, 0, "") if err != nil { t.Fatalf("Select: %v", err) } if decision.ExecutionProfileID != "fallback" || decision.Generation != 1 || decision.ProfileVersion != 4 { t.Fatalf("decision = %#v", decision) } if _, err := engine.Select("session-1", profile, 0, ""); !errors.Is(err, ErrStaleGeneration) { t.Fatalf("stale select error = %v, want %v", err, ErrStaleGeneration) } engine.Circuits().Open("open", now.Add(-time.Second), routingerr.CodeProviderUnavailable) decision, err = engine.Select("session-1", profile, 1, "fallback") if err != nil { t.Fatalf("Try next select: %v", err) } if decision.ExecutionProfileID != "open" || decision.Generation != 2 { t.Fatalf("try next decision = %#v", decision) } } func TestEnginePreferenceKeepsRetryOnCurrentCandidate(t *testing.T) { engine := NewEngine() profile := Profile{ ID: "dynamic-1", Candidates: []Candidate{ {ID: "first", Enabled: true}, {ID: "second", Enabled: true}, }, } decision, err := engine.SelectContextWithPreference( context.Background(), "session-1", profile, 0, "", "second", ) if err != nil { t.Fatalf("SelectContextWithPreference: %v", err) } if decision.ExecutionProfileID != "second" { t.Fatalf("preferred decision = %#v", decision) } } func TestEngineAppliesCandidateErrorActions(t *testing.T) { profile := Profile{ ID: "dynamic-1", Version: 1, Candidates: []Candidate{ {ID: "first", Enabled: true, BindingKey: "first", Rules: map[string]Action{ string(routingerr.CodeQuotaLimited): ActionTryNext, string(routingerr.CodeRateLimited): ActionRetrySame, string(routingerr.CodeTask): ActionStop, }}, {ID: "second", Enabled: true, BindingKey: "second"}, }, } engine := NewEngine() if got := engine.ActionFor(profile, "first", routingerr.CodeQuotaLimited); got != ActionTryNext { t.Fatalf("quota action = %q, want %q", got, ActionTryNext) } if got := engine.ActionFor(profile, "first", routingerr.CodeRateLimited); got != ActionRetrySame { t.Fatalf("rate action = %q, want %q", got, ActionRetrySame) } if got := engine.ActionFor(profile, "first", routingerr.CodeTask); got != ActionStop { t.Fatalf("task action = %q, want %q", got, ActionStop) } if got := engine.ActionFor(profile, "second", routingerr.CodeQuotaLimited); got != ActionStop { t.Fatalf("unconfigured action = %q, want %q", got, ActionStop) } } // Contract coverage: Codex's complete usage-limit envelope must reach the // existing hard-error policy without adding a provider branch to the engine. func TestEngineRoutesCodexUsageLimitThroughHardPolicy(t *testing.T) { failure := routingerr.Classify(routingerr.Input{ Phase: routingerr.PhasePromptSend, ProviderID: "codex-acp", Stderr: `{"code":-32603,"message":"Internal error","data":{"codexErrorInfo":"usageLimitExceeded","message":"You've hit your usage limit. Visit https://chatgpt.com/codex/settings/usage to purchase more credits or try again at Sep 1st, 2026 3:14 PM."}}`, }) if failure.Code != routingerr.CodeQuotaLimited || failure.Class != routingerr.ClassHard { t.Fatalf("classification = %+v, want hard quota_limited", failure) } profile := Profile{ ID: "dynamic-1", Version: 1, Candidates: []Candidate{ {ID: "codex", Enabled: true, BindingKey: "codex", Policies: routingpolicy.DefaultDocument()}, {ID: "fallback", Enabled: true, BindingKey: "fallback", Policies: routingpolicy.DefaultDocument()}, }, } engine := NewEngine() initial, err := engine.Select("session-1", profile, 0, "") if err != nil { t.Fatalf("Select: %v", err) } decision, err := engine.ApplyFailure( "session-1", profile, initial.Generation, initial.ExecutionProfileID, failure, ) if err != nil { t.Fatalf("ApplyFailure: %v", err) } if decision.ExecutionProfileID != "fallback" || decision.Reason != "policy_skip" { t.Fatalf("decision = %#v, want fallback selected by hard policy", decision) } } func TestEngineRetrySameKeepsTheFailedCandidate(t *testing.T) { profile := Profile{ ID: "dynamic-1", Version: 1, Candidates: []Candidate{ {ID: "first", Enabled: true, Rules: map[string]Action{ string(routingerr.CodeRateLimited): ActionRetrySame, }}, {ID: "second", Enabled: true}, }, } engine := NewEngine() initial, err := engine.Select("session-1", profile, 0, "") if err != nil { t.Fatalf("initial Select: %v", err) } initialDecision, err := engine.ApplyFailure( "session-1", profile, initial.Generation, initial.ExecutionProfileID, &routingerr.Error{Code: routingerr.CodeRateLimited}, ) if err != nil { t.Fatalf("ApplyFailure: %v", err) } if initialDecision.ExecutionProfileID != "first" || initialDecision.Generation != 2 { t.Fatalf("retry decision = %#v, want first at generation 2", initialDecision) } } func TestEnginePersistsPolicyWaitAndResumesWithGenerationFence(t *testing.T) { now := time.Unix(100, 0).UTC() engine := NewEngine(WithClock(func() time.Time { return now })) document := routingpolicy.DefaultDocument() document.Transient.Retry = routingpolicy.RetryPolicy{ Enabled: true, MaxRetries: 2, InitialIntervalSeconds: 5, } profile := Profile{ ID: "dynamic-policy", Version: 1, Candidates: []Candidate{ {ID: "first", Enabled: true, Policies: document}, {ID: "second", Enabled: true}, }, } initial, err := engine.Select("policy-session", profile, 0, "") if err != nil { t.Fatalf("initial Select: %v", err) } decision, err := engine.ApplyFailure( "policy-session", profile, initial.Generation, initial.ExecutionProfileID, &routingerr.Error{ Code: routingerr.CodeRateLimited, Class: routingerr.ClassTransient, FallbackAllowed: true, }, ) if !errors.Is(err, ErrRecoveryPending) { t.Fatalf("ApplyFailure error = %v, want pending recovery", err) } if decision.Status != "retry_wait" || decision.Generation != initial.Generation { t.Fatalf("pending decision = %#v", decision) } state, ok := engine.State("policy-session") if !ok || state.PolicyStateJSON == "" { t.Fatalf("policy state was not persisted: %#v, ok=%v", state, ok) } now = now.Add(5 * time.Second) due, err := engine.ResumePending(context.Background(), "policy-session", initial.Generation) if err != nil { t.Fatalf("ResumePending: %v", err) } if due.Status != "retrying" || due.ExecutionProfileID != "first" { t.Fatalf("due decision = %#v", due) } if _, err := engine.ResumePending(context.Background(), "policy-session", initial.Generation-1); !errors.Is(err, ErrStaleGeneration) { t.Fatalf("stale ResumePending error = %v", err) } } func TestEngineManualRetryNowBypassesPolicyDeadlineWithoutAdvancingGeneration(t *testing.T) { now := time.Unix(300, 0).UTC() engine := NewEngine(WithClock(func() time.Time { return now })) document := routingpolicy.DefaultDocument() document.Transient.Retry = routingpolicy.RetryPolicy{Enabled: true, MaxRetries: 1, InitialIntervalSeconds: 60} profile := Profile{ID: "manual-policy", Version: 1, Candidates: []Candidate{ {ID: "first", Enabled: true, Policies: document}, {ID: "second", Enabled: true}, }} initial, err := engine.Select("manual-session", profile, 0, "") if err != nil { t.Fatalf("initial Select: %v", err) } pending, err := engine.ApplyFailure( "manual-session", profile, initial.Generation, initial.ExecutionProfileID, &routingerr.Error{Code: routingerr.CodeRateLimited, Class: routingerr.ClassTransient, FallbackAllowed: true}, ) if !errors.Is(err, ErrRecoveryPending) || pending.Status != "retry_wait" { t.Fatalf("pending decision = %#v, error = %v", pending, err) } manual, err := engine.ResumePendingNow(context.Background(), "manual-session", initial.Generation) if err != nil { t.Fatalf("ResumePendingNow: %v", err) } if manual.Generation != initial.Generation || manual.ExecutionProfileID != "first" || manual.Status != "retrying" { t.Fatalf("manual decision = %#v", manual) } } func TestEngineManualSkipRecordsItsReason(t *testing.T) { engine := NewEngine() profile := Profile{ID: "manual-skip", Version: 1, Candidates: []Candidate{ {ID: "first", Enabled: true}, {ID: "second", Enabled: true}, }} initial, err := engine.Select("skip-session", profile, 0, "") if err != nil { t.Fatalf("initial Select: %v", err) } next, err := engine.SelectContextWithReason( context.Background(), "skip-session", profile, initial.Generation, initial.ExecutionProfileID, "manual_skip", ) if err != nil { t.Fatalf("manual skip: %v", err) } if next.ExecutionProfileID != "second" || next.Reason != "manual_skip" { t.Fatalf("manual skip decision = %#v", next) } } func TestEngineCancelPendingStopsWithoutAdvancingGeneration(t *testing.T) { engine := NewEngine() document := routingpolicy.DefaultDocument() document.Transient.Retry = routingpolicy.RetryPolicy{Enabled: true, MaxRetries: 1, InitialIntervalSeconds: 5} profile := Profile{ID: "cancel-policy", Version: 1, Candidates: []Candidate{ {ID: "first", Enabled: true, Policies: document}, }} initial, err := engine.Select("cancel-session", profile, 0, "") if err != nil { t.Fatalf("initial Select: %v", err) } pending, err := engine.ApplyFailure( "cancel-session", profile, initial.Generation, initial.ExecutionProfileID, &routingerr.Error{Code: routingerr.CodeRateLimited, Class: routingerr.ClassTransient, FallbackAllowed: true}, ) if !errors.Is(err, ErrRecoveryPending) || pending.Status != "retry_wait" { t.Fatalf("pending decision = %#v, error = %v", pending, err) } stopped, err := engine.CancelPending(context.Background(), "cancel-session", initial.Generation, "manual_stop") if err != nil { t.Fatalf("CancelPending: %v", err) } if stopped.Status != "action_required" || stopped.Generation != initial.Generation || stopped.Reason != "manual_stop" { t.Fatalf("stopped decision = %#v", stopped) } } func TestEngineMarkActiveCompletesStartingPhase(t *testing.T) { engine := NewEngine() profile := Profile{ID: "dynamic-active", Candidates: []Candidate{{ID: "first", Enabled: true}}} initial, err := engine.Select("active-session", profile, 0, "") if err != nil { t.Fatalf("initial Select: %v", err) } state, ok := engine.State("active-session") if !ok || state.Status != routeStatusStarting { t.Fatalf("precondition: state = %#v, ok=%v, want starting", state, ok) } if err := engine.MarkActive(context.Background(), "active-session", initial.Generation); err != nil { t.Fatalf("MarkActive: %v", err) } state, ok = engine.State("active-session") if !ok || state.Status != routeStatusActive || state.Generation != initial.Generation { t.Fatalf("state after MarkActive = %#v, ok=%v", state, ok) } // A repeat call (e.g. from a duplicate callback) is a no-op, not an error. if err := engine.MarkActive(context.Background(), "active-session", initial.Generation); err != nil { t.Fatalf("repeat MarkActive: %v", err) } if err := engine.MarkActive(context.Background(), "active-session", initial.Generation-1); !errors.Is(err, ErrStaleGeneration) { t.Fatalf("stale MarkActive error = %v, want %v", err, ErrStaleGeneration) } if err := engine.MarkActive(context.Background(), "missing-session", 1); !errors.Is(err, ErrRouteStateNotFound) { t.Fatalf("missing session MarkActive error = %v, want %v", err, ErrRouteStateNotFound) } } func TestEngineMarkActiveIsNoOpWhenRouteAlreadyLeftStarting(t *testing.T) { engine := NewEngine() profile := Profile{ID: "dynamic-active-skip", Candidates: []Candidate{ {ID: "first", Enabled: true}, {ID: "second", Enabled: true}, }} initial, err := engine.Select("active-skip-session", profile, 0, "") if err != nil { t.Fatalf("initial Select: %v", err) } next, err := engine.SelectContextWithReason( context.Background(), "active-skip-session", profile, initial.Generation, initial.ExecutionProfileID, "manual_skip", ) if err != nil { t.Fatalf("manual skip: %v", err) } // A late MarkActive call for the superseded generation must not resurrect // it as active over the newer claim. if err := engine.MarkActive(context.Background(), "active-skip-session", initial.Generation); !errors.Is(err, ErrStaleGeneration) { t.Fatalf("MarkActive for superseded generation error = %v, want %v", err, ErrStaleGeneration) } state, ok := engine.State("active-skip-session") if !ok || state.Generation != next.Generation || state.Status != routeStatusStarting { t.Fatalf("state after stale MarkActive = %#v, ok=%v", state, ok) } } func TestEngineMarkActionRequiredFromStartingAndFencesGeneration(t *testing.T) { engine := NewEngine() profile := Profile{ID: "dynamic-action-required", Candidates: []Candidate{{ID: "first", Enabled: true}}} initial, err := engine.Select("action-required-session", profile, 0, "") if err != nil { t.Fatalf("initial Select: %v", err) } // A route sitting in "starting" (the exact stranded case this exists to // fix) must be acceptable input. decision, err := engine.MarkActionRequired(context.Background(), "action-required-session", initial.Generation, "launch_failed") if err != nil { t.Fatalf("MarkActionRequired: %v", err) } if decision.Status != routeStatusActionRequired || decision.Reason != "launch_failed" || decision.Generation != initial.Generation { t.Fatalf("decision = %#v", decision) } state, ok := engine.State("action-required-session") if !ok || state.Status != routeStatusActionRequired { t.Fatalf("state after MarkActionRequired = %#v, ok=%v", state, ok) } // A second call (a different failure path re-declaring the same route) is // idempotent, not an error, and keeps its own reason. decision, err = engine.MarkActionRequired(context.Background(), "action-required-session", initial.Generation, "second_reason") if err != nil { t.Fatalf("repeat MarkActionRequired: %v", err) } if decision.Reason != "second_reason" || decision.Status != routeStatusActionRequired { t.Fatalf("repeat decision = %#v", decision) } if _, err := engine.MarkActionRequired(context.Background(), "action-required-session", initial.Generation-1, "stale"); !errors.Is(err, ErrStaleGeneration) { t.Fatalf("stale MarkActionRequired error = %v, want %v", err, ErrStaleGeneration) } if _, err := engine.MarkActionRequired(context.Background(), "missing-session", 1, "missing"); !errors.Is(err, ErrRouteStateNotFound) { t.Fatalf("missing session MarkActionRequired error = %v, want %v", err, ErrRouteStateNotFound) } } // TestEngineMarkActionRequiredIsNoOpOnceRouteIsActive is the regression test // for review finding F1: a route a launch already carried to "active" must // not be demoted to action_required by an unrelated later failure (for // example a permission-denied or resume-corrupted error the classifier // forbids fallback for), which would surface a "Try next provider" recovery // banner on a healthy session. func TestEngineMarkActionRequiredIsNoOpOnceRouteIsActive(t *testing.T) { engine := NewEngine() profile := Profile{ID: "dynamic-action-required-active", Candidates: []Candidate{{ID: "first", Enabled: true}}} initial, err := engine.Select("active-route-session", profile, 0, "") if err != nil { t.Fatalf("initial Select: %v", err) } if err := engine.MarkActive(context.Background(), "active-route-session", initial.Generation); err != nil { t.Fatalf("MarkActive: %v", err) } decision, err := engine.MarkActionRequired(context.Background(), "active-route-session", initial.Generation, "permission_denied") if err != nil { t.Fatalf("MarkActionRequired on active route: %v", err) } if decision.Status != routeStatusActive { t.Fatalf("decision.Status = %q, want unchanged %q", decision.Status, routeStatusActive) } state, ok := engine.State("active-route-session") if !ok || state.Status != routeStatusActive { t.Fatalf("state after no-op MarkActionRequired = %#v, ok=%v, want unchanged active", state, ok) } } // TestEngineMarkActionRequiredFromRetryingTransitionsAtSameGeneration is the // regression test restoring the deferred recovery guard in // LaunchDynamicRouteAction for the timer-recovery path: persistDynamicPolicyRecovery // leaves a resumed route at "retrying" (resume never advances the generation), // so a launch that fails after that resume must still reach action_required // instead of sticking at "retrying" with a dead Retry/Stop banner. func TestEngineMarkActionRequiredFromRetryingTransitionsAtSameGeneration(t *testing.T) { engine := NewEngine() document := routingpolicy.DefaultDocument() document.Transient.Retry = routingpolicy.RetryPolicy{Enabled: true, MaxRetries: 1, InitialIntervalSeconds: 60} profile := Profile{ID: "dynamic-retrying-action-required", Candidates: []Candidate{ {ID: "first", Enabled: true, Policies: document}, }} initial, err := engine.Select("retrying-action-required-session", profile, 0, "") if err != nil { t.Fatalf("initial Select: %v", err) } if _, err := engine.ApplyFailure( "retrying-action-required-session", profile, initial.Generation, initial.ExecutionProfileID, &routingerr.Error{Code: routingerr.CodeRateLimited, Class: routingerr.ClassTransient, FallbackAllowed: true}, ); !errors.Is(err, ErrRecoveryPending) { t.Fatalf("ApplyFailure: %v", err) } resumed, err := engine.ResumePendingNow(context.Background(), "retrying-action-required-session", initial.Generation) if err != nil { t.Fatalf("ResumePendingNow: %v", err) } if resumed.Status != routeStatusRetrying { t.Fatalf("precondition: resumed status = %q, want %q", resumed.Status, routeStatusRetrying) } decision, err := engine.MarkActionRequired( context.Background(), "retrying-action-required-session", initial.Generation, "route_action_launch_failed", ) if err != nil { t.Fatalf("MarkActionRequired from retrying: %v", err) } if decision.Status != routeStatusActionRequired || decision.Generation != initial.Generation { t.Fatalf("decision = %#v", decision) } state, ok := engine.State("retrying-action-required-session") if !ok || state.Status != routeStatusActionRequired || state.Generation != initial.Generation { t.Fatalf("state after MarkActionRequired = %#v, ok=%v", state, ok) } } // TestEngineMarkActiveFromRetryingThenActionRequiredIsNoOp is the regression // test for review finding F1: a resumed route (status "retrying") that then // launches successfully must reach "active" via MarkActive, exactly as a // freshly claimed "starting" route does. Without this, a resumed route stays // at "retrying" forever, and MarkActionRequired's starting-or-retrying guard // (widened for the resumed-then-FAILED case) would wrongly demote a healthy, // successfully launched route on any later unrelated failure. func TestEngineMarkActiveFromRetryingThenActionRequiredIsNoOp(t *testing.T) { engine := NewEngine() document := routingpolicy.DefaultDocument() document.Transient.Retry = routingpolicy.RetryPolicy{Enabled: true, MaxRetries: 1, InitialIntervalSeconds: 60} profile := Profile{ID: "dynamic-retrying-active", Candidates: []Candidate{ {ID: "first", Enabled: true, Policies: document}, }} initial, err := engine.Select("retrying-active-session", profile, 0, "") if err != nil { t.Fatalf("initial Select: %v", err) } if _, err := engine.ApplyFailure( "retrying-active-session", profile, initial.Generation, initial.ExecutionProfileID, &routingerr.Error{Code: routingerr.CodeRateLimited, Class: routingerr.ClassTransient, FallbackAllowed: true}, ); !errors.Is(err, ErrRecoveryPending) { t.Fatalf("ApplyFailure: %v", err) } resumed, err := engine.ResumePendingNow(context.Background(), "retrying-active-session", initial.Generation) if err != nil { t.Fatalf("ResumePendingNow: %v", err) } if resumed.Status != routeStatusRetrying { t.Fatalf("precondition: resumed status = %q, want %q", resumed.Status, routeStatusRetrying) } if err := engine.MarkActive(context.Background(), "retrying-active-session", initial.Generation); err != nil { t.Fatalf("MarkActive from retrying: %v", err) } state, ok := engine.State("retrying-active-session") if !ok || state.Status != routeStatusActive || state.Generation != initial.Generation { t.Fatalf("state after MarkActive = %#v, ok=%v, want active", state, ok) } // A later, unrelated failure that reaches the catch-all guard must not // demote this healthy, successfully launched route. decision, err := engine.MarkActionRequired( context.Background(), "retrying-active-session", initial.Generation, "unrelated_declined_failure", ) if err != nil { t.Fatalf("MarkActionRequired on active route: %v", err) } if decision.Status != routeStatusActive { t.Fatalf("decision.Status = %q, want unchanged %q", decision.Status, routeStatusActive) } state, ok = engine.State("retrying-active-session") if !ok || state.Status != routeStatusActive { t.Fatalf("state after no-op MarkActionRequired = %#v, ok=%v, want unchanged active", state, ok) } } func TestEngineMarkRecoveryActionRequiredUnsticksManualRecoveryAfterFailedLaunch(t *testing.T) { engine := NewEngine() document := routingpolicy.DefaultDocument() document.Transient.Retry = routingpolicy.RetryPolicy{Enabled: true, MaxRetries: 1, InitialIntervalSeconds: 60} profile := Profile{ID: "launch-fail-policy", Version: 1, Candidates: []Candidate{{ID: "first", Enabled: true, Policies: document}}} initial, err := engine.Select("launch-fail-session", profile, 0, "") if err != nil { t.Fatalf("initial Select: %v", err) } ctx := context.Background() if _, err := engine.ApplyFailure( "launch-fail-session", profile, initial.Generation, initial.ExecutionProfileID, &routingerr.Error{Code: routingerr.CodeRateLimited, Class: routingerr.ClassTransient, FallbackAllowed: true}, ); !errors.Is(err, ErrRecoveryPending) { t.Fatalf("ApplyFailure error = %v, want recovery pending", err) } if _, err := engine.ResumePendingNow(ctx, "launch-fail-session", initial.Generation); err != nil { t.Fatalf("ResumePendingNow: %v", err) } if _, err := engine.ResumePendingNow(ctx, "launch-fail-session", initial.Generation); !errors.Is(err, ErrRecoveryPending) { t.Fatalf("ResumePendingNow while stuck at retrying = %v, want recovery pending", err) } if _, err := engine.CancelPending(ctx, "launch-fail-session", initial.Generation, "manual_stop"); !errors.Is(err, ErrRecoveryPending) { t.Fatalf("CancelPending while stuck at retrying = %v, want recovery pending", err) } if reclaimed, err := engine.ReclaimRetrying(ctx, "launch-fail-session", initial.Generation); !errors.Is(err, ErrRecoveryPending) || reclaimed { t.Fatalf("ReclaimRetrying for owned claim = reclaimed %v, error %v, want recovery pending", reclaimed, err) } if err := engine.MarkRecoveryActionRequired(ctx, "launch-fail-session", initial.Generation); err != nil { t.Fatalf("MarkRecoveryActionRequired: %v", err) } state, exists := engine.State("launch-fail-session") if !exists || state.Status != routeStatusActionRequired || state.Generation != initial.Generation { t.Fatalf("state after MarkRecoveryActionRequired = %#v, exists=%v", state, exists) } if stopped, err := engine.CancelPending(ctx, "launch-fail-session", initial.Generation, "manual_stop"); err != nil { t.Fatalf("CancelPending after sync: %v", err) } else if stopped.Status != routeStatusActionRequired { t.Fatalf("stopped decision = %#v", stopped) } } func TestEngineMarkRecoveryActionRequiredIsNoOpWhenStateMovedOn(t *testing.T) { engine := NewEngine() profile := Profile{ID: "no-op-policy", Candidates: []Candidate{{ID: "first", Enabled: true}}} ctx := context.Background() initial, err := engine.Select("no-op-session", profile, 0, "") if err != nil { t.Fatalf("initial Select: %v", err) } if err := engine.MarkRecoveryActionRequired(ctx, "no-op-session", initial.Generation); err != nil { t.Fatalf("MarkRecoveryActionRequired on non-retrying state: %v", err) } state, _ := engine.State("no-op-session") if state.Status == routeStatusActionRequired { t.Fatalf("state unexpectedly advanced to action_required: %#v", state) } if err := engine.MarkRecoveryActionRequired(ctx, "no-op-session", initial.Generation+1); err != nil { t.Fatalf("MarkRecoveryActionRequired on stale generation: %v", err) } if err := engine.MarkRecoveryActionRequired(ctx, "unknown-session", 1); err != nil { t.Fatalf("MarkRecoveryActionRequired on unknown session: %v", err) } } func TestBindingFingerprinterUsesOpaqueStableHMACKeys(t *testing.T) { key := []byte("installation-secret") fingerprinter := NewBindingFingerprinter(key) descriptor := CredentialBindingDescriptor{ Version: 1, AgentFamilyID: "claude-acp", AuthenticationMethod: "subscription", CredentialSourceKind: "credential_file", CredentialLocator: "home:claude", ExecutorNamespace: "local", AuthorizationScope: "messages", WorkspaceScope: "workspace-1", } a := fingerprinter.Fingerprint(descriptor) b := fingerprinter.Fingerprint(descriptor) if a == "" || a != b || len(a) != 64 { t.Fatalf("fingerprints = %q and %q", a, b) } changed := descriptor changed.WorkspaceScope = "workspace-2" if a == fingerprinter.Fingerprint(changed) { t.Fatal("different binding scopes shared a fingerprint") } if containsSensitive(a, "installation-secret") || containsSensitive(a, "home:claude") { t.Fatal("fingerprint exposed descriptor material") } if fallback := fingerprinter.FallbackFingerprint("profile-1"); fallback != "profile:profile-1" { t.Fatalf("fallback fingerprint = %q", fallback) } } func TestCredentialBindingResolverCanonicalizesAndFallsBackToProfileScope(t *testing.T) { resolver := NewCredentialBindingResolver([]byte("installation-secret")) base := CredentialBindingDescriptor{ Version: 1, AgentFamilyID: "Claude-ACP", AuthenticationMethod: "Subscription", CredentialSourceKind: "credential_file", CredentialLocator: " home:claude ", } if resolver.Fingerprint(base) != resolver.Fingerprint(CredentialBindingDescriptor{ Version: 1, AgentFamilyID: "claude-acp", AuthenticationMethod: "subscription", CredentialSourceKind: "credential_file", CredentialLocator: "home:claude", }) { t.Fatal("canonical descriptors produced different fingerprints") } if got := resolver.Resolve(CredentialBindingDescriptor{}, "profile-1"); got != "profile:profile-1" { t.Fatalf("fallback binding = %q", got) } } func TestCircuitProbeLeaseIsExclusive(t *testing.T) { now := time.Unix(200, 0) registry := NewCircuitRegistry(WithCircuitClock(func() time.Time { return now })) registry.Open("provider:claude", now.Add(-time.Second), routingerr.CodeQuotaLimited) lease, ok := registry.AcquireProbe("provider:claude", time.Minute) if !ok || lease.Key != "provider:claude" { t.Fatalf("first probe lease = %#v, ok=%v", lease, ok) } if _, ok := registry.AcquireProbe("provider:claude", time.Minute); ok { t.Fatal("second worker acquired the same half-open probe") } now = now.Add(2 * time.Minute) lease2, ok := registry.AcquireProbe("provider:claude", time.Minute) if !ok { t.Fatal("expired probe lease was not replaced") } registry.ReleaseProbe(lease, true, time.Second) if _, ok := registry.AcquireProbe("provider:claude", time.Minute); ok { t.Fatal("stale probe lease changed the current half-open circuit") } registry.ReleaseProbe(lease2, false, time.Second) now = now.Add(2 * time.Second) if _, ok := registry.AcquireProbe("provider:claude", time.Minute); !ok { t.Fatal("probe lease was not released after failure backoff") } } func TestEngineUsesExclusiveProbeForExpiredCircuit(t *testing.T) { now := time.Unix(500, 0) engine := NewEngine(WithClock(func() time.Time { return now })) engine.Circuits().Open("shared-binding", now.Add(-time.Second), routingerr.CodeQuotaLimited) profile := Profile{ID: "dynamic-probe", Candidates: []Candidate{ {ID: "first", Enabled: true, BindingKey: "shared-binding"}, {ID: "second", Enabled: true, BindingKey: "other"}, }} first, err := engine.Select("probe-session-1", profile, 0, "") if err != nil || first.ExecutionProfileID != "first" { t.Fatalf("first selection = %#v, %v; want probe candidate", first, err) } second, err := engine.Select("probe-session-2", profile, 0, "") if err != nil || second.ExecutionProfileID != "second" { t.Fatalf("second selection = %#v, %v; want other candidate while probe is held", second, err) } engine.ReleaseProbe(first, true) } func containsSensitive(value, sensitive string) bool { return sensitive != "" && strings.Contains(value, sensitive) }