fix: retry transient Cursor RetriableError failures (#3765)
This commit is contained in:
+122
-16
@@ -1,6 +1,7 @@
|
||||
package routingerr
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
@@ -9,10 +10,18 @@ const realCursorRetriableStreamReset = "Error: RetriableError: HTTP/2 stream clo
|
||||
|
||||
const leadingCanceledCursorRetriableStreamReset = "Error: RetriableError: [canceled] HTTP/2 stream closed with error code CANCEL (0x8)"
|
||||
|
||||
const cursorRetriablePingTimeout = "Error: RetriableError: [unavailable] PING timed out"
|
||||
|
||||
const cursorRetriableConnectionStalled = "Error: RetriableError: Connection stalled"
|
||||
|
||||
const wantCursorRetriableStreamResetRuleID = "cursor.retriable_stream_reset.v1"
|
||||
|
||||
func matchCursorRuntimeEnvironmentRules(text string) (*Error, bool) {
|
||||
return matchRuntimeEnvironmentRulesForProvider(cursorRetriableProviderID, text, text)
|
||||
}
|
||||
|
||||
func TestMatchRuntimeEnvironmentRules_CursorRetriable(t *testing.T) {
|
||||
got, ok := matchRuntimeEnvironmentRules(realCursorRetriableStreamReset)
|
||||
got, ok := matchCursorRuntimeEnvironmentRules(realCursorRetriableStreamReset)
|
||||
if !ok {
|
||||
t.Fatal("expected Cursor stream-reset match")
|
||||
}
|
||||
@@ -25,6 +34,17 @@ func TestMatchRuntimeEnvironmentRules_CursorRetriable(t *testing.T) {
|
||||
if got.Confidence != ConfHigh {
|
||||
t.Fatalf("Confidence = %q, want %q", got.Confidence, ConfHigh)
|
||||
}
|
||||
for _, message := range []string{cursorRetriablePingTimeout, cursorRetriableConnectionStalled} {
|
||||
t.Run(message, func(t *testing.T) {
|
||||
got, ok := matchCursorRuntimeEnvironmentRules(message)
|
||||
if !ok {
|
||||
t.Fatalf("expected Cursor RetriableError match for %q", message)
|
||||
}
|
||||
if got.Code != CodeAgentTransportLost || got.ClassifierRule != wantCursorRetriableStreamResetRuleID {
|
||||
t.Fatalf("match for %q = %+v, want agent_transport_lost with rule %q", message, got, wantCursorRetriableStreamResetRuleID)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
@@ -39,36 +59,61 @@ func TestMatchRuntimeEnvironmentRules_CursorRetriable(t *testing.T) {
|
||||
text: "Error: Retriable: HTTP/2 stream closed with error code CANCEL (0x8)",
|
||||
},
|
||||
{
|
||||
name: "partial transport fingerprint",
|
||||
text: "Error: RetriableError: HTTP/2 stream reset with error code CANCEL",
|
||||
},
|
||||
{
|
||||
name: "unrelated explanation before fingerprint",
|
||||
text: "Error: RetriableError: explanation for CANCEL (0x8)",
|
||||
},
|
||||
{
|
||||
name: "missing transport fingerprint",
|
||||
text: "Error: RetriableError: provider is busy",
|
||||
name: "empty diagnostic suffix",
|
||||
text: "Error: RetriableError:",
|
||||
},
|
||||
{
|
||||
name: "context cancellation",
|
||||
text: "context canceled: " + realCursorRetriableStreamReset,
|
||||
},
|
||||
{
|
||||
name: "underscore-delimited context cancellation",
|
||||
text: "Error: RetriableError: context canceled_by_client",
|
||||
},
|
||||
{
|
||||
name: "context deadline exceeded",
|
||||
text: "Error: RetriableError: context deadline exceeded",
|
||||
},
|
||||
{
|
||||
name: "cancel escalation",
|
||||
text: "Error: RetriableError: cancel escalated",
|
||||
},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
if got, ok := matchRuntimeEnvironmentRules(tc.text); ok {
|
||||
if got, ok := matchCursorRuntimeEnvironmentRules(tc.text); ok {
|
||||
t.Fatalf("unexpected match: %+v", got)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
withCursorCancellationToken := realCursorRetriableStreamReset + " [canceled]"
|
||||
got, ok = matchRuntimeEnvironmentRules(withCursorCancellationToken)
|
||||
got, ok = matchCursorRuntimeEnvironmentRules(withCursorCancellationToken)
|
||||
if !ok || got.Code != CodeAgentTransportLost {
|
||||
t.Fatalf("bracketed cancellation token classified as ok=%v err=%v, want transport loss", ok, got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMatchRuntimeEnvironmentRules_CursorRetriableUsesAdapterBounds(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
suffix string
|
||||
want bool
|
||||
}{
|
||||
{name: "256 ASCII bytes", suffix: strings.Repeat("x", 256), want: true},
|
||||
{name: "257 ASCII bytes", suffix: strings.Repeat("x", 257), want: false},
|
||||
{name: "128 multibyte characters at 256 bytes", suffix: strings.Repeat("é", 128), want: true},
|
||||
{name: "129 multibyte characters over 256 bytes", suffix: strings.Repeat("é", 129), want: false},
|
||||
{name: "unicode whitespace only", suffix: "\u2003\u2003", want: false},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
_, ok := matchCursorRuntimeEnvironmentRules("Error: RetriableError: " + tc.suffix)
|
||||
if ok != tc.want {
|
||||
t.Fatalf("matchRuntimeEnvironmentRules(%q) = %v, want %v", tc.suffix, ok, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestClassifyCursorRetriable(t *testing.T) {
|
||||
resetInjection()
|
||||
|
||||
@@ -148,15 +193,66 @@ func TestClassifyCursorRetriableLeadingCanceled(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestClassifyCursorRetriableVariants(t *testing.T) {
|
||||
for _, message := range []string{cursorRetriablePingTimeout, cursorRetriableConnectionStalled} {
|
||||
t.Run(message, func(t *testing.T) {
|
||||
resetInjection()
|
||||
e := Classify(Input{Phase: PhasePromptSend, ProviderID: "cursor-acp", Stderr: message})
|
||||
if e.Code != CodeAgentTransportLost || e.Class != ClassTransient || !e.AutoRetryable {
|
||||
t.Fatalf("Classify(%q) = %+v, want transient auto-retryable transport loss", message, e)
|
||||
}
|
||||
if e.ClassifierRule != wantCursorRetriableStreamResetRuleID {
|
||||
t.Fatalf("ClassifierRule = %q, want %q", e.ClassifierRule, wantCursorRetriableStreamResetRuleID)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestClassifyCursorRetriableDoesNotCrossProviders(t *testing.T) {
|
||||
for _, providerID := range []string{"claude-acp", "codex-acp", "opencode-acp", "unknown-provider"} {
|
||||
t.Run(providerID, func(t *testing.T) {
|
||||
resetInjection()
|
||||
e := Classify(Input{
|
||||
Phase: PhasePromptSend,
|
||||
ProviderID: providerID,
|
||||
Stderr: cursorRetriablePingTimeout,
|
||||
})
|
||||
if e.Code == CodeAgentTransportLost || e.ClassifierRule == wantCursorRetriableStreamResetRuleID {
|
||||
t.Fatalf("Classify(%q) = %+v, want no Cursor transport-loss match", providerID, e)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestClassifyCursorRetriableRejectsOversizedSanitizedSuffix(t *testing.T) {
|
||||
resetInjection()
|
||||
message := "Error: RetriableError: " + strings.Repeat("x", 257)
|
||||
e := Classify(Input{
|
||||
Phase: PhasePromptSend,
|
||||
ProviderID: "cursor-acp",
|
||||
Stderr: message,
|
||||
})
|
||||
if e.Code == CodeAgentTransportLost || e.ClassifierRule == wantCursorRetriableStreamResetRuleID {
|
||||
t.Fatalf("Classify(oversized redacted suffix) = %+v, want no Cursor transport-loss match", e)
|
||||
}
|
||||
}
|
||||
|
||||
func TestClassifyCursorRetriableCancellationRemainsManual(t *testing.T) {
|
||||
resetInjection()
|
||||
|
||||
for _, message := range []string{
|
||||
"context canceled: " + leadingCanceledCursorRetriableStreamReset,
|
||||
"cancel escalated: " + leadingCanceledCursorRetriableStreamReset,
|
||||
"Error: RetriableError: context canceled",
|
||||
"Error: RetriableError: context canceled_by_client",
|
||||
"Error: RetriableError: context deadline exceeded",
|
||||
"Error: RetriableError: cancel escalated",
|
||||
} {
|
||||
t.Run(message, func(t *testing.T) {
|
||||
e := Classify(Input{Phase: PhasePromptSend, Stderr: message})
|
||||
e := Classify(Input{Phase: PhasePromptSend, ProviderID: cursorRetriableProviderID, Stderr: message})
|
||||
if e != nil && e.AutoRetryable {
|
||||
t.Fatalf("Classify(%q) = %+v, want non-auto-retryable cancellation", message, e)
|
||||
}
|
||||
if got := Decide(ContextKanban, e, time.Time{}); got != DecisionManual {
|
||||
t.Fatalf("Decide(ContextKanban, Classify(%q)) = %q, want %q", message, got, DecisionManual)
|
||||
}
|
||||
@@ -180,6 +276,16 @@ func TestIsTransientProviderError_Cursor(t *testing.T) {
|
||||
text: realCursorRetriableStreamReset + " [canceled]",
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "PING timed out",
|
||||
text: cursorRetriablePingTimeout,
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "connection stalled",
|
||||
text: cursorRetriableConnectionStalled,
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "context canceled",
|
||||
text: "context canceled: " + realCursorRetriableStreamReset,
|
||||
@@ -192,8 +298,8 @@ func TestIsTransientProviderError_Cursor(t *testing.T) {
|
||||
},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
if got := IsTransientProviderError(tc.text); got != tc.want {
|
||||
t.Errorf("IsTransientProviderError(%q) = %v, want %v", tc.text, got, tc.want)
|
||||
if got := IsTransientProviderErrorForProvider(cursorRetriableProviderID, tc.text); got != tc.want {
|
||||
t.Errorf("IsTransientProviderErrorForProvider(%q, %q) = %v, want %v", cursorRetriableProviderID, tc.text, got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
@@ -166,7 +166,8 @@ const statusOverloaded = 529
|
||||
// Classify always returns a non-nil *Error, even for an unmatched or empty
|
||||
// input; callers may dereference the result without a nil check.
|
||||
func Classify(in Input) *Error {
|
||||
excerpt := Sanitize(in.Stderr + "\n" + in.Stdout)
|
||||
rawText := in.Stderr + "\n" + in.Stdout
|
||||
excerpt := Sanitize(rawText)
|
||||
if e := classifyInjection(in, excerpt); e != nil {
|
||||
return e
|
||||
}
|
||||
@@ -188,24 +189,24 @@ func Classify(in Input) *Error {
|
||||
e.RawExcerpt = excerpt
|
||||
return applyInvariants(e)
|
||||
}
|
||||
if e, ok := matchRuntimeEnvironmentRules(excerpt); ok {
|
||||
if e, ok := matchRuntimeEnvironmentRulesForProvider(in.ProviderID, excerpt, rawText); ok {
|
||||
e.Phase = in.Phase
|
||||
e.ExitCode = in.ExitCode
|
||||
e.ResetHint = in.ResetHint
|
||||
if e.Code == CodeNpxCacheCorrupted {
|
||||
// Preserve the legacy path for the path-aware remediation guard;
|
||||
// the path is validated again before deletion.
|
||||
e.RemediationPath = extractNpxCachePath(in.Stderr + "\n" + in.Stdout)
|
||||
e.RemediationPath = extractNpxCachePath(rawText)
|
||||
}
|
||||
e.RawExcerpt = excerpt
|
||||
return applyInvariants(e)
|
||||
}
|
||||
if e, ok := matchLegacyRuntimeEnvironmentRules(in.Stderr + "\n" + in.Stdout); ok {
|
||||
if e, ok := matchLegacyRuntimeEnvironmentRulesForProvider(in.ProviderID, rawText, rawText); ok {
|
||||
e.Phase = in.Phase
|
||||
e.ExitCode = in.ExitCode
|
||||
e.ResetHint = in.ResetHint
|
||||
if e.Code == CodeNpxCacheCorrupted {
|
||||
e.RemediationPath = extractNpxCachePath(in.Stderr + "\n" + in.Stdout)
|
||||
e.RemediationPath = extractNpxCachePath(rawText)
|
||||
}
|
||||
e.RawExcerpt = excerpt
|
||||
return applyInvariants(e)
|
||||
|
||||
@@ -1,6 +1,9 @@
|
||||
package routingerr
|
||||
|
||||
import "regexp"
|
||||
import (
|
||||
"regexp"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// runtimeEnvironmentRules match failures caused by the local execution
|
||||
// environment (npm/npx cache state, missing binaries' install steps,
|
||||
@@ -9,8 +12,10 @@ import "regexp"
|
||||
// fallback, so a specific environment fingerprint wins over the
|
||||
// low-confidence "phase.prestart.unknown" verdict.
|
||||
//
|
||||
// Each entry pairs a regex matcher with a builder that can extract
|
||||
// signal-specific metadata (e.g. RemediationPath) from the raw text.
|
||||
// Each entry pairs a matcher with a builder that can extract signal-specific
|
||||
// metadata (e.g. RemediationPath) from the raw text. Most matchers are regular
|
||||
// expressions; rules that share an adapter's byte-level contract can provide a
|
||||
// custom matcher instead.
|
||||
var runtimeEnvironmentRules = []runtimeRule{
|
||||
{
|
||||
// npm 9 and npm 10 use different casing for the error prefix. Require
|
||||
@@ -89,14 +94,17 @@ var runtimeEnvironmentRules = []runtimeRule{
|
||||
},
|
||||
},
|
||||
{
|
||||
// Cursor emits this exact control prefix as an assistant message chunk
|
||||
// when its upstream HTTP/2 stream resets. Keep the fingerprint anchored
|
||||
// to the control frame and bounded so user-authored prose cannot turn
|
||||
// into an automatic retry.
|
||||
id: cursorRetriableStreamResetRuleID,
|
||||
pattern: cursorRetriableStreamResetRe,
|
||||
// Cursor emits this control prefix as an assistant message chunk for
|
||||
// transient provider failures. Keep the fingerprint anchored to the
|
||||
// control frame and bounded so user-authored prose cannot turn into an
|
||||
// automatic retry. The custom matcher shares the adapter's byte and
|
||||
// Unicode-whitespace contract.
|
||||
id: cursorRetriableStreamResetRuleID,
|
||||
providerID: cursorRetriableProviderID,
|
||||
match: matchCursorRetriableStreamReset,
|
||||
rawMatch: matchCursorRetriableStreamReset,
|
||||
build: func(text string) *Error {
|
||||
if cancellationOrDeadlineRe.MatchString(text) {
|
||||
if IsCursorRetriableCancellation(text) {
|
||||
return nil
|
||||
}
|
||||
return &Error{
|
||||
@@ -163,13 +171,40 @@ const transportLostRuleID = "acp.transport_lost.v1"
|
||||
// carries the signature.
|
||||
var transportLostRe = regexp.MustCompile(`(?i)peer disconnected|connection closed`)
|
||||
|
||||
// cursorRetriableStreamResetRe matches Cursor's complete control diagnostic.
|
||||
// Requiring the whole normalized message prevents ordinary provider prose from
|
||||
// combining the control prefix with an unrelated transport fragment.
|
||||
var cursorRetriableStreamResetRe = regexp.MustCompile(
|
||||
`(?i)^\s*Error:\s*RetriableError:\s*(?:\[canceled\]\s+)?HTTP/2 stream closed with error code CANCEL \(0x8\)(?:\s+\[canceled\])?\s*$`,
|
||||
const (
|
||||
cursorRetriableStreamResetPrefix = "Error: RetriableError:"
|
||||
cursorRetriableStreamResetMaxTail = 256
|
||||
cursorRetriableProviderID = "cursor-acp"
|
||||
)
|
||||
|
||||
// matchCursorRetriableStreamReset matches Cursor's complete bounded control
|
||||
// diagnostic using the same trim, prefix, byte-limit, and cancellation-veto
|
||||
// rules as the ACP adapter. The byte limit prevents the classifier from
|
||||
// accepting a diagnostic that the adapter would reject after multibyte text is
|
||||
// measured.
|
||||
func matchCursorRetriableStreamReset(text string) bool {
|
||||
trimmed := strings.TrimSpace(text)
|
||||
if len(trimmed) < len(cursorRetriableStreamResetPrefix) ||
|
||||
!strings.EqualFold(trimmed[:len(cursorRetriableStreamResetPrefix)], cursorRetriableStreamResetPrefix) {
|
||||
return false
|
||||
}
|
||||
suffix := strings.TrimSpace(trimmed[len(cursorRetriableStreamResetPrefix):])
|
||||
if suffix == "" || len(suffix) > cursorRetriableStreamResetMaxTail || IsCursorRetriableCancellation(suffix) {
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// IsCursorRetriableCancellation reports whether Cursor's RetriableError
|
||||
// suffix carries cancellation or deadline evidence that must not be retried.
|
||||
// The adapter uses this same predicate before it suppresses diagnostic output.
|
||||
func IsCursorRetriableCancellation(text string) bool {
|
||||
lower := strings.ToLower(text)
|
||||
return strings.Contains(lower, "context canceled") ||
|
||||
strings.Contains(lower, "context deadline exceeded") ||
|
||||
strings.Contains(lower, "cancel escalated")
|
||||
}
|
||||
|
||||
// overloadedRe matches the transient 529 Overloaded signature: either the
|
||||
// numeric code adjacent to "overloaded" on a single line (in either order), or
|
||||
// the Anthropic "overloaded_error" type token. Kept tight ([^\n] stays within
|
||||
@@ -182,15 +217,28 @@ var overloadedRe = regexp.MustCompile(`(?i)\b529\b[^\n]*overloaded|overloaded[^\
|
||||
// bare 5xx numbers and generic "internal error" prose.
|
||||
var gatewayServerFailureRe = regexp.MustCompile(`(?i)\bAPI\s+Error:\s*(?:500\s+Internal(?:\s+Server)?\s+Error|502\s+Bad\s+Gateway|504\s+Gateway\s+Timeout)\b`)
|
||||
|
||||
// IsTransientProviderError reports whether a provider error is eligible for a
|
||||
// short same-provider retry. It is intentionally backed by Classify so new
|
||||
// provider-neutral fingerprints (capacity and network failures) become
|
||||
// available to callers outside the adapter without another allow-list.
|
||||
// IsTransientProviderError reports whether a provider-neutral error is eligible
|
||||
// for a short same-provider retry. Provider-specific callers must use
|
||||
// IsTransientProviderErrorForProvider so a signature cannot cross adapters.
|
||||
func IsTransientProviderError(message string) bool {
|
||||
return isTransientProviderError(Classify(Input{Phase: PhasePromptSend, Stderr: message}))
|
||||
}
|
||||
|
||||
// IsTransientProviderErrorForProvider reports whether a provider error is
|
||||
// eligible for a short same-provider retry while preserving provider-specific
|
||||
// catalogue scope.
|
||||
func IsTransientProviderErrorForProvider(providerID, message string) bool {
|
||||
if message == "" {
|
||||
return false
|
||||
}
|
||||
e := Classify(Input{Phase: PhasePromptSend, Stderr: message})
|
||||
return isTransientProviderError(Classify(Input{
|
||||
Phase: PhasePromptSend,
|
||||
ProviderID: providerID,
|
||||
Stderr: message,
|
||||
}))
|
||||
}
|
||||
|
||||
func isTransientProviderError(e *Error) bool {
|
||||
if e == nil || !e.AutoRetryable || e.UserAction {
|
||||
return false
|
||||
}
|
||||
@@ -219,7 +267,10 @@ func IsResumeCorrupted(message string) bool {
|
||||
|
||||
type runtimeRule struct {
|
||||
id string
|
||||
providerID string
|
||||
pattern *regexp.Regexp
|
||||
match func(text string) bool
|
||||
rawMatch func(text string) bool
|
||||
build func(text string) *Error
|
||||
sanitizedOnly bool
|
||||
}
|
||||
@@ -242,14 +293,22 @@ func extractNpxCachePath(text string) string {
|
||||
}
|
||||
|
||||
func matchRuntimeEnvironmentRules(text string) (*Error, bool) {
|
||||
return matchRuntimeRules(text, false)
|
||||
return matchRuntimeRules(text, text, "", false)
|
||||
}
|
||||
|
||||
func matchLegacyRuntimeEnvironmentRules(text string) (*Error, bool) {
|
||||
return matchRuntimeRules(text, true)
|
||||
return matchRuntimeRules(text, text, "", true)
|
||||
}
|
||||
|
||||
func matchRuntimeRules(text string, legacyOnly bool) (*Error, bool) {
|
||||
func matchRuntimeEnvironmentRulesForProvider(providerID, text, rawText string) (*Error, bool) {
|
||||
return matchRuntimeRules(text, rawText, providerID, false)
|
||||
}
|
||||
|
||||
func matchLegacyRuntimeEnvironmentRulesForProvider(providerID, text, rawText string) (*Error, bool) {
|
||||
return matchRuntimeRules(text, rawText, providerID, true)
|
||||
}
|
||||
|
||||
func matchRuntimeRules(text, rawText, providerID string, legacyOnly bool) (*Error, bool) {
|
||||
if text == "" {
|
||||
return nil, false
|
||||
}
|
||||
@@ -257,7 +316,19 @@ func matchRuntimeRules(text string, legacyOnly bool) (*Error, bool) {
|
||||
if legacyOnly && r.sanitizedOnly {
|
||||
continue
|
||||
}
|
||||
if !r.pattern.MatchString(text) {
|
||||
if r.providerID != "" && r.providerID != providerID {
|
||||
continue
|
||||
}
|
||||
if r.rawMatch != nil && !r.rawMatch(rawText) {
|
||||
continue
|
||||
}
|
||||
var matched bool
|
||||
if r.match != nil {
|
||||
matched = r.match(text)
|
||||
} else {
|
||||
matched = r.pattern != nil && r.pattern.MatchString(text)
|
||||
}
|
||||
if !matched {
|
||||
continue
|
||||
}
|
||||
if e := r.build(text); e != nil {
|
||||
|
||||
@@ -4,32 +4,31 @@ import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
|
||||
"github.com/kandev/kandev/internal/agent/runtime/routingerr"
|
||||
"github.com/kandev/kandev/internal/agentctl/types/streams"
|
||||
)
|
||||
|
||||
const (
|
||||
cursorRetriableStreamResetPrefix = "Error: RetriableError:"
|
||||
cursorRetriableStreamResetMessage = "Error: RetriableError: HTTP/2 stream closed with error code CANCEL (0x8)"
|
||||
cursorRetriableStreamResetLeadingCanceledMessage = "Error: RetriableError: [canceled] HTTP/2 stream closed with error code CANCEL (0x8)"
|
||||
cursorRetriableStreamResetMaxTail = 256
|
||||
cursorRetriableStreamResetPrefix = "Error: RetriableError:"
|
||||
cursorRetriableStreamResetMessage = "Error: RetriableError: HTTP/2 stream closed with error code CANCEL (0x8)"
|
||||
cursorRetriableStreamResetMaxTail = 256
|
||||
)
|
||||
|
||||
// isCursorRetriableStreamReset recognizes Cursor's complete transport control
|
||||
// chunk. The prefix check keeps the common per-token path allocation-free;
|
||||
// requiring the complete diagnostic prevents ordinary provider prose from
|
||||
// combining the control prefix with an unrelated transport fragment.
|
||||
// isCursorRetriableStreamReset recognizes Cursor's bounded RetriableError
|
||||
// control chunk. The prefix check keeps the common per-token path
|
||||
// allocation-free; requiring a non-empty bounded suffix prevents a partial
|
||||
// marker from becoming transport evidence.
|
||||
func isCursorRetriableStreamReset(text string) bool {
|
||||
trimmed := strings.TrimSpace(text)
|
||||
if len(trimmed) < len(cursorRetriableStreamResetPrefix) ||
|
||||
!strings.EqualFold(trimmed[:len(cursorRetriableStreamResetPrefix)], cursorRetriableStreamResetPrefix) {
|
||||
return false
|
||||
}
|
||||
if len(trimmed)-len(cursorRetriableStreamResetPrefix) > cursorRetriableStreamResetMaxTail {
|
||||
suffix := strings.TrimSpace(trimmed[len(cursorRetriableStreamResetPrefix):])
|
||||
if suffix == "" || len(suffix) > cursorRetriableStreamResetMaxTail || routingerr.IsCursorRetriableCancellation(suffix) {
|
||||
return false
|
||||
}
|
||||
return strings.EqualFold(trimmed, cursorRetriableStreamResetMessage) ||
|
||||
strings.EqualFold(trimmed, cursorRetriableStreamResetLeadingCanceledMessage) ||
|
||||
strings.EqualFold(trimmed, cursorRetriableStreamResetMessage+" [canceled]")
|
||||
return true
|
||||
}
|
||||
|
||||
type cursorTaskMeta struct {
|
||||
|
||||
+117
-12
@@ -2,6 +2,7 @@ package acp
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -14,6 +15,16 @@ const cursorRetriableStreamResetChunk = "Error: RetriableError: HTTP/2 stream cl
|
||||
|
||||
const cursorRetriableStreamResetLeadingCanceledChunk = "Error: RetriableError: [canceled] HTTP/2 stream closed with error code CANCEL (0x8)"
|
||||
|
||||
const cursorRetriablePingTimeoutChunk = "Error: RetriableError: [unavailable] PING timed out"
|
||||
|
||||
const cursorRetriableConnectionStalledChunk = "Error: RetriableError: Connection stalled"
|
||||
|
||||
const cursorRetriableContextCanceledChunk = "Error: RetriableError: context canceled"
|
||||
|
||||
const cursorRetriableDeadlineExceededChunk = "Error: RetriableError: context deadline exceeded"
|
||||
|
||||
const cursorRetriableCancelEscalatedChunk = "Error: RetriableError: cancel escalated"
|
||||
|
||||
func cursorMessageNotification(sessionID, text string) acpsdk.SessionNotification {
|
||||
return makeNotification(sessionID, acpsdk.SessionUpdate{
|
||||
AgentMessageChunk: &acpsdk.SessionUpdateAgentMessageChunk{
|
||||
@@ -57,6 +68,47 @@ func TestCursorRetriableStreamResetSuppressesLeadingCanceledCurrentChunk(t *test
|
||||
}
|
||||
}
|
||||
|
||||
func TestCursorRetriableErrorSuppressesObservedDiagnostics(t *testing.T) {
|
||||
for _, chunk := range []string{
|
||||
cursorRetriablePingTimeoutChunk,
|
||||
cursorRetriableConnectionStalledChunk,
|
||||
} {
|
||||
t.Run(chunk, func(t *testing.T) {
|
||||
a, turn := newCursorPromptTurn(t, 7)
|
||||
|
||||
a.handleACPUpdate(cursorMessageNotification("session-1", chunk), 7)
|
||||
|
||||
if !turn.cursorRetriableFailure() {
|
||||
t.Fatalf("Cursor diagnostic %q did not set retriable marker", chunk)
|
||||
}
|
||||
if events := drainEvents(a); len(events) != 0 {
|
||||
t.Fatalf("Cursor diagnostic %q emitted %d events: %+v", chunk, len(events), events)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestCursorRetriableStreamResetUsesByteBoundsAndUnicodeWhitespace(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
suffix string
|
||||
want bool
|
||||
}{
|
||||
{name: "256 ASCII bytes", suffix: strings.Repeat("x", 256), want: true},
|
||||
{name: "257 ASCII bytes", suffix: strings.Repeat("x", 257), want: false},
|
||||
{name: "128 multibyte characters at 256 bytes", suffix: strings.Repeat("é", 128), want: true},
|
||||
{name: "129 multibyte characters over 256 bytes", suffix: strings.Repeat("é", 129), want: false},
|
||||
{name: "unicode whitespace only", suffix: "\u2003\u2003", want: false},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
text := cursorRetriableStreamResetPrefix + " " + tc.suffix
|
||||
if got := isCursorRetriableStreamReset(text); got != tc.want {
|
||||
t.Fatalf("isCursorRetriableStreamReset(%q) = %v, want %v", tc.suffix, got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestCursorRetriableStreamResetRejectsUnrelatedEvidence(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
@@ -69,21 +121,11 @@ func TestCursorRetriableStreamResetRejectsUnrelatedEvidence(t *testing.T) {
|
||||
generation: 7,
|
||||
text: "The provider returned Error: RetriableError: HTTP/2 stream closed with error code CANCEL (0x8)",
|
||||
},
|
||||
{
|
||||
name: "unrelated explanation before fingerprint",
|
||||
generation: 7,
|
||||
text: "Error: RetriableError: explanation for CANCEL (0x8)",
|
||||
},
|
||||
{
|
||||
name: "partial marker",
|
||||
generation: 7,
|
||||
text: "Error: RetriableError:",
|
||||
},
|
||||
{
|
||||
name: "missing transport fingerprint",
|
||||
generation: 7,
|
||||
text: "Error: RetriableError: provider is busy",
|
||||
},
|
||||
{
|
||||
name: "stale generation",
|
||||
generation: 6,
|
||||
@@ -161,7 +203,7 @@ func TestCursorRetriableStreamResetNotClearedByToolUpdate(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestSendPromptCursorRetriableStreamResetSettlesAfterNotificationBarrier(t *testing.T) {
|
||||
func TestSendPromptCursorRetriableErrorSettlesAfterNotificationBarrier(t *testing.T) {
|
||||
a, fake, conn := setupHandoffFakeAgent(t)
|
||||
a.agentID = acpcompat.CursorAgentID
|
||||
a.normalizer = NewNormalizer(acpcompat.CursorAgentID)
|
||||
@@ -184,7 +226,7 @@ func TestSendPromptCursorRetriableStreamResetSettlesAfterNotificationBarrier(t *
|
||||
t.Fatal("prompt did not reach the fake Cursor agent")
|
||||
}
|
||||
|
||||
sendCapturedUpdate(t, conn, `{"sessionId":"session-handoff","update":{"sessionUpdate":"agent_message_chunk","content":{"type":"text","text":"`+cursorRetriableStreamResetChunk+`"}}}`)
|
||||
sendCapturedUpdate(t, conn, `{"sessionId":"session-handoff","update":{"sessionUpdate":"agent_message_chunk","content":{"type":"text","text":"`+cursorRetriablePingTimeoutChunk+`"}}}`)
|
||||
fake.releasePrompts()
|
||||
|
||||
select {
|
||||
@@ -221,6 +263,69 @@ func TestSendPromptCursorRetriableStreamResetSettlesAfterNotificationBarrier(t *
|
||||
}
|
||||
}
|
||||
|
||||
func TestSendPromptCursorCancellationRetriableDiagnosticsRemainOrdinaryOutput(t *testing.T) {
|
||||
for _, chunk := range []string{
|
||||
cursorRetriableContextCanceledChunk,
|
||||
cursorRetriableDeadlineExceededChunk,
|
||||
cursorRetriableCancelEscalatedChunk,
|
||||
} {
|
||||
t.Run(chunk, func(t *testing.T) {
|
||||
a, fake, conn := setupHandoffFakeAgent(t)
|
||||
a.agentID = acpcompat.CursorAgentID
|
||||
a.normalizer = NewNormalizer(acpcompat.CursorAgentID)
|
||||
a.dialect = newACPDialect(acpcompat.CursorAgentID)
|
||||
|
||||
ctx := context.Background()
|
||||
if err := a.Initialize(ctx); err != nil {
|
||||
t.Fatalf("initialize: %v", err)
|
||||
}
|
||||
if _, err := a.NewSession(ctx, nil); err != nil {
|
||||
t.Fatalf("new session: %v", err)
|
||||
}
|
||||
_ = drainEvents(a)
|
||||
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- a.Prompt(ctx, "try this", nil, 7) }()
|
||||
select {
|
||||
case <-fake.entered:
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("prompt did not reach the fake Cursor agent")
|
||||
}
|
||||
|
||||
sendCapturedUpdate(t, conn, `{"sessionId":"session-handoff","update":{"sessionUpdate":"agent_message_chunk","content":{"type":"text","text":"`+chunk+`"}}}`)
|
||||
fake.releasePrompts()
|
||||
|
||||
select {
|
||||
case err := <-done:
|
||||
if err != nil {
|
||||
t.Fatalf("Prompt returned error: %v", err)
|
||||
}
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("prompt did not settle")
|
||||
}
|
||||
|
||||
events := drainEvents(a)
|
||||
var messageChunks, completes, errors int
|
||||
for _, event := range events {
|
||||
switch event.Type {
|
||||
case streams.EventTypeMessageChunk:
|
||||
messageChunks++
|
||||
if event.Text != chunk {
|
||||
t.Fatalf("message chunk = %q, want %q", event.Text, chunk)
|
||||
}
|
||||
case streams.EventTypeComplete:
|
||||
completes++
|
||||
case streams.EventTypeError:
|
||||
errors++
|
||||
}
|
||||
}
|
||||
if messageChunks != 1 || completes != 1 || errors != 0 {
|
||||
t.Fatalf("events = %+v, want one ordinary chunk, one complete, and no error", events)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSendPromptCursorProgressSupersedesRetriableMarker(t *testing.T) {
|
||||
a, fake, conn := setupHandoffFakeAgent(t)
|
||||
a.agentID = acpcompat.CursorAgentID
|
||||
|
||||
@@ -18,6 +18,7 @@ func TestHandleTransientFailure_ReplayRequiresPromptAttemptEvidence(t *testing.T
|
||||
TaskID: "t1",
|
||||
SessionID: "s1",
|
||||
AgentExecutionID: "execution-1",
|
||||
AgentID: "cursor-acp",
|
||||
PromptGeneration: 7,
|
||||
EvidenceKnown: true,
|
||||
ErrorMessage: cursorTransportLostDiagnostic,
|
||||
@@ -53,6 +54,7 @@ func TestHandleTransientFailure_ReplayRejectsOutputOrToolEvidence(t *testing.T)
|
||||
data.TaskID = "t1"
|
||||
data.SessionID = "s1"
|
||||
data.AgentExecutionID = "execution-1"
|
||||
data.AgentID = "cursor-acp"
|
||||
data.PromptGeneration = 7
|
||||
data.EvidenceKnown = true
|
||||
data.ErrorMessage = cursorTransportLostDiagnostic
|
||||
@@ -73,6 +75,7 @@ func TestHandleTransientFailure_ReplayUsesCurrentPromptAttempt(t *testing.T) {
|
||||
TaskID: "t1",
|
||||
SessionID: "s1",
|
||||
AgentExecutionID: "execution-1",
|
||||
AgentID: "cursor-acp",
|
||||
PromptGeneration: 7,
|
||||
ErrorMessage: cursorTransportLostDiagnostic,
|
||||
}
|
||||
@@ -124,6 +127,7 @@ func TestPromptAttemptEvidence_ObservesThoughtAndToolActivity(t *testing.T) {
|
||||
TaskID: "t1",
|
||||
SessionID: "s1",
|
||||
AgentExecutionID: "execution-1",
|
||||
AgentID: "cursor-acp",
|
||||
PromptGeneration: 7,
|
||||
ErrorMessage: cursorTransportLostDiagnostic,
|
||||
}
|
||||
@@ -145,6 +149,7 @@ func TestPromptAttemptEvidence_RejectsReplacedAttempt(t *testing.T) {
|
||||
TaskID: "t1",
|
||||
SessionID: "s1",
|
||||
AgentExecutionID: "execution-1",
|
||||
AgentID: "cursor-acp",
|
||||
PromptGeneration: 7,
|
||||
ErrorMessage: cursorTransportLostDiagnostic,
|
||||
}
|
||||
|
||||
@@ -12,6 +12,8 @@ import (
|
||||
// production: the upstream provider connection drops mid-turn on prompt-send.
|
||||
const realPeerDisconnected = `{"code":-32603,"message":"Internal error","data":{"error":"peer disconnected before response"}}`
|
||||
|
||||
const cursorRetriableConnectionStalled = "Error: RetriableError: Connection stalled"
|
||||
|
||||
func TestHandleTransientFailure_TransportLostSchedulesRetry(t *testing.T) {
|
||||
svc, mc := newTransientTestService(t)
|
||||
t.Cleanup(svc.cancelAllTransientRetries)
|
||||
@@ -45,6 +47,34 @@ func TestHandleTransientFailure_TransportLostSchedulesRetry(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleTransientFailure_CursorRetriableErrorReplaySchedulesRetry(t *testing.T) {
|
||||
svc, mc := newTransientTestService(t)
|
||||
t.Cleanup(svc.cancelAllTransientRetries)
|
||||
armTransientPromptEvidence(svc)
|
||||
|
||||
took := svc.handleTransientFailure(context.Background(), watcher.AgentEventData{
|
||||
TaskID: "t1",
|
||||
SessionID: "s1",
|
||||
AgentExecutionID: "execution-1",
|
||||
AgentID: "cursor-acp",
|
||||
PromptGeneration: 7,
|
||||
ErrorMessage: cursorRetriableConnectionStalled,
|
||||
})
|
||||
if !took {
|
||||
t.Fatal("handleTransientFailure = false, want true for Cursor RetriableError")
|
||||
}
|
||||
|
||||
if _, ok := svc.transientRetries.Load("s1"); !ok {
|
||||
t.Fatal("expected a transient retry entry for Cursor RetriableError")
|
||||
}
|
||||
if len(mc.sessionMessages) != 1 {
|
||||
t.Fatalf("expected 1 status message, got %d", len(mc.sessionMessages))
|
||||
}
|
||||
if got := mc.sessionMessages[0].metadata["failure_code"]; got != "agent_transport_lost" {
|
||||
t.Errorf("failure_code = %v, want agent_transport_lost", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTransientFailureLabelAndExhaustedMessage_TransportLost(t *testing.T) {
|
||||
classified := &routingerr.Error{Code: routingerr.CodeAgentTransportLost}
|
||||
|
||||
|
||||
@@ -0,0 +1,104 @@
|
||||
---
|
||||
created: 2026-09-17
|
||||
status: implemented
|
||||
requirements:
|
||||
- REQ-PLATFORM-PROVIDER-ERROR-RECOVERY-001
|
||||
system_design:
|
||||
- ../../specs/platform/system-design/provider-error-recovery.md
|
||||
legacy_specs: []
|
||||
---
|
||||
|
||||
# Implementation Plan: Cursor RetriableError Expansion
|
||||
|
||||
## Overview
|
||||
|
||||
Extend the existing Cursor ACP transient-error projection to recognize any
|
||||
bounded, non-empty diagnostic suffix after Cursor's `Error: RetriableError:`
|
||||
prefix. This covers the observed `[unavailable] PING timed out` and
|
||||
`Connection stalled` messages while retaining the existing structured error,
|
||||
same-provider retry owner, retry budget, and effect-safety gate.
|
||||
|
||||
The work is one sequential backend slice because the ACP evidence matcher and
|
||||
the routing catalogue must agree on the same signature contract before the
|
||||
existing retry path can use it.
|
||||
|
||||
## Scope
|
||||
|
||||
### In scope
|
||||
|
||||
- Broaden current-prompt Cursor ACP control-frame recognition from one HTTP/2
|
||||
suffix to any bounded, non-empty `RetriableError` suffix, while vetoing
|
||||
cancellation, deadline, and retry-escalation signatures before projection.
|
||||
- Keep control-frame suppression, notification-barrier ordering, stable safe
|
||||
provider-error projection, and Cursor-owned progress clearing unchanged.
|
||||
- Broaden the existing Cursor catalogue rule to classify the same prefix-based
|
||||
diagnostics as high-confidence transient `agent_transport_lost` errors. Use
|
||||
the same Unicode-trimmed 256-byte suffix contract in both layers.
|
||||
- Add positive coverage for the reported messages and negative coverage for
|
||||
prose, partial markers, other adapters, stale generations, and cancellation.
|
||||
- Prove that broad classification still cannot schedule replay without current
|
||||
prompt evidence showing no output or tool activity.
|
||||
|
||||
### Out of scope
|
||||
|
||||
- Generic scanning of arbitrary assistant prose or other adapters.
|
||||
- Signatures that exist only in ACP `RequestError.Data`.
|
||||
- New retry counters, delays, policy settings, provider switching, or UI.
|
||||
- Changes to the generic `transportLostRe` rule or the existing cancellation
|
||||
veto.
|
||||
|
||||
## Technical approach
|
||||
|
||||
- Update `isCursorRetriableStreamReset` in
|
||||
`apps/backend/internal/agentctl/server/adapter/transport/acp/dialect_cursor.go`
|
||||
to retain Cursor identity, anchored prefix, bounded tail, and non-empty
|
||||
suffix checks while accepting the reported variants.
|
||||
- Keep `adapter_updates.go` generation and role fencing, `adapter_prompt.go`
|
||||
notification-barrier settlement, and the fixed sanitized provider diagnostic.
|
||||
- Update `cursor.retriable_stream_reset.v1` in
|
||||
`apps/backend/internal/agent/runtime/routingerr/runtime_rules.go` to use the
|
||||
same prefix-and-tail contract. Retain the rule ID and map to the existing
|
||||
`CodeAgentTransportLost` invariants.
|
||||
- Preserve `promptAttemptPreResultSafe` as the authorization boundary. A
|
||||
classification match alone remains insufficient for automatic replay.
|
||||
|
||||
## Tests
|
||||
|
||||
| Acceptance criterion | Evidence |
|
||||
| --- | --- |
|
||||
| `AC-PLATFORM-PROVIDER-ERROR-RECOVERY-001.12` | ACP dialect tests for `PING timed out`, `Connection stalled`, suppression, barrier settlement, cancellation vetoes, byte bounds, and anchored negative cases. |
|
||||
| `AC-PLATFORM-PROVIDER-ERROR-RECOVERY-001.13` | Existing ACP progress-clear and later-marker re-arm tests, extended with a non-HTTP/2 suffix. |
|
||||
| `AC-PLATFORM-PROVIDER-ERROR-RECOVERY-001.14` | Routing catalogue tests for both reported suffixes, stable rule identity, transient invariants, collision priority, cancellation veto, and adapter-aligned byte/Unicode-whitespace bounds. |
|
||||
| `AC-PLATFORM-PROVIDER-ERROR-RECOVERY-001.15` | Existing orchestrator transport-loss replay tests, including safe and unsafe prompt evidence. |
|
||||
|
||||
## E2E tests
|
||||
|
||||
No new browser flow is required. The retry banner and manual recovery surface
|
||||
are unchanged; this package changes only which backend Cursor diagnostics enter
|
||||
the existing recovery path. Backend integration tests provide the new coverage
|
||||
for the diagnostic-to-retry decision.
|
||||
|
||||
## Work orders
|
||||
|
||||
- [x] [Task 01: Broaden Cursor RetriableError matching](task-01-broaden-cursor-retriable-errors.md)
|
||||
|
||||
## Verification results
|
||||
|
||||
- Focused ACP, routing, and orchestrator race-enabled suites passed.
|
||||
- Full ACP adapter race suite passed.
|
||||
- Full routing catalogue race suite passed.
|
||||
- The post-barrier ACP regression suite passes for context cancellation,
|
||||
deadline, and retry-escalation suffixes without projecting an auto-retryable
|
||||
transport error.
|
||||
- Full orchestrator race suite passed.
|
||||
- `make -C apps/backend lint` passed with 0 issues.
|
||||
- Specification catalog validation, specification tests, full specification
|
||||
lint, `gofmt` checks, and `git diff --check` passed.
|
||||
|
||||
## Risks
|
||||
|
||||
- A suffix matcher that is not anchored can classify ordinary provider prose.
|
||||
- A matcher that accepts an empty or oversized suffix can reintroduce partial
|
||||
diagnostic false positives.
|
||||
- Broad classification must not bypass the existing prompt identity and
|
||||
no-output/no-effect replay fence.
|
||||
+136
@@ -0,0 +1,136 @@
|
||||
---
|
||||
id: "01-broaden-cursor-retriable-errors"
|
||||
title: "Broaden Cursor RetriableError matching"
|
||||
status: done
|
||||
wave: 1
|
||||
depends_on: []
|
||||
plan: "plan.md"
|
||||
requirements:
|
||||
- REQ-PLATFORM-PROVIDER-ERROR-RECOVERY-001
|
||||
acceptance_criteria:
|
||||
- AC-PLATFORM-PROVIDER-ERROR-RECOVERY-001.12
|
||||
- AC-PLATFORM-PROVIDER-ERROR-RECOVERY-001.13
|
||||
- AC-PLATFORM-PROVIDER-ERROR-RECOVERY-001.14
|
||||
- AC-PLATFORM-PROVIDER-ERROR-RECOVERY-001.15
|
||||
system_design:
|
||||
- ../../specs/platform/system-design/provider-error-recovery.md
|
||||
---
|
||||
|
||||
# Task 01: Broaden Cursor RetriableError Matching
|
||||
|
||||
## Summary
|
||||
|
||||
Make the Cursor ACP transport and shared routing catalogue recognize any
|
||||
bounded, non-empty diagnostic after the anchored `Error: RetriableError:`
|
||||
prefix. The reported `[unavailable] PING timed out` and `Connection stalled`
|
||||
messages shall enter the existing transient transport-loss path.
|
||||
|
||||
The current notification barrier, stable safe provider error, same-provider
|
||||
retry owner, and pre-result effect-safety gate remain authoritative.
|
||||
|
||||
## In scope
|
||||
|
||||
- Update the Cursor ACP matcher and its focused tests.
|
||||
- Update the existing Cursor routing rule and its focused tests.
|
||||
- Add end-to-end backend assertions that broad matches still settle through one
|
||||
structured error and only safe prompt evidence schedules replay.
|
||||
|
||||
## Out of scope
|
||||
|
||||
- New retry policy or provider-switch behavior.
|
||||
- Other ACP adapters or arbitrary prose scanning.
|
||||
- ACP `RequestError.Data` classification.
|
||||
- Frontend, localization, persistence, and public API changes.
|
||||
|
||||
## Acceptance
|
||||
|
||||
- A current Cursor message beginning with `Error: RetriableError:` and carrying
|
||||
either reported suffix is suppressed and becomes one post-barrier structured
|
||||
provider error; progress before settlement clears the marker. Cancellation,
|
||||
deadline, and retry-escalation suffixes remain ordinary output.
|
||||
- The routing catalogue maps both suffixes to high-confidence transient
|
||||
`agent_transport_lost` with the existing rule identity and cancellation
|
||||
veto. The adapter and catalogue use the same Unicode-trimmed 256-byte bound;
|
||||
prose before the prefix, empty/partial markers, stale generations, and other
|
||||
adapters do not match.
|
||||
- Automatic replay still requires the current execution and prompt generation
|
||||
with known pre-result evidence and no output or tool effect; broad matching
|
||||
alone cannot authorize replay.
|
||||
|
||||
## Verification
|
||||
|
||||
```bash
|
||||
(cd apps/backend && go test -race ./internal/agentctl/server/adapter/transport/acp -run 'Test(CursorRetriable|ObserveCursorRetriable|SendPrompt.*Cursor)')
|
||||
(cd apps/backend && go test -race ./internal/agent/runtime/routingerr -run 'Test(ClassifyCursorRetriable|MatchRuntimeEnvironmentRules_CursorRetriable|IsTransientProviderError_Cursor)')
|
||||
(cd apps/backend && go test -race -tags fts5 ./internal/orchestrator -run 'Test(HandleTransientFailure.*Replay|PromptAttemptEvidence|CursorTransportLost)')
|
||||
python3 scripts/list-docs.py validate
|
||||
python3 scripts/lint-spec-files.test.py
|
||||
python3 scripts/lint-spec-files.py --all
|
||||
git diff --check -- docs/specs docs/plans/cursor-retriable-error-expansion
|
||||
```
|
||||
|
||||
## Files likely touched
|
||||
|
||||
- `apps/backend/internal/agentctl/server/adapter/transport/acp/dialect_cursor.go`
|
||||
- `apps/backend/internal/agentctl/server/adapter/transport/acp/dialect_cursor_retriable_test.go`
|
||||
- `apps/backend/internal/agent/runtime/routingerr/runtime_rules.go`
|
||||
- `apps/backend/internal/agent/runtime/routingerr/cursor_retriable_stream_reset_test.go`
|
||||
- `apps/backend/internal/orchestrator/event_handlers_transient_transport_lost_test.go`
|
||||
- `apps/backend/internal/orchestrator/event_handlers_transient_replay_safety_test.go`
|
||||
|
||||
## Dependencies
|
||||
|
||||
The completed `docs/plans/cursor-retriable-stream-reset/` package supplies the
|
||||
existing projection, classifier, and retry-owner contracts. No new dependency
|
||||
or schema migration is required.
|
||||
|
||||
## Risks
|
||||
|
||||
- An unanchored or unbounded match could turn provider prose into retry
|
||||
evidence.
|
||||
- A classifier match without the existing evidence fence could replay a turn
|
||||
after output or a tool side effect.
|
||||
- The stable `cursor.retriable_stream_reset.v1` rule identity must not be
|
||||
changed while its accepted suffix set expands.
|
||||
|
||||
## Parallelism
|
||||
|
||||
`sequential`
|
||||
|
||||
## Inputs
|
||||
|
||||
- Provider Error Recovery criteria `.12`, `.13`, `.14`, and `.15`.
|
||||
- Cursor projection and retry-safety sections in
|
||||
`docs/specs/platform/system-design/provider-error-recovery.md`.
|
||||
- Existing Cursor ACP, routing catalogue, and orchestrator transport-loss
|
||||
tests.
|
||||
|
||||
## Results
|
||||
|
||||
- Broadened the Cursor ACP matcher to accept any bounded, non-empty suffix
|
||||
after the anchored `Error: RetriableError:` prefix.
|
||||
- Broadened `cursor.retriable_stream_reset.v1` to classify the same diagnostics
|
||||
as high-confidence transient `agent_transport_lost` failures while retaining
|
||||
its stable rule identity and cancellation veto.
|
||||
- Scoped the catalogue rule to `cursor-acp`, applied the byte bound to raw
|
||||
diagnostics before sanitization, and shared the cancellation predicate
|
||||
between the adapter and classifier.
|
||||
- Added coverage for `[unavailable] PING timed out`, `Connection stalled`,
|
||||
structured post-barrier settlement, cancellation vetoes at the adapter
|
||||
boundary, byte and Unicode-whitespace limits, anchored negatives, and
|
||||
existing safe retry scheduling.
|
||||
- Verification passed:
|
||||
|
||||
```text
|
||||
(cd apps/backend && go test -race ./internal/agentctl/server/adapter/transport/acp -run 'Test(CursorRetriable|ObserveCursorRetriable|SendPrompt.*Cursor)' -count=1)
|
||||
(cd apps/backend && go test -race ./internal/agent/runtime/routingerr -run 'Test(ClassifyCursorRetriable|MatchRuntimeEnvironmentRules_CursorRetriable|IsTransientProviderError_Cursor)' -count=1)
|
||||
(cd apps/backend && go test -race -tags fts5 ./internal/orchestrator -run 'Test(HandleTransientFailure.*Replay|PromptAttemptEvidence|CursorTransportLost)' -count=1)
|
||||
(cd apps/backend && go test -race ./internal/agentctl/server/adapter/transport/acp -count=1)
|
||||
(cd apps/backend && go test -race ./internal/agent/runtime/routingerr -count=1)
|
||||
(cd apps/backend && go test -race -tags fts5 ./internal/orchestrator -count=1)
|
||||
make -C apps/backend lint
|
||||
python3 scripts/list-docs.py validate
|
||||
python3 scripts/lint-spec-files.test.py
|
||||
python3 scripts/lint-spec-files.py --all
|
||||
git diff --check
|
||||
```
|
||||
@@ -10,8 +10,6 @@ owners:
|
||||
|
||||
## Overview
|
||||
|
||||
Agent CLIs and providers report equivalent failures through different ACP frames, HTTP metadata, process exits, and diagnostic strings. Capacity, network, subscription, and quota failures need different recovery behavior, but orchestration code must not branch on provider names or raw prose.
|
||||
|
||||
A provider can also abandon a partially streamed response and retry it inside
|
||||
the same prompt. That retry must replace the abandoned attempt without leaving
|
||||
duplicate transcript rows or stale recovery context.
|
||||
@@ -43,10 +41,10 @@ duplicate transcript rows or stale recovery context.
|
||||
- **AC-PLATFORM-PROVIDER-ERROR-RECOVERY-001.9:** While an eligible interactive transient retry is pending, desktop and mobile task chat shall show the retry reason, provider, attempt count, countdown, and Cancel action only while the backend owns that retry.
|
||||
- **AC-PLATFORM-PROVIDER-ERROR-RECOVERY-001.10:** When an interactive transient retry succeeds, is exhausted, reaches a terminal state, is stopped, or is cancelled, the system shall attempt to durably retire every outstanding retry notice for the session so that a successful cleanup prevents the notice from reappearing after the session becomes idle, the task changes, the page reloads, or another viewer observes the session. If listing or deletion fails, the system shall log and swallow the failure so a later authorized cleanup can retry the remaining notices.
|
||||
- **AC-PLATFORM-PROVIDER-ERROR-RECOVERY-001.11:** After authorization, cancelling an interactive transient retry shall attempt to retire outstanding retry notices even when the retry loop has already ended. Cancelling an active loop shall also expose manual Resume and Start fresh recovery actions. Listing or deletion failures follow AC-PLATFORM-PROVIDER-ERROR-RECOVERY-001.10. A denied or foreign task-session pair shall not expose whether retry state or retry notices exist.
|
||||
- **AC-PLATFORM-PROVIDER-ERROR-RECOVERY-001.12:** When Cursor reports a current-prompt `RetriableError` control frame and then settles without later provider progress, Kandev shall suppress the control frame. After Kandev processes all earlier prompt notifications, it shall publish exactly one structured provider failure. Similar prose, partial signatures, stale prompt evidence, and non-Cursor output shall remain ordinary agent output.
|
||||
- **AC-PLATFORM-PROVIDER-ERROR-RECOVERY-001.12:** A current Cursor frame beginning, after trimming, with case-insensitive `Error: RetriableError:` and a non-empty suffix of at most 256 UTF-8 bytes shall be suppressed when no later progress occurs, then yield one structured failure after the barrier. `[unavailable] PING timed out` and `Connection stalled` are valid. Cancellation, deadline, and retry-escalation signatures remain ordinary output. Prose before the prefix, empty/partial markers, stale evidence, and non-Cursor output stay ordinary output.
|
||||
- **AC-PLATFORM-PROVIDER-ERROR-RECOVERY-001.13:** When Cursor resumes generation before the prompt settles, Kandev shall treat the earlier matching frame as internal retry telemetry. Kandev shall not start a retry from the superseded frame. A later matching terminal frame for the same prompt can become new failure evidence.
|
||||
- **AC-PLATFORM-PROVIDER-ERROR-RECOVERY-001.14:** A terminal Cursor `RetriableError` HTTP/2 stream reset shall classify as the existing transient agent transport-loss cause. The bracketed word `canceled` alone shall not classify the failure as an operator or context cancellation.
|
||||
- **AC-PLATFORM-PROVIDER-ERROR-RECOVERY-001.15:** Kandev shall automatically recover a terminal Cursor stream reset only when current-turn evidence proves that the attempt had no output or tool activity. Eligible recovery shall retain the selected provider. It shall use the existing same-provider retry owner, attempt limit, and backoff. Missing evidence, output, or tool activity shall stop automatic replay and expose manual recovery.
|
||||
- **AC-PLATFORM-PROVIDER-ERROR-RECOVERY-001.14:** A terminal Cursor diagnostic beginning with case-insensitive `Error: RetriableError:` and carrying a non-empty suffix of at most 256 UTF-8 bytes shall classify as transient `agent_transport_lost`. Both layers share Unicode trim and byte bound. `[canceled]` stays eligible; context cancellation, deadline, and retry escalation retain the veto.
|
||||
- **AC-PLATFORM-PROVIDER-ERROR-RECOVERY-001.15:** Kandev shall automatically recover a terminal Cursor `RetriableError` only when current-turn evidence proves that the attempt had no output or tool activity. Eligible recovery shall retain the selected provider. It shall use the existing same-provider retry owner, attempt limit, and backoff. Missing evidence, output, or tool activity shall stop automatic replay and expose manual recovery.
|
||||
- **AC-PLATFORM-PROVIDER-ERROR-RECOVERY-001.16:** When an ACP adapter emits a current-prompt diagnostic message and then returns a matching structured `session/prompt` provider error, Kandev shall retain the diagnostic for the transcript but treat it as pre-result evidence only when both messages classify to the same high-confidence, fallback-eligible semantic code and no later output or tool activity exists. A mismatch, stale generation, low-confidence classification, later output, or tool activity shall keep the diagnostic as ordinary output and stop automatic recovery.
|
||||
|
||||
- **AC-PLATFORM-PROVIDER-ERROR-RECOVERY-001.17:** The catalogue shall carry a raw ACP replay fixture for every required cell of the replay matrix. The gateway enumeration is `claude-direct`, `teamclaude`, `openrouter`, `litellm`. The case enumeration is `matched`, `mismatched`, `later-output`, `queued`, `prose-matched`, `prose-tool`, `prose-output`. The first four cases are required for every gateway. The three `prose-` cases are required only for a gateway the matrix declares classifying, because a prose case asserts nothing when the terminal envelope does not classify. A required cell with no fixture shall fail the fixture test, and a test shall not skip a required cell. Two fixtures sharing a case shall not carry equivalent frames, compared as canonically re-encoded values rather than as raw file bytes.
|
||||
@@ -62,7 +60,7 @@ duplicate transcript rows or stale recovery context.
|
||||
- **AC-PLATFORM-PROVIDER-ERROR-RECOVERY-001.26:** When a supported adapter receives exact structured evidence that the active provider abandoned the current response attempt and will retry it internally, Kandev shall accept a response-attempt retry boundary within that prompt generation. One prompt generation may contain multiple sequential boundaries; each boundary applies only to assistant and thinking records created after the preceding boundary or commit boundary. Malformed evidence, a false retry flag, an unsupported adapter, a zero or stale prompt generation, and terminal provider errors shall not create that boundary or remove transcript content.
|
||||
- **AC-PLATFORM-PROVIDER-ERROR-RECOVERY-001.27:** After Kandev accepts a response-attempt retry boundary, it shall durably remove assistant and thinking messages created by the abandoned attempt. Connected desktop and mobile chats shall remove those messages before later replacement output, and a reload or another viewer shall see only the replacement response.
|
||||
- **AC-PLATFORM-PROVIDER-ERROR-RECOVERY-001.28:** Response-attempt cleanup shall preserve messages, tool activity, permission state, and turns committed before the active provider attempt. A repeated retry boundary with no new output shall be a no-op, and abandoned assistant text that has not crossed a history boundary shall not enter later fallback resume context. The boundary shall not erase prompt-level output or effect evidence used to decide whether a separate Kandev-owned replay is safe.
|
||||
- **AC-PLATFORM-PROVIDER-ERROR-RECOVERY-001.29:** The primary launch prompt shall receive credential-only, unbounded redaction every attempt including attempt 0, never the diagnostic tier reserved for provider output and `FailureReason`.
|
||||
- **AC-PLATFORM-PROVIDER-ERROR-RECOVERY-001.29:** The primary launch prompt shall use credential-only, unbounded redaction on every attempt, including attempt 0. Provider output and `FailureReason` shall use the diagnostic tier.
|
||||
|
||||
## Out of scope
|
||||
|
||||
|
||||
@@ -74,9 +74,8 @@ workspace modes.
|
||||
|
||||
#### Cursor normal-completion failure projection
|
||||
|
||||
`cursor-agent` can report an upstream HTTP/2 stream reset as an ordinary
|
||||
`agent_message_chunk`. It can later return a successful `session/prompt`
|
||||
response. The ACP transport therefore owns a Cursor-specific evidence
|
||||
`cursor-agent` can report an upstream transient failure as an ordinary
|
||||
`agent_message_chunk`; the ACP transport therefore owns a Cursor-specific evidence
|
||||
projection that mirrors the existing Codex capacity projection. It does not add
|
||||
generic content scanning to orchestration.
|
||||
|
||||
@@ -86,16 +85,17 @@ checks in order:
|
||||
1. The adapter identity is `cursor-acp`.
|
||||
2. The normalized event is a non-empty assistant message chunk for a non-zero
|
||||
prompt generation that matches the active turn.
|
||||
3. After trimming leading and trailing whitespace, the chunk begins with
|
||||
`Error: RetriableError:`.
|
||||
4. A case-insensitive bounded match finds `RetriableError` and either
|
||||
`http/2 stream closed` or `CANCEL (0x8)`.
|
||||
3. After trimming leading and trailing whitespace, the chunk begins with the
|
||||
case-insensitive prefix `Error: RetriableError:`.
|
||||
4. The text after that prefix contains a non-empty suffix of at most 256 bytes
|
||||
after Unicode whitespace trimming. It need not describe an HTTP/2 reset;
|
||||
`[unavailable] PING timed out` and `Connection stalled` are valid examples.
|
||||
Context cancellation, deadline, and retry escalation are vetoed.
|
||||
|
||||
The identity, event-type, and prefix checks precede the full fingerprint. The
|
||||
prefix check uses `strings.HasPrefix(strings.TrimSpace(text), ...)`, so ordinary
|
||||
per-token traffic does not allocate a normalized copy. Prose that mentions the
|
||||
error after other text, a partial `RetriableError`, a stale generation, and the
|
||||
same text from another adapter do not match.
|
||||
Identity, event type, and prefix checks precede the suffix check. Both layers
|
||||
share the case-insensitive prefix, Unicode trim, and byte bound. Prose before
|
||||
the prefix, an empty suffix, cancellation signatures, stale generations, and
|
||||
other adapters do not match.
|
||||
|
||||
A match sets pending evidence on the active `promptTurnState` under its existing
|
||||
evidence mutex. The observer suppresses the control chunk. A later non-empty
|
||||
@@ -114,17 +114,15 @@ The valid `ProviderError` uses source `cursor_acp`, provider ID `cursor-acp`, an
|
||||
a UTC occurrence time. The adapter does not copy raw assistant text into the
|
||||
structured diagnostic.
|
||||
|
||||
The deterministic catalogue adds the dedicated rule
|
||||
`cursor.retriable_stream_reset.v1` before the generic transport-loss rule. The
|
||||
rule requires the complete normalized Cursor diagnostic
|
||||
`Error: RetriableError: HTTP/2 stream closed with error code CANCEL (0x8)`
|
||||
(with the existing bracketed `canceled` decoration allowed). It maps to
|
||||
`agent_transport_lost` with high confidence and class `transient`.
|
||||
It retains the existing `AutoRetryable` invariant. The rule applies the shared
|
||||
context-cancellation veto. Cursor's `[canceled]` token does not satisfy that
|
||||
veto. It lacks `context canceled`, `context deadline exceeded`, or
|
||||
`cancel escalated`. The deliberately narrow `transportLostRe` remains
|
||||
unchanged.
|
||||
The existing `cursor.retriable_stream_reset.v1` rule keeps its position before
|
||||
generic transport loss and now accepts any complete normalized Cursor
|
||||
diagnostic with the `Error: RetriableError:` prefix and bounded, non-empty
|
||||
suffix, including the adapter's safe diagnostic and the observed `PING timed
|
||||
out` and `Connection stalled` variants. Its stable ID preserves history. It
|
||||
maps to high-confidence transient `agent_transport_lost`, retains
|
||||
`AutoRetryable`, and applies the existing cancellation veto: Cursor's
|
||||
`[canceled]` does not satisfy it, while context cancellation, deadline, and
|
||||
retry escalation do. The narrow `transportLostRe` remains unchanged.
|
||||
|
||||
### Error classes
|
||||
|
||||
|
||||
Reference in New Issue
Block a user