fix: retain session failures across recovery surfaces (#3669)

This commit is contained in:
Carlos Florêncio
2026-09-15 09:36:47 +01:00
committed by GitHub
parent 54e2e21f32
commit eeeb6f5586
101 changed files with 4057 additions and 1704 deletions
@@ -111,23 +111,6 @@ type sessionMetadataKeyStateSetter interface {
) (bool, error)
}
// sessionMetadataErrorCASWriter is the optional persistence seam for
// execution-scoped bootstrap errors. Production repositories implement both
// operations atomically; legacy test stores can fall back to the ordinary
// metadata setter after the executor's current-execution fence.
type sessionMetadataErrorCASWriter interface {
SetSessionMetadataKeyIfAbsent(
ctx context.Context,
sessionID, key string,
value interface{},
) (bool, error)
SetSessionMetadataKeyIfStamp(
ctx context.Context,
sessionID, key, expectedStamp string,
value interface{},
) (bool, error)
}
// bootstrapFailureCommitter is the atomic repository boundary for an
// asynchronous process-start failure. The expected state and error stamp are
// captured immediately before the commit; the repository must also require
@@ -831,6 +814,16 @@ type BootstrapFailureTransitionFunc func(
errorValue models.LastAgentError,
) (changed bool, finalState models.TaskSessionState, err error)
// BootstrapFailureMessageRepairFunc retries the idempotent transcript write
// after the state admission has already succeeded. The repair is deliberately
// separate from the state commit so a transient message-store failure cannot
// make an accepted bootstrap failure disappear from session history.
type BootstrapFailureMessageRepairFunc func(
ctx context.Context,
taskID, sessionID, agentExecutionID string,
errorValue models.LastAgentError,
) error
// SessionStartingFunc is called when the executor has prepared/resumed an
// execution and needs to mark the session STARTING while preserving other
// session-row updates such as metadata. expectedState is the state observed
@@ -983,6 +976,9 @@ type Executor struct {
// typed error, FAILED state, and corresponding publication as one ownership
// decision.
onBootstrapFailureTransition BootstrapFailureTransitionFunc
// Retry hook for the chronological transcript entry when the state commit
// succeeds but the first idempotent message write fails.
onBootstrapFailureMessageRepair BootstrapFailureMessageRepairFunc
// Callback for STARTING writes that carry full session-row changes. Set by
// the orchestrator so launch/resume/model-switch transitions serialize with
@@ -1304,6 +1300,12 @@ func (e *Executor) SetOnBootstrapFailureTransition(fn BootstrapFailureTransition
e.onBootstrapFailureTransition = fn
}
// SetOnBootstrapFailureMessageRepair wires the bounded repair hook for a
// bootstrap failure whose session admission already succeeded.
func (e *Executor) SetOnBootstrapFailureMessageRepair(fn BootstrapFailureMessageRepairFunc) {
e.onBootstrapFailureMessageRepair = fn
}
// SetOnSessionStarting sets a callback for full session-row STARTING updates.
func (e *Executor) SetOnSessionStarting(fn SessionStartingFunc) {
e.onSessionStarting = fn
@@ -285,6 +285,17 @@ func (e *Executor) handleAgentProcessStartFailure(
zap.String("session_id", sessionID),
zap.String("agent_execution_id", agentExecutionID),
zap.Error(transitionErr))
if changed && finalState == models.TaskSessionStateFailed && e.onBootstrapFailureMessageRepair != nil {
if repairErr := e.onBootstrapFailureMessageRepair(
ctx, taskID, sessionID, agentExecutionID, errorValue,
); repairErr != nil {
e.logger.Warn("failed to repair bootstrap failure history",
zap.String("task_id", taskID),
zap.String("session_id", sessionID),
zap.String("agent_execution_id", agentExecutionID),
zap.Error(repairErr))
}
}
} else if !changed {
// An ownership or compare-and-set miss means a newer execution won the
// race. Do not
@@ -170,6 +170,9 @@ func TestBuildLastAgentErrorSanitizesRepositoryPreparationDetails(t *testing.T)
if !strings.Contains(errorValue.Details, "repo-back") || !strings.Contains(errorValue.Details, "backend") {
t.Fatalf("launch details = %q, want repository identity", errorValue.Details)
}
if errorValue.Phase != models.LaunchErrorPhaseBootstrap {
t.Fatalf("launch phase = %q, want bootstrap", errorValue.Phase)
}
if strings.Contains(errorValue.Details, "ghp_abcdefghijklmnopqrstuvwxyz1234567890AB") ||
strings.Contains(errorValue.Details, "user:") {
t.Fatalf("launch details exposed credential-bearing URL: %q", errorValue.Details)
@@ -235,6 +238,27 @@ func TestBootstrapFailureProjection(t *testing.T) {
exec.SetOnSessionStateChange(func(ctx context.Context, _, sessionID string, state models.TaskSessionState, message string) error {
return repo.UpdateTaskSessionState(ctx, sessionID, state, message)
})
exec.SetOnBootstrapFailureTransition(func(
ctx context.Context,
taskID, sessionID, _ string,
_ models.TaskSessionState,
_ string,
errorValue models.LastAgentError,
) (bool, models.TaskSessionState, error) {
changed, _, err := repo.CommitBootstrapFailureIfCurrentExecution(
ctx,
taskID,
sessionID,
"exec-1",
models.TaskSessionStateStarting,
"",
errorValue,
)
if err != nil || !changed {
return changed, models.TaskSessionStateStarting, err
}
return true, models.TaskSessionStateFailed, nil
})
exec.runAgentProcessAsync(
context.Background(), "task-1", "session-1", "exec-1",
@@ -273,6 +297,54 @@ func TestBootstrapFailureProjection(t *testing.T) {
}
}
func TestBootstrapFailureHistoryRepairRunsAfterAcceptedCommitError(t *testing.T) {
repo := newMockRepository()
repo.sessions["session-repair"] = &models.TaskSession{
ID: "session-repair",
TaskID: "task-repair",
State: models.TaskSessionStateStarting,
AgentExecutionID: "exec-repair",
}
repo.tasks["task-repair"] = &models.Task{ID: "task-repair", State: v1.TaskStateReview}
exec := newTestExecutor(t, &mockAgentManager{}, repo)
var repairCalls int
exec.SetOnBootstrapFailureTransition(func(
context.Context,
string,
string,
string,
models.TaskSessionState,
string,
models.LastAgentError,
) (bool, models.TaskSessionState, error) {
return true, models.TaskSessionStateFailed, errors.New("transcript write failed after admission")
})
exec.SetOnBootstrapFailureMessageRepair(func(
context.Context,
string,
string,
string,
models.LastAgentError,
) error {
repairCalls++
return nil
})
exec.handleAgentProcessStartFailure(
context.Background(),
"task-repair",
"session-repair",
"exec-repair",
errors.New("agent process failed to start"),
false,
false,
)
if repairCalls != 1 {
t.Fatalf("bootstrap history repair calls = %d, want 1", repairCalls)
}
}
func TestBootstrapFailureSuccessorFence(t *testing.T) {
repo := newMockRepository()
successor := models.LastAgentError{
@@ -544,6 +544,45 @@ func (m *mockRepository) UpdateTaskSessionStateIfCurrent(
return true, now, nil
}
// CommitBootstrapFailureIfCurrentExecution mirrors the production admission
// boundary for executor tests. The fake has no separate executor table, so the
// session's execution identity is the equivalent ownership fence.
func (m *mockRepository) CommitBootstrapFailureIfCurrentExecution(
_ context.Context,
taskID, sessionID, agentExecutionID string,
expectedState models.TaskSessionState,
expectedStamp string,
errorValue models.LastAgentError,
) (bool, time.Time, error) {
m.mu.Lock()
defer m.mu.Unlock()
session, ok := m.sessions[sessionID]
if !ok || session == nil || (session.TaskID != "" && session.TaskID != taskID) || session.State != expectedState {
return false, time.Time{}, nil
}
if agentExecutionID != "" && session.AgentExecutionID != "" && session.AgentExecutionID != agentExecutionID {
return false, time.Time{}, nil
}
current, hasCurrent := models.LoadLastAgentError(session.Metadata)
if expectedStamp == "" {
if hasCurrent {
return false, time.Time{}, nil
}
} else if !hasCurrent || current.Stamp() != expectedStamp {
return false, time.Time{}, nil
}
if session.Metadata == nil {
session.Metadata = make(map[string]interface{})
}
session.Metadata[models.SessionMetaKeyLastAgentError] = errorValue
now := time.Now().UTC()
session.State = models.TaskSessionStateFailed
session.ErrorMessage = errorValue.Message
session.CompletedAt = &now
session.UpdatedAt = now
return true, now, nil
}
func (m *mockRepository) UpdateTaskSessionIfCurrentStateRemovingMetadataKeys(
ctx context.Context,
session *models.TaskSession,
@@ -2135,6 +2135,27 @@ func TestRunAgentProcessAsync_CleansUpOnStartFailure(t *testing.T) {
exec.SetOnSessionStateChange(func(ctx context.Context, taskID, sessionID string, state models.TaskSessionState, errorMessage string) error {
return repo.UpdateTaskSessionState(ctx, sessionID, state, errorMessage)
})
exec.SetOnBootstrapFailureTransition(func(
ctx context.Context,
taskID, sessionID, _ string,
_ models.TaskSessionState,
_ string,
errorValue models.LastAgentError,
) (bool, models.TaskSessionState, error) {
changed, _, err := repo.CommitBootstrapFailureIfCurrentExecution(
ctx,
taskID,
sessionID,
"exec-456",
models.TaskSessionStateStarting,
"",
errorValue,
)
if err != nil || !changed {
return changed, models.TaskSessionStateStarting, err
}
return true, models.TaskSessionStateFailed, nil
})
exec.SetOnTaskStateChange(func(ctx context.Context, taskID string, state v1.TaskState) error {
return repo.UpdateTaskState(ctx, taskID, state)
})
@@ -2310,6 +2331,15 @@ func TestHandleAgentProcessStartFailure_CancellationDuringCallbackStopsUnclaimed
transitionCalls.Add(1)
return false, models.TaskSessionStateCancelled, nil
})
exec.SetOnBootstrapFailureTransition(func(
ctx context.Context,
taskID, sessionID, _ string,
_ models.TaskSessionState,
_ string,
errorValue models.LastAgentError,
) (bool, models.TaskSessionState, error) {
return exec.transitionSessionState(ctx, taskID, sessionID, models.TaskSessionStateFailed, errorValue.Message)
})
exec.SetOnExecutionCleanupClaim(func(sessionID, executionID string) bool {
if sessionID != "session-123" || executionID != "exec-456" {
t.Fatalf("cleanup claim = (%q, %q), want (session-123, exec-456)", sessionID, executionID)
@@ -2654,6 +2684,30 @@ func newRunAgentProcessAsyncFailureFixture(t *testing.T) *runAgentProcessAsyncFa
}
return repo.UpdateTaskSessionState(ctx, sessionID, state, errorMessage)
})
f.exec.SetOnBootstrapFailureTransition(func(
ctx context.Context,
taskID, sessionID, _ string,
_ models.TaskSessionState,
_ string,
errorValue models.LastAgentError,
) (bool, models.TaskSessionState, error) {
changed, _, err := repo.CommitBootstrapFailureIfCurrentExecution(
ctx,
taskID,
sessionID,
"exec-456",
models.TaskSessionStateStarting,
"",
errorValue,
)
if changed {
f.sessionFailedSeen = true
}
if err != nil || !changed {
return changed, models.TaskSessionStateStarting, err
}
return true, models.TaskSessionStateFailed, nil
})
f.exec.SetOnAgentStartFailed(func(ctx context.Context, taskID, sessionID, agentExecutionID string, err error, fromResume bool) bool {
f.startFailedCalls++
f.lastFromResume = fromResume
@@ -102,6 +102,8 @@ func (e *Executor) buildLastAgentError(
return models.LastAgentError{
Message: classification.message,
OccurredAt: occurredAt,
Scope: models.ErrorScopeSession,
Phase: models.LaunchErrorPhaseBootstrap,
Code: classification.code,
Details: details,
RecoveryActions: func() []string {
@@ -167,6 +169,7 @@ func (e *Executor) buildBootstrapLastAgentError(
return models.LastAgentError{
Message: classification.message,
OccurredAt: occurredAt,
Scope: models.ErrorScopeSession,
AgentExecutionID: agentExecutionID,
ExecutionID: agentExecutionID,
Phase: models.LaunchErrorPhaseBootstrap,
@@ -306,8 +309,8 @@ func (e *Executor) bootstrapFailureExpectation(
// commitBootstrapFailure commits the correlated failure projection and the
// FAILED transition through the atomic orchestrator callback or repository
// capability. Legacy test stores retain the older CAS plus state-transition
// fallback, but the production SQL repository uses the guarded commit.
// capability. Stores without an execution-fenced commit fail closed so a
// bootstrap failure cannot overwrite a successor execution.
func (e *Executor) commitBootstrapFailure(
ctx context.Context,
taskID, sessionID, agentExecutionID string,
@@ -352,43 +355,7 @@ func (e *Executor) commitBootstrapFailure(
return true, models.TaskSessionStateFailed, nil
}
// Legacy stores cannot atomically fence the executor row and session write.
// Keep their existing behavior for focused test doubles and older adapters;
// the production repository always takes the guarded path above.
writer, supportsCAS := e.repo.(sessionMetadataErrorCASWriter)
var persisted bool
if supportsCAS {
if expectedStamp != "" {
persisted, err = writer.SetSessionMetadataKeyIfStamp(
ctx,
sessionID,
models.SessionMetaKeyLastAgentError,
expectedStamp,
errorValue,
)
} else {
persisted, err = writer.SetSessionMetadataKeyIfAbsent(
ctx,
sessionID,
models.SessionMetaKeyLastAgentError,
errorValue,
)
}
} else {
err = e.repo.SetSessionMetadataKey(
ctx, sessionID, models.SessionMetaKeyLastAgentError, errorValue,
)
persisted = err == nil
}
if err != nil {
return false, session.State, err
}
if !persisted {
return false, session.State, nil
}
changed, finalState, err := e.transitionSessionState(
ctx, taskID, sessionID, models.TaskSessionStateFailed, errorValue.Message,
return false, session.State, fmt.Errorf(
"bootstrap failure requires an execution-fenced repository commit",
)
return changed, finalState, err
}