fix: retain session failures across recovery surfaces (#3669)
This commit is contained in:
@@ -111,23 +111,6 @@ type sessionMetadataKeyStateSetter interface {
|
||||
) (bool, error)
|
||||
}
|
||||
|
||||
// sessionMetadataErrorCASWriter is the optional persistence seam for
|
||||
// execution-scoped bootstrap errors. Production repositories implement both
|
||||
// operations atomically; legacy test stores can fall back to the ordinary
|
||||
// metadata setter after the executor's current-execution fence.
|
||||
type sessionMetadataErrorCASWriter interface {
|
||||
SetSessionMetadataKeyIfAbsent(
|
||||
ctx context.Context,
|
||||
sessionID, key string,
|
||||
value interface{},
|
||||
) (bool, error)
|
||||
SetSessionMetadataKeyIfStamp(
|
||||
ctx context.Context,
|
||||
sessionID, key, expectedStamp string,
|
||||
value interface{},
|
||||
) (bool, error)
|
||||
}
|
||||
|
||||
// bootstrapFailureCommitter is the atomic repository boundary for an
|
||||
// asynchronous process-start failure. The expected state and error stamp are
|
||||
// captured immediately before the commit; the repository must also require
|
||||
@@ -831,6 +814,16 @@ type BootstrapFailureTransitionFunc func(
|
||||
errorValue models.LastAgentError,
|
||||
) (changed bool, finalState models.TaskSessionState, err error)
|
||||
|
||||
// BootstrapFailureMessageRepairFunc retries the idempotent transcript write
|
||||
// after the state admission has already succeeded. The repair is deliberately
|
||||
// separate from the state commit so a transient message-store failure cannot
|
||||
// make an accepted bootstrap failure disappear from session history.
|
||||
type BootstrapFailureMessageRepairFunc func(
|
||||
ctx context.Context,
|
||||
taskID, sessionID, agentExecutionID string,
|
||||
errorValue models.LastAgentError,
|
||||
) error
|
||||
|
||||
// SessionStartingFunc is called when the executor has prepared/resumed an
|
||||
// execution and needs to mark the session STARTING while preserving other
|
||||
// session-row updates such as metadata. expectedState is the state observed
|
||||
@@ -983,6 +976,9 @@ type Executor struct {
|
||||
// typed error, FAILED state, and corresponding publication as one ownership
|
||||
// decision.
|
||||
onBootstrapFailureTransition BootstrapFailureTransitionFunc
|
||||
// Retry hook for the chronological transcript entry when the state commit
|
||||
// succeeds but the first idempotent message write fails.
|
||||
onBootstrapFailureMessageRepair BootstrapFailureMessageRepairFunc
|
||||
|
||||
// Callback for STARTING writes that carry full session-row changes. Set by
|
||||
// the orchestrator so launch/resume/model-switch transitions serialize with
|
||||
@@ -1304,6 +1300,12 @@ func (e *Executor) SetOnBootstrapFailureTransition(fn BootstrapFailureTransition
|
||||
e.onBootstrapFailureTransition = fn
|
||||
}
|
||||
|
||||
// SetOnBootstrapFailureMessageRepair wires the bounded repair hook for a
|
||||
// bootstrap failure whose session admission already succeeded.
|
||||
func (e *Executor) SetOnBootstrapFailureMessageRepair(fn BootstrapFailureMessageRepairFunc) {
|
||||
e.onBootstrapFailureMessageRepair = fn
|
||||
}
|
||||
|
||||
// SetOnSessionStarting sets a callback for full session-row STARTING updates.
|
||||
func (e *Executor) SetOnSessionStarting(fn SessionStartingFunc) {
|
||||
e.onSessionStarting = fn
|
||||
|
||||
@@ -285,6 +285,17 @@ func (e *Executor) handleAgentProcessStartFailure(
|
||||
zap.String("session_id", sessionID),
|
||||
zap.String("agent_execution_id", agentExecutionID),
|
||||
zap.Error(transitionErr))
|
||||
if changed && finalState == models.TaskSessionStateFailed && e.onBootstrapFailureMessageRepair != nil {
|
||||
if repairErr := e.onBootstrapFailureMessageRepair(
|
||||
ctx, taskID, sessionID, agentExecutionID, errorValue,
|
||||
); repairErr != nil {
|
||||
e.logger.Warn("failed to repair bootstrap failure history",
|
||||
zap.String("task_id", taskID),
|
||||
zap.String("session_id", sessionID),
|
||||
zap.String("agent_execution_id", agentExecutionID),
|
||||
zap.Error(repairErr))
|
||||
}
|
||||
}
|
||||
} else if !changed {
|
||||
// An ownership or compare-and-set miss means a newer execution won the
|
||||
// race. Do not
|
||||
|
||||
+72
@@ -170,6 +170,9 @@ func TestBuildLastAgentErrorSanitizesRepositoryPreparationDetails(t *testing.T)
|
||||
if !strings.Contains(errorValue.Details, "repo-back") || !strings.Contains(errorValue.Details, "backend") {
|
||||
t.Fatalf("launch details = %q, want repository identity", errorValue.Details)
|
||||
}
|
||||
if errorValue.Phase != models.LaunchErrorPhaseBootstrap {
|
||||
t.Fatalf("launch phase = %q, want bootstrap", errorValue.Phase)
|
||||
}
|
||||
if strings.Contains(errorValue.Details, "ghp_abcdefghijklmnopqrstuvwxyz1234567890AB") ||
|
||||
strings.Contains(errorValue.Details, "user:") {
|
||||
t.Fatalf("launch details exposed credential-bearing URL: %q", errorValue.Details)
|
||||
@@ -235,6 +238,27 @@ func TestBootstrapFailureProjection(t *testing.T) {
|
||||
exec.SetOnSessionStateChange(func(ctx context.Context, _, sessionID string, state models.TaskSessionState, message string) error {
|
||||
return repo.UpdateTaskSessionState(ctx, sessionID, state, message)
|
||||
})
|
||||
exec.SetOnBootstrapFailureTransition(func(
|
||||
ctx context.Context,
|
||||
taskID, sessionID, _ string,
|
||||
_ models.TaskSessionState,
|
||||
_ string,
|
||||
errorValue models.LastAgentError,
|
||||
) (bool, models.TaskSessionState, error) {
|
||||
changed, _, err := repo.CommitBootstrapFailureIfCurrentExecution(
|
||||
ctx,
|
||||
taskID,
|
||||
sessionID,
|
||||
"exec-1",
|
||||
models.TaskSessionStateStarting,
|
||||
"",
|
||||
errorValue,
|
||||
)
|
||||
if err != nil || !changed {
|
||||
return changed, models.TaskSessionStateStarting, err
|
||||
}
|
||||
return true, models.TaskSessionStateFailed, nil
|
||||
})
|
||||
|
||||
exec.runAgentProcessAsync(
|
||||
context.Background(), "task-1", "session-1", "exec-1",
|
||||
@@ -273,6 +297,54 @@ func TestBootstrapFailureProjection(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestBootstrapFailureHistoryRepairRunsAfterAcceptedCommitError(t *testing.T) {
|
||||
repo := newMockRepository()
|
||||
repo.sessions["session-repair"] = &models.TaskSession{
|
||||
ID: "session-repair",
|
||||
TaskID: "task-repair",
|
||||
State: models.TaskSessionStateStarting,
|
||||
AgentExecutionID: "exec-repair",
|
||||
}
|
||||
repo.tasks["task-repair"] = &models.Task{ID: "task-repair", State: v1.TaskStateReview}
|
||||
exec := newTestExecutor(t, &mockAgentManager{}, repo)
|
||||
var repairCalls int
|
||||
exec.SetOnBootstrapFailureTransition(func(
|
||||
context.Context,
|
||||
string,
|
||||
string,
|
||||
string,
|
||||
models.TaskSessionState,
|
||||
string,
|
||||
models.LastAgentError,
|
||||
) (bool, models.TaskSessionState, error) {
|
||||
return true, models.TaskSessionStateFailed, errors.New("transcript write failed after admission")
|
||||
})
|
||||
exec.SetOnBootstrapFailureMessageRepair(func(
|
||||
context.Context,
|
||||
string,
|
||||
string,
|
||||
string,
|
||||
models.LastAgentError,
|
||||
) error {
|
||||
repairCalls++
|
||||
return nil
|
||||
})
|
||||
|
||||
exec.handleAgentProcessStartFailure(
|
||||
context.Background(),
|
||||
"task-repair",
|
||||
"session-repair",
|
||||
"exec-repair",
|
||||
errors.New("agent process failed to start"),
|
||||
false,
|
||||
false,
|
||||
)
|
||||
|
||||
if repairCalls != 1 {
|
||||
t.Fatalf("bootstrap history repair calls = %d, want 1", repairCalls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBootstrapFailureSuccessorFence(t *testing.T) {
|
||||
repo := newMockRepository()
|
||||
successor := models.LastAgentError{
|
||||
|
||||
@@ -544,6 +544,45 @@ func (m *mockRepository) UpdateTaskSessionStateIfCurrent(
|
||||
return true, now, nil
|
||||
}
|
||||
|
||||
// CommitBootstrapFailureIfCurrentExecution mirrors the production admission
|
||||
// boundary for executor tests. The fake has no separate executor table, so the
|
||||
// session's execution identity is the equivalent ownership fence.
|
||||
func (m *mockRepository) CommitBootstrapFailureIfCurrentExecution(
|
||||
_ context.Context,
|
||||
taskID, sessionID, agentExecutionID string,
|
||||
expectedState models.TaskSessionState,
|
||||
expectedStamp string,
|
||||
errorValue models.LastAgentError,
|
||||
) (bool, time.Time, error) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
session, ok := m.sessions[sessionID]
|
||||
if !ok || session == nil || (session.TaskID != "" && session.TaskID != taskID) || session.State != expectedState {
|
||||
return false, time.Time{}, nil
|
||||
}
|
||||
if agentExecutionID != "" && session.AgentExecutionID != "" && session.AgentExecutionID != agentExecutionID {
|
||||
return false, time.Time{}, nil
|
||||
}
|
||||
current, hasCurrent := models.LoadLastAgentError(session.Metadata)
|
||||
if expectedStamp == "" {
|
||||
if hasCurrent {
|
||||
return false, time.Time{}, nil
|
||||
}
|
||||
} else if !hasCurrent || current.Stamp() != expectedStamp {
|
||||
return false, time.Time{}, nil
|
||||
}
|
||||
if session.Metadata == nil {
|
||||
session.Metadata = make(map[string]interface{})
|
||||
}
|
||||
session.Metadata[models.SessionMetaKeyLastAgentError] = errorValue
|
||||
now := time.Now().UTC()
|
||||
session.State = models.TaskSessionStateFailed
|
||||
session.ErrorMessage = errorValue.Message
|
||||
session.CompletedAt = &now
|
||||
session.UpdatedAt = now
|
||||
return true, now, nil
|
||||
}
|
||||
|
||||
func (m *mockRepository) UpdateTaskSessionIfCurrentStateRemovingMetadataKeys(
|
||||
ctx context.Context,
|
||||
session *models.TaskSession,
|
||||
|
||||
@@ -2135,6 +2135,27 @@ func TestRunAgentProcessAsync_CleansUpOnStartFailure(t *testing.T) {
|
||||
exec.SetOnSessionStateChange(func(ctx context.Context, taskID, sessionID string, state models.TaskSessionState, errorMessage string) error {
|
||||
return repo.UpdateTaskSessionState(ctx, sessionID, state, errorMessage)
|
||||
})
|
||||
exec.SetOnBootstrapFailureTransition(func(
|
||||
ctx context.Context,
|
||||
taskID, sessionID, _ string,
|
||||
_ models.TaskSessionState,
|
||||
_ string,
|
||||
errorValue models.LastAgentError,
|
||||
) (bool, models.TaskSessionState, error) {
|
||||
changed, _, err := repo.CommitBootstrapFailureIfCurrentExecution(
|
||||
ctx,
|
||||
taskID,
|
||||
sessionID,
|
||||
"exec-456",
|
||||
models.TaskSessionStateStarting,
|
||||
"",
|
||||
errorValue,
|
||||
)
|
||||
if err != nil || !changed {
|
||||
return changed, models.TaskSessionStateStarting, err
|
||||
}
|
||||
return true, models.TaskSessionStateFailed, nil
|
||||
})
|
||||
exec.SetOnTaskStateChange(func(ctx context.Context, taskID string, state v1.TaskState) error {
|
||||
return repo.UpdateTaskState(ctx, taskID, state)
|
||||
})
|
||||
@@ -2310,6 +2331,15 @@ func TestHandleAgentProcessStartFailure_CancellationDuringCallbackStopsUnclaimed
|
||||
transitionCalls.Add(1)
|
||||
return false, models.TaskSessionStateCancelled, nil
|
||||
})
|
||||
exec.SetOnBootstrapFailureTransition(func(
|
||||
ctx context.Context,
|
||||
taskID, sessionID, _ string,
|
||||
_ models.TaskSessionState,
|
||||
_ string,
|
||||
errorValue models.LastAgentError,
|
||||
) (bool, models.TaskSessionState, error) {
|
||||
return exec.transitionSessionState(ctx, taskID, sessionID, models.TaskSessionStateFailed, errorValue.Message)
|
||||
})
|
||||
exec.SetOnExecutionCleanupClaim(func(sessionID, executionID string) bool {
|
||||
if sessionID != "session-123" || executionID != "exec-456" {
|
||||
t.Fatalf("cleanup claim = (%q, %q), want (session-123, exec-456)", sessionID, executionID)
|
||||
@@ -2654,6 +2684,30 @@ func newRunAgentProcessAsyncFailureFixture(t *testing.T) *runAgentProcessAsyncFa
|
||||
}
|
||||
return repo.UpdateTaskSessionState(ctx, sessionID, state, errorMessage)
|
||||
})
|
||||
f.exec.SetOnBootstrapFailureTransition(func(
|
||||
ctx context.Context,
|
||||
taskID, sessionID, _ string,
|
||||
_ models.TaskSessionState,
|
||||
_ string,
|
||||
errorValue models.LastAgentError,
|
||||
) (bool, models.TaskSessionState, error) {
|
||||
changed, _, err := repo.CommitBootstrapFailureIfCurrentExecution(
|
||||
ctx,
|
||||
taskID,
|
||||
sessionID,
|
||||
"exec-456",
|
||||
models.TaskSessionStateStarting,
|
||||
"",
|
||||
errorValue,
|
||||
)
|
||||
if changed {
|
||||
f.sessionFailedSeen = true
|
||||
}
|
||||
if err != nil || !changed {
|
||||
return changed, models.TaskSessionStateStarting, err
|
||||
}
|
||||
return true, models.TaskSessionStateFailed, nil
|
||||
})
|
||||
f.exec.SetOnAgentStartFailed(func(ctx context.Context, taskID, sessionID, agentExecutionID string, err error, fromResume bool) bool {
|
||||
f.startFailedCalls++
|
||||
f.lastFromResume = fromResume
|
||||
|
||||
@@ -102,6 +102,8 @@ func (e *Executor) buildLastAgentError(
|
||||
return models.LastAgentError{
|
||||
Message: classification.message,
|
||||
OccurredAt: occurredAt,
|
||||
Scope: models.ErrorScopeSession,
|
||||
Phase: models.LaunchErrorPhaseBootstrap,
|
||||
Code: classification.code,
|
||||
Details: details,
|
||||
RecoveryActions: func() []string {
|
||||
@@ -167,6 +169,7 @@ func (e *Executor) buildBootstrapLastAgentError(
|
||||
return models.LastAgentError{
|
||||
Message: classification.message,
|
||||
OccurredAt: occurredAt,
|
||||
Scope: models.ErrorScopeSession,
|
||||
AgentExecutionID: agentExecutionID,
|
||||
ExecutionID: agentExecutionID,
|
||||
Phase: models.LaunchErrorPhaseBootstrap,
|
||||
@@ -306,8 +309,8 @@ func (e *Executor) bootstrapFailureExpectation(
|
||||
|
||||
// commitBootstrapFailure commits the correlated failure projection and the
|
||||
// FAILED transition through the atomic orchestrator callback or repository
|
||||
// capability. Legacy test stores retain the older CAS plus state-transition
|
||||
// fallback, but the production SQL repository uses the guarded commit.
|
||||
// capability. Stores without an execution-fenced commit fail closed so a
|
||||
// bootstrap failure cannot overwrite a successor execution.
|
||||
func (e *Executor) commitBootstrapFailure(
|
||||
ctx context.Context,
|
||||
taskID, sessionID, agentExecutionID string,
|
||||
@@ -352,43 +355,7 @@ func (e *Executor) commitBootstrapFailure(
|
||||
return true, models.TaskSessionStateFailed, nil
|
||||
}
|
||||
|
||||
// Legacy stores cannot atomically fence the executor row and session write.
|
||||
// Keep their existing behavior for focused test doubles and older adapters;
|
||||
// the production repository always takes the guarded path above.
|
||||
writer, supportsCAS := e.repo.(sessionMetadataErrorCASWriter)
|
||||
var persisted bool
|
||||
if supportsCAS {
|
||||
if expectedStamp != "" {
|
||||
persisted, err = writer.SetSessionMetadataKeyIfStamp(
|
||||
ctx,
|
||||
sessionID,
|
||||
models.SessionMetaKeyLastAgentError,
|
||||
expectedStamp,
|
||||
errorValue,
|
||||
)
|
||||
} else {
|
||||
persisted, err = writer.SetSessionMetadataKeyIfAbsent(
|
||||
ctx,
|
||||
sessionID,
|
||||
models.SessionMetaKeyLastAgentError,
|
||||
errorValue,
|
||||
)
|
||||
}
|
||||
} else {
|
||||
err = e.repo.SetSessionMetadataKey(
|
||||
ctx, sessionID, models.SessionMetaKeyLastAgentError, errorValue,
|
||||
)
|
||||
persisted = err == nil
|
||||
}
|
||||
if err != nil {
|
||||
return false, session.State, err
|
||||
}
|
||||
if !persisted {
|
||||
return false, session.State, nil
|
||||
}
|
||||
|
||||
changed, finalState, err := e.transitionSessionState(
|
||||
ctx, taskID, sessionID, models.TaskSessionStateFailed, errorValue.Message,
|
||||
return false, session.State, fmt.Errorf(
|
||||
"bootstrap failure requires an execution-fenced repository commit",
|
||||
)
|
||||
return changed, finalState, err
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user