fix(backend): unblock Resume for FAILED/CANCELLED task sessions (#670)

This commit is contained in:
Carlos Florêncio
2026-04-21 21:04:35 +01:00
committed by GitHub
parent e61e09cc1e
commit 86b3099631
5 changed files with 346 additions and 11 deletions
@@ -182,6 +182,10 @@ type mockRepository struct {
executors map[string]*models.Executor
executorsRunning map[string]*models.ExecutorRunning
// Optional hook to inject behavior into GetTaskSession (e.g. simulate a
// transient DB error); if nil, the default map lookup is used.
getTaskSessionFunc func(ctx context.Context, id string) (*models.TaskSession, error)
// Track calls for verification
createTaskSessionCalls []*models.TaskSession
updateTaskSessionCalls []*models.TaskSession
@@ -227,6 +231,12 @@ func (m *mockRepository) CreateTaskSession(ctx context.Context, session *models.
}
func (m *mockRepository) GetTaskSession(ctx context.Context, id string) (*models.TaskSession, error) {
m.mu.Lock()
fn := m.getTaskSessionFunc
m.mu.Unlock()
if fn != nil {
return fn(ctx, id)
}
m.mu.Lock()
defer m.mu.Unlock()
if session, ok := m.sessions[id]; ok {
@@ -22,6 +22,14 @@ func isAgentAlreadyRunningError(err error) bool {
return err != nil && strings.Contains(err.Error(), "already has an agent running")
}
// isTerminalSessionState reports whether a session state implies the agent
// process is no longer running. Stale in-memory execution or agentctl status
// for these states should be cleaned up rather than trusted.
func isTerminalSessionState(state models.TaskSessionState) bool {
return state == models.TaskSessionStateFailed ||
state == models.TaskSessionStateCancelled
}
// repoInfo holds resolved repository details for agent launch.
type repoInfo struct {
RepositoryID string
@@ -246,6 +254,18 @@ func (e *Executor) ResumeSession(ctx context.Context, session *models.TaskSessio
zap.String("resume_token", req.ACPSessionID),
zap.Bool("use_worktree", req.UseWorktree))
// Force-cleanup any stale in-memory execution / agentctl state for terminal-state
// sessions. Their agent process is dead by definition, so "already running" signals
// from the execution store or agentctl's "starting" status are stale and would
// otherwise block the relaunch.
if isTerminalSessionState(session.State) {
if cleanupErr := e.agentManager.CleanupStaleExecutionBySessionID(ctx, session.ID); cleanupErr != nil {
e.logger.Warn("failed to force-cleanup stale execution before terminal-state resume",
zap.String("session_id", session.ID),
zap.Error(cleanupErr))
}
}
req.Env = e.applyPreferredShellEnv(ctx, req.ExecutorType, req.Env)
resp, err := e.agentManager.LaunchAgent(ctx, req)
@@ -253,8 +273,11 @@ func (e *Executor) ResumeSession(ctx context.Context, session *models.TaskSessio
// "already has an agent running" fires both for live executions (a concurrent
// resume raced us) and stale ones (agent never started or exited without
// cleanup). Probe liveness before deciding what to do — otherwise we'd kill a
// healthy agent mid-prompt.
if e.agentManager.IsAgentRunningForSession(ctx, session.ID) {
// healthy agent mid-prompt. For terminal states the agent is dead by definition,
// so skip the probe and go straight to cleanup+retry — this avoids a silent
// regression to ErrExecutionAlreadyRunning if the preemptive cleanup above
// failed and agentctl still reports a stale "starting" status.
if !isTerminalSessionState(session.State) && e.agentManager.IsAgentRunningForSession(ctx, session.ID) {
e.logger.Info("resume race: agent already running for session, returning ErrExecutionAlreadyRunning",
zap.String("task_id", task.ID),
zap.String("session_id", session.ID))
@@ -350,9 +373,34 @@ func (e *Executor) validateAndLockResume(ctx context.Context, session *models.Ta
return nil, func() {}, ErrNoAgentProfileID
}
if existing, ok := e.GetExecutionBySession(session.ID); ok && existing != nil {
// Re-read session state after acquiring the lock. The caller fetched the
// session before the lock, so on concurrent resumes the state may be stale —
// the first request could have already transitioned FAILED → STARTING, and
// a stale FAILED state here would wrongly make isTerminalSessionState bypass
// the live-execution guard and cleanup the live agent the first request just
// registered, launching a duplicate. If the re-read fails, abort rather than
// proceeding with uncertain state — silently falling back to the stale state
// would reintroduce the exact race this re-read prevents.
fresh, fetchErr := e.repo.GetTaskSession(ctx, session.ID)
if fetchErr != nil {
unlock()
return nil, func() {}, ErrExecutionAlreadyRunning
e.logger.Warn("failed to re-read session state inside lock; aborting resume to avoid duplicate agent",
zap.String("session_id", session.ID),
zap.Error(fetchErr))
return nil, func() {}, fetchErr
}
if fresh != nil {
session.State = fresh.State
}
// Skip the "already running" rejection for terminal-state sessions — the agent
// process is dead by definition, and ResumeSession will force-cleanup stale
// state before the relaunch.
if !isTerminalSessionState(session.State) {
if existing, ok := e.GetExecutionBySession(session.ID); ok && existing != nil {
unlock()
return nil, func() {}, ErrExecutionAlreadyRunning
}
}
return task, unlock, nil
@@ -290,3 +290,232 @@ func TestResumeSession_StaleExecutionCleansUpAndRetries(t *testing.T) {
t.Errorf("expected LaunchAgent called twice, got %d", agentMgr.launchAgentCallCount)
}
}
// TestResumeSession_FailedStateForceCleansUpStaleState covers the FAILED-task
// resume scenario where agentctl wrongly reports "starting" and the executionStore
// still tracks a stale AgentExecution. The pre-launch cleanup should wipe stale
// state so the fresh LaunchAgent succeeds without hitting the "resume race" path —
// and crucially without probing IsAgentRunningForSession (which would return true
// for the stale status).
func TestResumeSession_FailedStateForceCleansUpStaleState(t *testing.T) {
repo := newMockRepository()
setupLiveResumeTestFixture(repo)
repo.sessions["sess-1"].State = models.TaskSessionStateFailed
agentMgr := &mockAgentManager{
launchAgentFunc: func(_ context.Context, _ *LaunchAgentRequest) (*LaunchAgentResponse, error) {
return &LaunchAgentResponse{
AgentExecutionID: "exec-new",
Status: v1.AgentStatusStarting,
}, nil
},
// Returns true if accidentally called; the assertion below proves it is not.
isAgentRunningForSessionFunc: func(_ context.Context, _ string) bool { return true },
}
exec := newTestExecutor(t, agentMgr, repo)
execution, err := exec.ResumeSession(context.Background(), repo.sessions["sess-1"], true)
if err != nil {
t.Fatalf("expected success, got: %v", err)
}
if execution == nil || execution.AgentExecutionID != "exec-new" {
t.Fatalf("expected fresh execution exec-new, got %+v", execution)
}
if agentMgr.cleanupStaleExecutionCallCount != 1 {
t.Errorf("expected stale-cleanup called exactly once before LaunchAgent, got %d",
agentMgr.cleanupStaleExecutionCallCount)
}
if agentMgr.launchAgentCallCount != 1 {
t.Errorf("expected LaunchAgent called once (no retry), got %d", agentMgr.launchAgentCallCount)
}
// Terminal-state resume must bypass the stale "starting" liveness probe
// entirely — otherwise it would wrongly re-trigger ErrExecutionAlreadyRunning.
if len(agentMgr.isAgentRunningForSessionCallArgs) != 0 {
t.Errorf("expected IsAgentRunningForSession NOT called for terminal-state resume, got %v",
agentMgr.isAgentRunningForSessionCallArgs)
}
}
// TestResumeSession_CancelledStateForceCleansUpStaleState mirrors the FAILED
// scenario for CANCELLED sessions: stale state must not block the relaunch.
func TestResumeSession_CancelledStateForceCleansUpStaleState(t *testing.T) {
repo := newMockRepository()
setupLiveResumeTestFixture(repo)
repo.sessions["sess-1"].State = models.TaskSessionStateCancelled
agentMgr := &mockAgentManager{
launchAgentFunc: func(_ context.Context, _ *LaunchAgentRequest) (*LaunchAgentResponse, error) {
return &LaunchAgentResponse{
AgentExecutionID: "exec-new",
Status: v1.AgentStatusStarting,
}, nil
},
isAgentRunningForSessionFunc: func(_ context.Context, _ string) bool { return true },
}
exec := newTestExecutor(t, agentMgr, repo)
if _, err := exec.ResumeSession(context.Background(), repo.sessions["sess-1"], true); err != nil {
t.Fatalf("expected success, got: %v", err)
}
if agentMgr.cleanupStaleExecutionCallCount != 1 {
t.Errorf("expected stale-cleanup called once, got %d", agentMgr.cleanupStaleExecutionCallCount)
}
if len(agentMgr.isAgentRunningForSessionCallArgs) != 0 {
t.Errorf("expected IsAgentRunningForSession NOT called for CANCELLED resume, got %v",
agentMgr.isAgentRunningForSessionCallArgs)
}
}
// TestResumeSession_TerminalStateSkipsLivenessProbeOnFallback covers the
// residual path where the preemptive CleanupStaleExecutionBySessionID is not
// enough (e.g. the first LaunchAgent still returns "already has an agent running"
// because a stale executionStore entry survived). For terminal-state sessions
// the fallback block MUST skip the IsAgentRunningForSession probe and go
// straight to cleanup+retry, otherwise a stale agentctl "starting" status
// silently regresses to ErrExecutionAlreadyRunning — the original bug.
func TestResumeSession_TerminalStateSkipsLivenessProbeOnFallback(t *testing.T) {
repo := newMockRepository()
setupLiveResumeTestFixture(repo)
repo.sessions["sess-1"].State = models.TaskSessionStateFailed
var launchCalls int
agentMgr := &mockAgentManager{
launchAgentFunc: func(_ context.Context, req *LaunchAgentRequest) (*LaunchAgentResponse, error) {
launchCalls++
if launchCalls == 1 {
return nil, fmt.Errorf("session %q already has an agent running (execution: %s)", req.SessionID, "exec-stale")
}
return &LaunchAgentResponse{
AgentExecutionID: "exec-new",
Status: v1.AgentStatusStarting,
}, nil
},
// Live-probe would return true (stale "starting") — if we wrongly call
// it, the test fails because err is ErrExecutionAlreadyRunning.
isAgentRunningForSessionFunc: func(_ context.Context, _ string) bool { return true },
}
exec := newTestExecutor(t, agentMgr, repo)
execution, err := exec.ResumeSession(context.Background(), repo.sessions["sess-1"], true)
if err != nil {
t.Fatalf("expected success after fallback cleanup+retry, got: %v", err)
}
if execution == nil || execution.AgentExecutionID != "exec-new" {
t.Fatalf("expected fresh execution exec-new, got %+v", execution)
}
if agentMgr.launchAgentCallCount != 2 {
t.Errorf("expected LaunchAgent called twice (first fails, second succeeds), got %d", agentMgr.launchAgentCallCount)
}
// 1 preemptive + 1 fallback cleanup.
if agentMgr.cleanupStaleExecutionCallCount != 2 {
t.Errorf("expected stale-cleanup called twice, got %d", agentMgr.cleanupStaleExecutionCallCount)
}
if len(agentMgr.isAgentRunningForSessionCallArgs) != 0 {
t.Errorf("expected IsAgentRunningForSession NOT called for terminal-state fallback, got %v",
agentMgr.isAgentRunningForSessionCallArgs)
}
}
// TestResumeSession_ConcurrentResumeReReadsFreshState exercises the concurrent-
// resume race: the caller's session object has a stale State=FAILED, but a
// concurrent resume already transitioned the DB to STARTING. validateAndLockResume
// MUST re-read the session state inside the lock — otherwise isTerminalSessionState
// wrongly bypasses the live-execution guard, CleanupStaleExecutionBySessionID
// wipes the live AgentExecution, and LaunchAgent launches a duplicate agent.
func TestResumeSession_ConcurrentResumeReReadsFreshState(t *testing.T) {
repo := newMockRepository()
setupLiveResumeTestFixture(repo)
// Caller's in-memory session carries a stale FAILED state.
callerSession := *repo.sessions["sess-1"]
callerSession.State = models.TaskSessionStateFailed
// DB truth: a concurrent resume already transitioned this session to STARTING.
repo.sessions["sess-1"].State = models.TaskSessionStateStarting
repo.sessions["sess-1"].UpdatedAt = time.Now().UTC()
repo.sessions["sess-1"].AgentExecutionID = "exec-live"
agentMgr := &mockAgentManager{
launchAgentFunc: func(_ context.Context, _ *LaunchAgentRequest) (*LaunchAgentResponse, error) {
t.Fatal("LaunchAgent must not be called — the live execution guard must reject the duplicate resume")
return nil, nil
},
// Live execution was registered by the first resume; probe returns true.
isAgentRunningForSessionFunc: func(_ context.Context, _ string) bool { return true },
}
exec := newTestExecutor(t, agentMgr, repo)
_, err := exec.ResumeSession(context.Background(), &callerSession, true)
if !errors.Is(err, ErrExecutionAlreadyRunning) {
t.Fatalf("expected ErrExecutionAlreadyRunning (live agent protected), got: %v", err)
}
if agentMgr.cleanupStaleExecutionCallCount != 0 {
t.Errorf("cleanup must NOT be called when a live execution is detected, got %d",
agentMgr.cleanupStaleExecutionCallCount)
}
}
// TestResumeSession_AbortsIfSessionReReadFails locks down the abort-on-error
// behavior of the in-lock session re-read. If GetTaskSession fails transiently,
// silently falling back to the caller's (potentially stale) session.State would
// reintroduce the concurrent-resume duplicate-launch race. The resume MUST
// return the fetch error without calling LaunchAgent or CleanupStaleExecutionBySessionID.
func TestResumeSession_AbortsIfSessionReReadFails(t *testing.T) {
repo := newMockRepository()
setupLiveResumeTestFixture(repo)
repo.sessions["sess-1"].State = models.TaskSessionStateFailed
wantErr := fmt.Errorf("transient DB error")
var callCount int
repo.getTaskSessionFunc = func(_ context.Context, _ string) (*models.TaskSession, error) {
callCount++
// First call is the caller's pre-lock fetch (via the service, not
// reached directly in this test). The re-read inside the lock is the
// only GetTaskSession the executor makes, so fail it unconditionally.
return nil, wantErr
}
agentMgr := &mockAgentManager{
launchAgentFunc: func(_ context.Context, _ *LaunchAgentRequest) (*LaunchAgentResponse, error) {
t.Fatal("LaunchAgent must not be called when the session re-read fails")
return nil, nil
},
}
exec := newTestExecutor(t, agentMgr, repo)
_, err := exec.ResumeSession(context.Background(), repo.sessions["sess-1"], true)
if !errors.Is(err, wantErr) {
t.Fatalf("expected transient DB error to be returned, got: %v", err)
}
if agentMgr.launchAgentCallCount != 0 {
t.Errorf("expected LaunchAgent NOT called, got %d", agentMgr.launchAgentCallCount)
}
if agentMgr.cleanupStaleExecutionCallCount != 0 {
t.Errorf("expected CleanupStaleExecutionBySessionID NOT called, got %d",
agentMgr.cleanupStaleExecutionCallCount)
}
if callCount == 0 {
t.Error("expected the in-lock session re-read to be called at least once")
}
}
// TestIsTerminalSessionState is a small unit test for the helper that drives
// both the preemptive cleanup and the validateAndLockResume carve-out.
func TestIsTerminalSessionState(t *testing.T) {
cases := []struct {
state models.TaskSessionState
want bool
}{
{models.TaskSessionStateFailed, true},
{models.TaskSessionStateCancelled, true},
{models.TaskSessionStateWaitingForInput, false},
{models.TaskSessionStateRunning, false},
{models.TaskSessionStateStarting, false},
{models.TaskSessionStateCompleted, false},
}
for _, c := range cases {
if got := isTerminalSessionState(c.state); got != c.want {
t.Errorf("isTerminalSessionState(%q) = %v, want %v", c.state, got, c.want)
}
}
}
@@ -593,14 +593,12 @@ func (s *Service) ResumeTaskSession(ctx context.Context, taskID, sessionID strin
}
// Completed sessions cannot be restarted — they require a new session.
// Failed sessions clear the resume token so the agent starts fresh (the old
// process crashed). Cancelled sessions keep the token so --resume restores
// the conversation when the user re-starts a manually stopped session.
switch session.State {
case models.TaskSessionStateCompleted:
// Failed and cancelled sessions keep the resume token so the relaunched
// agent continues the previous conversation (via ACP session/load for
// native-resume agents, or --resume on CLI). Users who want a fresh start
// after a failure can invoke RecoverSession with action="fresh_start".
if session.State == models.TaskSessionStateCompleted {
return nil, fmt.Errorf("session is completed and cannot be resumed; create a new session instead")
case models.TaskSessionStateFailed:
s.clearResumeToken(ctx, sessionID)
}
// Use context.WithoutCancel to prevent WebSocket request timeout from canceling the resume.
@@ -615,6 +615,56 @@ func TestResumeTaskSession_ArchivedDuringLaunch(t *testing.T) {
}
}
// TestResumeTaskSession_FailedKeepsResumeToken verifies that resuming a FAILED
// session preserves the ACP resume token so the relaunched agent restores the
// prior conversation via ACP session/load (for native-resume agents).
// Regression test for the "Resume blocked by stale state" bug where FAILED sessions
// couldn't be restarted at all; the fix also changes policy to keep the token
// (previously it was cleared to force a fresh agent).
func TestResumeTaskSession_FailedKeepsResumeToken(t *testing.T) {
ctx := context.Background()
repo := setupTestRepo(t)
taskRepo := newMockTaskRepo()
// Agent launch succeeds so the resume path does not unwind and mark the task
// FAILED, which would exercise a separate state-mutation code path.
agentMgr := &mockAgentManager{
launchAgentFunc: func(_ context.Context, _ *executor.LaunchAgentRequest) (*executor.LaunchAgentResponse, error) {
return &executor.LaunchAgentResponse{
AgentExecutionID: "exec-new",
Status: v1.AgentStatusStarting,
}, nil
},
}
svc := createTestServiceWithAgent(repo, newMockStepGetter(), taskRepo, agentMgr)
svc.executor = executor.NewExecutor(agentMgr, repo, testLogger(), executor.ExecutorConfig{})
seedTaskAndSession(t, repo, "task1", "session1", models.TaskSessionStateFailed)
session, _ := repo.GetTaskSession(ctx, "session1")
session.AgentProfileID = "profile-1"
_ = repo.UpdateTaskSession(ctx, session)
now := time.Now().UTC()
_ = repo.UpsertExecutorRunning(ctx, &models.ExecutorRunning{
ID: "er1", SessionID: "session1", TaskID: "task1",
ResumeToken: "acp-session-xyz",
Resumable: true,
CreatedAt: now, UpdatedAt: now,
})
if _, err := svc.ResumeTaskSession(ctx, "task1", "session1"); err != nil {
t.Fatalf("ResumeTaskSession on FAILED session returned: %v", err)
}
er, err := repo.GetExecutorRunningBySessionID(ctx, "session1")
if err != nil || er == nil {
t.Fatalf("ExecutorRunning lookup failed: %v (nil=%v)", err, er == nil)
}
if er.ResumeToken != "acp-session-xyz" {
t.Errorf("expected resume token to be preserved on FAILED resume, got %q", er.ResumeToken)
}
}
// --- CompleteTask ---
func TestCompleteTask_UpdatesTaskState(t *testing.T) {