From 86b3099631dc4c7f88691662c071fb0715b60dfd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Flor=C3=AAncio?= Date: Tue, 21 Apr 2026 21:04:35 +0100 Subject: [PATCH] fix(backend): unblock Resume for FAILED/CANCELLED task sessions (#670) --- .../executor/executor_mocks_test.go | 10 + .../orchestrator/executor/executor_resume.go | 56 ++++- .../executor/executor_resume_test.go | 229 ++++++++++++++++++ .../internal/orchestrator/task_operations.go | 12 +- .../orchestrator/task_operations_test.go | 50 ++++ 5 files changed, 346 insertions(+), 11 deletions(-) diff --git a/apps/backend/internal/orchestrator/executor/executor_mocks_test.go b/apps/backend/internal/orchestrator/executor/executor_mocks_test.go index 79b2bf81b4..991cc2c95b 100644 --- a/apps/backend/internal/orchestrator/executor/executor_mocks_test.go +++ b/apps/backend/internal/orchestrator/executor/executor_mocks_test.go @@ -182,6 +182,10 @@ type mockRepository struct { executors map[string]*models.Executor executorsRunning map[string]*models.ExecutorRunning + // Optional hook to inject behavior into GetTaskSession (e.g. simulate a + // transient DB error); if nil, the default map lookup is used. + getTaskSessionFunc func(ctx context.Context, id string) (*models.TaskSession, error) + // Track calls for verification createTaskSessionCalls []*models.TaskSession updateTaskSessionCalls []*models.TaskSession @@ -227,6 +231,12 @@ func (m *mockRepository) CreateTaskSession(ctx context.Context, session *models. } func (m *mockRepository) GetTaskSession(ctx context.Context, id string) (*models.TaskSession, error) { + m.mu.Lock() + fn := m.getTaskSessionFunc + m.mu.Unlock() + if fn != nil { + return fn(ctx, id) + } m.mu.Lock() defer m.mu.Unlock() if session, ok := m.sessions[id]; ok { diff --git a/apps/backend/internal/orchestrator/executor/executor_resume.go b/apps/backend/internal/orchestrator/executor/executor_resume.go index 1321560fc3..d48a20fd8b 100644 --- a/apps/backend/internal/orchestrator/executor/executor_resume.go +++ b/apps/backend/internal/orchestrator/executor/executor_resume.go @@ -22,6 +22,14 @@ func isAgentAlreadyRunningError(err error) bool { return err != nil && strings.Contains(err.Error(), "already has an agent running") } +// isTerminalSessionState reports whether a session state implies the agent +// process is no longer running. Stale in-memory execution or agentctl status +// for these states should be cleaned up rather than trusted. +func isTerminalSessionState(state models.TaskSessionState) bool { + return state == models.TaskSessionStateFailed || + state == models.TaskSessionStateCancelled +} + // repoInfo holds resolved repository details for agent launch. type repoInfo struct { RepositoryID string @@ -246,6 +254,18 @@ func (e *Executor) ResumeSession(ctx context.Context, session *models.TaskSessio zap.String("resume_token", req.ACPSessionID), zap.Bool("use_worktree", req.UseWorktree)) + // Force-cleanup any stale in-memory execution / agentctl state for terminal-state + // sessions. Their agent process is dead by definition, so "already running" signals + // from the execution store or agentctl's "starting" status are stale and would + // otherwise block the relaunch. + if isTerminalSessionState(session.State) { + if cleanupErr := e.agentManager.CleanupStaleExecutionBySessionID(ctx, session.ID); cleanupErr != nil { + e.logger.Warn("failed to force-cleanup stale execution before terminal-state resume", + zap.String("session_id", session.ID), + zap.Error(cleanupErr)) + } + } + req.Env = e.applyPreferredShellEnv(ctx, req.ExecutorType, req.Env) resp, err := e.agentManager.LaunchAgent(ctx, req) @@ -253,8 +273,11 @@ func (e *Executor) ResumeSession(ctx context.Context, session *models.TaskSessio // "already has an agent running" fires both for live executions (a concurrent // resume raced us) and stale ones (agent never started or exited without // cleanup). Probe liveness before deciding what to do — otherwise we'd kill a - // healthy agent mid-prompt. - if e.agentManager.IsAgentRunningForSession(ctx, session.ID) { + // healthy agent mid-prompt. For terminal states the agent is dead by definition, + // so skip the probe and go straight to cleanup+retry — this avoids a silent + // regression to ErrExecutionAlreadyRunning if the preemptive cleanup above + // failed and agentctl still reports a stale "starting" status. + if !isTerminalSessionState(session.State) && e.agentManager.IsAgentRunningForSession(ctx, session.ID) { e.logger.Info("resume race: agent already running for session, returning ErrExecutionAlreadyRunning", zap.String("task_id", task.ID), zap.String("session_id", session.ID)) @@ -350,9 +373,34 @@ func (e *Executor) validateAndLockResume(ctx context.Context, session *models.Ta return nil, func() {}, ErrNoAgentProfileID } - if existing, ok := e.GetExecutionBySession(session.ID); ok && existing != nil { + // Re-read session state after acquiring the lock. The caller fetched the + // session before the lock, so on concurrent resumes the state may be stale — + // the first request could have already transitioned FAILED → STARTING, and + // a stale FAILED state here would wrongly make isTerminalSessionState bypass + // the live-execution guard and cleanup the live agent the first request just + // registered, launching a duplicate. If the re-read fails, abort rather than + // proceeding with uncertain state — silently falling back to the stale state + // would reintroduce the exact race this re-read prevents. + fresh, fetchErr := e.repo.GetTaskSession(ctx, session.ID) + if fetchErr != nil { unlock() - return nil, func() {}, ErrExecutionAlreadyRunning + e.logger.Warn("failed to re-read session state inside lock; aborting resume to avoid duplicate agent", + zap.String("session_id", session.ID), + zap.Error(fetchErr)) + return nil, func() {}, fetchErr + } + if fresh != nil { + session.State = fresh.State + } + + // Skip the "already running" rejection for terminal-state sessions — the agent + // process is dead by definition, and ResumeSession will force-cleanup stale + // state before the relaunch. + if !isTerminalSessionState(session.State) { + if existing, ok := e.GetExecutionBySession(session.ID); ok && existing != nil { + unlock() + return nil, func() {}, ErrExecutionAlreadyRunning + } } return task, unlock, nil diff --git a/apps/backend/internal/orchestrator/executor/executor_resume_test.go b/apps/backend/internal/orchestrator/executor/executor_resume_test.go index 37114724b0..2765a41b7d 100644 --- a/apps/backend/internal/orchestrator/executor/executor_resume_test.go +++ b/apps/backend/internal/orchestrator/executor/executor_resume_test.go @@ -290,3 +290,232 @@ func TestResumeSession_StaleExecutionCleansUpAndRetries(t *testing.T) { t.Errorf("expected LaunchAgent called twice, got %d", agentMgr.launchAgentCallCount) } } + +// TestResumeSession_FailedStateForceCleansUpStaleState covers the FAILED-task +// resume scenario where agentctl wrongly reports "starting" and the executionStore +// still tracks a stale AgentExecution. The pre-launch cleanup should wipe stale +// state so the fresh LaunchAgent succeeds without hitting the "resume race" path — +// and crucially without probing IsAgentRunningForSession (which would return true +// for the stale status). +func TestResumeSession_FailedStateForceCleansUpStaleState(t *testing.T) { + repo := newMockRepository() + setupLiveResumeTestFixture(repo) + repo.sessions["sess-1"].State = models.TaskSessionStateFailed + + agentMgr := &mockAgentManager{ + launchAgentFunc: func(_ context.Context, _ *LaunchAgentRequest) (*LaunchAgentResponse, error) { + return &LaunchAgentResponse{ + AgentExecutionID: "exec-new", + Status: v1.AgentStatusStarting, + }, nil + }, + // Returns true if accidentally called; the assertion below proves it is not. + isAgentRunningForSessionFunc: func(_ context.Context, _ string) bool { return true }, + } + exec := newTestExecutor(t, agentMgr, repo) + + execution, err := exec.ResumeSession(context.Background(), repo.sessions["sess-1"], true) + if err != nil { + t.Fatalf("expected success, got: %v", err) + } + if execution == nil || execution.AgentExecutionID != "exec-new" { + t.Fatalf("expected fresh execution exec-new, got %+v", execution) + } + if agentMgr.cleanupStaleExecutionCallCount != 1 { + t.Errorf("expected stale-cleanup called exactly once before LaunchAgent, got %d", + agentMgr.cleanupStaleExecutionCallCount) + } + if agentMgr.launchAgentCallCount != 1 { + t.Errorf("expected LaunchAgent called once (no retry), got %d", agentMgr.launchAgentCallCount) + } + // Terminal-state resume must bypass the stale "starting" liveness probe + // entirely — otherwise it would wrongly re-trigger ErrExecutionAlreadyRunning. + if len(agentMgr.isAgentRunningForSessionCallArgs) != 0 { + t.Errorf("expected IsAgentRunningForSession NOT called for terminal-state resume, got %v", + agentMgr.isAgentRunningForSessionCallArgs) + } +} + +// TestResumeSession_CancelledStateForceCleansUpStaleState mirrors the FAILED +// scenario for CANCELLED sessions: stale state must not block the relaunch. +func TestResumeSession_CancelledStateForceCleansUpStaleState(t *testing.T) { + repo := newMockRepository() + setupLiveResumeTestFixture(repo) + repo.sessions["sess-1"].State = models.TaskSessionStateCancelled + + agentMgr := &mockAgentManager{ + launchAgentFunc: func(_ context.Context, _ *LaunchAgentRequest) (*LaunchAgentResponse, error) { + return &LaunchAgentResponse{ + AgentExecutionID: "exec-new", + Status: v1.AgentStatusStarting, + }, nil + }, + isAgentRunningForSessionFunc: func(_ context.Context, _ string) bool { return true }, + } + exec := newTestExecutor(t, agentMgr, repo) + + if _, err := exec.ResumeSession(context.Background(), repo.sessions["sess-1"], true); err != nil { + t.Fatalf("expected success, got: %v", err) + } + if agentMgr.cleanupStaleExecutionCallCount != 1 { + t.Errorf("expected stale-cleanup called once, got %d", agentMgr.cleanupStaleExecutionCallCount) + } + if len(agentMgr.isAgentRunningForSessionCallArgs) != 0 { + t.Errorf("expected IsAgentRunningForSession NOT called for CANCELLED resume, got %v", + agentMgr.isAgentRunningForSessionCallArgs) + } +} + +// TestResumeSession_TerminalStateSkipsLivenessProbeOnFallback covers the +// residual path where the preemptive CleanupStaleExecutionBySessionID is not +// enough (e.g. the first LaunchAgent still returns "already has an agent running" +// because a stale executionStore entry survived). For terminal-state sessions +// the fallback block MUST skip the IsAgentRunningForSession probe and go +// straight to cleanup+retry, otherwise a stale agentctl "starting" status +// silently regresses to ErrExecutionAlreadyRunning — the original bug. +func TestResumeSession_TerminalStateSkipsLivenessProbeOnFallback(t *testing.T) { + repo := newMockRepository() + setupLiveResumeTestFixture(repo) + repo.sessions["sess-1"].State = models.TaskSessionStateFailed + + var launchCalls int + agentMgr := &mockAgentManager{ + launchAgentFunc: func(_ context.Context, req *LaunchAgentRequest) (*LaunchAgentResponse, error) { + launchCalls++ + if launchCalls == 1 { + return nil, fmt.Errorf("session %q already has an agent running (execution: %s)", req.SessionID, "exec-stale") + } + return &LaunchAgentResponse{ + AgentExecutionID: "exec-new", + Status: v1.AgentStatusStarting, + }, nil + }, + // Live-probe would return true (stale "starting") — if we wrongly call + // it, the test fails because err is ErrExecutionAlreadyRunning. + isAgentRunningForSessionFunc: func(_ context.Context, _ string) bool { return true }, + } + exec := newTestExecutor(t, agentMgr, repo) + + execution, err := exec.ResumeSession(context.Background(), repo.sessions["sess-1"], true) + if err != nil { + t.Fatalf("expected success after fallback cleanup+retry, got: %v", err) + } + if execution == nil || execution.AgentExecutionID != "exec-new" { + t.Fatalf("expected fresh execution exec-new, got %+v", execution) + } + if agentMgr.launchAgentCallCount != 2 { + t.Errorf("expected LaunchAgent called twice (first fails, second succeeds), got %d", agentMgr.launchAgentCallCount) + } + // 1 preemptive + 1 fallback cleanup. + if agentMgr.cleanupStaleExecutionCallCount != 2 { + t.Errorf("expected stale-cleanup called twice, got %d", agentMgr.cleanupStaleExecutionCallCount) + } + if len(agentMgr.isAgentRunningForSessionCallArgs) != 0 { + t.Errorf("expected IsAgentRunningForSession NOT called for terminal-state fallback, got %v", + agentMgr.isAgentRunningForSessionCallArgs) + } +} + +// TestResumeSession_ConcurrentResumeReReadsFreshState exercises the concurrent- +// resume race: the caller's session object has a stale State=FAILED, but a +// concurrent resume already transitioned the DB to STARTING. validateAndLockResume +// MUST re-read the session state inside the lock — otherwise isTerminalSessionState +// wrongly bypasses the live-execution guard, CleanupStaleExecutionBySessionID +// wipes the live AgentExecution, and LaunchAgent launches a duplicate agent. +func TestResumeSession_ConcurrentResumeReReadsFreshState(t *testing.T) { + repo := newMockRepository() + setupLiveResumeTestFixture(repo) + + // Caller's in-memory session carries a stale FAILED state. + callerSession := *repo.sessions["sess-1"] + callerSession.State = models.TaskSessionStateFailed + + // DB truth: a concurrent resume already transitioned this session to STARTING. + repo.sessions["sess-1"].State = models.TaskSessionStateStarting + repo.sessions["sess-1"].UpdatedAt = time.Now().UTC() + repo.sessions["sess-1"].AgentExecutionID = "exec-live" + + agentMgr := &mockAgentManager{ + launchAgentFunc: func(_ context.Context, _ *LaunchAgentRequest) (*LaunchAgentResponse, error) { + t.Fatal("LaunchAgent must not be called — the live execution guard must reject the duplicate resume") + return nil, nil + }, + // Live execution was registered by the first resume; probe returns true. + isAgentRunningForSessionFunc: func(_ context.Context, _ string) bool { return true }, + } + exec := newTestExecutor(t, agentMgr, repo) + + _, err := exec.ResumeSession(context.Background(), &callerSession, true) + if !errors.Is(err, ErrExecutionAlreadyRunning) { + t.Fatalf("expected ErrExecutionAlreadyRunning (live agent protected), got: %v", err) + } + if agentMgr.cleanupStaleExecutionCallCount != 0 { + t.Errorf("cleanup must NOT be called when a live execution is detected, got %d", + agentMgr.cleanupStaleExecutionCallCount) + } +} + +// TestResumeSession_AbortsIfSessionReReadFails locks down the abort-on-error +// behavior of the in-lock session re-read. If GetTaskSession fails transiently, +// silently falling back to the caller's (potentially stale) session.State would +// reintroduce the concurrent-resume duplicate-launch race. The resume MUST +// return the fetch error without calling LaunchAgent or CleanupStaleExecutionBySessionID. +func TestResumeSession_AbortsIfSessionReReadFails(t *testing.T) { + repo := newMockRepository() + setupLiveResumeTestFixture(repo) + repo.sessions["sess-1"].State = models.TaskSessionStateFailed + + wantErr := fmt.Errorf("transient DB error") + var callCount int + repo.getTaskSessionFunc = func(_ context.Context, _ string) (*models.TaskSession, error) { + callCount++ + // First call is the caller's pre-lock fetch (via the service, not + // reached directly in this test). The re-read inside the lock is the + // only GetTaskSession the executor makes, so fail it unconditionally. + return nil, wantErr + } + + agentMgr := &mockAgentManager{ + launchAgentFunc: func(_ context.Context, _ *LaunchAgentRequest) (*LaunchAgentResponse, error) { + t.Fatal("LaunchAgent must not be called when the session re-read fails") + return nil, nil + }, + } + exec := newTestExecutor(t, agentMgr, repo) + + _, err := exec.ResumeSession(context.Background(), repo.sessions["sess-1"], true) + if !errors.Is(err, wantErr) { + t.Fatalf("expected transient DB error to be returned, got: %v", err) + } + if agentMgr.launchAgentCallCount != 0 { + t.Errorf("expected LaunchAgent NOT called, got %d", agentMgr.launchAgentCallCount) + } + if agentMgr.cleanupStaleExecutionCallCount != 0 { + t.Errorf("expected CleanupStaleExecutionBySessionID NOT called, got %d", + agentMgr.cleanupStaleExecutionCallCount) + } + if callCount == 0 { + t.Error("expected the in-lock session re-read to be called at least once") + } +} + +// TestIsTerminalSessionState is a small unit test for the helper that drives +// both the preemptive cleanup and the validateAndLockResume carve-out. +func TestIsTerminalSessionState(t *testing.T) { + cases := []struct { + state models.TaskSessionState + want bool + }{ + {models.TaskSessionStateFailed, true}, + {models.TaskSessionStateCancelled, true}, + {models.TaskSessionStateWaitingForInput, false}, + {models.TaskSessionStateRunning, false}, + {models.TaskSessionStateStarting, false}, + {models.TaskSessionStateCompleted, false}, + } + for _, c := range cases { + if got := isTerminalSessionState(c.state); got != c.want { + t.Errorf("isTerminalSessionState(%q) = %v, want %v", c.state, got, c.want) + } + } +} diff --git a/apps/backend/internal/orchestrator/task_operations.go b/apps/backend/internal/orchestrator/task_operations.go index 46ceaf6698..ddb12ddd94 100644 --- a/apps/backend/internal/orchestrator/task_operations.go +++ b/apps/backend/internal/orchestrator/task_operations.go @@ -593,14 +593,12 @@ func (s *Service) ResumeTaskSession(ctx context.Context, taskID, sessionID strin } // Completed sessions cannot be restarted — they require a new session. - // Failed sessions clear the resume token so the agent starts fresh (the old - // process crashed). Cancelled sessions keep the token so --resume restores - // the conversation when the user re-starts a manually stopped session. - switch session.State { - case models.TaskSessionStateCompleted: + // Failed and cancelled sessions keep the resume token so the relaunched + // agent continues the previous conversation (via ACP session/load for + // native-resume agents, or --resume on CLI). Users who want a fresh start + // after a failure can invoke RecoverSession with action="fresh_start". + if session.State == models.TaskSessionStateCompleted { return nil, fmt.Errorf("session is completed and cannot be resumed; create a new session instead") - case models.TaskSessionStateFailed: - s.clearResumeToken(ctx, sessionID) } // Use context.WithoutCancel to prevent WebSocket request timeout from canceling the resume. diff --git a/apps/backend/internal/orchestrator/task_operations_test.go b/apps/backend/internal/orchestrator/task_operations_test.go index 3cac7cf512..031f2f2ff3 100644 --- a/apps/backend/internal/orchestrator/task_operations_test.go +++ b/apps/backend/internal/orchestrator/task_operations_test.go @@ -615,6 +615,56 @@ func TestResumeTaskSession_ArchivedDuringLaunch(t *testing.T) { } } +// TestResumeTaskSession_FailedKeepsResumeToken verifies that resuming a FAILED +// session preserves the ACP resume token so the relaunched agent restores the +// prior conversation via ACP session/load (for native-resume agents). +// Regression test for the "Resume blocked by stale state" bug where FAILED sessions +// couldn't be restarted at all; the fix also changes policy to keep the token +// (previously it was cleared to force a fresh agent). +func TestResumeTaskSession_FailedKeepsResumeToken(t *testing.T) { + ctx := context.Background() + repo := setupTestRepo(t) + taskRepo := newMockTaskRepo() + + // Agent launch succeeds so the resume path does not unwind and mark the task + // FAILED, which would exercise a separate state-mutation code path. + agentMgr := &mockAgentManager{ + launchAgentFunc: func(_ context.Context, _ *executor.LaunchAgentRequest) (*executor.LaunchAgentResponse, error) { + return &executor.LaunchAgentResponse{ + AgentExecutionID: "exec-new", + Status: v1.AgentStatusStarting, + }, nil + }, + } + svc := createTestServiceWithAgent(repo, newMockStepGetter(), taskRepo, agentMgr) + svc.executor = executor.NewExecutor(agentMgr, repo, testLogger(), executor.ExecutorConfig{}) + + seedTaskAndSession(t, repo, "task1", "session1", models.TaskSessionStateFailed) + session, _ := repo.GetTaskSession(ctx, "session1") + session.AgentProfileID = "profile-1" + _ = repo.UpdateTaskSession(ctx, session) + + now := time.Now().UTC() + _ = repo.UpsertExecutorRunning(ctx, &models.ExecutorRunning{ + ID: "er1", SessionID: "session1", TaskID: "task1", + ResumeToken: "acp-session-xyz", + Resumable: true, + CreatedAt: now, UpdatedAt: now, + }) + + if _, err := svc.ResumeTaskSession(ctx, "task1", "session1"); err != nil { + t.Fatalf("ResumeTaskSession on FAILED session returned: %v", err) + } + + er, err := repo.GetExecutorRunningBySessionID(ctx, "session1") + if err != nil || er == nil { + t.Fatalf("ExecutorRunning lookup failed: %v (nil=%v)", err, er == nil) + } + if er.ResumeToken != "acp-session-xyz" { + t.Errorf("expected resume token to be preserved on FAILED resume, got %q", er.ResumeToken) + } +} + // --- CompleteTask --- func TestCompleteTask_UpdatesTaskState(t *testing.T) {