fix(backend): unblock Resume for FAILED/CANCELLED task sessions (#670)
This commit is contained in:
@@ -182,6 +182,10 @@ type mockRepository struct {
|
||||
executors map[string]*models.Executor
|
||||
executorsRunning map[string]*models.ExecutorRunning
|
||||
|
||||
// Optional hook to inject behavior into GetTaskSession (e.g. simulate a
|
||||
// transient DB error); if nil, the default map lookup is used.
|
||||
getTaskSessionFunc func(ctx context.Context, id string) (*models.TaskSession, error)
|
||||
|
||||
// Track calls for verification
|
||||
createTaskSessionCalls []*models.TaskSession
|
||||
updateTaskSessionCalls []*models.TaskSession
|
||||
@@ -227,6 +231,12 @@ func (m *mockRepository) CreateTaskSession(ctx context.Context, session *models.
|
||||
}
|
||||
|
||||
func (m *mockRepository) GetTaskSession(ctx context.Context, id string) (*models.TaskSession, error) {
|
||||
m.mu.Lock()
|
||||
fn := m.getTaskSessionFunc
|
||||
m.mu.Unlock()
|
||||
if fn != nil {
|
||||
return fn(ctx, id)
|
||||
}
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
if session, ok := m.sessions[id]; ok {
|
||||
|
||||
@@ -22,6 +22,14 @@ func isAgentAlreadyRunningError(err error) bool {
|
||||
return err != nil && strings.Contains(err.Error(), "already has an agent running")
|
||||
}
|
||||
|
||||
// isTerminalSessionState reports whether a session state implies the agent
|
||||
// process is no longer running. Stale in-memory execution or agentctl status
|
||||
// for these states should be cleaned up rather than trusted.
|
||||
func isTerminalSessionState(state models.TaskSessionState) bool {
|
||||
return state == models.TaskSessionStateFailed ||
|
||||
state == models.TaskSessionStateCancelled
|
||||
}
|
||||
|
||||
// repoInfo holds resolved repository details for agent launch.
|
||||
type repoInfo struct {
|
||||
RepositoryID string
|
||||
@@ -246,6 +254,18 @@ func (e *Executor) ResumeSession(ctx context.Context, session *models.TaskSessio
|
||||
zap.String("resume_token", req.ACPSessionID),
|
||||
zap.Bool("use_worktree", req.UseWorktree))
|
||||
|
||||
// Force-cleanup any stale in-memory execution / agentctl state for terminal-state
|
||||
// sessions. Their agent process is dead by definition, so "already running" signals
|
||||
// from the execution store or agentctl's "starting" status are stale and would
|
||||
// otherwise block the relaunch.
|
||||
if isTerminalSessionState(session.State) {
|
||||
if cleanupErr := e.agentManager.CleanupStaleExecutionBySessionID(ctx, session.ID); cleanupErr != nil {
|
||||
e.logger.Warn("failed to force-cleanup stale execution before terminal-state resume",
|
||||
zap.String("session_id", session.ID),
|
||||
zap.Error(cleanupErr))
|
||||
}
|
||||
}
|
||||
|
||||
req.Env = e.applyPreferredShellEnv(ctx, req.ExecutorType, req.Env)
|
||||
|
||||
resp, err := e.agentManager.LaunchAgent(ctx, req)
|
||||
@@ -253,8 +273,11 @@ func (e *Executor) ResumeSession(ctx context.Context, session *models.TaskSessio
|
||||
// "already has an agent running" fires both for live executions (a concurrent
|
||||
// resume raced us) and stale ones (agent never started or exited without
|
||||
// cleanup). Probe liveness before deciding what to do — otherwise we'd kill a
|
||||
// healthy agent mid-prompt.
|
||||
if e.agentManager.IsAgentRunningForSession(ctx, session.ID) {
|
||||
// healthy agent mid-prompt. For terminal states the agent is dead by definition,
|
||||
// so skip the probe and go straight to cleanup+retry — this avoids a silent
|
||||
// regression to ErrExecutionAlreadyRunning if the preemptive cleanup above
|
||||
// failed and agentctl still reports a stale "starting" status.
|
||||
if !isTerminalSessionState(session.State) && e.agentManager.IsAgentRunningForSession(ctx, session.ID) {
|
||||
e.logger.Info("resume race: agent already running for session, returning ErrExecutionAlreadyRunning",
|
||||
zap.String("task_id", task.ID),
|
||||
zap.String("session_id", session.ID))
|
||||
@@ -350,9 +373,34 @@ func (e *Executor) validateAndLockResume(ctx context.Context, session *models.Ta
|
||||
return nil, func() {}, ErrNoAgentProfileID
|
||||
}
|
||||
|
||||
if existing, ok := e.GetExecutionBySession(session.ID); ok && existing != nil {
|
||||
// Re-read session state after acquiring the lock. The caller fetched the
|
||||
// session before the lock, so on concurrent resumes the state may be stale —
|
||||
// the first request could have already transitioned FAILED → STARTING, and
|
||||
// a stale FAILED state here would wrongly make isTerminalSessionState bypass
|
||||
// the live-execution guard and cleanup the live agent the first request just
|
||||
// registered, launching a duplicate. If the re-read fails, abort rather than
|
||||
// proceeding with uncertain state — silently falling back to the stale state
|
||||
// would reintroduce the exact race this re-read prevents.
|
||||
fresh, fetchErr := e.repo.GetTaskSession(ctx, session.ID)
|
||||
if fetchErr != nil {
|
||||
unlock()
|
||||
return nil, func() {}, ErrExecutionAlreadyRunning
|
||||
e.logger.Warn("failed to re-read session state inside lock; aborting resume to avoid duplicate agent",
|
||||
zap.String("session_id", session.ID),
|
||||
zap.Error(fetchErr))
|
||||
return nil, func() {}, fetchErr
|
||||
}
|
||||
if fresh != nil {
|
||||
session.State = fresh.State
|
||||
}
|
||||
|
||||
// Skip the "already running" rejection for terminal-state sessions — the agent
|
||||
// process is dead by definition, and ResumeSession will force-cleanup stale
|
||||
// state before the relaunch.
|
||||
if !isTerminalSessionState(session.State) {
|
||||
if existing, ok := e.GetExecutionBySession(session.ID); ok && existing != nil {
|
||||
unlock()
|
||||
return nil, func() {}, ErrExecutionAlreadyRunning
|
||||
}
|
||||
}
|
||||
|
||||
return task, unlock, nil
|
||||
|
||||
@@ -290,3 +290,232 @@ func TestResumeSession_StaleExecutionCleansUpAndRetries(t *testing.T) {
|
||||
t.Errorf("expected LaunchAgent called twice, got %d", agentMgr.launchAgentCallCount)
|
||||
}
|
||||
}
|
||||
|
||||
// TestResumeSession_FailedStateForceCleansUpStaleState covers the FAILED-task
|
||||
// resume scenario where agentctl wrongly reports "starting" and the executionStore
|
||||
// still tracks a stale AgentExecution. The pre-launch cleanup should wipe stale
|
||||
// state so the fresh LaunchAgent succeeds without hitting the "resume race" path —
|
||||
// and crucially without probing IsAgentRunningForSession (which would return true
|
||||
// for the stale status).
|
||||
func TestResumeSession_FailedStateForceCleansUpStaleState(t *testing.T) {
|
||||
repo := newMockRepository()
|
||||
setupLiveResumeTestFixture(repo)
|
||||
repo.sessions["sess-1"].State = models.TaskSessionStateFailed
|
||||
|
||||
agentMgr := &mockAgentManager{
|
||||
launchAgentFunc: func(_ context.Context, _ *LaunchAgentRequest) (*LaunchAgentResponse, error) {
|
||||
return &LaunchAgentResponse{
|
||||
AgentExecutionID: "exec-new",
|
||||
Status: v1.AgentStatusStarting,
|
||||
}, nil
|
||||
},
|
||||
// Returns true if accidentally called; the assertion below proves it is not.
|
||||
isAgentRunningForSessionFunc: func(_ context.Context, _ string) bool { return true },
|
||||
}
|
||||
exec := newTestExecutor(t, agentMgr, repo)
|
||||
|
||||
execution, err := exec.ResumeSession(context.Background(), repo.sessions["sess-1"], true)
|
||||
if err != nil {
|
||||
t.Fatalf("expected success, got: %v", err)
|
||||
}
|
||||
if execution == nil || execution.AgentExecutionID != "exec-new" {
|
||||
t.Fatalf("expected fresh execution exec-new, got %+v", execution)
|
||||
}
|
||||
if agentMgr.cleanupStaleExecutionCallCount != 1 {
|
||||
t.Errorf("expected stale-cleanup called exactly once before LaunchAgent, got %d",
|
||||
agentMgr.cleanupStaleExecutionCallCount)
|
||||
}
|
||||
if agentMgr.launchAgentCallCount != 1 {
|
||||
t.Errorf("expected LaunchAgent called once (no retry), got %d", agentMgr.launchAgentCallCount)
|
||||
}
|
||||
// Terminal-state resume must bypass the stale "starting" liveness probe
|
||||
// entirely — otherwise it would wrongly re-trigger ErrExecutionAlreadyRunning.
|
||||
if len(agentMgr.isAgentRunningForSessionCallArgs) != 0 {
|
||||
t.Errorf("expected IsAgentRunningForSession NOT called for terminal-state resume, got %v",
|
||||
agentMgr.isAgentRunningForSessionCallArgs)
|
||||
}
|
||||
}
|
||||
|
||||
// TestResumeSession_CancelledStateForceCleansUpStaleState mirrors the FAILED
|
||||
// scenario for CANCELLED sessions: stale state must not block the relaunch.
|
||||
func TestResumeSession_CancelledStateForceCleansUpStaleState(t *testing.T) {
|
||||
repo := newMockRepository()
|
||||
setupLiveResumeTestFixture(repo)
|
||||
repo.sessions["sess-1"].State = models.TaskSessionStateCancelled
|
||||
|
||||
agentMgr := &mockAgentManager{
|
||||
launchAgentFunc: func(_ context.Context, _ *LaunchAgentRequest) (*LaunchAgentResponse, error) {
|
||||
return &LaunchAgentResponse{
|
||||
AgentExecutionID: "exec-new",
|
||||
Status: v1.AgentStatusStarting,
|
||||
}, nil
|
||||
},
|
||||
isAgentRunningForSessionFunc: func(_ context.Context, _ string) bool { return true },
|
||||
}
|
||||
exec := newTestExecutor(t, agentMgr, repo)
|
||||
|
||||
if _, err := exec.ResumeSession(context.Background(), repo.sessions["sess-1"], true); err != nil {
|
||||
t.Fatalf("expected success, got: %v", err)
|
||||
}
|
||||
if agentMgr.cleanupStaleExecutionCallCount != 1 {
|
||||
t.Errorf("expected stale-cleanup called once, got %d", agentMgr.cleanupStaleExecutionCallCount)
|
||||
}
|
||||
if len(agentMgr.isAgentRunningForSessionCallArgs) != 0 {
|
||||
t.Errorf("expected IsAgentRunningForSession NOT called for CANCELLED resume, got %v",
|
||||
agentMgr.isAgentRunningForSessionCallArgs)
|
||||
}
|
||||
}
|
||||
|
||||
// TestResumeSession_TerminalStateSkipsLivenessProbeOnFallback covers the
|
||||
// residual path where the preemptive CleanupStaleExecutionBySessionID is not
|
||||
// enough (e.g. the first LaunchAgent still returns "already has an agent running"
|
||||
// because a stale executionStore entry survived). For terminal-state sessions
|
||||
// the fallback block MUST skip the IsAgentRunningForSession probe and go
|
||||
// straight to cleanup+retry, otherwise a stale agentctl "starting" status
|
||||
// silently regresses to ErrExecutionAlreadyRunning — the original bug.
|
||||
func TestResumeSession_TerminalStateSkipsLivenessProbeOnFallback(t *testing.T) {
|
||||
repo := newMockRepository()
|
||||
setupLiveResumeTestFixture(repo)
|
||||
repo.sessions["sess-1"].State = models.TaskSessionStateFailed
|
||||
|
||||
var launchCalls int
|
||||
agentMgr := &mockAgentManager{
|
||||
launchAgentFunc: func(_ context.Context, req *LaunchAgentRequest) (*LaunchAgentResponse, error) {
|
||||
launchCalls++
|
||||
if launchCalls == 1 {
|
||||
return nil, fmt.Errorf("session %q already has an agent running (execution: %s)", req.SessionID, "exec-stale")
|
||||
}
|
||||
return &LaunchAgentResponse{
|
||||
AgentExecutionID: "exec-new",
|
||||
Status: v1.AgentStatusStarting,
|
||||
}, nil
|
||||
},
|
||||
// Live-probe would return true (stale "starting") — if we wrongly call
|
||||
// it, the test fails because err is ErrExecutionAlreadyRunning.
|
||||
isAgentRunningForSessionFunc: func(_ context.Context, _ string) bool { return true },
|
||||
}
|
||||
exec := newTestExecutor(t, agentMgr, repo)
|
||||
|
||||
execution, err := exec.ResumeSession(context.Background(), repo.sessions["sess-1"], true)
|
||||
if err != nil {
|
||||
t.Fatalf("expected success after fallback cleanup+retry, got: %v", err)
|
||||
}
|
||||
if execution == nil || execution.AgentExecutionID != "exec-new" {
|
||||
t.Fatalf("expected fresh execution exec-new, got %+v", execution)
|
||||
}
|
||||
if agentMgr.launchAgentCallCount != 2 {
|
||||
t.Errorf("expected LaunchAgent called twice (first fails, second succeeds), got %d", agentMgr.launchAgentCallCount)
|
||||
}
|
||||
// 1 preemptive + 1 fallback cleanup.
|
||||
if agentMgr.cleanupStaleExecutionCallCount != 2 {
|
||||
t.Errorf("expected stale-cleanup called twice, got %d", agentMgr.cleanupStaleExecutionCallCount)
|
||||
}
|
||||
if len(agentMgr.isAgentRunningForSessionCallArgs) != 0 {
|
||||
t.Errorf("expected IsAgentRunningForSession NOT called for terminal-state fallback, got %v",
|
||||
agentMgr.isAgentRunningForSessionCallArgs)
|
||||
}
|
||||
}
|
||||
|
||||
// TestResumeSession_ConcurrentResumeReReadsFreshState exercises the concurrent-
|
||||
// resume race: the caller's session object has a stale State=FAILED, but a
|
||||
// concurrent resume already transitioned the DB to STARTING. validateAndLockResume
|
||||
// MUST re-read the session state inside the lock — otherwise isTerminalSessionState
|
||||
// wrongly bypasses the live-execution guard, CleanupStaleExecutionBySessionID
|
||||
// wipes the live AgentExecution, and LaunchAgent launches a duplicate agent.
|
||||
func TestResumeSession_ConcurrentResumeReReadsFreshState(t *testing.T) {
|
||||
repo := newMockRepository()
|
||||
setupLiveResumeTestFixture(repo)
|
||||
|
||||
// Caller's in-memory session carries a stale FAILED state.
|
||||
callerSession := *repo.sessions["sess-1"]
|
||||
callerSession.State = models.TaskSessionStateFailed
|
||||
|
||||
// DB truth: a concurrent resume already transitioned this session to STARTING.
|
||||
repo.sessions["sess-1"].State = models.TaskSessionStateStarting
|
||||
repo.sessions["sess-1"].UpdatedAt = time.Now().UTC()
|
||||
repo.sessions["sess-1"].AgentExecutionID = "exec-live"
|
||||
|
||||
agentMgr := &mockAgentManager{
|
||||
launchAgentFunc: func(_ context.Context, _ *LaunchAgentRequest) (*LaunchAgentResponse, error) {
|
||||
t.Fatal("LaunchAgent must not be called — the live execution guard must reject the duplicate resume")
|
||||
return nil, nil
|
||||
},
|
||||
// Live execution was registered by the first resume; probe returns true.
|
||||
isAgentRunningForSessionFunc: func(_ context.Context, _ string) bool { return true },
|
||||
}
|
||||
exec := newTestExecutor(t, agentMgr, repo)
|
||||
|
||||
_, err := exec.ResumeSession(context.Background(), &callerSession, true)
|
||||
if !errors.Is(err, ErrExecutionAlreadyRunning) {
|
||||
t.Fatalf("expected ErrExecutionAlreadyRunning (live agent protected), got: %v", err)
|
||||
}
|
||||
if agentMgr.cleanupStaleExecutionCallCount != 0 {
|
||||
t.Errorf("cleanup must NOT be called when a live execution is detected, got %d",
|
||||
agentMgr.cleanupStaleExecutionCallCount)
|
||||
}
|
||||
}
|
||||
|
||||
// TestResumeSession_AbortsIfSessionReReadFails locks down the abort-on-error
|
||||
// behavior of the in-lock session re-read. If GetTaskSession fails transiently,
|
||||
// silently falling back to the caller's (potentially stale) session.State would
|
||||
// reintroduce the concurrent-resume duplicate-launch race. The resume MUST
|
||||
// return the fetch error without calling LaunchAgent or CleanupStaleExecutionBySessionID.
|
||||
func TestResumeSession_AbortsIfSessionReReadFails(t *testing.T) {
|
||||
repo := newMockRepository()
|
||||
setupLiveResumeTestFixture(repo)
|
||||
repo.sessions["sess-1"].State = models.TaskSessionStateFailed
|
||||
|
||||
wantErr := fmt.Errorf("transient DB error")
|
||||
var callCount int
|
||||
repo.getTaskSessionFunc = func(_ context.Context, _ string) (*models.TaskSession, error) {
|
||||
callCount++
|
||||
// First call is the caller's pre-lock fetch (via the service, not
|
||||
// reached directly in this test). The re-read inside the lock is the
|
||||
// only GetTaskSession the executor makes, so fail it unconditionally.
|
||||
return nil, wantErr
|
||||
}
|
||||
|
||||
agentMgr := &mockAgentManager{
|
||||
launchAgentFunc: func(_ context.Context, _ *LaunchAgentRequest) (*LaunchAgentResponse, error) {
|
||||
t.Fatal("LaunchAgent must not be called when the session re-read fails")
|
||||
return nil, nil
|
||||
},
|
||||
}
|
||||
exec := newTestExecutor(t, agentMgr, repo)
|
||||
|
||||
_, err := exec.ResumeSession(context.Background(), repo.sessions["sess-1"], true)
|
||||
if !errors.Is(err, wantErr) {
|
||||
t.Fatalf("expected transient DB error to be returned, got: %v", err)
|
||||
}
|
||||
if agentMgr.launchAgentCallCount != 0 {
|
||||
t.Errorf("expected LaunchAgent NOT called, got %d", agentMgr.launchAgentCallCount)
|
||||
}
|
||||
if agentMgr.cleanupStaleExecutionCallCount != 0 {
|
||||
t.Errorf("expected CleanupStaleExecutionBySessionID NOT called, got %d",
|
||||
agentMgr.cleanupStaleExecutionCallCount)
|
||||
}
|
||||
if callCount == 0 {
|
||||
t.Error("expected the in-lock session re-read to be called at least once")
|
||||
}
|
||||
}
|
||||
|
||||
// TestIsTerminalSessionState is a small unit test for the helper that drives
|
||||
// both the preemptive cleanup and the validateAndLockResume carve-out.
|
||||
func TestIsTerminalSessionState(t *testing.T) {
|
||||
cases := []struct {
|
||||
state models.TaskSessionState
|
||||
want bool
|
||||
}{
|
||||
{models.TaskSessionStateFailed, true},
|
||||
{models.TaskSessionStateCancelled, true},
|
||||
{models.TaskSessionStateWaitingForInput, false},
|
||||
{models.TaskSessionStateRunning, false},
|
||||
{models.TaskSessionStateStarting, false},
|
||||
{models.TaskSessionStateCompleted, false},
|
||||
}
|
||||
for _, c := range cases {
|
||||
if got := isTerminalSessionState(c.state); got != c.want {
|
||||
t.Errorf("isTerminalSessionState(%q) = %v, want %v", c.state, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -593,14 +593,12 @@ func (s *Service) ResumeTaskSession(ctx context.Context, taskID, sessionID strin
|
||||
}
|
||||
|
||||
// Completed sessions cannot be restarted — they require a new session.
|
||||
// Failed sessions clear the resume token so the agent starts fresh (the old
|
||||
// process crashed). Cancelled sessions keep the token so --resume restores
|
||||
// the conversation when the user re-starts a manually stopped session.
|
||||
switch session.State {
|
||||
case models.TaskSessionStateCompleted:
|
||||
// Failed and cancelled sessions keep the resume token so the relaunched
|
||||
// agent continues the previous conversation (via ACP session/load for
|
||||
// native-resume agents, or --resume on CLI). Users who want a fresh start
|
||||
// after a failure can invoke RecoverSession with action="fresh_start".
|
||||
if session.State == models.TaskSessionStateCompleted {
|
||||
return nil, fmt.Errorf("session is completed and cannot be resumed; create a new session instead")
|
||||
case models.TaskSessionStateFailed:
|
||||
s.clearResumeToken(ctx, sessionID)
|
||||
}
|
||||
|
||||
// Use context.WithoutCancel to prevent WebSocket request timeout from canceling the resume.
|
||||
|
||||
@@ -615,6 +615,56 @@ func TestResumeTaskSession_ArchivedDuringLaunch(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestResumeTaskSession_FailedKeepsResumeToken verifies that resuming a FAILED
|
||||
// session preserves the ACP resume token so the relaunched agent restores the
|
||||
// prior conversation via ACP session/load (for native-resume agents).
|
||||
// Regression test for the "Resume blocked by stale state" bug where FAILED sessions
|
||||
// couldn't be restarted at all; the fix also changes policy to keep the token
|
||||
// (previously it was cleared to force a fresh agent).
|
||||
func TestResumeTaskSession_FailedKeepsResumeToken(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
repo := setupTestRepo(t)
|
||||
taskRepo := newMockTaskRepo()
|
||||
|
||||
// Agent launch succeeds so the resume path does not unwind and mark the task
|
||||
// FAILED, which would exercise a separate state-mutation code path.
|
||||
agentMgr := &mockAgentManager{
|
||||
launchAgentFunc: func(_ context.Context, _ *executor.LaunchAgentRequest) (*executor.LaunchAgentResponse, error) {
|
||||
return &executor.LaunchAgentResponse{
|
||||
AgentExecutionID: "exec-new",
|
||||
Status: v1.AgentStatusStarting,
|
||||
}, nil
|
||||
},
|
||||
}
|
||||
svc := createTestServiceWithAgent(repo, newMockStepGetter(), taskRepo, agentMgr)
|
||||
svc.executor = executor.NewExecutor(agentMgr, repo, testLogger(), executor.ExecutorConfig{})
|
||||
|
||||
seedTaskAndSession(t, repo, "task1", "session1", models.TaskSessionStateFailed)
|
||||
session, _ := repo.GetTaskSession(ctx, "session1")
|
||||
session.AgentProfileID = "profile-1"
|
||||
_ = repo.UpdateTaskSession(ctx, session)
|
||||
|
||||
now := time.Now().UTC()
|
||||
_ = repo.UpsertExecutorRunning(ctx, &models.ExecutorRunning{
|
||||
ID: "er1", SessionID: "session1", TaskID: "task1",
|
||||
ResumeToken: "acp-session-xyz",
|
||||
Resumable: true,
|
||||
CreatedAt: now, UpdatedAt: now,
|
||||
})
|
||||
|
||||
if _, err := svc.ResumeTaskSession(ctx, "task1", "session1"); err != nil {
|
||||
t.Fatalf("ResumeTaskSession on FAILED session returned: %v", err)
|
||||
}
|
||||
|
||||
er, err := repo.GetExecutorRunningBySessionID(ctx, "session1")
|
||||
if err != nil || er == nil {
|
||||
t.Fatalf("ExecutorRunning lookup failed: %v (nil=%v)", err, er == nil)
|
||||
}
|
||||
if er.ResumeToken != "acp-session-xyz" {
|
||||
t.Errorf("expected resume token to be preserved on FAILED resume, got %q", er.ResumeToken)
|
||||
}
|
||||
}
|
||||
|
||||
// --- CompleteTask ---
|
||||
|
||||
func TestCompleteTask_UpdatesTaskState(t *testing.T) {
|
||||
|
||||
Reference in New Issue
Block a user