fix(backend): unstick agent session when cancel times out (#651)

This commit is contained in:
Carlos Florêncio
2026-04-20 20:02:19 +01:00
committed by GitHub
parent d209d2ce2a
commit ad00b90d2a
7 changed files with 450 additions and 24 deletions
@@ -88,8 +88,25 @@ func (m *Manager) PromptAgent(ctx context.Context, executionID string, prompt st
return m.sessionManager.SendPrompt(ctx, execution, prompt, true, attachments)
}
// cancelWaitTimeout bounds how long CancelAgent waits for the in-flight SendPrompt
// to exit after the ACP cancel was acknowledged by the agent.
// Exposed as a var (not const) so tests can shorten it without fake clocks.
var cancelWaitTimeout = 10 * time.Second
// cancelEscalationTimeout bounds the post-escalation wait after we inject a synthetic
// error onto promptDoneCh to unblock a stuck SendPrompt.
var cancelEscalationTimeout = 2 * time.Second
// CancelAgent interrupts the current agent turn without terminating the process,
// allowing the user to send a new prompt.
//
// When the agent subprocess accepts the ACP cancel but never publishes a `complete`
// event (and the update stream stays open), the in-flight SendPrompt would otherwise
// block forever. After cancelWaitTimeout we escalate by signalling promptDoneCh with
// a synthetic error so SendPrompt returns and its deferred cleanup closes
// promptFinished normally. Callers receive ErrCancelEscalated to signal that local
// lifecycle state was reconciled but the agent never confirmed the cancel, and
// higher layers (Service.CancelAgent) must still reconcile DB state.
func (m *Manager) CancelAgent(ctx context.Context, executionID string) error {
execution, exists := m.executionStore.Get(executionID)
if !exists {
@@ -126,21 +143,75 @@ func (m *Manager) CancelAgent(ctx context.Context, executionID string) error {
ch := execution.promptFinished
execution.promptFinishedMu.Unlock()
if ch != nil {
select {
case <-ch:
m.logger.Debug("in-flight prompt finished after cancel",
zap.String("execution_id", executionID))
case <-time.After(10 * time.Second):
m.logger.Warn("timed out waiting for in-flight prompt to finish after cancel",
zap.String("execution_id", executionID))
return fmt.Errorf("timed out waiting for in-flight prompt to finish after cancel")
case <-ctx.Done():
return ctx.Err()
}
if ch == nil {
return nil
}
return nil
select {
case <-ch:
m.logger.Debug("in-flight prompt finished after cancel",
zap.String("execution_id", executionID))
return nil
case <-time.After(cancelWaitTimeout):
return m.escalateStuckCancel(ctx, execution, ch)
case <-ctx.Done():
return ctx.Err()
}
}
// escalateStuckCancel unblocks a SendPrompt that is stuck waiting for a completion
// event the agent will never emit. It injects a synthetic error onto promptDoneCh
// (non-blocking; the channel is buffered size 1), waits briefly for SendPrompt to
// exit and close promptFinished, and marks the execution ready so the workflow
// leaves the running state. Returns ErrCancelEscalated so callers know the ACP
// cancel did not get a clean acknowledgement.
func (m *Manager) escalateStuckCancel(ctx context.Context, execution *AgentExecution, ch <-chan struct{}) error {
m.logger.Warn("timed out waiting for in-flight prompt to finish after cancel; escalating",
zap.String("execution_id", execution.ID),
zap.String("session_id", execution.SessionID))
select {
case execution.promptDoneCh <- PromptCompletionSignal{
IsError: true,
Error: "cancel escalated: agent did not complete turn within timeout",
}:
default:
// Channel already has a pending signal; SendPrompt will pick that up instead.
}
select {
case <-ch:
m.logger.Info("in-flight prompt released after cancel escalation",
zap.String("execution_id", execution.ID))
case <-time.After(cancelEscalationTimeout):
m.logger.Warn("in-flight prompt did not release after cancel escalation",
zap.String("execution_id", execution.ID))
case <-ctx.Done():
// Fall through to MarkReady/drain below — once the synthetic signal is
// queued, the cleanup must survive the caller's context cancellation
// or the execution leaks in the Running state and the stale signal
// breaks the next PromptAgent call.
}
if err := m.MarkReady(execution.ID); err != nil {
m.logger.Warn("failed to mark execution ready after cancel escalation",
zap.String("execution_id", execution.ID),
zap.Error(err))
}
// Drain any stale signal that may have been left if the agent completed
// concurrently with the escalation timeout (Go's select is non-deterministic
// when both promptFinished and time.After are ready, so our send above may
// have landed on a channel that SendPrompt had just drained).
select {
case <-execution.promptDoneCh:
default:
}
if err := ctx.Err(); err != nil {
return err
}
return ErrCancelEscalated
}
// SetSessionMode changes the session mode for a running agent.
@@ -283,6 +354,14 @@ func (m *Manager) ResetAgentContext(ctx context.Context, executionID string) err
// with no agent to drive it.
var ErrNoExecutionForSession = errors.New("no execution for session")
// ErrCancelEscalated is returned by CancelAgent when the agent subprocess accepted
// the ACP cancel but did not publish a completion event before the timeout. The
// lifecycle manager has locally unblocked the in-flight SendPrompt and marked the
// execution ready, but the agent never acknowledged the cancel. Callers should
// still reconcile session-level state (e.g. transition the task session to
// WAITING_FOR_INPUT) — the user's intent was unambiguous.
var ErrCancelEscalated = errors.New("cancel escalated: agent did not acknowledge within timeout")
// CancelAgentBySessionID cancels the current agent turn for a specific session.
// Returns ErrNoExecutionForSession (wrapped) when no execution is tracked for the session.
func (m *Manager) CancelAgentBySessionID(ctx context.Context, sessionID string) error {
@@ -0,0 +1,195 @@
package lifecycle
import (
"context"
"testing"
"time"
"github.com/stretchr/testify/require"
agentctlClient "github.com/kandev/kandev/internal/agentctl/client"
"github.com/kandev/kandev/internal/events"
v1 "github.com/kandev/kandev/pkg/api/v1"
ws "github.com/kandev/kandev/pkg/websocket"
)
// TestManager_CancelAgent_EscalatesWhenAgentHangs reproduces the stuck-turn bug:
// the agent accepts the ACP cancel but never publishes a `complete` event, so the
// in-flight SendPrompt would block forever. The manager must escalate by
// unblocking SendPrompt via promptDoneCh, marking the execution ready, and
// returning ErrCancelEscalated so higher layers can still reconcile DB state.
func TestManager_CancelAgent_EscalatesWhenAgentHangs(t *testing.T) {
prevWait := cancelWaitTimeout
prevEsc := cancelEscalationTimeout
cancelWaitTimeout = 50 * time.Millisecond
cancelEscalationTimeout = 50 * time.Millisecond
t.Cleanup(func() {
cancelWaitTimeout = prevWait
cancelEscalationTimeout = prevEsc
})
// Mock agentctl: ack agent.cancel but never emit a completion event.
mock := newMockAgentServer(t)
t.Cleanup(func() { mock.server.Close() })
mock.handler = func(msg ws.Message) *ws.Message {
if msg.Action == "agent.cancel" {
resp, _ := ws.NewResponse(msg.ID, msg.Action, map[string]interface{}{
"success": true,
})
return resp
}
return mock.defaultHandler(msg)
}
client := createTestClient(t, mock.server.URL)
t.Cleanup(client.Close)
// Establish the agent stream so the client can send agent.cancel.
streamCtx, streamCancel := context.WithCancel(context.Background())
t.Cleanup(streamCancel)
require.NoError(t, client.StreamUpdates(streamCtx, func(_ agentctlClient.AgentEvent) {}, nil, nil))
select {
case <-mock.wsConnected:
case <-time.After(2 * time.Second):
t.Fatal("mock server did not see WS connection")
}
mgr := newTestManager()
promptFinished := make(chan struct{})
exec := &AgentExecution{
ID: "exec-cancel-hang",
TaskID: "task-1",
SessionID: "session-1",
AgentProfileID: "profile-1",
Status: v1.AgentStatusRunning,
WorkspacePath: "/workspace",
agentctl: client,
promptDoneCh: make(chan PromptCompletionSignal, 1),
promptFinished: promptFinished,
}
mgr.executionStore.Add(exec)
// Simulate the in-flight SendPrompt: it blocks reading promptDoneCh, and on
// signal closes promptFinished (the same cleanup beginPromptBarrier's deferred
// closer does).
sendPromptDone := make(chan struct{})
var signal PromptCompletionSignal
go func() {
defer close(sendPromptDone)
signal = <-exec.promptDoneCh
close(promptFinished)
}()
// Tight bounds: escalation window is cancelWaitTimeout + cancelEscalationTimeout
// (100 ms). Use channel-based synchronization per CLAUDE.md — synctest cannot be
// used here because the HTTP mock server spawns goroutines outside its bubble.
ctx, cancel := context.WithTimeout(context.Background(), 500*time.Millisecond)
defer cancel()
err := mgr.CancelAgent(ctx, exec.ID)
require.ErrorIs(t, err, ErrCancelEscalated)
select {
case <-sendPromptDone:
case <-time.After(200 * time.Millisecond):
t.Fatal("simulated SendPrompt did not release after cancel escalation")
}
require.True(t, signal.IsError, "escalation signal must carry IsError=true")
require.Contains(t, signal.Error, "cancel escalated")
updated, found := mgr.executionStore.Get(exec.ID)
require.True(t, found)
require.Equal(t, v1.AgentStatusReady, updated.Status,
"execution must be marked ready after cancel escalation so the workflow can proceed")
mockBus, ok := mgr.eventBus.(*MockEventBus)
require.True(t, ok)
var sawReady bool
for _, ev := range mockBus.PublishedEvents {
if ev.Type == events.AgentReady {
sawReady = true
break
}
}
require.True(t, sawReady, "expected AgentReady event after cancel escalation")
}
// TestManager_CancelAgent_EscalationCleanupSurvivesCtxCancel covers the case where
// the caller's context is cancelled during the post-escalation wait. Once the
// synthetic signal has been queued on promptDoneCh, the cleanup (MarkReady + drain)
// must still run — otherwise the execution leaks in Running state and the stale
// signal breaks the next PromptAgent call.
func TestManager_CancelAgent_EscalationCleanupSurvivesCtxCancel(t *testing.T) {
prevWait := cancelWaitTimeout
prevEsc := cancelEscalationTimeout
cancelWaitTimeout = 20 * time.Millisecond
// Long enough that ctx.Done() fires first during the post-escalation wait.
cancelEscalationTimeout = 500 * time.Millisecond
t.Cleanup(func() {
cancelWaitTimeout = prevWait
cancelEscalationTimeout = prevEsc
})
mock := newMockAgentServer(t)
t.Cleanup(func() { mock.server.Close() })
mock.handler = func(msg ws.Message) *ws.Message {
if msg.Action == "agent.cancel" {
resp, _ := ws.NewResponse(msg.ID, msg.Action, map[string]interface{}{
"success": true,
})
return resp
}
return mock.defaultHandler(msg)
}
client := createTestClient(t, mock.server.URL)
t.Cleanup(client.Close)
streamCtx, streamCancel := context.WithCancel(context.Background())
t.Cleanup(streamCancel)
require.NoError(t, client.StreamUpdates(streamCtx, func(_ agentctlClient.AgentEvent) {}, nil, nil))
select {
case <-mock.wsConnected:
case <-time.After(2 * time.Second):
t.Fatal("mock server did not see WS connection")
}
mgr := newTestManager()
// promptFinished is deliberately never closed — simulates a SendPrompt that
// is blocked on something other than promptDoneCh (so escalation can't
// release it in time, and our ctx will cancel first).
exec := &AgentExecution{
ID: "exec-cancel-ctx",
TaskID: "task-1",
SessionID: "session-1",
Status: v1.AgentStatusRunning,
WorkspacePath: "/workspace",
agentctl: client,
promptDoneCh: make(chan PromptCompletionSignal, 1),
promptFinished: make(chan struct{}),
}
mgr.executionStore.Add(exec)
// Cancel the caller's context after escalation starts but before the
// post-escalation wait completes.
ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond)
defer cancel()
err := mgr.CancelAgent(ctx, exec.ID)
require.ErrorIs(t, err, context.DeadlineExceeded,
"returns ctx error, not ErrCancelEscalated, when caller ctx cancelled")
// Critical invariant: cleanup ran despite ctx cancellation.
updated, ok := mgr.executionStore.Get(exec.ID)
require.True(t, ok)
require.Equal(t, v1.AgentStatusReady, updated.Status,
"execution must be marked ready even when caller ctx is cancelled mid-escalation")
select {
case sig := <-exec.promptDoneCh:
t.Fatalf("stale signal must be drained after escalation; got: %+v", sig)
default:
}
}
@@ -477,6 +477,12 @@ func (sm *SessionManager) waitForPromptDone(ctx context.Context, execution *Agen
sm.logger.Error("prompt completed with error",
zap.String("execution_id", execution.ID),
zap.String("error", signal.Error))
// Wrap the cancel-escalation sentinel so PromptTask can identify it and
// skip the REVIEW task-state transition — the user is cancelling, not
// hitting a real agent failure.
if strings.HasPrefix(signal.Error, "cancel escalated") {
return nil, fmt.Errorf("agent error: %s: %w", signal.Error, ErrCancelEscalated)
}
return nil, fmt.Errorf("agent error: %s", signal.Error)
}
@@ -95,3 +95,71 @@ func TestOrchestratorCancelAfterAgentCrash_UnsticksSession(t *testing.T) {
assert.True(t, foundCancel,
"expected a 'Turn cancelled by user' status message to be recorded")
}
// TestOrchestratorCancelWhenAgentHangs_UnsticksSession covers the second path to a stuck
// session: the agent subprocess is alive and acknowledges the ACP cancel, but never
// publishes a completion event. The lifecycle manager's CancelAgent escalates and returns
// ErrCancelEscalated. The orchestrator service must treat that as a soft-fail (reconcile
// DB state) — otherwise the UI shows "agent is running" forever and cancel stays broken.
func TestOrchestratorCancelWhenAgentHangs_UnsticksSession(t *testing.T) {
ts := NewOrchestratorTestServer(t)
defer ts.Close()
taskID := ts.CreateTestTask(t, "augment-agent", 2)
client := NewOrchestratorWSClient(t, ts.Server.URL)
defer client.Close()
startResp, err := client.SendRequest("start-1", ws.ActionOrchestratorStart, map[string]interface{}{
"task_id": taskID,
"agent_profile_id": "augment-agent",
})
require.NoError(t, err)
require.Equal(t, ws.MessageTypeResponse, startResp.Type, "start should succeed")
var startPayload map[string]interface{}
require.NoError(t, startResp.ParsePayload(&startPayload))
sessionID, _ := startPayload["session_id"].(string)
require.NotEmpty(t, sessionID)
require.NoError(t, ts.TaskRepo.UpdateTaskSessionState(
context.Background(), sessionID, models.TaskSessionStateRunning, "",
))
// Mark the simulated agent as hung: it's tracked, but CancelAgent will return
// ErrCancelEscalated to model the real lifecycle escalation.
ts.AgentManager.MarkAgentHungForSession(sessionID)
cancelResp, err := client.SendRequest("cancel-1", ws.ActionAgentCancel, map[string]interface{}{
"session_id": sessionID,
})
require.NoError(t, err)
assert.Equal(t, ws.MessageTypeResponse, cancelResp.Type,
"cancel should succeed even when lifecycle escalated after an unresponsive agent; got: %+v", cancelResp)
deadline := time.Now().Add(2 * time.Second)
var finalState models.TaskSessionState
for time.Now().Before(deadline) {
session, gErr := ts.TaskRepo.GetTaskSession(context.Background(), sessionID)
require.NoError(t, gErr)
finalState = session.State
if finalState == models.TaskSessionStateWaitingForInput {
break
}
time.Sleep(25 * time.Millisecond)
}
assert.Equal(t, models.TaskSessionStateWaitingForInput, finalState,
"session should unstick to WAITING_FOR_INPUT after cancel escalation")
msgs, err := ts.TaskRepo.ListMessages(context.Background(), sessionID)
require.NoError(t, err)
foundCancel := false
for _, m := range msgs {
if m.Type == models.MessageTypeStatus && m.Content == "Turn cancelled by user" {
foundCancel = true
break
}
}
assert.True(t, foundCancel,
"expected a 'Turn cancelled by user' status message to be recorded")
}
@@ -47,6 +47,9 @@ type simulatedInstance struct {
status v1.AgentStatus
statusMu sync.Mutex // Protects status field
stopCh chan struct{}
// hung indicates the agent acknowledged an ACP cancel but never emitted a
// completion event, so CancelAgent should return lifecycle.ErrCancelEscalated.
hung bool
}
// NewSimulatedAgentManager creates a new simulated agent manager
@@ -392,6 +395,10 @@ func (s *SimulatedAgentManagerClient) IsAgentRunningForSession(ctx context.Conte
// Returns lifecycle.ErrNoExecutionForSession when no live execution exists for the session
// (for example, after CrashAgentForSession has been used to simulate a crash), matching the
// real lifecycle adapter's behaviour.
//
// If the session has been marked "hung" via MarkAgentHungForSession, the call returns
// lifecycle.ErrCancelEscalated to model the real manager's escalation path when the agent
// accepts an ACP cancel but never publishes a completion event.
func (s *SimulatedAgentManagerClient) CancelAgent(ctx context.Context, sessionID string) error {
s.logger.Info("simulated: cancelling agent turn",
zap.String("session_id", sessionID))
@@ -399,13 +406,32 @@ func (s *SimulatedAgentManagerClient) CancelAgent(ctx context.Context, sessionID
s.mu.Lock()
defer s.mu.Unlock()
for _, inst := range s.instances {
if inst.sessionID == sessionID {
return nil
if inst.sessionID != sessionID {
continue
}
if inst.hung {
return fmt.Errorf("session %q: %w", sessionID, lifecycle.ErrCancelEscalated)
}
return nil
}
return fmt.Errorf("session %q: %w", sessionID, lifecycle.ErrNoExecutionForSession)
}
// MarkAgentHungForSession marks the simulated agent for a session as "hung" so subsequent
// CancelAgent calls return lifecycle.ErrCancelEscalated. This models a real agent that
// acknowledged the ACP cancel but never emitted a completion event, forcing the lifecycle
// manager to escalate.
func (s *SimulatedAgentManagerClient) MarkAgentHungForSession(sessionID string) {
s.mu.Lock()
defer s.mu.Unlock()
for _, inst := range s.instances {
if inst.sessionID == sessionID {
inst.hung = true
return
}
}
}
// CrashAgentForSession simulates an agent subprocess crashing mid-turn.
// It stops the agent simulation goroutine and removes the instance from the tracking map,
// so subsequent lookups (CancelAgent, IsAgentRunningForSession) behave as if the execution
@@ -1539,7 +1539,11 @@ func (s *Service) PromptTask(ctx context.Context, taskID, sessionID string, prom
// allowWakeFromWaiting=false — this is a revert away from RUNNING, never a
// wake transition; the flag only matters when going WAITING_FOR_INPUT → RUNNING.
s.updateTaskSessionState(ctx, taskID, sessionID, previousSessionState, "", false)
if !isTransientPromptError(err) {
// ErrCancelEscalated means the user cancelled and the lifecycle manager had to
// force-unblock a hung agent. That is not a "review this failure" condition —
// Service.CancelAgent reconciles DB state (WAITING_FOR_INPUT, cancel message,
// complete turn) already.
if !isTransientPromptError(err) && !errors.Is(err, lifecycle.ErrCancelEscalated) {
_ = s.taskRepo.UpdateTaskState(ctx, taskID, v1.TaskStateReview)
}
s.completeTurnForSession(ctx, sessionID)
@@ -1666,16 +1670,25 @@ func (s *Service) CancelAgent(ctx context.Context, sessionID string) error {
}
if err := s.agentManager.CancelAgent(ctx, sessionID); err != nil {
if !errors.Is(err, lifecycle.ErrNoExecutionForSession) {
switch {
case errors.Is(err, lifecycle.ErrNoExecutionForSession):
// The session was live but there is no execution to cancel — the agent process
// crashed, exited, or never re-registered after a backend restart. Log at error
// level so operators notice the stuck state; we still reconcile DB state below
// so the UI unsticks.
s.logger.Error("agent process appears to have crashed: no live execution for session on cancel",
zap.String("session_id", sessionID),
zap.Error(err))
case errors.Is(err, lifecycle.ErrCancelEscalated):
// The agent accepted the ACP cancel but never published a completion event.
// The lifecycle manager already unblocked the in-flight prompt and marked the
// execution ready; reconcile DB state below so the UI unsticks.
s.logger.Warn("agent did not acknowledge cancel; reconciling session state",
zap.String("session_id", sessionID),
zap.Error(err))
default:
return fmt.Errorf("cancel agent: %w", err)
}
// The session was live but there is no execution to cancel — the agent process
// crashed, exited, or never re-registered after a backend restart. Log at error
// level so operators notice the stuck state; we still reconcile DB state below
// so the UI unsticks.
s.logger.Error("agent process appears to have crashed: no live execution for session on cancel",
zap.String("session_id", sessionID),
zap.Error(err))
}
// Transition to WAITING_FOR_INPUT so the user can send a new prompt
@@ -119,6 +119,45 @@ func TestPromptTask_TransientErrorDoesNotMoveTaskToReview(t *testing.T) {
}
}
// TestPromptTask_CancelEscalatedDoesNotMoveTaskToReview ensures that when the
// lifecycle manager force-unblocks a hung agent (returning ErrCancelEscalated
// wrapped in the agent-error format), PromptTask recognises it as a cancel and
// leaves the task state untouched — the user cancelled, this is not a failure
// the reviewer needs to look at. Service.CancelAgent reconciles session state
// separately; PromptTask must not race ahead with UpdateTaskState(REVIEW).
func TestPromptTask_CancelEscalatedDoesNotMoveTaskToReview(t *testing.T) {
repo := setupTestRepo(t)
taskRepo := newMockTaskRepo()
agentMgr := &mockAgentManager{
isAgentRunning: true,
promptErr: fmt.Errorf("agent error: cancel escalated: agent did not complete turn within timeout: %w", lifecycle.ErrCancelEscalated),
}
svc := createTestServiceWithAgent(repo, newMockStepGetter(), taskRepo, agentMgr)
svc.executor = executor.NewExecutor(agentMgr, repo, testLogger(), executor.ExecutorConfig{})
seedTaskAndSession(t, repo, "task1", "session1", models.TaskSessionStateWaitingForInput)
session, err := repo.GetTaskSession(context.Background(), "session1")
if err != nil {
t.Fatalf("failed to load session: %v", err)
}
session.AgentExecutionID = "exec-1"
if err := repo.UpdateTaskSession(context.Background(), session); err != nil {
t.Fatalf("failed to update session: %v", err)
}
_, err = svc.PromptTask(context.Background(), "task1", "session1", "hello", "", false, nil)
if err == nil {
t.Fatal("expected cancel-escalated error to bubble up from PromptTask")
}
if !errors.Is(err, lifecycle.ErrCancelEscalated) {
t.Fatalf("expected ErrCancelEscalated, got: %v", err)
}
if got, ok := taskRepo.updatedStates["task1"]; ok && got == v1.TaskStateReview {
t.Fatalf("expected task state to avoid REVIEW on cancel escalation, got %q", got)
}
}
// TestPromptTask_ExecutionNotFoundRevertsStateAndBroadcasts ensures that when
// Prompt returns executor.ErrExecutionNotFound, PromptTask reverts the session
// state via the broadcasting wrapper (not a direct repo write), so the WS