fix(backend): unstick agent session when cancel times out (#651)
This commit is contained in:
@@ -88,8 +88,25 @@ func (m *Manager) PromptAgent(ctx context.Context, executionID string, prompt st
|
||||
return m.sessionManager.SendPrompt(ctx, execution, prompt, true, attachments)
|
||||
}
|
||||
|
||||
// cancelWaitTimeout bounds how long CancelAgent waits for the in-flight SendPrompt
|
||||
// to exit after the ACP cancel was acknowledged by the agent.
|
||||
// Exposed as a var (not const) so tests can shorten it without fake clocks.
|
||||
var cancelWaitTimeout = 10 * time.Second
|
||||
|
||||
// cancelEscalationTimeout bounds the post-escalation wait after we inject a synthetic
|
||||
// error onto promptDoneCh to unblock a stuck SendPrompt.
|
||||
var cancelEscalationTimeout = 2 * time.Second
|
||||
|
||||
// CancelAgent interrupts the current agent turn without terminating the process,
|
||||
// allowing the user to send a new prompt.
|
||||
//
|
||||
// When the agent subprocess accepts the ACP cancel but never publishes a `complete`
|
||||
// event (and the update stream stays open), the in-flight SendPrompt would otherwise
|
||||
// block forever. After cancelWaitTimeout we escalate by signalling promptDoneCh with
|
||||
// a synthetic error so SendPrompt returns and its deferred cleanup closes
|
||||
// promptFinished normally. Callers receive ErrCancelEscalated to signal that local
|
||||
// lifecycle state was reconciled but the agent never confirmed the cancel, and
|
||||
// higher layers (Service.CancelAgent) must still reconcile DB state.
|
||||
func (m *Manager) CancelAgent(ctx context.Context, executionID string) error {
|
||||
execution, exists := m.executionStore.Get(executionID)
|
||||
if !exists {
|
||||
@@ -126,21 +143,75 @@ func (m *Manager) CancelAgent(ctx context.Context, executionID string) error {
|
||||
ch := execution.promptFinished
|
||||
execution.promptFinishedMu.Unlock()
|
||||
|
||||
if ch != nil {
|
||||
select {
|
||||
case <-ch:
|
||||
m.logger.Debug("in-flight prompt finished after cancel",
|
||||
zap.String("execution_id", executionID))
|
||||
case <-time.After(10 * time.Second):
|
||||
m.logger.Warn("timed out waiting for in-flight prompt to finish after cancel",
|
||||
zap.String("execution_id", executionID))
|
||||
return fmt.Errorf("timed out waiting for in-flight prompt to finish after cancel")
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
}
|
||||
if ch == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
return nil
|
||||
select {
|
||||
case <-ch:
|
||||
m.logger.Debug("in-flight prompt finished after cancel",
|
||||
zap.String("execution_id", executionID))
|
||||
return nil
|
||||
case <-time.After(cancelWaitTimeout):
|
||||
return m.escalateStuckCancel(ctx, execution, ch)
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
}
|
||||
}
|
||||
|
||||
// escalateStuckCancel unblocks a SendPrompt that is stuck waiting for a completion
|
||||
// event the agent will never emit. It injects a synthetic error onto promptDoneCh
|
||||
// (non-blocking; the channel is buffered size 1), waits briefly for SendPrompt to
|
||||
// exit and close promptFinished, and marks the execution ready so the workflow
|
||||
// leaves the running state. Returns ErrCancelEscalated so callers know the ACP
|
||||
// cancel did not get a clean acknowledgement.
|
||||
func (m *Manager) escalateStuckCancel(ctx context.Context, execution *AgentExecution, ch <-chan struct{}) error {
|
||||
m.logger.Warn("timed out waiting for in-flight prompt to finish after cancel; escalating",
|
||||
zap.String("execution_id", execution.ID),
|
||||
zap.String("session_id", execution.SessionID))
|
||||
|
||||
select {
|
||||
case execution.promptDoneCh <- PromptCompletionSignal{
|
||||
IsError: true,
|
||||
Error: "cancel escalated: agent did not complete turn within timeout",
|
||||
}:
|
||||
default:
|
||||
// Channel already has a pending signal; SendPrompt will pick that up instead.
|
||||
}
|
||||
|
||||
select {
|
||||
case <-ch:
|
||||
m.logger.Info("in-flight prompt released after cancel escalation",
|
||||
zap.String("execution_id", execution.ID))
|
||||
case <-time.After(cancelEscalationTimeout):
|
||||
m.logger.Warn("in-flight prompt did not release after cancel escalation",
|
||||
zap.String("execution_id", execution.ID))
|
||||
case <-ctx.Done():
|
||||
// Fall through to MarkReady/drain below — once the synthetic signal is
|
||||
// queued, the cleanup must survive the caller's context cancellation
|
||||
// or the execution leaks in the Running state and the stale signal
|
||||
// breaks the next PromptAgent call.
|
||||
}
|
||||
|
||||
if err := m.MarkReady(execution.ID); err != nil {
|
||||
m.logger.Warn("failed to mark execution ready after cancel escalation",
|
||||
zap.String("execution_id", execution.ID),
|
||||
zap.Error(err))
|
||||
}
|
||||
|
||||
// Drain any stale signal that may have been left if the agent completed
|
||||
// concurrently with the escalation timeout (Go's select is non-deterministic
|
||||
// when both promptFinished and time.After are ready, so our send above may
|
||||
// have landed on a channel that SendPrompt had just drained).
|
||||
select {
|
||||
case <-execution.promptDoneCh:
|
||||
default:
|
||||
}
|
||||
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
return ErrCancelEscalated
|
||||
}
|
||||
|
||||
// SetSessionMode changes the session mode for a running agent.
|
||||
@@ -283,6 +354,14 @@ func (m *Manager) ResetAgentContext(ctx context.Context, executionID string) err
|
||||
// with no agent to drive it.
|
||||
var ErrNoExecutionForSession = errors.New("no execution for session")
|
||||
|
||||
// ErrCancelEscalated is returned by CancelAgent when the agent subprocess accepted
|
||||
// the ACP cancel but did not publish a completion event before the timeout. The
|
||||
// lifecycle manager has locally unblocked the in-flight SendPrompt and marked the
|
||||
// execution ready, but the agent never acknowledged the cancel. Callers should
|
||||
// still reconcile session-level state (e.g. transition the task session to
|
||||
// WAITING_FOR_INPUT) — the user's intent was unambiguous.
|
||||
var ErrCancelEscalated = errors.New("cancel escalated: agent did not acknowledge within timeout")
|
||||
|
||||
// CancelAgentBySessionID cancels the current agent turn for a specific session.
|
||||
// Returns ErrNoExecutionForSession (wrapped) when no execution is tracked for the session.
|
||||
func (m *Manager) CancelAgentBySessionID(ctx context.Context, sessionID string) error {
|
||||
|
||||
@@ -0,0 +1,195 @@
|
||||
package lifecycle
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
agentctlClient "github.com/kandev/kandev/internal/agentctl/client"
|
||||
"github.com/kandev/kandev/internal/events"
|
||||
v1 "github.com/kandev/kandev/pkg/api/v1"
|
||||
ws "github.com/kandev/kandev/pkg/websocket"
|
||||
)
|
||||
|
||||
// TestManager_CancelAgent_EscalatesWhenAgentHangs reproduces the stuck-turn bug:
|
||||
// the agent accepts the ACP cancel but never publishes a `complete` event, so the
|
||||
// in-flight SendPrompt would block forever. The manager must escalate by
|
||||
// unblocking SendPrompt via promptDoneCh, marking the execution ready, and
|
||||
// returning ErrCancelEscalated so higher layers can still reconcile DB state.
|
||||
func TestManager_CancelAgent_EscalatesWhenAgentHangs(t *testing.T) {
|
||||
prevWait := cancelWaitTimeout
|
||||
prevEsc := cancelEscalationTimeout
|
||||
cancelWaitTimeout = 50 * time.Millisecond
|
||||
cancelEscalationTimeout = 50 * time.Millisecond
|
||||
t.Cleanup(func() {
|
||||
cancelWaitTimeout = prevWait
|
||||
cancelEscalationTimeout = prevEsc
|
||||
})
|
||||
|
||||
// Mock agentctl: ack agent.cancel but never emit a completion event.
|
||||
mock := newMockAgentServer(t)
|
||||
t.Cleanup(func() { mock.server.Close() })
|
||||
mock.handler = func(msg ws.Message) *ws.Message {
|
||||
if msg.Action == "agent.cancel" {
|
||||
resp, _ := ws.NewResponse(msg.ID, msg.Action, map[string]interface{}{
|
||||
"success": true,
|
||||
})
|
||||
return resp
|
||||
}
|
||||
return mock.defaultHandler(msg)
|
||||
}
|
||||
|
||||
client := createTestClient(t, mock.server.URL)
|
||||
t.Cleanup(client.Close)
|
||||
|
||||
// Establish the agent stream so the client can send agent.cancel.
|
||||
streamCtx, streamCancel := context.WithCancel(context.Background())
|
||||
t.Cleanup(streamCancel)
|
||||
require.NoError(t, client.StreamUpdates(streamCtx, func(_ agentctlClient.AgentEvent) {}, nil, nil))
|
||||
select {
|
||||
case <-mock.wsConnected:
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("mock server did not see WS connection")
|
||||
}
|
||||
|
||||
mgr := newTestManager()
|
||||
|
||||
promptFinished := make(chan struct{})
|
||||
exec := &AgentExecution{
|
||||
ID: "exec-cancel-hang",
|
||||
TaskID: "task-1",
|
||||
SessionID: "session-1",
|
||||
AgentProfileID: "profile-1",
|
||||
Status: v1.AgentStatusRunning,
|
||||
WorkspacePath: "/workspace",
|
||||
agentctl: client,
|
||||
promptDoneCh: make(chan PromptCompletionSignal, 1),
|
||||
promptFinished: promptFinished,
|
||||
}
|
||||
mgr.executionStore.Add(exec)
|
||||
|
||||
// Simulate the in-flight SendPrompt: it blocks reading promptDoneCh, and on
|
||||
// signal closes promptFinished (the same cleanup beginPromptBarrier's deferred
|
||||
// closer does).
|
||||
sendPromptDone := make(chan struct{})
|
||||
var signal PromptCompletionSignal
|
||||
go func() {
|
||||
defer close(sendPromptDone)
|
||||
signal = <-exec.promptDoneCh
|
||||
close(promptFinished)
|
||||
}()
|
||||
|
||||
// Tight bounds: escalation window is cancelWaitTimeout + cancelEscalationTimeout
|
||||
// (100 ms). Use channel-based synchronization per CLAUDE.md — synctest cannot be
|
||||
// used here because the HTTP mock server spawns goroutines outside its bubble.
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 500*time.Millisecond)
|
||||
defer cancel()
|
||||
|
||||
err := mgr.CancelAgent(ctx, exec.ID)
|
||||
require.ErrorIs(t, err, ErrCancelEscalated)
|
||||
|
||||
select {
|
||||
case <-sendPromptDone:
|
||||
case <-time.After(200 * time.Millisecond):
|
||||
t.Fatal("simulated SendPrompt did not release after cancel escalation")
|
||||
}
|
||||
require.True(t, signal.IsError, "escalation signal must carry IsError=true")
|
||||
require.Contains(t, signal.Error, "cancel escalated")
|
||||
|
||||
updated, found := mgr.executionStore.Get(exec.ID)
|
||||
require.True(t, found)
|
||||
require.Equal(t, v1.AgentStatusReady, updated.Status,
|
||||
"execution must be marked ready after cancel escalation so the workflow can proceed")
|
||||
|
||||
mockBus, ok := mgr.eventBus.(*MockEventBus)
|
||||
require.True(t, ok)
|
||||
var sawReady bool
|
||||
for _, ev := range mockBus.PublishedEvents {
|
||||
if ev.Type == events.AgentReady {
|
||||
sawReady = true
|
||||
break
|
||||
}
|
||||
}
|
||||
require.True(t, sawReady, "expected AgentReady event after cancel escalation")
|
||||
}
|
||||
|
||||
// TestManager_CancelAgent_EscalationCleanupSurvivesCtxCancel covers the case where
|
||||
// the caller's context is cancelled during the post-escalation wait. Once the
|
||||
// synthetic signal has been queued on promptDoneCh, the cleanup (MarkReady + drain)
|
||||
// must still run — otherwise the execution leaks in Running state and the stale
|
||||
// signal breaks the next PromptAgent call.
|
||||
func TestManager_CancelAgent_EscalationCleanupSurvivesCtxCancel(t *testing.T) {
|
||||
prevWait := cancelWaitTimeout
|
||||
prevEsc := cancelEscalationTimeout
|
||||
cancelWaitTimeout = 20 * time.Millisecond
|
||||
// Long enough that ctx.Done() fires first during the post-escalation wait.
|
||||
cancelEscalationTimeout = 500 * time.Millisecond
|
||||
t.Cleanup(func() {
|
||||
cancelWaitTimeout = prevWait
|
||||
cancelEscalationTimeout = prevEsc
|
||||
})
|
||||
|
||||
mock := newMockAgentServer(t)
|
||||
t.Cleanup(func() { mock.server.Close() })
|
||||
mock.handler = func(msg ws.Message) *ws.Message {
|
||||
if msg.Action == "agent.cancel" {
|
||||
resp, _ := ws.NewResponse(msg.ID, msg.Action, map[string]interface{}{
|
||||
"success": true,
|
||||
})
|
||||
return resp
|
||||
}
|
||||
return mock.defaultHandler(msg)
|
||||
}
|
||||
|
||||
client := createTestClient(t, mock.server.URL)
|
||||
t.Cleanup(client.Close)
|
||||
|
||||
streamCtx, streamCancel := context.WithCancel(context.Background())
|
||||
t.Cleanup(streamCancel)
|
||||
require.NoError(t, client.StreamUpdates(streamCtx, func(_ agentctlClient.AgentEvent) {}, nil, nil))
|
||||
select {
|
||||
case <-mock.wsConnected:
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("mock server did not see WS connection")
|
||||
}
|
||||
|
||||
mgr := newTestManager()
|
||||
|
||||
// promptFinished is deliberately never closed — simulates a SendPrompt that
|
||||
// is blocked on something other than promptDoneCh (so escalation can't
|
||||
// release it in time, and our ctx will cancel first).
|
||||
exec := &AgentExecution{
|
||||
ID: "exec-cancel-ctx",
|
||||
TaskID: "task-1",
|
||||
SessionID: "session-1",
|
||||
Status: v1.AgentStatusRunning,
|
||||
WorkspacePath: "/workspace",
|
||||
agentctl: client,
|
||||
promptDoneCh: make(chan PromptCompletionSignal, 1),
|
||||
promptFinished: make(chan struct{}),
|
||||
}
|
||||
mgr.executionStore.Add(exec)
|
||||
|
||||
// Cancel the caller's context after escalation starts but before the
|
||||
// post-escalation wait completes.
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond)
|
||||
defer cancel()
|
||||
|
||||
err := mgr.CancelAgent(ctx, exec.ID)
|
||||
require.ErrorIs(t, err, context.DeadlineExceeded,
|
||||
"returns ctx error, not ErrCancelEscalated, when caller ctx cancelled")
|
||||
|
||||
// Critical invariant: cleanup ran despite ctx cancellation.
|
||||
updated, ok := mgr.executionStore.Get(exec.ID)
|
||||
require.True(t, ok)
|
||||
require.Equal(t, v1.AgentStatusReady, updated.Status,
|
||||
"execution must be marked ready even when caller ctx is cancelled mid-escalation")
|
||||
|
||||
select {
|
||||
case sig := <-exec.promptDoneCh:
|
||||
t.Fatalf("stale signal must be drained after escalation; got: %+v", sig)
|
||||
default:
|
||||
}
|
||||
}
|
||||
@@ -477,6 +477,12 @@ func (sm *SessionManager) waitForPromptDone(ctx context.Context, execution *Agen
|
||||
sm.logger.Error("prompt completed with error",
|
||||
zap.String("execution_id", execution.ID),
|
||||
zap.String("error", signal.Error))
|
||||
// Wrap the cancel-escalation sentinel so PromptTask can identify it and
|
||||
// skip the REVIEW task-state transition — the user is cancelling, not
|
||||
// hitting a real agent failure.
|
||||
if strings.HasPrefix(signal.Error, "cancel escalated") {
|
||||
return nil, fmt.Errorf("agent error: %s: %w", signal.Error, ErrCancelEscalated)
|
||||
}
|
||||
return nil, fmt.Errorf("agent error: %s", signal.Error)
|
||||
}
|
||||
|
||||
|
||||
@@ -95,3 +95,71 @@ func TestOrchestratorCancelAfterAgentCrash_UnsticksSession(t *testing.T) {
|
||||
assert.True(t, foundCancel,
|
||||
"expected a 'Turn cancelled by user' status message to be recorded")
|
||||
}
|
||||
|
||||
// TestOrchestratorCancelWhenAgentHangs_UnsticksSession covers the second path to a stuck
|
||||
// session: the agent subprocess is alive and acknowledges the ACP cancel, but never
|
||||
// publishes a completion event. The lifecycle manager's CancelAgent escalates and returns
|
||||
// ErrCancelEscalated. The orchestrator service must treat that as a soft-fail (reconcile
|
||||
// DB state) — otherwise the UI shows "agent is running" forever and cancel stays broken.
|
||||
func TestOrchestratorCancelWhenAgentHangs_UnsticksSession(t *testing.T) {
|
||||
ts := NewOrchestratorTestServer(t)
|
||||
defer ts.Close()
|
||||
|
||||
taskID := ts.CreateTestTask(t, "augment-agent", 2)
|
||||
|
||||
client := NewOrchestratorWSClient(t, ts.Server.URL)
|
||||
defer client.Close()
|
||||
|
||||
startResp, err := client.SendRequest("start-1", ws.ActionOrchestratorStart, map[string]interface{}{
|
||||
"task_id": taskID,
|
||||
"agent_profile_id": "augment-agent",
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, ws.MessageTypeResponse, startResp.Type, "start should succeed")
|
||||
|
||||
var startPayload map[string]interface{}
|
||||
require.NoError(t, startResp.ParsePayload(&startPayload))
|
||||
sessionID, _ := startPayload["session_id"].(string)
|
||||
require.NotEmpty(t, sessionID)
|
||||
|
||||
require.NoError(t, ts.TaskRepo.UpdateTaskSessionState(
|
||||
context.Background(), sessionID, models.TaskSessionStateRunning, "",
|
||||
))
|
||||
|
||||
// Mark the simulated agent as hung: it's tracked, but CancelAgent will return
|
||||
// ErrCancelEscalated to model the real lifecycle escalation.
|
||||
ts.AgentManager.MarkAgentHungForSession(sessionID)
|
||||
|
||||
cancelResp, err := client.SendRequest("cancel-1", ws.ActionAgentCancel, map[string]interface{}{
|
||||
"session_id": sessionID,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, ws.MessageTypeResponse, cancelResp.Type,
|
||||
"cancel should succeed even when lifecycle escalated after an unresponsive agent; got: %+v", cancelResp)
|
||||
|
||||
deadline := time.Now().Add(2 * time.Second)
|
||||
var finalState models.TaskSessionState
|
||||
for time.Now().Before(deadline) {
|
||||
session, gErr := ts.TaskRepo.GetTaskSession(context.Background(), sessionID)
|
||||
require.NoError(t, gErr)
|
||||
finalState = session.State
|
||||
if finalState == models.TaskSessionStateWaitingForInput {
|
||||
break
|
||||
}
|
||||
time.Sleep(25 * time.Millisecond)
|
||||
}
|
||||
assert.Equal(t, models.TaskSessionStateWaitingForInput, finalState,
|
||||
"session should unstick to WAITING_FOR_INPUT after cancel escalation")
|
||||
|
||||
msgs, err := ts.TaskRepo.ListMessages(context.Background(), sessionID)
|
||||
require.NoError(t, err)
|
||||
foundCancel := false
|
||||
for _, m := range msgs {
|
||||
if m.Type == models.MessageTypeStatus && m.Content == "Turn cancelled by user" {
|
||||
foundCancel = true
|
||||
break
|
||||
}
|
||||
}
|
||||
assert.True(t, foundCancel,
|
||||
"expected a 'Turn cancelled by user' status message to be recorded")
|
||||
}
|
||||
|
||||
@@ -47,6 +47,9 @@ type simulatedInstance struct {
|
||||
status v1.AgentStatus
|
||||
statusMu sync.Mutex // Protects status field
|
||||
stopCh chan struct{}
|
||||
// hung indicates the agent acknowledged an ACP cancel but never emitted a
|
||||
// completion event, so CancelAgent should return lifecycle.ErrCancelEscalated.
|
||||
hung bool
|
||||
}
|
||||
|
||||
// NewSimulatedAgentManager creates a new simulated agent manager
|
||||
@@ -392,6 +395,10 @@ func (s *SimulatedAgentManagerClient) IsAgentRunningForSession(ctx context.Conte
|
||||
// Returns lifecycle.ErrNoExecutionForSession when no live execution exists for the session
|
||||
// (for example, after CrashAgentForSession has been used to simulate a crash), matching the
|
||||
// real lifecycle adapter's behaviour.
|
||||
//
|
||||
// If the session has been marked "hung" via MarkAgentHungForSession, the call returns
|
||||
// lifecycle.ErrCancelEscalated to model the real manager's escalation path when the agent
|
||||
// accepts an ACP cancel but never publishes a completion event.
|
||||
func (s *SimulatedAgentManagerClient) CancelAgent(ctx context.Context, sessionID string) error {
|
||||
s.logger.Info("simulated: cancelling agent turn",
|
||||
zap.String("session_id", sessionID))
|
||||
@@ -399,13 +406,32 @@ func (s *SimulatedAgentManagerClient) CancelAgent(ctx context.Context, sessionID
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
for _, inst := range s.instances {
|
||||
if inst.sessionID == sessionID {
|
||||
return nil
|
||||
if inst.sessionID != sessionID {
|
||||
continue
|
||||
}
|
||||
if inst.hung {
|
||||
return fmt.Errorf("session %q: %w", sessionID, lifecycle.ErrCancelEscalated)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("session %q: %w", sessionID, lifecycle.ErrNoExecutionForSession)
|
||||
}
|
||||
|
||||
// MarkAgentHungForSession marks the simulated agent for a session as "hung" so subsequent
|
||||
// CancelAgent calls return lifecycle.ErrCancelEscalated. This models a real agent that
|
||||
// acknowledged the ACP cancel but never emitted a completion event, forcing the lifecycle
|
||||
// manager to escalate.
|
||||
func (s *SimulatedAgentManagerClient) MarkAgentHungForSession(sessionID string) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
for _, inst := range s.instances {
|
||||
if inst.sessionID == sessionID {
|
||||
inst.hung = true
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// CrashAgentForSession simulates an agent subprocess crashing mid-turn.
|
||||
// It stops the agent simulation goroutine and removes the instance from the tracking map,
|
||||
// so subsequent lookups (CancelAgent, IsAgentRunningForSession) behave as if the execution
|
||||
|
||||
@@ -1539,7 +1539,11 @@ func (s *Service) PromptTask(ctx context.Context, taskID, sessionID string, prom
|
||||
// allowWakeFromWaiting=false — this is a revert away from RUNNING, never a
|
||||
// wake transition; the flag only matters when going WAITING_FOR_INPUT → RUNNING.
|
||||
s.updateTaskSessionState(ctx, taskID, sessionID, previousSessionState, "", false)
|
||||
if !isTransientPromptError(err) {
|
||||
// ErrCancelEscalated means the user cancelled and the lifecycle manager had to
|
||||
// force-unblock a hung agent. That is not a "review this failure" condition —
|
||||
// Service.CancelAgent reconciles DB state (WAITING_FOR_INPUT, cancel message,
|
||||
// complete turn) already.
|
||||
if !isTransientPromptError(err) && !errors.Is(err, lifecycle.ErrCancelEscalated) {
|
||||
_ = s.taskRepo.UpdateTaskState(ctx, taskID, v1.TaskStateReview)
|
||||
}
|
||||
s.completeTurnForSession(ctx, sessionID)
|
||||
@@ -1666,16 +1670,25 @@ func (s *Service) CancelAgent(ctx context.Context, sessionID string) error {
|
||||
}
|
||||
|
||||
if err := s.agentManager.CancelAgent(ctx, sessionID); err != nil {
|
||||
if !errors.Is(err, lifecycle.ErrNoExecutionForSession) {
|
||||
switch {
|
||||
case errors.Is(err, lifecycle.ErrNoExecutionForSession):
|
||||
// The session was live but there is no execution to cancel — the agent process
|
||||
// crashed, exited, or never re-registered after a backend restart. Log at error
|
||||
// level so operators notice the stuck state; we still reconcile DB state below
|
||||
// so the UI unsticks.
|
||||
s.logger.Error("agent process appears to have crashed: no live execution for session on cancel",
|
||||
zap.String("session_id", sessionID),
|
||||
zap.Error(err))
|
||||
case errors.Is(err, lifecycle.ErrCancelEscalated):
|
||||
// The agent accepted the ACP cancel but never published a completion event.
|
||||
// The lifecycle manager already unblocked the in-flight prompt and marked the
|
||||
// execution ready; reconcile DB state below so the UI unsticks.
|
||||
s.logger.Warn("agent did not acknowledge cancel; reconciling session state",
|
||||
zap.String("session_id", sessionID),
|
||||
zap.Error(err))
|
||||
default:
|
||||
return fmt.Errorf("cancel agent: %w", err)
|
||||
}
|
||||
// The session was live but there is no execution to cancel — the agent process
|
||||
// crashed, exited, or never re-registered after a backend restart. Log at error
|
||||
// level so operators notice the stuck state; we still reconcile DB state below
|
||||
// so the UI unsticks.
|
||||
s.logger.Error("agent process appears to have crashed: no live execution for session on cancel",
|
||||
zap.String("session_id", sessionID),
|
||||
zap.Error(err))
|
||||
}
|
||||
|
||||
// Transition to WAITING_FOR_INPUT so the user can send a new prompt
|
||||
|
||||
@@ -119,6 +119,45 @@ func TestPromptTask_TransientErrorDoesNotMoveTaskToReview(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestPromptTask_CancelEscalatedDoesNotMoveTaskToReview ensures that when the
|
||||
// lifecycle manager force-unblocks a hung agent (returning ErrCancelEscalated
|
||||
// wrapped in the agent-error format), PromptTask recognises it as a cancel and
|
||||
// leaves the task state untouched — the user cancelled, this is not a failure
|
||||
// the reviewer needs to look at. Service.CancelAgent reconciles session state
|
||||
// separately; PromptTask must not race ahead with UpdateTaskState(REVIEW).
|
||||
func TestPromptTask_CancelEscalatedDoesNotMoveTaskToReview(t *testing.T) {
|
||||
repo := setupTestRepo(t)
|
||||
taskRepo := newMockTaskRepo()
|
||||
agentMgr := &mockAgentManager{
|
||||
isAgentRunning: true,
|
||||
promptErr: fmt.Errorf("agent error: cancel escalated: agent did not complete turn within timeout: %w", lifecycle.ErrCancelEscalated),
|
||||
}
|
||||
svc := createTestServiceWithAgent(repo, newMockStepGetter(), taskRepo, agentMgr)
|
||||
svc.executor = executor.NewExecutor(agentMgr, repo, testLogger(), executor.ExecutorConfig{})
|
||||
|
||||
seedTaskAndSession(t, repo, "task1", "session1", models.TaskSessionStateWaitingForInput)
|
||||
session, err := repo.GetTaskSession(context.Background(), "session1")
|
||||
if err != nil {
|
||||
t.Fatalf("failed to load session: %v", err)
|
||||
}
|
||||
session.AgentExecutionID = "exec-1"
|
||||
if err := repo.UpdateTaskSession(context.Background(), session); err != nil {
|
||||
t.Fatalf("failed to update session: %v", err)
|
||||
}
|
||||
|
||||
_, err = svc.PromptTask(context.Background(), "task1", "session1", "hello", "", false, nil)
|
||||
if err == nil {
|
||||
t.Fatal("expected cancel-escalated error to bubble up from PromptTask")
|
||||
}
|
||||
if !errors.Is(err, lifecycle.ErrCancelEscalated) {
|
||||
t.Fatalf("expected ErrCancelEscalated, got: %v", err)
|
||||
}
|
||||
|
||||
if got, ok := taskRepo.updatedStates["task1"]; ok && got == v1.TaskStateReview {
|
||||
t.Fatalf("expected task state to avoid REVIEW on cancel escalation, got %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestPromptTask_ExecutionNotFoundRevertsStateAndBroadcasts ensures that when
|
||||
// Prompt returns executor.ErrExecutionNotFound, PromptTask reverts the session
|
||||
// state via the broadcasting wrapper (not a direct repo write), so the WS
|
||||
|
||||
Reference in New Issue
Block a user