fix: add stalled agent recovery (#2035)

This commit is contained in:
Carlos Florêncio
2026-07-29 14:57:05 +01:00
committed by GitHub
parent 1f19dbe6d5
commit b734d3f8a5
29 changed files with 1381 additions and 25 deletions
@@ -24,6 +24,22 @@ type AgentEventPayload struct {
PromptGeneration uint64 `json:"prompt_generation,omitempty"`
}
// AgentStalledPayload describes a prompt that has stopped receiving agent events.
// It is advisory only; the lifecycle remains running until the provider completes
// or the user cancels the turn.
type AgentStalledPayload struct {
AgentExecutionID string `json:"agent_execution_id"`
TaskID string `json:"task_id"`
SessionID string `json:"session_id"`
PromptGeneration uint64 `json:"prompt_generation"`
LastActivityAt time.Time `json:"last_activity_at"`
StalledFor time.Duration `json:"stalled_for"`
ToolCallID string `json:"tool_call_id,omitempty"`
ToolName string `json:"tool_name,omitempty"`
ToolTitle string `json:"tool_title,omitempty"`
ToolStatus string `json:"tool_status,omitempty"`
}
// AgentctlEventPayload is the payload for agentctl lifecycle events (starting, ready, error).
type AgentctlEventPayload struct {
TaskID string `json:"task_id"`
@@ -34,6 +34,39 @@ func (p *EventPublisher) PublishAgentEvent(ctx context.Context, eventType string
p.publishAgentEventPayload(ctx, eventType, newAgentEventPayload(execution))
}
// PublishAgentStalled publishes one advisory inactivity signal for a prompt.
func (p *EventPublisher) PublishAgentStalled(
ctx context.Context,
execution *AgentExecution,
promptGeneration uint64,
lastActivityAt time.Time,
stalledFor time.Duration,
) {
if p.eventBus == nil {
return
}
payload := AgentStalledPayload{
AgentExecutionID: execution.ID,
TaskID: execution.TaskID,
SessionID: execution.SessionID,
PromptGeneration: promptGeneration,
LastActivityAt: lastActivityAt,
StalledFor: stalledFor,
}
if tool := execution.activeToolSnapshot(); tool != nil {
payload.ToolCallID = tool.ToolCallID
payload.ToolName = tool.Name
payload.ToolTitle = tool.Title
payload.ToolStatus = tool.Status
}
event := bus.NewEvent(events.AgentStalled, "agent-manager", payload)
if err := p.eventBus.Publish(ctx, events.AgentStalled, event); err != nil {
p.logger.Error("failed to publish stalled agent event",
zap.String("execution_id", execution.ID),
zap.Error(err))
}
}
// publishAgentEventPayload publishes an immutable agent lifecycle snapshot.
func (p *EventPublisher) publishAgentEventPayload(ctx context.Context, eventType string, payload AgentEventPayload) {
if p.eventBus == nil {
@@ -220,6 +220,7 @@ func (s *ExecutionStore) BeginPrompt(executionID string) (uint64, error) {
func beginExecutionPrompt(execution *AgentExecution) uint64 {
execution.promptGeneration++
execution.Status = v1.AgentStatusRunning
execution.resetActiveTool()
return execution.promptGeneration
}
@@ -119,3 +119,19 @@ func TestExecutionStore_BeginPromptAlwaysAdvancesGeneration(t *testing.T) {
t.Fatal("replacement prompt must create generation 2 while already running")
}
}
func TestExecutionStore_BeginPromptClearsActiveTopLevelTool(t *testing.T) {
store := NewExecutionStore()
exec := &AgentExecution{ID: "exec-1", SessionID: "session-1"}
if err := store.Add(exec); err != nil {
t.Fatalf("Add: %v", err)
}
exec.setActiveTool(activeTopLevelTool{ToolCallID: "tool-1", Name: "shell"})
if _, err := store.BeginPrompt(exec.ID); err != nil {
t.Fatalf("BeginPrompt: %v", err)
}
if got := exec.activeToolSnapshot(); got != nil {
t.Fatalf("active tool after BeginPrompt = %#v, want nil", got)
}
}
@@ -346,6 +346,12 @@ func (m *Manager) handleCompleteEvent(execution *AgentExecution, event *agentctl
// separate DB rows mid-sentence, breaking markdown that spans the boundary.
func (m *Manager) handleToolCallEvent(execution *AgentExecution, event agentctl.AgentEvent) agentctl.AgentEvent {
if event.ParentToolCallID == "" {
execution.setActiveTool(activeTopLevelTool{
ToolCallID: event.ToolCallID,
Name: event.ToolName,
Title: event.ToolTitle,
Status: event.ToolStatus,
})
// flushMessageBuffer publishes any remaining buffered content through
// the streaming path itself and always returns "".
m.flushMessageBuffer(execution)
@@ -365,6 +371,9 @@ func (m *Manager) handleToolCallEvent(execution *AgentExecution, event agentctl.
// handleToolUpdateEvent stores completed tool results in session history.
func (m *Manager) handleToolUpdateEvent(execution *AgentExecution, event agentctl.AgentEvent) {
if event.ParentToolCallID == "" && isTerminalToolUpdate(event) {
execution.clearActiveTool(event.ToolCallID)
}
if m.historyManager != nil && execution.historyEnabled && execution.SessionID != "" && event.ToolStatus == toolStatusComplete {
m.flushAssistantHistory(execution)
if err := m.historyManager.AppendToolResult(execution.SessionID, event); err != nil {
@@ -1392,6 +1392,47 @@ func TestHandleAgentEvent_UpdatesLastActivityAt(t *testing.T) {
}
}
func TestHandleAgentEvent_TracksActiveTopLevelTool(t *testing.T) {
mgr, _ := createTestManagerWithTracking()
execution := createTestExecution("exec-1", "task-1", "session-1")
mgr.handleAgentEvent(execution, agentctl.AgentEvent{
Type: "tool_call",
ToolCallID: "top-level-tool",
ToolName: "shell",
ToolTitle: "Start dev server",
ToolStatus: "in_progress",
})
if got := execution.activeToolSnapshot(); got == nil ||
got.ToolCallID != "top-level-tool" || got.Title != "Start dev server" {
t.Fatalf("active top-level tool = %#v, want Start dev server", got)
}
mgr.handleAgentEvent(execution, agentctl.AgentEvent{
Type: "tool_call",
ToolCallID: "subagent-tool",
ToolName: "shell",
ToolTitle: "Run subagent command",
ParentToolCallID: "top-level-tool",
ToolStatus: "in_progress",
})
if got := execution.activeToolSnapshot(); got == nil || got.ToolCallID != "top-level-tool" {
t.Fatalf("subagent tool replaced top-level tool: %#v", got)
}
mgr.handleAgentEvent(execution, agentctl.AgentEvent{
Type: "tool_update",
ToolCallID: "top-level-tool",
ToolStatus: toolStatusComplete,
})
if got := execution.activeToolSnapshot(); got != nil {
t.Fatalf("terminal update left active top-level tool: %#v", got)
}
}
// TestHandleAgentEvent_PromptDoneChDoesNotBlockWhenFull verifies that signaling
// promptDoneCh with a full channel (no receiver) doesn't block the event handler.
func TestHandleAgentEvent_PromptDoneChDoesNotBlockWhenFull(t *testing.T) {
@@ -565,8 +565,9 @@ func (sm *SessionManager) waitForPromptDone(
execution *AgentExecution,
promptGeneration uint64,
) (*PromptResult, error) {
stallTicker := time.NewTicker(30 * time.Second)
stallTicker := time.NewTicker(time.Minute)
defer stallTicker.Stop()
stallReported := false
for {
select {
@@ -618,11 +619,21 @@ func (sm *SessionManager) waitForPromptDone(
lastActivity := execution.lastActivityAt
execution.lastActivityAtMu.Unlock()
if elapsed > 5*time.Minute {
if elapsed >= 5*time.Minute && !stallReported {
sm.logger.Warn("agent stall detected: no events received",
zap.String("execution_id", execution.ID),
zap.Duration("elapsed_since_last_event", elapsed),
zap.Time("last_activity", lastActivity))
if sm.eventPublisher != nil {
sm.eventPublisher.PublishAgentStalled(
ctx,
execution,
promptGeneration,
lastActivity,
elapsed,
)
}
stallReported = true
}
}
}
@@ -11,6 +11,7 @@ import (
"strings"
"sync"
"testing"
"testing/synctest"
"time"
"github.com/coder/acp-go-sdk"
@@ -1734,6 +1735,58 @@ func TestWaitForPromptDone_IgnoresSupersededGenerationSignal(t *testing.T) {
}
}
func TestWaitForPromptDone_PublishesSingleStall(t *testing.T) {
synctest.Test(t, func(t *testing.T) {
eventBus := &MockEventBusWithTracking{}
sm := NewSessionManager(newSessionTestLogger(), make(chan struct{}))
sm.eventPublisher = NewEventPublisher(eventBus, newSessionTestLogger())
execution := &AgentExecution{
ID: "test-exec",
TaskID: "test-task",
SessionID: "test-session",
promptDoneCh: make(chan PromptCompletionSignal, 1),
}
execution.lastActivityAt = time.Now()
ctx, cancel := context.WithCancel(context.Background())
t.Cleanup(cancel)
waitResult := make(chan error, 1)
go func() {
_, err := sm.waitForPromptDone(ctx, execution, 7)
waitResult <- err
}()
time.Sleep(5 * time.Minute)
synctest.Wait()
if got := countPublishedEvents(eventBus, "agent.stalled"); got != 1 {
t.Fatalf("published stalled events = %d, want 1", got)
}
time.Sleep(2 * time.Minute)
synctest.Wait()
if got := countPublishedEvents(eventBus, "agent.stalled"); got != 1 {
t.Fatalf("published stalled events after later checks = %d, want 1", got)
}
cancel()
if err := <-waitResult; !errors.Is(err, context.Canceled) {
t.Fatalf("waitForPromptDone error = %v, want context canceled", err)
}
})
}
func countPublishedEvents(eventBus *MockEventBusWithTracking, subject string) int {
eventBus.mu.Lock()
defer eventBus.mu.Unlock()
var count int
for _, event := range eventBus.PublishedEvents {
if event.Subject == subject {
count++
}
}
return count
}
func TestSendPrompt_TriggerTimeCancelReleaseReturnsErrCancelEscalated(t *testing.T) {
mock := newMockAgentServer(t)
defer mock.Close()
@@ -178,6 +178,8 @@ type AgentExecution struct {
// Last time an agent event was received (for stall detection)
lastActivityAt time.Time
lastActivityAtMu sync.Mutex
activeTool *activeTopLevelTool
activeToolMu sync.RWMutex
// Fires once on the first agent event to publish AgentRunning.
firstActivityOnce sync.Once
@@ -187,6 +189,43 @@ type AgentExecution struct {
sessionSpanMu sync.RWMutex
}
type activeTopLevelTool struct {
ToolCallID string
Name string
Title string
Status string
}
func (e *AgentExecution) setActiveTool(tool activeTopLevelTool) {
e.activeToolMu.Lock()
e.activeTool = &tool
e.activeToolMu.Unlock()
}
func (e *AgentExecution) clearActiveTool(toolCallID string) {
e.activeToolMu.Lock()
if e.activeTool != nil && e.activeTool.ToolCallID == toolCallID {
e.activeTool = nil
}
e.activeToolMu.Unlock()
}
func (e *AgentExecution) resetActiveTool() {
e.activeToolMu.Lock()
e.activeTool = nil
e.activeToolMu.Unlock()
}
func (e *AgentExecution) activeToolSnapshot() *activeTopLevelTool {
e.activeToolMu.RLock()
defer e.activeToolMu.RUnlock()
if e.activeTool == nil {
return nil
}
tool := *e.activeTool
return &tool
}
// setRuntimeEnvironment stores a defensive copy of the effective task runtime
// environment. Callers must use RuntimeEnvironment when reading it.
func (e *AgentExecution) setRuntimeEnvironment(env map[string]string) {
+1
View File
@@ -168,6 +168,7 @@ const (
AgentReady = "agent.ready" // Agent finished a prompt turn, ready for follow-up
AgentCompleted = "agent.completed"
AgentFailed = "agent.failed"
AgentStalled = "agent.stalled"
AgentStopped = "agent.stopped"
AgentContextReset = "agent.context_reset" // Agent subprocess restarted with fresh ACP session
AgentACPSessionCreated = "agent.acp_session_created"
@@ -0,0 +1,87 @@
package orchestrator
import (
"context"
"fmt"
"strings"
"go.uber.org/zap"
"github.com/kandev/kandev/internal/agent/runtime/lifecycle"
"github.com/kandev/kandev/internal/task/models"
v1 "github.com/kandev/kandev/pkg/api/v1"
)
const (
stallCancelTurnButtonTestID = "stall-cancel-turn-button"
metaKeyActionVisibility = "action_visibility"
)
// handleAgentStalled persists an advisory recovery affordance without changing
// the prompt, session, or task lifecycle.
func (s *Service) handleAgentStalled(ctx context.Context, payload lifecycle.AgentStalledPayload) {
if s.messageCreator == nil || s.repo == nil || payload.TaskID == "" || payload.SessionID == "" {
return
}
session, err := s.repo.GetTaskSession(ctx, payload.SessionID)
if err != nil || session == nil || session.State != models.TaskSessionStateRunning {
return
}
generationOwner, ok := s.agentManager.(interface {
OwnsPromptGeneration(sessionID, executionID string, generation uint64) bool
})
if !ok || payload.PromptGeneration == 0 || !generationOwner.OwnsPromptGeneration(
payload.SessionID,
payload.AgentExecutionID,
payload.PromptGeneration,
) {
return
}
turnID, err := s.peekActiveTurnID(ctx, payload.SessionID)
if err != nil {
return
}
if s.turnService != nil && turnID == "" {
return
}
metadata := map[string]interface{}{
metaKeyActionVisibility: "running",
metaKeySessionID: payload.SessionID,
metaKeyTaskID: payload.TaskID,
"actions": []map[string]interface{}{{
actionMetaKeyType: "ws_request",
actionMetaKeyLabel: "Cancel turn",
actionMetaKeyTestID: stallCancelTurnButtonTestID,
"params": map[string]interface{}{
"method": "agent.cancel",
"payload": map[string]interface{}{"session_id": payload.SessionID},
},
}},
}
if err := s.messageCreator.CreateSessionMessage(
ctx,
payload.TaskID,
stallNoticeContent(payload),
payload.SessionID,
string(v1.MessageTypeStatus),
turnID,
metadata,
false,
); err != nil {
s.logger.Warn("failed to create agent stall notice",
zap.String("task_id", payload.TaskID),
zap.String("session_id", payload.SessionID),
zap.Error(err))
}
}
func stallNoticeContent(payload lifecycle.AgentStalledPayload) string {
tool := strings.TrimSpace(payload.ToolTitle)
if tool == "" {
tool = strings.TrimSpace(payload.ToolName)
}
if tool == "" {
return "Still waiting for the agent."
}
return fmt.Sprintf("Still waiting on %s.", tool)
}
@@ -0,0 +1,124 @@
package orchestrator
import (
"context"
"strings"
"testing"
"github.com/kandev/kandev/internal/agent/runtime/lifecycle"
"github.com/kandev/kandev/internal/task/models"
)
func TestHandleAgentStalled_PersistsNeutralRunningNotice(t *testing.T) {
ctx := context.Background()
repo := setupTestRepo(t)
seedSession(t, repo, "task-1", "session-1", "step-1")
before, err := repo.GetTaskSession(ctx, "session-1")
if err != nil {
t.Fatalf("get session before handling stall: %v", err)
}
agentMgr := &mockAgentManager{
repoForExecutionLookup: repo,
currentPromptExecutionID: "execution-1",
}
agentMgr.currentPromptGeneration.Store(7)
svc := createTestServiceWithScheduler(
repo,
newMockStepGetter(),
newMockTaskRepo(),
agentMgr,
)
svc.turnService = &repoTurnService{repo: repo}
activeTurn, err := svc.turnService.StartTurn(ctx, "session-1")
if err != nil {
t.Fatalf("start active turn: %v", err)
}
messages := &mockMessageCreator{}
svc.messageCreator = messages
svc.handleAgentStalled(ctx, lifecycle.AgentStalledPayload{
AgentExecutionID: "execution-1",
TaskID: "task-1",
SessionID: "session-1",
PromptGeneration: 7,
ToolName: "shell",
ToolTitle: "Start dev server",
ToolStatus: "in_progress",
})
if len(messages.sessionMessages) != 1 {
t.Fatalf("session messages = %d, want 1", len(messages.sessionMessages))
}
message := messages.sessionMessages[0]
if !strings.Contains(message.content, "Still waiting on Start dev server") {
t.Fatalf("notice content = %q, want sanitized tool title", message.content)
}
if message.metadata["action_visibility"] != "running" {
t.Fatalf("action visibility = %v, want running", message.metadata["action_visibility"])
}
if message.turnID != activeTurn.ID {
t.Fatalf("notice turn ID = %q, want active turn %q", message.turnID, activeTurn.ID)
}
if _, hasVariant := message.metadata["variant"]; hasVariant {
t.Fatalf("notice metadata unexpectedly set a warning/error variant: %#v", message.metadata)
}
actions, ok := message.metadata["actions"].([]map[string]interface{})
if !ok || len(actions) != 1 {
t.Fatalf("actions = %#v, want one cancel action", message.metadata["actions"])
}
action := actions[0]
if action["label"] != "Cancel turn" || action["test_id"] != "stall-cancel-turn-button" {
t.Fatalf("cancel action = %#v", action)
}
params, ok := action["params"].(map[string]interface{})
if !ok || params["method"] != "agent.cancel" {
t.Fatalf("cancel params = %#v, want agent.cancel", action["params"])
}
after, err := repo.GetTaskSession(ctx, "session-1")
if err != nil {
t.Fatalf("get session after handling stall: %v", err)
}
if after.State != before.State {
t.Fatalf("session state changed from %q to %q", before.State, after.State)
}
}
func TestHandleAgentStalled_RejectsSettledOrStalePrompt(t *testing.T) {
ctx := context.Background()
repo := setupTestRepo(t)
seedSession(t, repo, "task-1", "session-1", "step-1")
agentMgr := &mockAgentManager{currentPromptExecutionID: "execution-1"}
agentMgr.currentPromptGeneration.Store(7)
svc := createTestServiceWithScheduler(repo, newMockStepGetter(), newMockTaskRepo(), agentMgr)
messages := &mockMessageCreator{}
svc.messageCreator = messages
payload := lifecycle.AgentStalledPayload{
AgentExecutionID: "execution-1",
TaskID: "task-1",
SessionID: "session-1",
PromptGeneration: 7,
}
if err := repo.UpdateTaskSessionState(ctx, "session-1", models.TaskSessionStateWaitingForInput, ""); err != nil {
t.Fatalf("settle session: %v", err)
}
svc.handleAgentStalled(ctx, payload)
if len(messages.sessionMessages) != 0 {
t.Fatalf("settled session messages = %d, want 0", len(messages.sessionMessages))
}
if err := repo.UpdateTaskSessionState(ctx, "session-1", models.TaskSessionStateRunning, ""); err != nil {
t.Fatalf("resume session: %v", err)
}
agentMgr.currentPromptGeneration.Store(8)
svc.handleAgentStalled(ctx, payload)
if len(messages.sessionMessages) != 0 {
t.Fatalf("stale generation messages = %d, want 0", len(messages.sessionMessages))
}
}
func TestStallNoticeContentFallsBackWithoutTool(t *testing.T) {
if got := stallNoticeContent(lifecycle.AgentStalledPayload{}); got != "Still waiting for the agent." {
t.Fatalf("stallNoticeContent() = %q, want generic fallback", got)
}
}
@@ -675,6 +675,7 @@ func NewService(
OnAgentReady: s.handleAgentReady,
OnAgentCompleted: s.handleAgentCompleted,
OnAgentFailed: s.handleAgentFailed,
OnAgentStalled: s.handleAgentStalled,
OnAgentStopped: s.handleAgentStopped,
OnAgentStreamEvent: s.handleAgentStreamEvent,
OnACPSessionCreated: s.handleACPSessionCreated,
@@ -99,6 +99,7 @@ type EventHandlers struct {
OnAgentReady func(ctx context.Context, data AgentEventData) // Turn ended; agent idle waiting for follow-up
OnAgentCompleted func(ctx context.Context, data AgentEventData)
OnAgentFailed func(ctx context.Context, data AgentEventData)
OnAgentStalled func(ctx context.Context, payload lifecycle.AgentStalledPayload)
OnAgentStopped func(ctx context.Context, data AgentEventData)
OnACPSessionCreated func(ctx context.Context, data ACPSessionEventData)
@@ -162,6 +163,10 @@ func (w *Watcher) Start(ctx context.Context) error {
w.unsubscribeAll()
return err
}
if err := w.subscribeToAgentStallEvents(); err != nil {
w.unsubscribeAll()
return err
}
// Subscribe to ACP session events
if err := w.subscribeToACPSessionEvents(); err != nil {
@@ -320,6 +325,23 @@ func (w *Watcher) subscribeToAgentEvents() error {
return nil
}
func (w *Watcher) subscribeToAgentStallEvents() error {
if w.handlers.OnAgentStalled == nil {
return nil
}
sub, err := w.eventBus.QueueSubscribe(
events.AgentStalled,
w.queue,
w.createAgentStalledEventHandler(w.handlers.OnAgentStalled),
)
if err != nil {
w.logger.Error("failed to subscribe to agent stalled event", zap.Error(err))
return err
}
w.subscriptions = append(w.subscriptions, sub)
return nil
}
// subscribeToACPSessionEvents subscribes to ACP session lifecycle events
func (w *Watcher) subscribeToACPSessionEvents() error {
if w.handlers.OnACPSessionCreated == nil {
@@ -425,6 +447,20 @@ func (w *Watcher) createAgentEventHandler(handler func(ctx context.Context, data
}
}
func (w *Watcher) createAgentStalledEventHandler(
handler func(ctx context.Context, payload lifecycle.AgentStalledPayload),
) bus.EventHandler {
return func(ctx context.Context, event *bus.Event) error {
var payload lifecycle.AgentStalledPayload
if err := w.parseEventData(event.Data, &payload); err != nil {
w.logger.Error("failed to parse agent stalled event", zap.Error(err))
return nil
}
handler(ctx, payload)
return nil
}
}
// createACPSessionEventHandler creates a bus.EventHandler for ACP session events
func (w *Watcher) createACPSessionEventHandler(handler func(ctx context.Context, data ACPSessionEventData)) bus.EventHandler {
return func(ctx context.Context, event *bus.Event) error {
@@ -303,6 +303,48 @@ func TestAgentEventHandling(t *testing.T) {
})
}
func TestWatcherDispatchesAgentStalled(t *testing.T) {
synctest.Test(t, func(t *testing.T) {
eventBus := newMockEventBus()
var received lifecycle.AgentStalledPayload
var handled bool
var mu sync.Mutex
w := NewWatcher(eventBus, EventHandlers{
OnAgentStalled: func(_ context.Context, payload lifecycle.AgentStalledPayload) {
mu.Lock()
received = payload
handled = true
mu.Unlock()
},
}, "orchestrator-test", createTestLogger())
if err := w.Start(context.Background()); err != nil {
t.Fatalf("start watcher: %v", err)
}
t.Cleanup(func() { _ = w.Stop() })
event := bus.NewEvent(events.AgentStalled, "test", map[string]interface{}{
"agent_execution_id": "agent-456",
"task_id": "task-123",
"session_id": "session-789",
"prompt_generation": 7,
"tool_title": "Start dev server",
})
if err := eventBus.Publish(context.Background(), events.AgentStalled, event); err != nil {
t.Fatalf("publish stalled event: %v", err)
}
synctest.Wait()
mu.Lock()
defer mu.Unlock()
if !handled {
t.Fatal("OnAgentStalled handler was not called")
}
if received.SessionID != "session-789" || received.ToolTitle != "Start dev server" {
t.Fatalf("received stalled payload = %#v", received)
}
})
}
func TestAgentStreamEventHandling(t *testing.T) {
synctest.Test(t, func(t *testing.T) {
eventBus := newMockEventBus()
@@ -30,6 +30,7 @@ const CANCEL_TEST_ID = "recovery-cancel-retry-button";
const TECHNICAL_DETAILS = "Technical details";
const RECOVERY_MESSAGE = "Agent encountered an error";
const RESUME_TEST_ID = "recovery-resume-button";
const STALL_CANCEL_TEST_ID = "stall-cancel-turn-button";
function retryMessage(overrides: Partial<Message> = {}): Message {
return {
@@ -90,9 +91,32 @@ function recoveryMessage(withParams = false): Message {
} as Partial<Message>);
}
function stalledMessage(turnId = "turn-1"): Message {
return retryMessage({
turn_id: turnId,
content: "Still waiting on Start dev server.",
metadata: {
action_visibility: "running",
actions: [
{
type: "ws_request",
label: "Cancel turn",
test_id: STALL_CANCEL_TEST_ID,
params: { method: "agent.cancel", payload: { session_id: "sess-1" } },
},
],
},
} as Partial<Message>);
}
/** ActionMessage reads session state from the store (keyed by comment.session_id),
* so seed it via the provider instead of passing a prop. */
function renderAction(comment: Message, sessionState?: TaskSessionState, sessionError?: string) {
function renderAction(
comment: Message,
sessionState?: TaskSessionState,
sessionError?: string,
activeTurnId?: string,
) {
const initialState: Partial<AppState> = sessionState
? {
taskSessions: {
@@ -100,6 +124,10 @@ function renderAction(comment: Message, sessionState?: TaskSessionState, session
"sess-1": { state: sessionState, error_message: sessionError } as TaskSession,
},
},
turns: {
bySession: {},
activeBySession: activeTurnId ? { "sess-1": activeTurnId } : {},
},
}
: {};
return render(<ActionMessage comment={comment} />, {
@@ -172,6 +200,50 @@ describe("ActionMessage — transient retry (warning variant)", () => {
});
});
describe("ActionMessage — running stall notice", () => {
it("renders a neutral compact notice only while the session is running", () => {
renderAction(stalledMessage(), "RUNNING", undefined, "turn-1");
const notice = screen.getByTestId("running-action-notice");
expect(notice.className).toContain("text-muted-foreground");
expect(notice.className).not.toContain("text-amber");
expect(notice.className).not.toContain("text-red");
expect(notice.querySelector("svg")).toBeNull();
const button = screen.getByTestId(STALL_CANCEL_TEST_ID);
expect(button.className).toContain("min-h-11");
expect(button.className).not.toContain("w-full");
});
it("hides the running-only notice after the session settles", () => {
const { container } = renderAction(stalledMessage(), "WAITING_FOR_INPUT");
expect(container.firstChild).toBeNull();
});
it("sends agent.cancel when Cancel turn is activated", async () => {
renderAction(stalledMessage(), "RUNNING", undefined, "turn-1");
fireEvent.click(screen.getByTestId(STALL_CANCEL_TEST_ID));
await waitFor(() =>
expect(requestMock).toHaveBeenCalledWith("agent.cancel", { session_id: "sess-1" }),
);
});
it("hides an old notice when a later turn is active", () => {
const { container } = renderAction(stalledMessage("turn-1"), "RUNNING", undefined, "turn-2");
expect(container.firstChild).toBeNull();
});
it("hides a running-only notice without a turn ID", () => {
const message = stalledMessage();
message.turn_id = undefined;
const { container } = renderAction(message, "RUNNING", undefined, "turn-1");
expect(container.firstChild).toBeNull();
});
});
describe("ActionMessage — missing PR branch", () => {
it("renders a plain-language recovery panel with collapsed technical details", () => {
renderAction(
@@ -38,6 +38,7 @@ const ICON_MAP: Record<string, React.ElementType> = {
type ActionMeta = {
actions?: MessageAction[];
action_visibility?: "running";
variant?: string;
recovery_actions?: boolean;
is_auth_error?: boolean;
@@ -55,21 +56,45 @@ export const ActionMessage = memo(function ActionMessage({ comment }: { comment:
// Read session state from the store instead of receiving it as a prop, so a
// state transition doesn't re-render every message in the list (only the
// rare action messages that actually depend on it).
const sessionState = useAppStore((state) =>
comment.session_id
? (state.taskSessions.items[comment.session_id]?.state ?? undefined)
: undefined,
);
const sessionError = useAppStore((state) =>
comment.session_id
? (state.taskSessions.items[comment.session_id]?.error_message as string | undefined)
: undefined,
);
const [recoveryRequested, setRecoveryRequested] = useState(false);
const { sessionState, sessionError, activeTurnId } = useActionMessageSession(comment.session_id);
const metadata = comment.metadata as ActionMeta | undefined;
const isWarning = metadata?.variant === "warning";
const message = comment.content || "An error occurred";
if (metadata?.action_visibility === "running") {
if (sessionState !== "RUNNING" || !comment.turn_id || activeTurnId !== comment.turn_id) {
return null;
}
return (
<RunningActionNotice actions={metadata.actions} message={message} taskId={comment.task_id} />
);
}
return (
<SettledActionMessage
metadata={metadata}
message={message}
sessionError={sessionError}
sessionState={sessionState}
taskId={comment.task_id}
/>
);
});
function SettledActionMessage({
metadata,
message,
sessionError,
sessionState,
taskId,
}: {
metadata: ActionMeta | undefined;
message: string;
sessionError?: string;
sessionState?: TaskSessionState;
taskId?: string;
}) {
const [recoveryRequested, setRecoveryRequested] = useState(false);
// A waiting session can still need recovery, so only hide this persisted
// card after its own Resume request is acknowledged (or the session starts).
if (isSessionActive(sessionState) || (metadata?.recovery_actions && recoveryRequested))
@@ -79,17 +104,18 @@ export const ActionMessage = memo(function ActionMessage({ comment }: { comment:
return (
<MissingBranchRecovery
metadata={metadata}
taskId={comment.task_id}
taskId={taskId}
fallbackMessage={message}
technicalDetails={sessionError}
/>
);
}
const iconClass = isWarning ? "text-amber-500" : "text-red-500";
const textClass = isWarning
? "text-amber-600 dark:text-amber-400"
: "text-red-600 dark:text-red-400";
const iconClass = metadata?.variant === "warning" ? "text-amber-500" : "text-red-500";
const textClass =
metadata?.variant === "warning"
? "text-amber-600 dark:text-amber-400"
: "text-red-600 dark:text-red-400";
return (
<div className="w-full">
@@ -103,7 +129,7 @@ export const ActionMessage = memo(function ActionMessage({ comment }: { comment:
{metadata?.actions && metadata.actions.length > 0 && (
<ActionButtons
actions={metadata.actions}
taskId={comment.task_id}
taskId={taskId}
onRecoveryRequested={
metadata.recovery_actions ? () => setRecoveryRequested(true) : undefined
}
@@ -113,7 +139,42 @@ export const ActionMessage = memo(function ActionMessage({ comment }: { comment:
</div>
</div>
);
});
}
function useActionMessageSession(sessionId: Message["session_id"]) {
const sessionState = useAppStore((state) =>
sessionId ? (state.taskSessions.items[sessionId]?.state ?? undefined) : undefined,
);
const sessionError = useAppStore((state) =>
sessionId
? (state.taskSessions.items[sessionId]?.error_message as string | undefined)
: undefined,
);
const activeTurnId = useAppStore((state) =>
sessionId ? (state.turns.activeBySession[sessionId] ?? undefined) : undefined,
);
return { sessionState, sessionError, activeTurnId };
}
function RunningActionNotice({
actions,
message,
taskId,
}: {
actions?: MessageAction[];
message: string;
taskId?: string;
}) {
return (
<div
data-testid="running-action-notice"
className="flex min-w-0 items-center gap-2 py-1 text-muted-foreground"
>
<span className="min-w-0 flex-1 truncate text-xs">{message}</span>
{actions && actions.length > 0 && <ActionButtons actions={actions} taskId={taskId} compact />}
</div>
);
}
function MissingBranchRecovery({
metadata,
@@ -227,19 +288,28 @@ function ActionButtons({
actions,
taskId,
onRecoveryRequested,
compact = false,
}: {
actions: MessageAction[];
taskId?: string;
onRecoveryRequested?: () => void;
compact?: boolean;
}) {
return (
<div className="mt-2 flex flex-col gap-2 sm:flex-row sm:flex-wrap sm:items-center">
<div
className={cn(
compact
? "flex shrink-0 items-center"
: "mt-2 flex flex-col gap-2 sm:flex-row sm:flex-wrap sm:items-center",
)}
>
{actions.map((action, i) => (
<ActionButton
key={action.test_id ?? i}
action={action}
messageTaskId={taskId}
onCompleted={onRecoveryRequested}
compact={compact}
/>
))}
</div>
@@ -250,10 +320,12 @@ function ActionButton({
action,
messageTaskId,
onCompleted,
compact = false,
}: {
action: MessageAction;
messageTaskId?: string;
onCompleted?: () => void;
compact?: boolean;
}): ReactElement | null {
const [state, setState] = useState<"idle" | "busy" | "done" | "error">("idle");
const activeTaskId = useAppStore((s) => s.tasks.activeTaskId);
@@ -312,10 +384,12 @@ function ActionButton({
const button = (
<Button
variant="outline"
variant={compact ? "ghost" : "outline"}
size="sm"
className={cn(
"h-auto min-h-11 w-full gap-1.5 text-xs cursor-pointer sm:min-h-8 sm:w-auto",
compact
? "h-auto min-h-11 shrink-0 px-2 text-xs cursor-pointer sm:min-h-8"
: "h-auto min-h-11 w-full gap-1.5 text-xs cursor-pointer sm:min-h-8 sm:w-auto",
isDestructive && "text-destructive hover:text-destructive",
)}
disabled={disabled}
+3
View File
@@ -163,6 +163,9 @@ export type StatusMetadata = {
attempt?: number;
max_attempts?: number;
retry_in_seconds?: number;
// Running-only action notices are hidden once the session settles. They use
// compact neutral presentation instead of the normal recovery/error card.
action_visibility?: "running";
};
export type RecoveryAuthMethod = {
@@ -3,6 +3,7 @@ import { test, expect } from "../../fixtures/test-base";
import { assertNoDocumentHorizontalOverflow } from "../../helpers/layout-assertions";
import { seedIdleSession } from "../../helpers/session";
import { typeWhileBusy } from "../../helpers/type-while-busy";
import { SessionPage } from "../../pages/session-page";
test.describe("mobile: pause queue recovery", () => {
test.describe.configure({ retries: 1 });
@@ -49,4 +50,58 @@ test.describe("mobile: pause queue recovery", () => {
await expect(chat.getByTestId("queue-chip")).not.toBeVisible({ timeout: 30_000 });
await session.expectChatResponseVisible("simple mock response", 1, { timeout: 30_000 });
});
test("a compact stalled notice remains touch-safe without expanding to full width", async ({
testPage,
apiClient,
seedData,
}) => {
const task = await apiClient.createTask(
seedData.workspaceId,
"Mobile stalled notice recovery",
{
workflow_id: seedData.workflowId,
workflow_step_id: seedData.startStepId,
},
);
const { session_id: sessionId } = await apiClient.seedTaskSession(task.id, {
state: "RUNNING",
});
await apiClient.seedSessionMessage(sessionId, {
type: "status",
content: "Still waiting on Start dev server.",
metadata: {
action_visibility: "running",
actions: [
{
type: "ws_request",
label: "Cancel turn",
test_id: "stall-cancel-turn-button",
params: { method: "agent.cancel", payload: { session_id: sessionId } },
},
],
},
});
await testPage.goto(`/t/${task.id}`);
const session = new SessionPage(testPage);
await session.waitForLoad();
const notice = session.activeChat().getByTestId("running-action-notice");
const cancel = notice.getByTestId("stall-cancel-turn-button");
await expect(notice).toBeVisible();
const [noticeBox, cancelBox] = await Promise.all([notice.boundingBox(), cancel.boundingBox()]);
expect(noticeBox).not.toBeNull();
expect(cancelBox).not.toBeNull();
expect(noticeBox!.height).toBeLessThanOrEqual(56);
expect(cancelBox!.height).toBeGreaterThanOrEqual(44);
expect(cancelBox!.width).toBeLessThan(noticeBox!.width);
await assertNoDocumentHorizontalOverflow(testPage);
await cancel.tap();
await expect(session.idleInput()).toBeVisible({ timeout: 30_000 });
await expect(notice).not.toBeVisible();
await assertNoDocumentHorizontalOverflow(testPage);
});
});
@@ -141,4 +141,55 @@ test.describe("Pause → resume recovery", () => {
await session.expectChatResponseVisible("simple mock response", 1, { timeout: 30_000 });
await expect(session.idleInput()).toBeVisible({ timeout: 30_000 });
});
test("a compact stalled notice cancels a running session without navigation", async ({
testPage,
apiClient,
seedData,
}) => {
const task = await apiClient.createTask(seedData.workspaceId, "Stalled notice recovery", {
workflow_id: seedData.workflowId,
workflow_step_id: seedData.startStepId,
});
const { session_id: sessionId } = await apiClient.seedTaskSession(task.id, {
state: "RUNNING",
});
await apiClient.seedSessionMessage(sessionId, {
type: "status",
content: "Still waiting on Start dev server.",
metadata: {
action_visibility: "running",
actions: [
{
type: "ws_request",
label: "Cancel turn",
test_id: "stall-cancel-turn-button",
params: { method: "agent.cancel", payload: { session_id: sessionId } },
},
],
},
});
await testPage.goto(`/t/${task.id}`);
const session = new SessionPage(testPage);
await session.waitForLoad();
const sessionUrl = testPage.url();
const notice = session.activeChat().getByTestId("running-action-notice");
const cancel = notice.getByTestId("stall-cancel-turn-button");
await expect(notice).toBeVisible();
await expect(notice).toContainText("Still waiting on Start dev server.");
await expect(notice.locator("svg")).toHaveCount(0);
const [noticeBox, cancelBox] = await Promise.all([notice.boundingBox(), cancel.boundingBox()]);
expect(noticeBox).not.toBeNull();
expect(cancelBox).not.toBeNull();
expect(noticeBox!.height).toBeLessThanOrEqual(48);
expect(cancelBox!.width).toBeLessThan(noticeBox!.width);
await cancel.click();
await expect(session.idleInput()).toBeVisible({ timeout: 30_000 });
await expect(notice).not.toBeVisible();
expect(testPage.url()).toBe(sessionUrl);
});
});
@@ -0,0 +1,58 @@
# ADR-2026-07-29-agent-stall-user-controlled-recovery: Keep Agent Stall Recovery User Controlled
**Status:** accepted
**Date:** 2026-07-29
**Area:** backend, frontend, protocol
## Context
An ACP turn can stop emitting events while its process remains alive, most
concretely when a top-level shell tool starts a long-lived process and never
reaches a terminal status. Kandev already detects five minutes of inactivity,
but only logs the condition and leaves the session `RUNNING`. Automatically
timing out every quiet turn would also terminate legitimate long-running tools
whose providers emit no progress frames.
## Decision
Stall detection is advisory and recovery remains operator controlled:
- After five minutes without agent events, Kandev presents one neutral notice
for the active prompt generation and keeps the session `RUNNING`.
- The notice identifies the active top-level tool when that information is
available, without exposing raw command arguments, and provides a compact
neutral **Cancel turn** action. Its wording and styling do not imply that a
quiet long-running tool has failed.
- **Cancel turn** reuses the existing `agent.cancel` escalation path. That path
bounds the ACP acknowledgement wait, releases a hung prompt, and reconciles
the session to an input-ready state.
- Stall detection never automatically cancels, fails, or kills the agent
process. `RUNNING` sessions retain the conservative prompt-admission behavior
defined by
[ADR-2026-07-28-coarse-running-busy-signal](2026-07-28-coarse-running-busy-signal.md).
- The watchdog checks once per 60 seconds and logs only the first detected stall
for a prompt generation; it does not repeat a notice every check.
The observable behavior is specified in
[`docs/specs/agent-stall-recovery/spec.md`](../specs/agent-stall-recovery/spec.md).
## Consequences
Users get a reload-safe explanation and a direct recovery action without
Kandev guessing whether quiet work is expendable. Sessions do not become
promptable until the user cancels or the provider completes the turn, so queued
input and lifecycle ownership remain conservative. A genuinely abandoned turn
can remain `RUNNING` indefinitely if the user does not intervene.
Lifecycle must retain enough in-memory top-level tool identity to enrich the
notice, while the persisted notice message remains the durable UI record.
## Alternatives Considered
- **Automatic timeout.** Rejected because event silence is not proof that a
command is safe to terminate.
- **Warn, then automatically timeout after a grace period.** Rejected for the
same ambiguity; a longer delay reduces but does not remove destructive false
positives.
- **Continue logging only.** Rejected because logs do not make the session
recoverable or explain the blocked Resume action to the user.
+1
View File
@@ -84,3 +84,4 @@ Read individual ADRs for full context. Create new ones via `/record decision` or
| 2026-07-27-legacy-add-branch-live-rescan | [Preserve Live Rescan for Legacy Add Branch](2026-07-27-legacy-add-branch-live-rescan.md) | accepted | backend, protocol | 2026-07-27 |
| 2026-07-28-visible-wip-overflow-queues | [Separate Visible Queueing From WIP Admission](2026-07-28-visible-wip-overflow-queues.md) | proposed | backend, frontend, protocol, workflow | 2026-07-28 |
| 2026-07-28-coarse-running-busy-signal | [Restore Coarse Running Prompt Admission](2026-07-28-coarse-running-busy-signal.md) | accepted | backend, frontend, protocol | 2026-07-28 |
| 2026-07-29-agent-stall-user-controlled-recovery | [Keep Agent Stall Recovery User Controlled](2026-07-29-agent-stall-user-controlled-recovery.md) | accepted | backend, frontend, protocol | 2026-07-29 |
+203
View File
@@ -0,0 +1,203 @@
---
spec: docs/specs/agent-stall-recovery/spec.md
decision: docs/decisions/2026-07-29-agent-stall-user-controlled-recovery.md
created: 2026-07-29
status: draft
---
# Implementation Plan: Agent Stall Recovery
## Overview
Turn the lifecycle watchdog's existing inactivity observation into one
generation-scoped internal event, persist that event as an actionable notice,
and make the notice visible while the session is still `RUNNING`. The existing
cancel-escalation path remains the sole recovery mechanism; no timeout changes
session or process state automatically.
## Confirmed root cause
The issue's deterministic reproduction leaves a top-level shell tool
`in_progress` after it starts a long-lived process. No further ACP frames reach
Kandev, so `waitForPromptDone` never receives its completion signal. Its
detached context has no deadline, and the five-minute watchdog branch only logs
before looping. The existing `agent.cancel` path already bounds acknowledgement
waits, force-releases a stuck prompt, and reconciles the session to
`WAITING_FOR_INPUT`; detection is not connected to that user-facing recovery.
---
## Backend
### Track and report a stalled prompt
- Extend `AgentExecution` in
`apps/backend/internal/agent/runtime/lifecycle/types.go` with
mutex-protected active top-level tool identity.
- Update `handleToolCallEvent` and `handleToolUpdateEvent` in
`apps/backend/internal/agent/runtime/lifecycle/manager_events.go` to record a
top-level tool and clear it on terminal updates. Subagent-internal tool calls
do not replace the top-level identity.
- Update `waitForPromptDone` in
`apps/backend/internal/agent/runtime/lifecycle/session.go` to check once per
60 seconds, detect five minutes of inactivity, and publish `agent.stalled`
once per prompt generation. The payload carries task/session/execution IDs,
prompt generation, last-activity time, stalled duration, and optional tool
ID/name/title/status.
- Add the payload and publisher support in
`apps/backend/internal/agent/runtime/lifecycle/event_types.go`,
`apps/backend/internal/agent/runtime/lifecycle/events.go`, and
`apps/backend/internal/events/types.go`.
### Persist the actionable notice
- Extend `apps/backend/internal/orchestrator/watcher/watcher.go` with the typed
`agent.stalled` subscription and wire it in
`apps/backend/internal/orchestrator/service.go`.
- Add `apps/backend/internal/orchestrator/event_handlers_stall.go` to create a
neutral status message without changing task/session state.
- Use only the tool display title or name in copy. Metadata carries
`action_visibility: running` and one `ws_request` action for `agent.cancel`
with `session_id`, label **Cancel turn**, neutral styling, and stable
`stall-cancel-turn-button` test ID. The copy says Kandev is still waiting; it
does not assert that the tool failed.
- Validate that the session is still `RUNNING` and that the event's immutable
execution/prompt-generation identity is current before persisting. Assign
the notice the active `turn_id` without lazily creating a successor turn.
- If message persistence fails, log the error and leave the running prompt
untouched.
---
## Frontend
### Compact running-session notice
- Update
`apps/web/components/task/chat/messages/action-message.tsx` so ordinary
recovery messages keep their current settled-session visibility, while
`action_visibility: running` messages render only during `RUNNING` when their
persisted `turn_id` matches the active turn.
- Add the optional visibility metadata contract to
`apps/web/components/task/chat/types.ts`.
- Render a running-only message as one inline row: muted `text-xs` copy, no
alert icon, no warning/error color, no tinted background or border, and one
neutral **Cancel turn** button at the end.
- Extend the action-button presentation with a compact inline mode. The button
is content-width at every breakpoint, uses the existing small desktop
height, and retains a minimum 44px height on phones.
### Mobile design contract
- **Desktop outcome:** one low-emphasis inline status row in the chat footer
says Kandev is still waiting and exposes a neutral **Cancel turn** button.
- **Mobile entry point and outcome:** the same compact row appears in the active
chat footer; **Cancel turn** remains inline and content-width, has a 44px
touch height, and returns the session to input-ready state.
- **Nearest shipped exemplar:** `ActionMessage` supplies the message/action
behavior and `mobile-pause-resume-recovery.spec.ts` supplies the touch
cancellation flow. The existing stacked full-width mobile `ActionButtons`
geometry is intentionally not reused for this low-emphasis notice.
- **Presentation:** inline in the existing chat footer; no drawer or new
navigation layer is needed for a frequent, single-step recovery action.
- **Hierarchy and geometry:** notice copy truncates before the trailing action;
chat remains the sole scroll owner, existing safe-area behavior is retained,
the row does not introduce a card or stacked action area, and the action uses
the shared 44px mobile touch height without expanding to full width.
- **Shared logic:** message metadata, session-state visibility, and the
`agent.cancel` handler are identical across viewports.
---
## Tests
- **What:** a top-level `in_progress` tool is retained as stall context while
subagent tools do not replace it and terminal updates clear it.
**File:**
`apps/backend/internal/agent/runtime/lifecycle/manager_events_test.go`.
**How:** table-driven lifecycle event tests.
- **What:** five minutes without events publishes one generation-scoped
`agent.stalled` event, does not complete the prompt, and does not duplicate
on later checks.
**File:** `apps/backend/internal/agent/runtime/lifecycle/session_test.go`.
**How:** `testing/synctest` with the real ticker and event bus.
- **What:** the orchestrator persists one neutral notice with sanitized tool
copy and the exact `agent.cancel` action without transitioning session state.
**File:**
`apps/backend/internal/orchestrator/event_handlers_stall_test.go` and
`apps/backend/internal/orchestrator/watcher/watcher_test.go`.
**How:** direct handler test with the existing message-creator fake plus a
memory-bus watcher dispatch test.
- **What:** running-only action messages render as a compact neutral row during
`RUNNING`, hide after settlement, and do not change ordinary recovery-message
visibility.
**File:**
`apps/web/components/task/chat/messages/action-message.test.tsx`.
**How:** rendered store-backed component tests.
## E2E Tests
- **Scenario:** a running session with a persisted stall notice exposes
**Cancel turn**, and activating it makes the session input-ready without a
backend restart.
**File:**
`apps/web/e2e/tests/session/pause-resume-recovery.spec.ts`.
**What to verify:** neutral compact copy, action request, notice
disappearance, idle composer, and unchanged task URL.
- **Scenario:** the same recovery is reachable by touch on a phone viewport.
**File:**
`apps/web/e2e/tests/session/mobile-pause-resume-recovery.spec.ts`.
**What to verify:** the content-width action is at least 44px high, the seeded
notice remains one compact row, `.tap()` settles the session, and the document
has no horizontal overflow.
- Seed the `RUNNING` session and actionable status message through the existing
E2E-only `seedTaskSession` and `seedSessionMessage` helpers. The cancel path's
missing-live-execution reconciliation provides a deterministic end-to-end
recovery without waiting five wall-clock minutes; lifecycle timing remains
covered by the `synctest` regression.
## Implementation Waves And Parallel Candidates
Execution is sequential in the primary conversation. No task is marked
parallel-safe because each task consumes the event or metadata contract created
by the previous one.
Wave 1:
- [x] [Task 01: Detect and publish stalled prompts](task-01-detect-stalled-prompt.md)
Wave 2:
- [x] [Task 02: Persist an actionable stall notice](task-02-persist-stall-warning.md)
Wave 3:
- [x] [Task 03: Render the notice during running sessions](task-03-render-running-warning.md)
Wave 4:
- [x] [Task 04: Prove desktop and mobile recovery](task-04-e2e-stall-recovery.md)
## Required workflow verification
After all targeted task checks pass:
1. Run `make fmt`.
2. Run `make typecheck test lint`.
3. Commit the explicit changed paths with a Conventional Commit message.
These broad commands are included because the Improve workflow explicitly
requires them; the task-level commands remain the TDD evidence for each change.
## Risks and non-goals
- Event silence remains ambiguous, so the notice is advisory and cannot alter
lifecycle state by itself.
- Tool titles are already user-visible display data; raw command arguments must
not be copied into the notice.
- The notice is persisted once per prompt generation and assigned the active
turn ID so reloads retain it only for the affected turn. A delayed event is
rejected when the session has settled or execution/generation ownership has
moved on; a later `RUNNING` turn cannot resurrect the old notice.
- No schema migration, public HTTP endpoint, configurable timeout, automatic
process termination, or relaxed `RUNNING` prompt-admission rule is planned.
@@ -0,0 +1,59 @@
---
id: "01-detect-stalled-prompt"
title: "Detect and publish stalled prompts"
status: done
wave: 1
depends_on: []
plan: "plan.md"
spec: "../../specs/agent-stall-recovery/spec.md"
---
# Task 01: Detect and publish stalled prompts
## Acceptance
- A prompt with no agent events for five minutes publishes one
generation-scoped `agent.stalled` event and remains in progress.
- The event includes active top-level tool display identity when available;
subagent tools do not replace it and terminal tool updates clear it.
- The watchdog checks every 60 seconds and does not repeat the notice or log
for the same prompt generation.
## Verification
- `make -C apps/backend test` (regressions: `TestHandleAgentEvent_TracksActiveTopLevelTool` and `TestWaitForPromptDone_PublishesSingleStall` in `internal/agent/runtime/lifecycle`)
The `testing/synctest` regression must first fail because no `agent.stalled`
event is published, then pass without shortening production durations.
## Files likely touched
- `apps/backend/internal/agent/runtime/lifecycle/types.go`
- `apps/backend/internal/agent/runtime/lifecycle/manager_events.go`
- `apps/backend/internal/agent/runtime/lifecycle/manager_events_test.go`
- `apps/backend/internal/agent/runtime/lifecycle/session.go`
- `apps/backend/internal/agent/runtime/lifecycle/session_test.go`
- `apps/backend/internal/agent/runtime/lifecycle/event_types.go`
- `apps/backend/internal/agent/runtime/lifecycle/events.go`
- `apps/backend/internal/events/types.go`
## Dependencies
None.
## Parallelism
Sequential. Task 02 consumes the event and payload contract defined here.
## Inputs
- Spec sections `What`, `Failure modes`, and the first three scenarios
- ADR decision bullets for advisory detection and once-per-generation logging
- Existing `recordActivity`, `handleToolCallEvent`, `handleToolUpdateEvent`, and
`waitForPromptDone` patterns
## Output contract
Report the RED assertion, event payload shape, tool-tracking behavior, exact
test result, files changed, blockers, and risks. Mark this task `done` and
update its plan checkbox in the same conversation.
@@ -0,0 +1,56 @@
---
id: "02-persist-stall-warning"
title: "Persist an actionable stall notice"
status: done
wave: 2
depends_on: ["01-detect-stalled-prompt"]
plan: "plan.md"
spec: "../../specs/agent-stall-recovery/spec.md"
---
# Task 02: Persist an actionable stall notice
## Acceptance
- The orchestrator consumes `agent.stalled` and creates one neutral status
message without changing task, session, or process state.
- Copy says Kandev is still waiting, uses only the tool display title/name, and
falls back to a generic notice without asserting failure.
- Metadata exposes a running-only **Cancel turn** `agent.cancel` action with the
affected `session_id`, neutral styling, and stable test ID.
## Verification
- `make -C apps/backend test` (regressions: `TestHandleAgentStalled` in `internal/orchestrator` and `TestWatcher.*AgentStalled` in `internal/orchestrator/watcher`)
The handler test must first fail because no notice is persisted, then pass
with exact metadata assertions and unchanged session state.
## Files likely touched
- `apps/backend/internal/orchestrator/watcher/watcher.go`
- `apps/backend/internal/orchestrator/watcher/watcher_test.go`
- `apps/backend/internal/orchestrator/service.go`
- `apps/backend/internal/orchestrator/event_handlers_stall.go`
- `apps/backend/internal/orchestrator/event_handlers_stall_test.go`
## Dependencies
Task 01.
## Parallelism
Sequential. It consumes Task 01's event contract and defines Task 03's message
metadata contract.
## Inputs
- Spec scenarios for tool-specific and generic notices
- Plan section `Persist the actionable notice`
- Existing transient-retry and recoverable-failure action-message builders
## Output contract
Report the RED assertion, notice copy and metadata, proof that no state
transition occurred, exact test results, files changed, blockers, and risks.
Mark this task `done` and update its plan checkbox in the same conversation.
@@ -0,0 +1,57 @@
---
id: "03-render-running-warning"
title: "Render the notice during running sessions"
status: done
wave: 3
depends_on: ["02-persist-stall-warning"]
plan: "plan.md"
spec: "../../specs/agent-stall-recovery/spec.md"
---
# Task 03: Render the notice during running sessions
## Acceptance
- `action_visibility: running` messages render while the session is `RUNNING`
only when their `turn_id` matches the active turn, and hide after settlement
or when a later turn becomes active.
- Existing recovery messages keep their current visibility behavior.
- The running-only notice is one inline row with muted neutral copy, no alert
icon, warning/error color, tinted background, border, or stacked action area.
- **Cancel turn** is neutral and content-width at every breakpoint. It stays
compact on desktop and has a minimum 44px touch height on phones.
## Verification
- `cd apps && pnpm --filter @kandev/web test -- --run components/task/chat/messages/action-message.test.tsx`
The component regression must first fail because all action messages are hidden
during `RUNNING`, then pass with the neutral compact presentation without
weakening ordinary recovery visibility.
## Files likely touched
- `apps/web/components/task/chat/messages/action-message.tsx`
- `apps/web/components/task/chat/messages/action-message.test.tsx`
- `apps/web/components/task/chat/types.ts`
## Dependencies
Task 02.
## Parallelism
Sequential. It consumes the backend metadata contract and precedes E2E.
## Inputs
- Spec desktop/mobile notice scenarios
- Plan frontend section and mobile design contract
- Existing `ActionMessage`, `ActionButtons`, and mobile button sizing; do not
reuse the stacked full-width mobile action layout for this notice
## Output contract
Report the RED assertion, visibility rule, desktop/mobile presentation,
targeted test result, files changed, blockers, and risks. Mark this task `done`
and update its plan checkbox in the same conversation.
@@ -0,0 +1,63 @@
---
id: "04-e2e-stall-recovery"
title: "Prove desktop and mobile stall recovery"
status: done
wave: 4
depends_on:
- "02-persist-stall-warning"
- "03-render-running-warning"
plan: "plan.md"
spec: "../../specs/agent-stall-recovery/spec.md"
---
# Task 04: Prove desktop and mobile stall recovery
## Acceptance
- Desktop Playwright proves a running stalled session shows **Cancel turn** and
becomes input-ready after the action without navigation or restart. The
notice is a neutral single inline row rather than an alert card.
- Mobile Playwright performs the same recovery with `.tap()`, proves a minimum
44px content-width action, verifies the seeded notice remains a compact row,
and finds no document horizontal overflow.
- Tests use the E2E-only session/message seed helpers; they do not wait five
wall-clock minutes or add production test hooks.
## Verification
- `cd apps && pnpm --filter @kandev/web e2e:run tests/session/pause-resume-recovery.spec.ts`
- `cd apps && pnpm --filter @kandev/web e2e:run tests/session/mobile-pause-resume-recovery.spec.ts -- --project=mobile-chrome`
Each E2E scenario must first fail because the running-only notice is hidden,
then pass against freshly built backend and Vite artifacts.
The browser scenarios seed the persisted notice for deterministic coverage;
the real five-minute watchdog boundary is covered by the lifecycle synctest.
## Files likely touched
- `apps/web/e2e/pages/session-page.ts`
- `apps/web/e2e/tests/session/pause-resume-recovery.spec.ts`
- `apps/web/e2e/tests/session/mobile-pause-resume-recovery.spec.ts`
## Dependencies
Tasks 02 and 03.
## Parallelism
Sequential. It validates the integrated backend metadata and frontend
presentation.
## Inputs
- Spec cancellation and phone scenarios
- Plan E2E section and mobile design contract
- Existing `seedTaskSession`, `seedSessionMessage`, and pause/recovery E2E
patterns
## Output contract
Report both RED failures, desktop and mobile user outcomes, exact managed-runner
results, rendered mobile inspection, files changed, blockers, and risks. Mark
this task `done` and update its plan checkbox in the same conversation.
+1
View File
@@ -155,6 +155,7 @@ System pages (Radarr/Sonarr-style) for status, disk usage, database maintenance,
| Spec | Status |
|---|---|
| [agent-resume-runtime-recovery](agent-resume-runtime-recovery/spec.md) | shipped |
| [agent-stall-recovery](agent-stall-recovery/spec.md) | draft |
| [auth](auth/spec.md) | building |
| [create-local-repository](create-local-repository/spec.md) | shipped |
| [workflow-cycle-guardrails](workflow-cycle-guardrails/spec.md) | building |
+93
View File
@@ -0,0 +1,93 @@
---
status: draft
created: 2026-07-29
owner: Kandev
---
# Agent Stall Recovery
Decision:
[ADR-2026-07-29-agent-stall-user-controlled-recovery](../../decisions/2026-07-29-agent-stall-user-controlled-recovery.md)
## Why
An agent turn can remain `RUNNING` forever after its active tool stops emitting
events. Users currently see only “Agent is running,” while Resume is correctly
blocked by the active-session guard and the backend repeats a diagnostic that
the UI never receives.
## Broken behavior
`waitForPromptDone` detects five minutes without events but only logs the
condition every 30 seconds. A top-level shell tool that remains `in_progress`
can therefore hold the prompt waiter and session state indefinitely even
though Kandev already has a bounded cancel-escalation path capable of releasing
the turn.
## What
- Kandev checks running prompts for inactivity once per 60 seconds.
- After five minutes without agent events, Kandev creates at most one
user-visible notice for that prompt generation.
- The notice says Kandev is still waiting and includes the active top-level
tool's display title or name when available. It does not assert that the tool
failed and does not include raw command arguments.
- The notice is a single compact inline row with muted neutral copy and a
neutral **Cancel turn** action. It has no warning/error colors, alert icon,
tinted background, or alert-card treatment.
- On phones, **Cancel turn** remains inline and content-width rather than
becoming a full-width row, while retaining a minimum 44px touch height.
Activating it uses the existing `agent.cancel` request.
- The notice remains visible and actionable while the affected prompt's
`turn_id` is the active turn in a `RUNNING` session, including after a page
reload. It is hidden when that turn settles or a later turn becomes active.
- Detection alone does not change task state, session state, prompt admission,
or process liveness.
- The backend logs the first stall detected for a prompt generation and does
not emit another notice or log entry on every watchdog check.
## Failure modes
- If active tool identity is unavailable, the generic notice and cancel action
still appear.
- If the agent acknowledges cancellation, normal cancellation reconciliation
makes the session input-ready.
- If the agent does not acknowledge cancellation, the existing bounded
cancel-escalation path releases the prompt and reconciles the session.
- If notice-message persistence fails, the failure is logged without changing
or terminating the running session.
## Scenarios
- **GIVEN** a running prompt whose top-level shell tool is `in_progress`,
**WHEN** no agent event arrives for five minutes, **THEN** one compact neutral
notice names that tool and offers **Cancel turn** while the session remains
`RUNNING`.
- **GIVEN** a running prompt with no known active tool, **WHEN** no agent event
arrives for five minutes, **THEN** one generic notice offers **Cancel turn**
while the session remains `RUNNING`.
- **GIVEN** a notice already exists for the active prompt generation,
**WHEN** subsequent watchdog checks observe the same stall, **THEN** Kandev
creates no duplicate notice and emits no repeated stall log.
- **GIVEN** a persisted notice belongs to an earlier prompt generation,
**WHEN** a later prompt is `RUNNING`, **THEN** the earlier notice is not
rendered and a delayed stall event cannot create a new notice for it.
- **GIVEN** a stall notice is visible, **WHEN** the user activates
**Cancel turn**, **THEN** the existing cancellation path settles the turn and
the session becomes available for new input without a backend restart.
- **GIVEN** a stall notice is visible on a phone viewport, **WHEN** the user taps
**Cancel turn**, **THEN** the same cancellation outcome is reachable through
an inline, content-width touch target of at least 44px.
- **GIVEN** a quiet but legitimate long-running turn, **WHEN** the inactivity
threshold passes and the user does not cancel, **THEN** Kandev leaves the
turn and process running.
## Out of scope
- Automatically timing out, failing, cancelling, or killing quiet turns.
- Making the inactivity threshold user-configurable.
- Provider-specific repair of shell commands that hold child-process streams
open.
- Treating event silence as proof that an agent process crashed.
- Allowing Resume or direct prompt admission while the session remains
`RUNNING`.