fix: add stalled agent recovery (#2035)
This commit is contained in:
@@ -24,6 +24,22 @@ type AgentEventPayload struct {
|
||||
PromptGeneration uint64 `json:"prompt_generation,omitempty"`
|
||||
}
|
||||
|
||||
// AgentStalledPayload describes a prompt that has stopped receiving agent events.
|
||||
// It is advisory only; the lifecycle remains running until the provider completes
|
||||
// or the user cancels the turn.
|
||||
type AgentStalledPayload struct {
|
||||
AgentExecutionID string `json:"agent_execution_id"`
|
||||
TaskID string `json:"task_id"`
|
||||
SessionID string `json:"session_id"`
|
||||
PromptGeneration uint64 `json:"prompt_generation"`
|
||||
LastActivityAt time.Time `json:"last_activity_at"`
|
||||
StalledFor time.Duration `json:"stalled_for"`
|
||||
ToolCallID string `json:"tool_call_id,omitempty"`
|
||||
ToolName string `json:"tool_name,omitempty"`
|
||||
ToolTitle string `json:"tool_title,omitempty"`
|
||||
ToolStatus string `json:"tool_status,omitempty"`
|
||||
}
|
||||
|
||||
// AgentctlEventPayload is the payload for agentctl lifecycle events (starting, ready, error).
|
||||
type AgentctlEventPayload struct {
|
||||
TaskID string `json:"task_id"`
|
||||
|
||||
@@ -34,6 +34,39 @@ func (p *EventPublisher) PublishAgentEvent(ctx context.Context, eventType string
|
||||
p.publishAgentEventPayload(ctx, eventType, newAgentEventPayload(execution))
|
||||
}
|
||||
|
||||
// PublishAgentStalled publishes one advisory inactivity signal for a prompt.
|
||||
func (p *EventPublisher) PublishAgentStalled(
|
||||
ctx context.Context,
|
||||
execution *AgentExecution,
|
||||
promptGeneration uint64,
|
||||
lastActivityAt time.Time,
|
||||
stalledFor time.Duration,
|
||||
) {
|
||||
if p.eventBus == nil {
|
||||
return
|
||||
}
|
||||
payload := AgentStalledPayload{
|
||||
AgentExecutionID: execution.ID,
|
||||
TaskID: execution.TaskID,
|
||||
SessionID: execution.SessionID,
|
||||
PromptGeneration: promptGeneration,
|
||||
LastActivityAt: lastActivityAt,
|
||||
StalledFor: stalledFor,
|
||||
}
|
||||
if tool := execution.activeToolSnapshot(); tool != nil {
|
||||
payload.ToolCallID = tool.ToolCallID
|
||||
payload.ToolName = tool.Name
|
||||
payload.ToolTitle = tool.Title
|
||||
payload.ToolStatus = tool.Status
|
||||
}
|
||||
event := bus.NewEvent(events.AgentStalled, "agent-manager", payload)
|
||||
if err := p.eventBus.Publish(ctx, events.AgentStalled, event); err != nil {
|
||||
p.logger.Error("failed to publish stalled agent event",
|
||||
zap.String("execution_id", execution.ID),
|
||||
zap.Error(err))
|
||||
}
|
||||
}
|
||||
|
||||
// publishAgentEventPayload publishes an immutable agent lifecycle snapshot.
|
||||
func (p *EventPublisher) publishAgentEventPayload(ctx context.Context, eventType string, payload AgentEventPayload) {
|
||||
if p.eventBus == nil {
|
||||
|
||||
@@ -220,6 +220,7 @@ func (s *ExecutionStore) BeginPrompt(executionID string) (uint64, error) {
|
||||
func beginExecutionPrompt(execution *AgentExecution) uint64 {
|
||||
execution.promptGeneration++
|
||||
execution.Status = v1.AgentStatusRunning
|
||||
execution.resetActiveTool()
|
||||
return execution.promptGeneration
|
||||
}
|
||||
|
||||
|
||||
@@ -119,3 +119,19 @@ func TestExecutionStore_BeginPromptAlwaysAdvancesGeneration(t *testing.T) {
|
||||
t.Fatal("replacement prompt must create generation 2 while already running")
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecutionStore_BeginPromptClearsActiveTopLevelTool(t *testing.T) {
|
||||
store := NewExecutionStore()
|
||||
exec := &AgentExecution{ID: "exec-1", SessionID: "session-1"}
|
||||
if err := store.Add(exec); err != nil {
|
||||
t.Fatalf("Add: %v", err)
|
||||
}
|
||||
exec.setActiveTool(activeTopLevelTool{ToolCallID: "tool-1", Name: "shell"})
|
||||
|
||||
if _, err := store.BeginPrompt(exec.ID); err != nil {
|
||||
t.Fatalf("BeginPrompt: %v", err)
|
||||
}
|
||||
if got := exec.activeToolSnapshot(); got != nil {
|
||||
t.Fatalf("active tool after BeginPrompt = %#v, want nil", got)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -346,6 +346,12 @@ func (m *Manager) handleCompleteEvent(execution *AgentExecution, event *agentctl
|
||||
// separate DB rows mid-sentence, breaking markdown that spans the boundary.
|
||||
func (m *Manager) handleToolCallEvent(execution *AgentExecution, event agentctl.AgentEvent) agentctl.AgentEvent {
|
||||
if event.ParentToolCallID == "" {
|
||||
execution.setActiveTool(activeTopLevelTool{
|
||||
ToolCallID: event.ToolCallID,
|
||||
Name: event.ToolName,
|
||||
Title: event.ToolTitle,
|
||||
Status: event.ToolStatus,
|
||||
})
|
||||
// flushMessageBuffer publishes any remaining buffered content through
|
||||
// the streaming path itself and always returns "".
|
||||
m.flushMessageBuffer(execution)
|
||||
@@ -365,6 +371,9 @@ func (m *Manager) handleToolCallEvent(execution *AgentExecution, event agentctl.
|
||||
|
||||
// handleToolUpdateEvent stores completed tool results in session history.
|
||||
func (m *Manager) handleToolUpdateEvent(execution *AgentExecution, event agentctl.AgentEvent) {
|
||||
if event.ParentToolCallID == "" && isTerminalToolUpdate(event) {
|
||||
execution.clearActiveTool(event.ToolCallID)
|
||||
}
|
||||
if m.historyManager != nil && execution.historyEnabled && execution.SessionID != "" && event.ToolStatus == toolStatusComplete {
|
||||
m.flushAssistantHistory(execution)
|
||||
if err := m.historyManager.AppendToolResult(execution.SessionID, event); err != nil {
|
||||
|
||||
@@ -1392,6 +1392,47 @@ func TestHandleAgentEvent_UpdatesLastActivityAt(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleAgentEvent_TracksActiveTopLevelTool(t *testing.T) {
|
||||
mgr, _ := createTestManagerWithTracking()
|
||||
execution := createTestExecution("exec-1", "task-1", "session-1")
|
||||
|
||||
mgr.handleAgentEvent(execution, agentctl.AgentEvent{
|
||||
Type: "tool_call",
|
||||
ToolCallID: "top-level-tool",
|
||||
ToolName: "shell",
|
||||
ToolTitle: "Start dev server",
|
||||
ToolStatus: "in_progress",
|
||||
})
|
||||
|
||||
if got := execution.activeToolSnapshot(); got == nil ||
|
||||
got.ToolCallID != "top-level-tool" || got.Title != "Start dev server" {
|
||||
t.Fatalf("active top-level tool = %#v, want Start dev server", got)
|
||||
}
|
||||
|
||||
mgr.handleAgentEvent(execution, agentctl.AgentEvent{
|
||||
Type: "tool_call",
|
||||
ToolCallID: "subagent-tool",
|
||||
ToolName: "shell",
|
||||
ToolTitle: "Run subagent command",
|
||||
ParentToolCallID: "top-level-tool",
|
||||
ToolStatus: "in_progress",
|
||||
})
|
||||
|
||||
if got := execution.activeToolSnapshot(); got == nil || got.ToolCallID != "top-level-tool" {
|
||||
t.Fatalf("subagent tool replaced top-level tool: %#v", got)
|
||||
}
|
||||
|
||||
mgr.handleAgentEvent(execution, agentctl.AgentEvent{
|
||||
Type: "tool_update",
|
||||
ToolCallID: "top-level-tool",
|
||||
ToolStatus: toolStatusComplete,
|
||||
})
|
||||
|
||||
if got := execution.activeToolSnapshot(); got != nil {
|
||||
t.Fatalf("terminal update left active top-level tool: %#v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestHandleAgentEvent_PromptDoneChDoesNotBlockWhenFull verifies that signaling
|
||||
// promptDoneCh with a full channel (no receiver) doesn't block the event handler.
|
||||
func TestHandleAgentEvent_PromptDoneChDoesNotBlockWhenFull(t *testing.T) {
|
||||
|
||||
@@ -565,8 +565,9 @@ func (sm *SessionManager) waitForPromptDone(
|
||||
execution *AgentExecution,
|
||||
promptGeneration uint64,
|
||||
) (*PromptResult, error) {
|
||||
stallTicker := time.NewTicker(30 * time.Second)
|
||||
stallTicker := time.NewTicker(time.Minute)
|
||||
defer stallTicker.Stop()
|
||||
stallReported := false
|
||||
|
||||
for {
|
||||
select {
|
||||
@@ -618,11 +619,21 @@ func (sm *SessionManager) waitForPromptDone(
|
||||
lastActivity := execution.lastActivityAt
|
||||
execution.lastActivityAtMu.Unlock()
|
||||
|
||||
if elapsed > 5*time.Minute {
|
||||
if elapsed >= 5*time.Minute && !stallReported {
|
||||
sm.logger.Warn("agent stall detected: no events received",
|
||||
zap.String("execution_id", execution.ID),
|
||||
zap.Duration("elapsed_since_last_event", elapsed),
|
||||
zap.Time("last_activity", lastActivity))
|
||||
if sm.eventPublisher != nil {
|
||||
sm.eventPublisher.PublishAgentStalled(
|
||||
ctx,
|
||||
execution,
|
||||
promptGeneration,
|
||||
lastActivity,
|
||||
elapsed,
|
||||
)
|
||||
}
|
||||
stallReported = true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -11,6 +11,7 @@ import (
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"testing/synctest"
|
||||
"time"
|
||||
|
||||
"github.com/coder/acp-go-sdk"
|
||||
@@ -1734,6 +1735,58 @@ func TestWaitForPromptDone_IgnoresSupersededGenerationSignal(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestWaitForPromptDone_PublishesSingleStall(t *testing.T) {
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
eventBus := &MockEventBusWithTracking{}
|
||||
sm := NewSessionManager(newSessionTestLogger(), make(chan struct{}))
|
||||
sm.eventPublisher = NewEventPublisher(eventBus, newSessionTestLogger())
|
||||
execution := &AgentExecution{
|
||||
ID: "test-exec",
|
||||
TaskID: "test-task",
|
||||
SessionID: "test-session",
|
||||
promptDoneCh: make(chan PromptCompletionSignal, 1),
|
||||
}
|
||||
execution.lastActivityAt = time.Now()
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
t.Cleanup(cancel)
|
||||
waitResult := make(chan error, 1)
|
||||
go func() {
|
||||
_, err := sm.waitForPromptDone(ctx, execution, 7)
|
||||
waitResult <- err
|
||||
}()
|
||||
|
||||
time.Sleep(5 * time.Minute)
|
||||
synctest.Wait()
|
||||
if got := countPublishedEvents(eventBus, "agent.stalled"); got != 1 {
|
||||
t.Fatalf("published stalled events = %d, want 1", got)
|
||||
}
|
||||
|
||||
time.Sleep(2 * time.Minute)
|
||||
synctest.Wait()
|
||||
if got := countPublishedEvents(eventBus, "agent.stalled"); got != 1 {
|
||||
t.Fatalf("published stalled events after later checks = %d, want 1", got)
|
||||
}
|
||||
|
||||
cancel()
|
||||
if err := <-waitResult; !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("waitForPromptDone error = %v, want context canceled", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func countPublishedEvents(eventBus *MockEventBusWithTracking, subject string) int {
|
||||
eventBus.mu.Lock()
|
||||
defer eventBus.mu.Unlock()
|
||||
var count int
|
||||
for _, event := range eventBus.PublishedEvents {
|
||||
if event.Subject == subject {
|
||||
count++
|
||||
}
|
||||
}
|
||||
return count
|
||||
}
|
||||
|
||||
func TestSendPrompt_TriggerTimeCancelReleaseReturnsErrCancelEscalated(t *testing.T) {
|
||||
mock := newMockAgentServer(t)
|
||||
defer mock.Close()
|
||||
|
||||
@@ -178,6 +178,8 @@ type AgentExecution struct {
|
||||
// Last time an agent event was received (for stall detection)
|
||||
lastActivityAt time.Time
|
||||
lastActivityAtMu sync.Mutex
|
||||
activeTool *activeTopLevelTool
|
||||
activeToolMu sync.RWMutex
|
||||
|
||||
// Fires once on the first agent event to publish AgentRunning.
|
||||
firstActivityOnce sync.Once
|
||||
@@ -187,6 +189,43 @@ type AgentExecution struct {
|
||||
sessionSpanMu sync.RWMutex
|
||||
}
|
||||
|
||||
type activeTopLevelTool struct {
|
||||
ToolCallID string
|
||||
Name string
|
||||
Title string
|
||||
Status string
|
||||
}
|
||||
|
||||
func (e *AgentExecution) setActiveTool(tool activeTopLevelTool) {
|
||||
e.activeToolMu.Lock()
|
||||
e.activeTool = &tool
|
||||
e.activeToolMu.Unlock()
|
||||
}
|
||||
|
||||
func (e *AgentExecution) clearActiveTool(toolCallID string) {
|
||||
e.activeToolMu.Lock()
|
||||
if e.activeTool != nil && e.activeTool.ToolCallID == toolCallID {
|
||||
e.activeTool = nil
|
||||
}
|
||||
e.activeToolMu.Unlock()
|
||||
}
|
||||
|
||||
func (e *AgentExecution) resetActiveTool() {
|
||||
e.activeToolMu.Lock()
|
||||
e.activeTool = nil
|
||||
e.activeToolMu.Unlock()
|
||||
}
|
||||
|
||||
func (e *AgentExecution) activeToolSnapshot() *activeTopLevelTool {
|
||||
e.activeToolMu.RLock()
|
||||
defer e.activeToolMu.RUnlock()
|
||||
if e.activeTool == nil {
|
||||
return nil
|
||||
}
|
||||
tool := *e.activeTool
|
||||
return &tool
|
||||
}
|
||||
|
||||
// setRuntimeEnvironment stores a defensive copy of the effective task runtime
|
||||
// environment. Callers must use RuntimeEnvironment when reading it.
|
||||
func (e *AgentExecution) setRuntimeEnvironment(env map[string]string) {
|
||||
|
||||
@@ -168,6 +168,7 @@ const (
|
||||
AgentReady = "agent.ready" // Agent finished a prompt turn, ready for follow-up
|
||||
AgentCompleted = "agent.completed"
|
||||
AgentFailed = "agent.failed"
|
||||
AgentStalled = "agent.stalled"
|
||||
AgentStopped = "agent.stopped"
|
||||
AgentContextReset = "agent.context_reset" // Agent subprocess restarted with fresh ACP session
|
||||
AgentACPSessionCreated = "agent.acp_session_created"
|
||||
|
||||
@@ -0,0 +1,87 @@
|
||||
package orchestrator
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"go.uber.org/zap"
|
||||
|
||||
"github.com/kandev/kandev/internal/agent/runtime/lifecycle"
|
||||
"github.com/kandev/kandev/internal/task/models"
|
||||
v1 "github.com/kandev/kandev/pkg/api/v1"
|
||||
)
|
||||
|
||||
const (
|
||||
stallCancelTurnButtonTestID = "stall-cancel-turn-button"
|
||||
metaKeyActionVisibility = "action_visibility"
|
||||
)
|
||||
|
||||
// handleAgentStalled persists an advisory recovery affordance without changing
|
||||
// the prompt, session, or task lifecycle.
|
||||
func (s *Service) handleAgentStalled(ctx context.Context, payload lifecycle.AgentStalledPayload) {
|
||||
if s.messageCreator == nil || s.repo == nil || payload.TaskID == "" || payload.SessionID == "" {
|
||||
return
|
||||
}
|
||||
session, err := s.repo.GetTaskSession(ctx, payload.SessionID)
|
||||
if err != nil || session == nil || session.State != models.TaskSessionStateRunning {
|
||||
return
|
||||
}
|
||||
generationOwner, ok := s.agentManager.(interface {
|
||||
OwnsPromptGeneration(sessionID, executionID string, generation uint64) bool
|
||||
})
|
||||
if !ok || payload.PromptGeneration == 0 || !generationOwner.OwnsPromptGeneration(
|
||||
payload.SessionID,
|
||||
payload.AgentExecutionID,
|
||||
payload.PromptGeneration,
|
||||
) {
|
||||
return
|
||||
}
|
||||
turnID, err := s.peekActiveTurnID(ctx, payload.SessionID)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
if s.turnService != nil && turnID == "" {
|
||||
return
|
||||
}
|
||||
metadata := map[string]interface{}{
|
||||
metaKeyActionVisibility: "running",
|
||||
metaKeySessionID: payload.SessionID,
|
||||
metaKeyTaskID: payload.TaskID,
|
||||
"actions": []map[string]interface{}{{
|
||||
actionMetaKeyType: "ws_request",
|
||||
actionMetaKeyLabel: "Cancel turn",
|
||||
actionMetaKeyTestID: stallCancelTurnButtonTestID,
|
||||
"params": map[string]interface{}{
|
||||
"method": "agent.cancel",
|
||||
"payload": map[string]interface{}{"session_id": payload.SessionID},
|
||||
},
|
||||
}},
|
||||
}
|
||||
if err := s.messageCreator.CreateSessionMessage(
|
||||
ctx,
|
||||
payload.TaskID,
|
||||
stallNoticeContent(payload),
|
||||
payload.SessionID,
|
||||
string(v1.MessageTypeStatus),
|
||||
turnID,
|
||||
metadata,
|
||||
false,
|
||||
); err != nil {
|
||||
s.logger.Warn("failed to create agent stall notice",
|
||||
zap.String("task_id", payload.TaskID),
|
||||
zap.String("session_id", payload.SessionID),
|
||||
zap.Error(err))
|
||||
}
|
||||
}
|
||||
|
||||
func stallNoticeContent(payload lifecycle.AgentStalledPayload) string {
|
||||
tool := strings.TrimSpace(payload.ToolTitle)
|
||||
if tool == "" {
|
||||
tool = strings.TrimSpace(payload.ToolName)
|
||||
}
|
||||
if tool == "" {
|
||||
return "Still waiting for the agent."
|
||||
}
|
||||
return fmt.Sprintf("Still waiting on %s.", tool)
|
||||
}
|
||||
@@ -0,0 +1,124 @@
|
||||
package orchestrator
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/kandev/kandev/internal/agent/runtime/lifecycle"
|
||||
"github.com/kandev/kandev/internal/task/models"
|
||||
)
|
||||
|
||||
func TestHandleAgentStalled_PersistsNeutralRunningNotice(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
repo := setupTestRepo(t)
|
||||
seedSession(t, repo, "task-1", "session-1", "step-1")
|
||||
before, err := repo.GetTaskSession(ctx, "session-1")
|
||||
if err != nil {
|
||||
t.Fatalf("get session before handling stall: %v", err)
|
||||
}
|
||||
agentMgr := &mockAgentManager{
|
||||
repoForExecutionLookup: repo,
|
||||
currentPromptExecutionID: "execution-1",
|
||||
}
|
||||
agentMgr.currentPromptGeneration.Store(7)
|
||||
svc := createTestServiceWithScheduler(
|
||||
repo,
|
||||
newMockStepGetter(),
|
||||
newMockTaskRepo(),
|
||||
agentMgr,
|
||||
)
|
||||
svc.turnService = &repoTurnService{repo: repo}
|
||||
activeTurn, err := svc.turnService.StartTurn(ctx, "session-1")
|
||||
if err != nil {
|
||||
t.Fatalf("start active turn: %v", err)
|
||||
}
|
||||
messages := &mockMessageCreator{}
|
||||
svc.messageCreator = messages
|
||||
|
||||
svc.handleAgentStalled(ctx, lifecycle.AgentStalledPayload{
|
||||
AgentExecutionID: "execution-1",
|
||||
TaskID: "task-1",
|
||||
SessionID: "session-1",
|
||||
PromptGeneration: 7,
|
||||
ToolName: "shell",
|
||||
ToolTitle: "Start dev server",
|
||||
ToolStatus: "in_progress",
|
||||
})
|
||||
|
||||
if len(messages.sessionMessages) != 1 {
|
||||
t.Fatalf("session messages = %d, want 1", len(messages.sessionMessages))
|
||||
}
|
||||
message := messages.sessionMessages[0]
|
||||
if !strings.Contains(message.content, "Still waiting on Start dev server") {
|
||||
t.Fatalf("notice content = %q, want sanitized tool title", message.content)
|
||||
}
|
||||
if message.metadata["action_visibility"] != "running" {
|
||||
t.Fatalf("action visibility = %v, want running", message.metadata["action_visibility"])
|
||||
}
|
||||
if message.turnID != activeTurn.ID {
|
||||
t.Fatalf("notice turn ID = %q, want active turn %q", message.turnID, activeTurn.ID)
|
||||
}
|
||||
if _, hasVariant := message.metadata["variant"]; hasVariant {
|
||||
t.Fatalf("notice metadata unexpectedly set a warning/error variant: %#v", message.metadata)
|
||||
}
|
||||
actions, ok := message.metadata["actions"].([]map[string]interface{})
|
||||
if !ok || len(actions) != 1 {
|
||||
t.Fatalf("actions = %#v, want one cancel action", message.metadata["actions"])
|
||||
}
|
||||
action := actions[0]
|
||||
if action["label"] != "Cancel turn" || action["test_id"] != "stall-cancel-turn-button" {
|
||||
t.Fatalf("cancel action = %#v", action)
|
||||
}
|
||||
params, ok := action["params"].(map[string]interface{})
|
||||
if !ok || params["method"] != "agent.cancel" {
|
||||
t.Fatalf("cancel params = %#v, want agent.cancel", action["params"])
|
||||
}
|
||||
|
||||
after, err := repo.GetTaskSession(ctx, "session-1")
|
||||
if err != nil {
|
||||
t.Fatalf("get session after handling stall: %v", err)
|
||||
}
|
||||
if after.State != before.State {
|
||||
t.Fatalf("session state changed from %q to %q", before.State, after.State)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleAgentStalled_RejectsSettledOrStalePrompt(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
repo := setupTestRepo(t)
|
||||
seedSession(t, repo, "task-1", "session-1", "step-1")
|
||||
agentMgr := &mockAgentManager{currentPromptExecutionID: "execution-1"}
|
||||
agentMgr.currentPromptGeneration.Store(7)
|
||||
svc := createTestServiceWithScheduler(repo, newMockStepGetter(), newMockTaskRepo(), agentMgr)
|
||||
messages := &mockMessageCreator{}
|
||||
svc.messageCreator = messages
|
||||
payload := lifecycle.AgentStalledPayload{
|
||||
AgentExecutionID: "execution-1",
|
||||
TaskID: "task-1",
|
||||
SessionID: "session-1",
|
||||
PromptGeneration: 7,
|
||||
}
|
||||
|
||||
if err := repo.UpdateTaskSessionState(ctx, "session-1", models.TaskSessionStateWaitingForInput, ""); err != nil {
|
||||
t.Fatalf("settle session: %v", err)
|
||||
}
|
||||
svc.handleAgentStalled(ctx, payload)
|
||||
if len(messages.sessionMessages) != 0 {
|
||||
t.Fatalf("settled session messages = %d, want 0", len(messages.sessionMessages))
|
||||
}
|
||||
if err := repo.UpdateTaskSessionState(ctx, "session-1", models.TaskSessionStateRunning, ""); err != nil {
|
||||
t.Fatalf("resume session: %v", err)
|
||||
}
|
||||
agentMgr.currentPromptGeneration.Store(8)
|
||||
svc.handleAgentStalled(ctx, payload)
|
||||
if len(messages.sessionMessages) != 0 {
|
||||
t.Fatalf("stale generation messages = %d, want 0", len(messages.sessionMessages))
|
||||
}
|
||||
}
|
||||
|
||||
func TestStallNoticeContentFallsBackWithoutTool(t *testing.T) {
|
||||
if got := stallNoticeContent(lifecycle.AgentStalledPayload{}); got != "Still waiting for the agent." {
|
||||
t.Fatalf("stallNoticeContent() = %q, want generic fallback", got)
|
||||
}
|
||||
}
|
||||
@@ -675,6 +675,7 @@ func NewService(
|
||||
OnAgentReady: s.handleAgentReady,
|
||||
OnAgentCompleted: s.handleAgentCompleted,
|
||||
OnAgentFailed: s.handleAgentFailed,
|
||||
OnAgentStalled: s.handleAgentStalled,
|
||||
OnAgentStopped: s.handleAgentStopped,
|
||||
OnAgentStreamEvent: s.handleAgentStreamEvent,
|
||||
OnACPSessionCreated: s.handleACPSessionCreated,
|
||||
|
||||
@@ -99,6 +99,7 @@ type EventHandlers struct {
|
||||
OnAgentReady func(ctx context.Context, data AgentEventData) // Turn ended; agent idle waiting for follow-up
|
||||
OnAgentCompleted func(ctx context.Context, data AgentEventData)
|
||||
OnAgentFailed func(ctx context.Context, data AgentEventData)
|
||||
OnAgentStalled func(ctx context.Context, payload lifecycle.AgentStalledPayload)
|
||||
OnAgentStopped func(ctx context.Context, data AgentEventData)
|
||||
OnACPSessionCreated func(ctx context.Context, data ACPSessionEventData)
|
||||
|
||||
@@ -162,6 +163,10 @@ func (w *Watcher) Start(ctx context.Context) error {
|
||||
w.unsubscribeAll()
|
||||
return err
|
||||
}
|
||||
if err := w.subscribeToAgentStallEvents(); err != nil {
|
||||
w.unsubscribeAll()
|
||||
return err
|
||||
}
|
||||
|
||||
// Subscribe to ACP session events
|
||||
if err := w.subscribeToACPSessionEvents(); err != nil {
|
||||
@@ -320,6 +325,23 @@ func (w *Watcher) subscribeToAgentEvents() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (w *Watcher) subscribeToAgentStallEvents() error {
|
||||
if w.handlers.OnAgentStalled == nil {
|
||||
return nil
|
||||
}
|
||||
sub, err := w.eventBus.QueueSubscribe(
|
||||
events.AgentStalled,
|
||||
w.queue,
|
||||
w.createAgentStalledEventHandler(w.handlers.OnAgentStalled),
|
||||
)
|
||||
if err != nil {
|
||||
w.logger.Error("failed to subscribe to agent stalled event", zap.Error(err))
|
||||
return err
|
||||
}
|
||||
w.subscriptions = append(w.subscriptions, sub)
|
||||
return nil
|
||||
}
|
||||
|
||||
// subscribeToACPSessionEvents subscribes to ACP session lifecycle events
|
||||
func (w *Watcher) subscribeToACPSessionEvents() error {
|
||||
if w.handlers.OnACPSessionCreated == nil {
|
||||
@@ -425,6 +447,20 @@ func (w *Watcher) createAgentEventHandler(handler func(ctx context.Context, data
|
||||
}
|
||||
}
|
||||
|
||||
func (w *Watcher) createAgentStalledEventHandler(
|
||||
handler func(ctx context.Context, payload lifecycle.AgentStalledPayload),
|
||||
) bus.EventHandler {
|
||||
return func(ctx context.Context, event *bus.Event) error {
|
||||
var payload lifecycle.AgentStalledPayload
|
||||
if err := w.parseEventData(event.Data, &payload); err != nil {
|
||||
w.logger.Error("failed to parse agent stalled event", zap.Error(err))
|
||||
return nil
|
||||
}
|
||||
handler(ctx, payload)
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// createACPSessionEventHandler creates a bus.EventHandler for ACP session events
|
||||
func (w *Watcher) createACPSessionEventHandler(handler func(ctx context.Context, data ACPSessionEventData)) bus.EventHandler {
|
||||
return func(ctx context.Context, event *bus.Event) error {
|
||||
|
||||
@@ -303,6 +303,48 @@ func TestAgentEventHandling(t *testing.T) {
|
||||
})
|
||||
}
|
||||
|
||||
func TestWatcherDispatchesAgentStalled(t *testing.T) {
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
eventBus := newMockEventBus()
|
||||
var received lifecycle.AgentStalledPayload
|
||||
var handled bool
|
||||
var mu sync.Mutex
|
||||
w := NewWatcher(eventBus, EventHandlers{
|
||||
OnAgentStalled: func(_ context.Context, payload lifecycle.AgentStalledPayload) {
|
||||
mu.Lock()
|
||||
received = payload
|
||||
handled = true
|
||||
mu.Unlock()
|
||||
},
|
||||
}, "orchestrator-test", createTestLogger())
|
||||
if err := w.Start(context.Background()); err != nil {
|
||||
t.Fatalf("start watcher: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { _ = w.Stop() })
|
||||
|
||||
event := bus.NewEvent(events.AgentStalled, "test", map[string]interface{}{
|
||||
"agent_execution_id": "agent-456",
|
||||
"task_id": "task-123",
|
||||
"session_id": "session-789",
|
||||
"prompt_generation": 7,
|
||||
"tool_title": "Start dev server",
|
||||
})
|
||||
if err := eventBus.Publish(context.Background(), events.AgentStalled, event); err != nil {
|
||||
t.Fatalf("publish stalled event: %v", err)
|
||||
}
|
||||
synctest.Wait()
|
||||
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
if !handled {
|
||||
t.Fatal("OnAgentStalled handler was not called")
|
||||
}
|
||||
if received.SessionID != "session-789" || received.ToolTitle != "Start dev server" {
|
||||
t.Fatalf("received stalled payload = %#v", received)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestAgentStreamEventHandling(t *testing.T) {
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
eventBus := newMockEventBus()
|
||||
|
||||
@@ -30,6 +30,7 @@ const CANCEL_TEST_ID = "recovery-cancel-retry-button";
|
||||
const TECHNICAL_DETAILS = "Technical details";
|
||||
const RECOVERY_MESSAGE = "Agent encountered an error";
|
||||
const RESUME_TEST_ID = "recovery-resume-button";
|
||||
const STALL_CANCEL_TEST_ID = "stall-cancel-turn-button";
|
||||
|
||||
function retryMessage(overrides: Partial<Message> = {}): Message {
|
||||
return {
|
||||
@@ -90,9 +91,32 @@ function recoveryMessage(withParams = false): Message {
|
||||
} as Partial<Message>);
|
||||
}
|
||||
|
||||
function stalledMessage(turnId = "turn-1"): Message {
|
||||
return retryMessage({
|
||||
turn_id: turnId,
|
||||
content: "Still waiting on Start dev server.",
|
||||
metadata: {
|
||||
action_visibility: "running",
|
||||
actions: [
|
||||
{
|
||||
type: "ws_request",
|
||||
label: "Cancel turn",
|
||||
test_id: STALL_CANCEL_TEST_ID,
|
||||
params: { method: "agent.cancel", payload: { session_id: "sess-1" } },
|
||||
},
|
||||
],
|
||||
},
|
||||
} as Partial<Message>);
|
||||
}
|
||||
|
||||
/** ActionMessage reads session state from the store (keyed by comment.session_id),
|
||||
* so seed it via the provider instead of passing a prop. */
|
||||
function renderAction(comment: Message, sessionState?: TaskSessionState, sessionError?: string) {
|
||||
function renderAction(
|
||||
comment: Message,
|
||||
sessionState?: TaskSessionState,
|
||||
sessionError?: string,
|
||||
activeTurnId?: string,
|
||||
) {
|
||||
const initialState: Partial<AppState> = sessionState
|
||||
? {
|
||||
taskSessions: {
|
||||
@@ -100,6 +124,10 @@ function renderAction(comment: Message, sessionState?: TaskSessionState, session
|
||||
"sess-1": { state: sessionState, error_message: sessionError } as TaskSession,
|
||||
},
|
||||
},
|
||||
turns: {
|
||||
bySession: {},
|
||||
activeBySession: activeTurnId ? { "sess-1": activeTurnId } : {},
|
||||
},
|
||||
}
|
||||
: {};
|
||||
return render(<ActionMessage comment={comment} />, {
|
||||
@@ -172,6 +200,50 @@ describe("ActionMessage — transient retry (warning variant)", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe("ActionMessage — running stall notice", () => {
|
||||
it("renders a neutral compact notice only while the session is running", () => {
|
||||
renderAction(stalledMessage(), "RUNNING", undefined, "turn-1");
|
||||
|
||||
const notice = screen.getByTestId("running-action-notice");
|
||||
expect(notice.className).toContain("text-muted-foreground");
|
||||
expect(notice.className).not.toContain("text-amber");
|
||||
expect(notice.className).not.toContain("text-red");
|
||||
expect(notice.querySelector("svg")).toBeNull();
|
||||
const button = screen.getByTestId(STALL_CANCEL_TEST_ID);
|
||||
expect(button.className).toContain("min-h-11");
|
||||
expect(button.className).not.toContain("w-full");
|
||||
});
|
||||
|
||||
it("hides the running-only notice after the session settles", () => {
|
||||
const { container } = renderAction(stalledMessage(), "WAITING_FOR_INPUT");
|
||||
expect(container.firstChild).toBeNull();
|
||||
});
|
||||
|
||||
it("sends agent.cancel when Cancel turn is activated", async () => {
|
||||
renderAction(stalledMessage(), "RUNNING", undefined, "turn-1");
|
||||
|
||||
fireEvent.click(screen.getByTestId(STALL_CANCEL_TEST_ID));
|
||||
|
||||
await waitFor(() =>
|
||||
expect(requestMock).toHaveBeenCalledWith("agent.cancel", { session_id: "sess-1" }),
|
||||
);
|
||||
});
|
||||
|
||||
it("hides an old notice when a later turn is active", () => {
|
||||
const { container } = renderAction(stalledMessage("turn-1"), "RUNNING", undefined, "turn-2");
|
||||
expect(container.firstChild).toBeNull();
|
||||
});
|
||||
|
||||
it("hides a running-only notice without a turn ID", () => {
|
||||
const message = stalledMessage();
|
||||
message.turn_id = undefined;
|
||||
|
||||
const { container } = renderAction(message, "RUNNING", undefined, "turn-1");
|
||||
|
||||
expect(container.firstChild).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe("ActionMessage — missing PR branch", () => {
|
||||
it("renders a plain-language recovery panel with collapsed technical details", () => {
|
||||
renderAction(
|
||||
|
||||
@@ -38,6 +38,7 @@ const ICON_MAP: Record<string, React.ElementType> = {
|
||||
|
||||
type ActionMeta = {
|
||||
actions?: MessageAction[];
|
||||
action_visibility?: "running";
|
||||
variant?: string;
|
||||
recovery_actions?: boolean;
|
||||
is_auth_error?: boolean;
|
||||
@@ -55,21 +56,45 @@ export const ActionMessage = memo(function ActionMessage({ comment }: { comment:
|
||||
// Read session state from the store instead of receiving it as a prop, so a
|
||||
// state transition doesn't re-render every message in the list (only the
|
||||
// rare action messages that actually depend on it).
|
||||
const sessionState = useAppStore((state) =>
|
||||
comment.session_id
|
||||
? (state.taskSessions.items[comment.session_id]?.state ?? undefined)
|
||||
: undefined,
|
||||
);
|
||||
const sessionError = useAppStore((state) =>
|
||||
comment.session_id
|
||||
? (state.taskSessions.items[comment.session_id]?.error_message as string | undefined)
|
||||
: undefined,
|
||||
);
|
||||
const [recoveryRequested, setRecoveryRequested] = useState(false);
|
||||
const { sessionState, sessionError, activeTurnId } = useActionMessageSession(comment.session_id);
|
||||
const metadata = comment.metadata as ActionMeta | undefined;
|
||||
const isWarning = metadata?.variant === "warning";
|
||||
const message = comment.content || "An error occurred";
|
||||
|
||||
if (metadata?.action_visibility === "running") {
|
||||
if (sessionState !== "RUNNING" || !comment.turn_id || activeTurnId !== comment.turn_id) {
|
||||
return null;
|
||||
}
|
||||
return (
|
||||
<RunningActionNotice actions={metadata.actions} message={message} taskId={comment.task_id} />
|
||||
);
|
||||
}
|
||||
|
||||
return (
|
||||
<SettledActionMessage
|
||||
metadata={metadata}
|
||||
message={message}
|
||||
sessionError={sessionError}
|
||||
sessionState={sessionState}
|
||||
taskId={comment.task_id}
|
||||
/>
|
||||
);
|
||||
});
|
||||
|
||||
function SettledActionMessage({
|
||||
metadata,
|
||||
message,
|
||||
sessionError,
|
||||
sessionState,
|
||||
taskId,
|
||||
}: {
|
||||
metadata: ActionMeta | undefined;
|
||||
message: string;
|
||||
sessionError?: string;
|
||||
sessionState?: TaskSessionState;
|
||||
taskId?: string;
|
||||
}) {
|
||||
const [recoveryRequested, setRecoveryRequested] = useState(false);
|
||||
|
||||
// A waiting session can still need recovery, so only hide this persisted
|
||||
// card after its own Resume request is acknowledged (or the session starts).
|
||||
if (isSessionActive(sessionState) || (metadata?.recovery_actions && recoveryRequested))
|
||||
@@ -79,17 +104,18 @@ export const ActionMessage = memo(function ActionMessage({ comment }: { comment:
|
||||
return (
|
||||
<MissingBranchRecovery
|
||||
metadata={metadata}
|
||||
taskId={comment.task_id}
|
||||
taskId={taskId}
|
||||
fallbackMessage={message}
|
||||
technicalDetails={sessionError}
|
||||
/>
|
||||
);
|
||||
}
|
||||
|
||||
const iconClass = isWarning ? "text-amber-500" : "text-red-500";
|
||||
const textClass = isWarning
|
||||
? "text-amber-600 dark:text-amber-400"
|
||||
: "text-red-600 dark:text-red-400";
|
||||
const iconClass = metadata?.variant === "warning" ? "text-amber-500" : "text-red-500";
|
||||
const textClass =
|
||||
metadata?.variant === "warning"
|
||||
? "text-amber-600 dark:text-amber-400"
|
||||
: "text-red-600 dark:text-red-400";
|
||||
|
||||
return (
|
||||
<div className="w-full">
|
||||
@@ -103,7 +129,7 @@ export const ActionMessage = memo(function ActionMessage({ comment }: { comment:
|
||||
{metadata?.actions && metadata.actions.length > 0 && (
|
||||
<ActionButtons
|
||||
actions={metadata.actions}
|
||||
taskId={comment.task_id}
|
||||
taskId={taskId}
|
||||
onRecoveryRequested={
|
||||
metadata.recovery_actions ? () => setRecoveryRequested(true) : undefined
|
||||
}
|
||||
@@ -113,7 +139,42 @@ export const ActionMessage = memo(function ActionMessage({ comment }: { comment:
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
function useActionMessageSession(sessionId: Message["session_id"]) {
|
||||
const sessionState = useAppStore((state) =>
|
||||
sessionId ? (state.taskSessions.items[sessionId]?.state ?? undefined) : undefined,
|
||||
);
|
||||
const sessionError = useAppStore((state) =>
|
||||
sessionId
|
||||
? (state.taskSessions.items[sessionId]?.error_message as string | undefined)
|
||||
: undefined,
|
||||
);
|
||||
const activeTurnId = useAppStore((state) =>
|
||||
sessionId ? (state.turns.activeBySession[sessionId] ?? undefined) : undefined,
|
||||
);
|
||||
return { sessionState, sessionError, activeTurnId };
|
||||
}
|
||||
|
||||
function RunningActionNotice({
|
||||
actions,
|
||||
message,
|
||||
taskId,
|
||||
}: {
|
||||
actions?: MessageAction[];
|
||||
message: string;
|
||||
taskId?: string;
|
||||
}) {
|
||||
return (
|
||||
<div
|
||||
data-testid="running-action-notice"
|
||||
className="flex min-w-0 items-center gap-2 py-1 text-muted-foreground"
|
||||
>
|
||||
<span className="min-w-0 flex-1 truncate text-xs">{message}</span>
|
||||
{actions && actions.length > 0 && <ActionButtons actions={actions} taskId={taskId} compact />}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function MissingBranchRecovery({
|
||||
metadata,
|
||||
@@ -227,19 +288,28 @@ function ActionButtons({
|
||||
actions,
|
||||
taskId,
|
||||
onRecoveryRequested,
|
||||
compact = false,
|
||||
}: {
|
||||
actions: MessageAction[];
|
||||
taskId?: string;
|
||||
onRecoveryRequested?: () => void;
|
||||
compact?: boolean;
|
||||
}) {
|
||||
return (
|
||||
<div className="mt-2 flex flex-col gap-2 sm:flex-row sm:flex-wrap sm:items-center">
|
||||
<div
|
||||
className={cn(
|
||||
compact
|
||||
? "flex shrink-0 items-center"
|
||||
: "mt-2 flex flex-col gap-2 sm:flex-row sm:flex-wrap sm:items-center",
|
||||
)}
|
||||
>
|
||||
{actions.map((action, i) => (
|
||||
<ActionButton
|
||||
key={action.test_id ?? i}
|
||||
action={action}
|
||||
messageTaskId={taskId}
|
||||
onCompleted={onRecoveryRequested}
|
||||
compact={compact}
|
||||
/>
|
||||
))}
|
||||
</div>
|
||||
@@ -250,10 +320,12 @@ function ActionButton({
|
||||
action,
|
||||
messageTaskId,
|
||||
onCompleted,
|
||||
compact = false,
|
||||
}: {
|
||||
action: MessageAction;
|
||||
messageTaskId?: string;
|
||||
onCompleted?: () => void;
|
||||
compact?: boolean;
|
||||
}): ReactElement | null {
|
||||
const [state, setState] = useState<"idle" | "busy" | "done" | "error">("idle");
|
||||
const activeTaskId = useAppStore((s) => s.tasks.activeTaskId);
|
||||
@@ -312,10 +384,12 @@ function ActionButton({
|
||||
|
||||
const button = (
|
||||
<Button
|
||||
variant="outline"
|
||||
variant={compact ? "ghost" : "outline"}
|
||||
size="sm"
|
||||
className={cn(
|
||||
"h-auto min-h-11 w-full gap-1.5 text-xs cursor-pointer sm:min-h-8 sm:w-auto",
|
||||
compact
|
||||
? "h-auto min-h-11 shrink-0 px-2 text-xs cursor-pointer sm:min-h-8"
|
||||
: "h-auto min-h-11 w-full gap-1.5 text-xs cursor-pointer sm:min-h-8 sm:w-auto",
|
||||
isDestructive && "text-destructive hover:text-destructive",
|
||||
)}
|
||||
disabled={disabled}
|
||||
|
||||
@@ -163,6 +163,9 @@ export type StatusMetadata = {
|
||||
attempt?: number;
|
||||
max_attempts?: number;
|
||||
retry_in_seconds?: number;
|
||||
// Running-only action notices are hidden once the session settles. They use
|
||||
// compact neutral presentation instead of the normal recovery/error card.
|
||||
action_visibility?: "running";
|
||||
};
|
||||
|
||||
export type RecoveryAuthMethod = {
|
||||
|
||||
@@ -3,6 +3,7 @@ import { test, expect } from "../../fixtures/test-base";
|
||||
import { assertNoDocumentHorizontalOverflow } from "../../helpers/layout-assertions";
|
||||
import { seedIdleSession } from "../../helpers/session";
|
||||
import { typeWhileBusy } from "../../helpers/type-while-busy";
|
||||
import { SessionPage } from "../../pages/session-page";
|
||||
|
||||
test.describe("mobile: pause queue recovery", () => {
|
||||
test.describe.configure({ retries: 1 });
|
||||
@@ -49,4 +50,58 @@ test.describe("mobile: pause queue recovery", () => {
|
||||
await expect(chat.getByTestId("queue-chip")).not.toBeVisible({ timeout: 30_000 });
|
||||
await session.expectChatResponseVisible("simple mock response", 1, { timeout: 30_000 });
|
||||
});
|
||||
|
||||
test("a compact stalled notice remains touch-safe without expanding to full width", async ({
|
||||
testPage,
|
||||
apiClient,
|
||||
seedData,
|
||||
}) => {
|
||||
const task = await apiClient.createTask(
|
||||
seedData.workspaceId,
|
||||
"Mobile stalled notice recovery",
|
||||
{
|
||||
workflow_id: seedData.workflowId,
|
||||
workflow_step_id: seedData.startStepId,
|
||||
},
|
||||
);
|
||||
const { session_id: sessionId } = await apiClient.seedTaskSession(task.id, {
|
||||
state: "RUNNING",
|
||||
});
|
||||
await apiClient.seedSessionMessage(sessionId, {
|
||||
type: "status",
|
||||
content: "Still waiting on Start dev server.",
|
||||
metadata: {
|
||||
action_visibility: "running",
|
||||
actions: [
|
||||
{
|
||||
type: "ws_request",
|
||||
label: "Cancel turn",
|
||||
test_id: "stall-cancel-turn-button",
|
||||
params: { method: "agent.cancel", payload: { session_id: sessionId } },
|
||||
},
|
||||
],
|
||||
},
|
||||
});
|
||||
|
||||
await testPage.goto(`/t/${task.id}`);
|
||||
const session = new SessionPage(testPage);
|
||||
await session.waitForLoad();
|
||||
const notice = session.activeChat().getByTestId("running-action-notice");
|
||||
const cancel = notice.getByTestId("stall-cancel-turn-button");
|
||||
|
||||
await expect(notice).toBeVisible();
|
||||
const [noticeBox, cancelBox] = await Promise.all([notice.boundingBox(), cancel.boundingBox()]);
|
||||
expect(noticeBox).not.toBeNull();
|
||||
expect(cancelBox).not.toBeNull();
|
||||
expect(noticeBox!.height).toBeLessThanOrEqual(56);
|
||||
expect(cancelBox!.height).toBeGreaterThanOrEqual(44);
|
||||
expect(cancelBox!.width).toBeLessThan(noticeBox!.width);
|
||||
await assertNoDocumentHorizontalOverflow(testPage);
|
||||
|
||||
await cancel.tap();
|
||||
|
||||
await expect(session.idleInput()).toBeVisible({ timeout: 30_000 });
|
||||
await expect(notice).not.toBeVisible();
|
||||
await assertNoDocumentHorizontalOverflow(testPage);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -141,4 +141,55 @@ test.describe("Pause → resume recovery", () => {
|
||||
await session.expectChatResponseVisible("simple mock response", 1, { timeout: 30_000 });
|
||||
await expect(session.idleInput()).toBeVisible({ timeout: 30_000 });
|
||||
});
|
||||
|
||||
test("a compact stalled notice cancels a running session without navigation", async ({
|
||||
testPage,
|
||||
apiClient,
|
||||
seedData,
|
||||
}) => {
|
||||
const task = await apiClient.createTask(seedData.workspaceId, "Stalled notice recovery", {
|
||||
workflow_id: seedData.workflowId,
|
||||
workflow_step_id: seedData.startStepId,
|
||||
});
|
||||
const { session_id: sessionId } = await apiClient.seedTaskSession(task.id, {
|
||||
state: "RUNNING",
|
||||
});
|
||||
await apiClient.seedSessionMessage(sessionId, {
|
||||
type: "status",
|
||||
content: "Still waiting on Start dev server.",
|
||||
metadata: {
|
||||
action_visibility: "running",
|
||||
actions: [
|
||||
{
|
||||
type: "ws_request",
|
||||
label: "Cancel turn",
|
||||
test_id: "stall-cancel-turn-button",
|
||||
params: { method: "agent.cancel", payload: { session_id: sessionId } },
|
||||
},
|
||||
],
|
||||
},
|
||||
});
|
||||
|
||||
await testPage.goto(`/t/${task.id}`);
|
||||
const session = new SessionPage(testPage);
|
||||
await session.waitForLoad();
|
||||
const sessionUrl = testPage.url();
|
||||
const notice = session.activeChat().getByTestId("running-action-notice");
|
||||
const cancel = notice.getByTestId("stall-cancel-turn-button");
|
||||
|
||||
await expect(notice).toBeVisible();
|
||||
await expect(notice).toContainText("Still waiting on Start dev server.");
|
||||
await expect(notice.locator("svg")).toHaveCount(0);
|
||||
const [noticeBox, cancelBox] = await Promise.all([notice.boundingBox(), cancel.boundingBox()]);
|
||||
expect(noticeBox).not.toBeNull();
|
||||
expect(cancelBox).not.toBeNull();
|
||||
expect(noticeBox!.height).toBeLessThanOrEqual(48);
|
||||
expect(cancelBox!.width).toBeLessThan(noticeBox!.width);
|
||||
|
||||
await cancel.click();
|
||||
|
||||
await expect(session.idleInput()).toBeVisible({ timeout: 30_000 });
|
||||
await expect(notice).not.toBeVisible();
|
||||
expect(testPage.url()).toBe(sessionUrl);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -0,0 +1,58 @@
|
||||
# ADR-2026-07-29-agent-stall-user-controlled-recovery: Keep Agent Stall Recovery User Controlled
|
||||
|
||||
**Status:** accepted
|
||||
**Date:** 2026-07-29
|
||||
**Area:** backend, frontend, protocol
|
||||
|
||||
## Context
|
||||
|
||||
An ACP turn can stop emitting events while its process remains alive, most
|
||||
concretely when a top-level shell tool starts a long-lived process and never
|
||||
reaches a terminal status. Kandev already detects five minutes of inactivity,
|
||||
but only logs the condition and leaves the session `RUNNING`. Automatically
|
||||
timing out every quiet turn would also terminate legitimate long-running tools
|
||||
whose providers emit no progress frames.
|
||||
|
||||
## Decision
|
||||
|
||||
Stall detection is advisory and recovery remains operator controlled:
|
||||
|
||||
- After five minutes without agent events, Kandev presents one neutral notice
|
||||
for the active prompt generation and keeps the session `RUNNING`.
|
||||
- The notice identifies the active top-level tool when that information is
|
||||
available, without exposing raw command arguments, and provides a compact
|
||||
neutral **Cancel turn** action. Its wording and styling do not imply that a
|
||||
quiet long-running tool has failed.
|
||||
- **Cancel turn** reuses the existing `agent.cancel` escalation path. That path
|
||||
bounds the ACP acknowledgement wait, releases a hung prompt, and reconciles
|
||||
the session to an input-ready state.
|
||||
- Stall detection never automatically cancels, fails, or kills the agent
|
||||
process. `RUNNING` sessions retain the conservative prompt-admission behavior
|
||||
defined by
|
||||
[ADR-2026-07-28-coarse-running-busy-signal](2026-07-28-coarse-running-busy-signal.md).
|
||||
- The watchdog checks once per 60 seconds and logs only the first detected stall
|
||||
for a prompt generation; it does not repeat a notice every check.
|
||||
|
||||
The observable behavior is specified in
|
||||
[`docs/specs/agent-stall-recovery/spec.md`](../specs/agent-stall-recovery/spec.md).
|
||||
|
||||
## Consequences
|
||||
|
||||
Users get a reload-safe explanation and a direct recovery action without
|
||||
Kandev guessing whether quiet work is expendable. Sessions do not become
|
||||
promptable until the user cancels or the provider completes the turn, so queued
|
||||
input and lifecycle ownership remain conservative. A genuinely abandoned turn
|
||||
can remain `RUNNING` indefinitely if the user does not intervene.
|
||||
|
||||
Lifecycle must retain enough in-memory top-level tool identity to enrich the
|
||||
notice, while the persisted notice message remains the durable UI record.
|
||||
|
||||
## Alternatives Considered
|
||||
|
||||
- **Automatic timeout.** Rejected because event silence is not proof that a
|
||||
command is safe to terminate.
|
||||
- **Warn, then automatically timeout after a grace period.** Rejected for the
|
||||
same ambiguity; a longer delay reduces but does not remove destructive false
|
||||
positives.
|
||||
- **Continue logging only.** Rejected because logs do not make the session
|
||||
recoverable or explain the blocked Resume action to the user.
|
||||
@@ -84,3 +84,4 @@ Read individual ADRs for full context. Create new ones via `/record decision` or
|
||||
| 2026-07-27-legacy-add-branch-live-rescan | [Preserve Live Rescan for Legacy Add Branch](2026-07-27-legacy-add-branch-live-rescan.md) | accepted | backend, protocol | 2026-07-27 |
|
||||
| 2026-07-28-visible-wip-overflow-queues | [Separate Visible Queueing From WIP Admission](2026-07-28-visible-wip-overflow-queues.md) | proposed | backend, frontend, protocol, workflow | 2026-07-28 |
|
||||
| 2026-07-28-coarse-running-busy-signal | [Restore Coarse Running Prompt Admission](2026-07-28-coarse-running-busy-signal.md) | accepted | backend, frontend, protocol | 2026-07-28 |
|
||||
| 2026-07-29-agent-stall-user-controlled-recovery | [Keep Agent Stall Recovery User Controlled](2026-07-29-agent-stall-user-controlled-recovery.md) | accepted | backend, frontend, protocol | 2026-07-29 |
|
||||
|
||||
@@ -0,0 +1,203 @@
|
||||
---
|
||||
spec: docs/specs/agent-stall-recovery/spec.md
|
||||
decision: docs/decisions/2026-07-29-agent-stall-user-controlled-recovery.md
|
||||
created: 2026-07-29
|
||||
status: draft
|
||||
---
|
||||
|
||||
# Implementation Plan: Agent Stall Recovery
|
||||
|
||||
## Overview
|
||||
|
||||
Turn the lifecycle watchdog's existing inactivity observation into one
|
||||
generation-scoped internal event, persist that event as an actionable notice,
|
||||
and make the notice visible while the session is still `RUNNING`. The existing
|
||||
cancel-escalation path remains the sole recovery mechanism; no timeout changes
|
||||
session or process state automatically.
|
||||
|
||||
## Confirmed root cause
|
||||
|
||||
The issue's deterministic reproduction leaves a top-level shell tool
|
||||
`in_progress` after it starts a long-lived process. No further ACP frames reach
|
||||
Kandev, so `waitForPromptDone` never receives its completion signal. Its
|
||||
detached context has no deadline, and the five-minute watchdog branch only logs
|
||||
before looping. The existing `agent.cancel` path already bounds acknowledgement
|
||||
waits, force-releases a stuck prompt, and reconciles the session to
|
||||
`WAITING_FOR_INPUT`; detection is not connected to that user-facing recovery.
|
||||
|
||||
---
|
||||
|
||||
## Backend
|
||||
|
||||
### Track and report a stalled prompt
|
||||
|
||||
- Extend `AgentExecution` in
|
||||
`apps/backend/internal/agent/runtime/lifecycle/types.go` with
|
||||
mutex-protected active top-level tool identity.
|
||||
- Update `handleToolCallEvent` and `handleToolUpdateEvent` in
|
||||
`apps/backend/internal/agent/runtime/lifecycle/manager_events.go` to record a
|
||||
top-level tool and clear it on terminal updates. Subagent-internal tool calls
|
||||
do not replace the top-level identity.
|
||||
- Update `waitForPromptDone` in
|
||||
`apps/backend/internal/agent/runtime/lifecycle/session.go` to check once per
|
||||
60 seconds, detect five minutes of inactivity, and publish `agent.stalled`
|
||||
once per prompt generation. The payload carries task/session/execution IDs,
|
||||
prompt generation, last-activity time, stalled duration, and optional tool
|
||||
ID/name/title/status.
|
||||
- Add the payload and publisher support in
|
||||
`apps/backend/internal/agent/runtime/lifecycle/event_types.go`,
|
||||
`apps/backend/internal/agent/runtime/lifecycle/events.go`, and
|
||||
`apps/backend/internal/events/types.go`.
|
||||
|
||||
### Persist the actionable notice
|
||||
|
||||
- Extend `apps/backend/internal/orchestrator/watcher/watcher.go` with the typed
|
||||
`agent.stalled` subscription and wire it in
|
||||
`apps/backend/internal/orchestrator/service.go`.
|
||||
- Add `apps/backend/internal/orchestrator/event_handlers_stall.go` to create a
|
||||
neutral status message without changing task/session state.
|
||||
- Use only the tool display title or name in copy. Metadata carries
|
||||
`action_visibility: running` and one `ws_request` action for `agent.cancel`
|
||||
with `session_id`, label **Cancel turn**, neutral styling, and stable
|
||||
`stall-cancel-turn-button` test ID. The copy says Kandev is still waiting; it
|
||||
does not assert that the tool failed.
|
||||
- Validate that the session is still `RUNNING` and that the event's immutable
|
||||
execution/prompt-generation identity is current before persisting. Assign
|
||||
the notice the active `turn_id` without lazily creating a successor turn.
|
||||
- If message persistence fails, log the error and leave the running prompt
|
||||
untouched.
|
||||
|
||||
---
|
||||
|
||||
## Frontend
|
||||
|
||||
### Compact running-session notice
|
||||
|
||||
- Update
|
||||
`apps/web/components/task/chat/messages/action-message.tsx` so ordinary
|
||||
recovery messages keep their current settled-session visibility, while
|
||||
`action_visibility: running` messages render only during `RUNNING` when their
|
||||
persisted `turn_id` matches the active turn.
|
||||
- Add the optional visibility metadata contract to
|
||||
`apps/web/components/task/chat/types.ts`.
|
||||
- Render a running-only message as one inline row: muted `text-xs` copy, no
|
||||
alert icon, no warning/error color, no tinted background or border, and one
|
||||
neutral **Cancel turn** button at the end.
|
||||
- Extend the action-button presentation with a compact inline mode. The button
|
||||
is content-width at every breakpoint, uses the existing small desktop
|
||||
height, and retains a minimum 44px height on phones.
|
||||
|
||||
### Mobile design contract
|
||||
|
||||
- **Desktop outcome:** one low-emphasis inline status row in the chat footer
|
||||
says Kandev is still waiting and exposes a neutral **Cancel turn** button.
|
||||
- **Mobile entry point and outcome:** the same compact row appears in the active
|
||||
chat footer; **Cancel turn** remains inline and content-width, has a 44px
|
||||
touch height, and returns the session to input-ready state.
|
||||
- **Nearest shipped exemplar:** `ActionMessage` supplies the message/action
|
||||
behavior and `mobile-pause-resume-recovery.spec.ts` supplies the touch
|
||||
cancellation flow. The existing stacked full-width mobile `ActionButtons`
|
||||
geometry is intentionally not reused for this low-emphasis notice.
|
||||
- **Presentation:** inline in the existing chat footer; no drawer or new
|
||||
navigation layer is needed for a frequent, single-step recovery action.
|
||||
- **Hierarchy and geometry:** notice copy truncates before the trailing action;
|
||||
chat remains the sole scroll owner, existing safe-area behavior is retained,
|
||||
the row does not introduce a card or stacked action area, and the action uses
|
||||
the shared 44px mobile touch height without expanding to full width.
|
||||
- **Shared logic:** message metadata, session-state visibility, and the
|
||||
`agent.cancel` handler are identical across viewports.
|
||||
|
||||
---
|
||||
|
||||
## Tests
|
||||
|
||||
- **What:** a top-level `in_progress` tool is retained as stall context while
|
||||
subagent tools do not replace it and terminal updates clear it.
|
||||
**File:**
|
||||
`apps/backend/internal/agent/runtime/lifecycle/manager_events_test.go`.
|
||||
**How:** table-driven lifecycle event tests.
|
||||
- **What:** five minutes without events publishes one generation-scoped
|
||||
`agent.stalled` event, does not complete the prompt, and does not duplicate
|
||||
on later checks.
|
||||
**File:** `apps/backend/internal/agent/runtime/lifecycle/session_test.go`.
|
||||
**How:** `testing/synctest` with the real ticker and event bus.
|
||||
- **What:** the orchestrator persists one neutral notice with sanitized tool
|
||||
copy and the exact `agent.cancel` action without transitioning session state.
|
||||
**File:**
|
||||
`apps/backend/internal/orchestrator/event_handlers_stall_test.go` and
|
||||
`apps/backend/internal/orchestrator/watcher/watcher_test.go`.
|
||||
**How:** direct handler test with the existing message-creator fake plus a
|
||||
memory-bus watcher dispatch test.
|
||||
- **What:** running-only action messages render as a compact neutral row during
|
||||
`RUNNING`, hide after settlement, and do not change ordinary recovery-message
|
||||
visibility.
|
||||
**File:**
|
||||
`apps/web/components/task/chat/messages/action-message.test.tsx`.
|
||||
**How:** rendered store-backed component tests.
|
||||
|
||||
## E2E Tests
|
||||
|
||||
- **Scenario:** a running session with a persisted stall notice exposes
|
||||
**Cancel turn**, and activating it makes the session input-ready without a
|
||||
backend restart.
|
||||
**File:**
|
||||
`apps/web/e2e/tests/session/pause-resume-recovery.spec.ts`.
|
||||
**What to verify:** neutral compact copy, action request, notice
|
||||
disappearance, idle composer, and unchanged task URL.
|
||||
- **Scenario:** the same recovery is reachable by touch on a phone viewport.
|
||||
**File:**
|
||||
`apps/web/e2e/tests/session/mobile-pause-resume-recovery.spec.ts`.
|
||||
**What to verify:** the content-width action is at least 44px high, the seeded
|
||||
notice remains one compact row, `.tap()` settles the session, and the document
|
||||
has no horizontal overflow.
|
||||
- Seed the `RUNNING` session and actionable status message through the existing
|
||||
E2E-only `seedTaskSession` and `seedSessionMessage` helpers. The cancel path's
|
||||
missing-live-execution reconciliation provides a deterministic end-to-end
|
||||
recovery without waiting five wall-clock minutes; lifecycle timing remains
|
||||
covered by the `synctest` regression.
|
||||
|
||||
## Implementation Waves And Parallel Candidates
|
||||
|
||||
Execution is sequential in the primary conversation. No task is marked
|
||||
parallel-safe because each task consumes the event or metadata contract created
|
||||
by the previous one.
|
||||
|
||||
Wave 1:
|
||||
|
||||
- [x] [Task 01: Detect and publish stalled prompts](task-01-detect-stalled-prompt.md)
|
||||
|
||||
Wave 2:
|
||||
|
||||
- [x] [Task 02: Persist an actionable stall notice](task-02-persist-stall-warning.md)
|
||||
|
||||
Wave 3:
|
||||
|
||||
- [x] [Task 03: Render the notice during running sessions](task-03-render-running-warning.md)
|
||||
|
||||
Wave 4:
|
||||
|
||||
- [x] [Task 04: Prove desktop and mobile recovery](task-04-e2e-stall-recovery.md)
|
||||
|
||||
## Required workflow verification
|
||||
|
||||
After all targeted task checks pass:
|
||||
|
||||
1. Run `make fmt`.
|
||||
2. Run `make typecheck test lint`.
|
||||
3. Commit the explicit changed paths with a Conventional Commit message.
|
||||
|
||||
These broad commands are included because the Improve workflow explicitly
|
||||
requires them; the task-level commands remain the TDD evidence for each change.
|
||||
|
||||
## Risks and non-goals
|
||||
|
||||
- Event silence remains ambiguous, so the notice is advisory and cannot alter
|
||||
lifecycle state by itself.
|
||||
- Tool titles are already user-visible display data; raw command arguments must
|
||||
not be copied into the notice.
|
||||
- The notice is persisted once per prompt generation and assigned the active
|
||||
turn ID so reloads retain it only for the affected turn. A delayed event is
|
||||
rejected when the session has settled or execution/generation ownership has
|
||||
moved on; a later `RUNNING` turn cannot resurrect the old notice.
|
||||
- No schema migration, public HTTP endpoint, configurable timeout, automatic
|
||||
process termination, or relaxed `RUNNING` prompt-admission rule is planned.
|
||||
@@ -0,0 +1,59 @@
|
||||
---
|
||||
id: "01-detect-stalled-prompt"
|
||||
title: "Detect and publish stalled prompts"
|
||||
status: done
|
||||
wave: 1
|
||||
depends_on: []
|
||||
plan: "plan.md"
|
||||
spec: "../../specs/agent-stall-recovery/spec.md"
|
||||
---
|
||||
|
||||
# Task 01: Detect and publish stalled prompts
|
||||
|
||||
## Acceptance
|
||||
|
||||
- A prompt with no agent events for five minutes publishes one
|
||||
generation-scoped `agent.stalled` event and remains in progress.
|
||||
- The event includes active top-level tool display identity when available;
|
||||
subagent tools do not replace it and terminal tool updates clear it.
|
||||
- The watchdog checks every 60 seconds and does not repeat the notice or log
|
||||
for the same prompt generation.
|
||||
|
||||
## Verification
|
||||
|
||||
- `make -C apps/backend test` (regressions: `TestHandleAgentEvent_TracksActiveTopLevelTool` and `TestWaitForPromptDone_PublishesSingleStall` in `internal/agent/runtime/lifecycle`)
|
||||
|
||||
The `testing/synctest` regression must first fail because no `agent.stalled`
|
||||
event is published, then pass without shortening production durations.
|
||||
|
||||
## Files likely touched
|
||||
|
||||
- `apps/backend/internal/agent/runtime/lifecycle/types.go`
|
||||
- `apps/backend/internal/agent/runtime/lifecycle/manager_events.go`
|
||||
- `apps/backend/internal/agent/runtime/lifecycle/manager_events_test.go`
|
||||
- `apps/backend/internal/agent/runtime/lifecycle/session.go`
|
||||
- `apps/backend/internal/agent/runtime/lifecycle/session_test.go`
|
||||
- `apps/backend/internal/agent/runtime/lifecycle/event_types.go`
|
||||
- `apps/backend/internal/agent/runtime/lifecycle/events.go`
|
||||
- `apps/backend/internal/events/types.go`
|
||||
|
||||
## Dependencies
|
||||
|
||||
None.
|
||||
|
||||
## Parallelism
|
||||
|
||||
Sequential. Task 02 consumes the event and payload contract defined here.
|
||||
|
||||
## Inputs
|
||||
|
||||
- Spec sections `What`, `Failure modes`, and the first three scenarios
|
||||
- ADR decision bullets for advisory detection and once-per-generation logging
|
||||
- Existing `recordActivity`, `handleToolCallEvent`, `handleToolUpdateEvent`, and
|
||||
`waitForPromptDone` patterns
|
||||
|
||||
## Output contract
|
||||
|
||||
Report the RED assertion, event payload shape, tool-tracking behavior, exact
|
||||
test result, files changed, blockers, and risks. Mark this task `done` and
|
||||
update its plan checkbox in the same conversation.
|
||||
@@ -0,0 +1,56 @@
|
||||
---
|
||||
id: "02-persist-stall-warning"
|
||||
title: "Persist an actionable stall notice"
|
||||
status: done
|
||||
wave: 2
|
||||
depends_on: ["01-detect-stalled-prompt"]
|
||||
plan: "plan.md"
|
||||
spec: "../../specs/agent-stall-recovery/spec.md"
|
||||
---
|
||||
|
||||
# Task 02: Persist an actionable stall notice
|
||||
|
||||
## Acceptance
|
||||
|
||||
- The orchestrator consumes `agent.stalled` and creates one neutral status
|
||||
message without changing task, session, or process state.
|
||||
- Copy says Kandev is still waiting, uses only the tool display title/name, and
|
||||
falls back to a generic notice without asserting failure.
|
||||
- Metadata exposes a running-only **Cancel turn** `agent.cancel` action with the
|
||||
affected `session_id`, neutral styling, and stable test ID.
|
||||
|
||||
## Verification
|
||||
|
||||
- `make -C apps/backend test` (regressions: `TestHandleAgentStalled` in `internal/orchestrator` and `TestWatcher.*AgentStalled` in `internal/orchestrator/watcher`)
|
||||
|
||||
The handler test must first fail because no notice is persisted, then pass
|
||||
with exact metadata assertions and unchanged session state.
|
||||
|
||||
## Files likely touched
|
||||
|
||||
- `apps/backend/internal/orchestrator/watcher/watcher.go`
|
||||
- `apps/backend/internal/orchestrator/watcher/watcher_test.go`
|
||||
- `apps/backend/internal/orchestrator/service.go`
|
||||
- `apps/backend/internal/orchestrator/event_handlers_stall.go`
|
||||
- `apps/backend/internal/orchestrator/event_handlers_stall_test.go`
|
||||
|
||||
## Dependencies
|
||||
|
||||
Task 01.
|
||||
|
||||
## Parallelism
|
||||
|
||||
Sequential. It consumes Task 01's event contract and defines Task 03's message
|
||||
metadata contract.
|
||||
|
||||
## Inputs
|
||||
|
||||
- Spec scenarios for tool-specific and generic notices
|
||||
- Plan section `Persist the actionable notice`
|
||||
- Existing transient-retry and recoverable-failure action-message builders
|
||||
|
||||
## Output contract
|
||||
|
||||
Report the RED assertion, notice copy and metadata, proof that no state
|
||||
transition occurred, exact test results, files changed, blockers, and risks.
|
||||
Mark this task `done` and update its plan checkbox in the same conversation.
|
||||
@@ -0,0 +1,57 @@
|
||||
---
|
||||
id: "03-render-running-warning"
|
||||
title: "Render the notice during running sessions"
|
||||
status: done
|
||||
wave: 3
|
||||
depends_on: ["02-persist-stall-warning"]
|
||||
plan: "plan.md"
|
||||
spec: "../../specs/agent-stall-recovery/spec.md"
|
||||
---
|
||||
|
||||
# Task 03: Render the notice during running sessions
|
||||
|
||||
## Acceptance
|
||||
|
||||
- `action_visibility: running` messages render while the session is `RUNNING`
|
||||
only when their `turn_id` matches the active turn, and hide after settlement
|
||||
or when a later turn becomes active.
|
||||
- Existing recovery messages keep their current visibility behavior.
|
||||
- The running-only notice is one inline row with muted neutral copy, no alert
|
||||
icon, warning/error color, tinted background, border, or stacked action area.
|
||||
- **Cancel turn** is neutral and content-width at every breakpoint. It stays
|
||||
compact on desktop and has a minimum 44px touch height on phones.
|
||||
|
||||
## Verification
|
||||
|
||||
- `cd apps && pnpm --filter @kandev/web test -- --run components/task/chat/messages/action-message.test.tsx`
|
||||
|
||||
The component regression must first fail because all action messages are hidden
|
||||
during `RUNNING`, then pass with the neutral compact presentation without
|
||||
weakening ordinary recovery visibility.
|
||||
|
||||
## Files likely touched
|
||||
|
||||
- `apps/web/components/task/chat/messages/action-message.tsx`
|
||||
- `apps/web/components/task/chat/messages/action-message.test.tsx`
|
||||
- `apps/web/components/task/chat/types.ts`
|
||||
|
||||
## Dependencies
|
||||
|
||||
Task 02.
|
||||
|
||||
## Parallelism
|
||||
|
||||
Sequential. It consumes the backend metadata contract and precedes E2E.
|
||||
|
||||
## Inputs
|
||||
|
||||
- Spec desktop/mobile notice scenarios
|
||||
- Plan frontend section and mobile design contract
|
||||
- Existing `ActionMessage`, `ActionButtons`, and mobile button sizing; do not
|
||||
reuse the stacked full-width mobile action layout for this notice
|
||||
|
||||
## Output contract
|
||||
|
||||
Report the RED assertion, visibility rule, desktop/mobile presentation,
|
||||
targeted test result, files changed, blockers, and risks. Mark this task `done`
|
||||
and update its plan checkbox in the same conversation.
|
||||
@@ -0,0 +1,63 @@
|
||||
---
|
||||
id: "04-e2e-stall-recovery"
|
||||
title: "Prove desktop and mobile stall recovery"
|
||||
status: done
|
||||
wave: 4
|
||||
depends_on:
|
||||
- "02-persist-stall-warning"
|
||||
- "03-render-running-warning"
|
||||
plan: "plan.md"
|
||||
spec: "../../specs/agent-stall-recovery/spec.md"
|
||||
---
|
||||
|
||||
# Task 04: Prove desktop and mobile stall recovery
|
||||
|
||||
## Acceptance
|
||||
|
||||
- Desktop Playwright proves a running stalled session shows **Cancel turn** and
|
||||
becomes input-ready after the action without navigation or restart. The
|
||||
notice is a neutral single inline row rather than an alert card.
|
||||
- Mobile Playwright performs the same recovery with `.tap()`, proves a minimum
|
||||
44px content-width action, verifies the seeded notice remains a compact row,
|
||||
and finds no document horizontal overflow.
|
||||
- Tests use the E2E-only session/message seed helpers; they do not wait five
|
||||
wall-clock minutes or add production test hooks.
|
||||
|
||||
## Verification
|
||||
|
||||
- `cd apps && pnpm --filter @kandev/web e2e:run tests/session/pause-resume-recovery.spec.ts`
|
||||
- `cd apps && pnpm --filter @kandev/web e2e:run tests/session/mobile-pause-resume-recovery.spec.ts -- --project=mobile-chrome`
|
||||
|
||||
Each E2E scenario must first fail because the running-only notice is hidden,
|
||||
then pass against freshly built backend and Vite artifacts.
|
||||
|
||||
The browser scenarios seed the persisted notice for deterministic coverage;
|
||||
the real five-minute watchdog boundary is covered by the lifecycle synctest.
|
||||
|
||||
## Files likely touched
|
||||
|
||||
- `apps/web/e2e/pages/session-page.ts`
|
||||
- `apps/web/e2e/tests/session/pause-resume-recovery.spec.ts`
|
||||
- `apps/web/e2e/tests/session/mobile-pause-resume-recovery.spec.ts`
|
||||
|
||||
## Dependencies
|
||||
|
||||
Tasks 02 and 03.
|
||||
|
||||
## Parallelism
|
||||
|
||||
Sequential. It validates the integrated backend metadata and frontend
|
||||
presentation.
|
||||
|
||||
## Inputs
|
||||
|
||||
- Spec cancellation and phone scenarios
|
||||
- Plan E2E section and mobile design contract
|
||||
- Existing `seedTaskSession`, `seedSessionMessage`, and pause/recovery E2E
|
||||
patterns
|
||||
|
||||
## Output contract
|
||||
|
||||
Report both RED failures, desktop and mobile user outcomes, exact managed-runner
|
||||
results, rendered mobile inspection, files changed, blockers, and risks. Mark
|
||||
this task `done` and update its plan checkbox in the same conversation.
|
||||
@@ -155,6 +155,7 @@ System pages (Radarr/Sonarr-style) for status, disk usage, database maintenance,
|
||||
| Spec | Status |
|
||||
|---|---|
|
||||
| [agent-resume-runtime-recovery](agent-resume-runtime-recovery/spec.md) | shipped |
|
||||
| [agent-stall-recovery](agent-stall-recovery/spec.md) | draft |
|
||||
| [auth](auth/spec.md) | building |
|
||||
| [create-local-repository](create-local-repository/spec.md) | shipped |
|
||||
| [workflow-cycle-guardrails](workflow-cycle-guardrails/spec.md) | building |
|
||||
|
||||
@@ -0,0 +1,93 @@
|
||||
---
|
||||
status: draft
|
||||
created: 2026-07-29
|
||||
owner: Kandev
|
||||
---
|
||||
|
||||
# Agent Stall Recovery
|
||||
|
||||
Decision:
|
||||
[ADR-2026-07-29-agent-stall-user-controlled-recovery](../../decisions/2026-07-29-agent-stall-user-controlled-recovery.md)
|
||||
|
||||
## Why
|
||||
|
||||
An agent turn can remain `RUNNING` forever after its active tool stops emitting
|
||||
events. Users currently see only “Agent is running,” while Resume is correctly
|
||||
blocked by the active-session guard and the backend repeats a diagnostic that
|
||||
the UI never receives.
|
||||
|
||||
## Broken behavior
|
||||
|
||||
`waitForPromptDone` detects five minutes without events but only logs the
|
||||
condition every 30 seconds. A top-level shell tool that remains `in_progress`
|
||||
can therefore hold the prompt waiter and session state indefinitely even
|
||||
though Kandev already has a bounded cancel-escalation path capable of releasing
|
||||
the turn.
|
||||
|
||||
## What
|
||||
|
||||
- Kandev checks running prompts for inactivity once per 60 seconds.
|
||||
- After five minutes without agent events, Kandev creates at most one
|
||||
user-visible notice for that prompt generation.
|
||||
- The notice says Kandev is still waiting and includes the active top-level
|
||||
tool's display title or name when available. It does not assert that the tool
|
||||
failed and does not include raw command arguments.
|
||||
- The notice is a single compact inline row with muted neutral copy and a
|
||||
neutral **Cancel turn** action. It has no warning/error colors, alert icon,
|
||||
tinted background, or alert-card treatment.
|
||||
- On phones, **Cancel turn** remains inline and content-width rather than
|
||||
becoming a full-width row, while retaining a minimum 44px touch height.
|
||||
Activating it uses the existing `agent.cancel` request.
|
||||
- The notice remains visible and actionable while the affected prompt's
|
||||
`turn_id` is the active turn in a `RUNNING` session, including after a page
|
||||
reload. It is hidden when that turn settles or a later turn becomes active.
|
||||
- Detection alone does not change task state, session state, prompt admission,
|
||||
or process liveness.
|
||||
- The backend logs the first stall detected for a prompt generation and does
|
||||
not emit another notice or log entry on every watchdog check.
|
||||
|
||||
## Failure modes
|
||||
|
||||
- If active tool identity is unavailable, the generic notice and cancel action
|
||||
still appear.
|
||||
- If the agent acknowledges cancellation, normal cancellation reconciliation
|
||||
makes the session input-ready.
|
||||
- If the agent does not acknowledge cancellation, the existing bounded
|
||||
cancel-escalation path releases the prompt and reconciles the session.
|
||||
- If notice-message persistence fails, the failure is logged without changing
|
||||
or terminating the running session.
|
||||
|
||||
## Scenarios
|
||||
|
||||
- **GIVEN** a running prompt whose top-level shell tool is `in_progress`,
|
||||
**WHEN** no agent event arrives for five minutes, **THEN** one compact neutral
|
||||
notice names that tool and offers **Cancel turn** while the session remains
|
||||
`RUNNING`.
|
||||
- **GIVEN** a running prompt with no known active tool, **WHEN** no agent event
|
||||
arrives for five minutes, **THEN** one generic notice offers **Cancel turn**
|
||||
while the session remains `RUNNING`.
|
||||
- **GIVEN** a notice already exists for the active prompt generation,
|
||||
**WHEN** subsequent watchdog checks observe the same stall, **THEN** Kandev
|
||||
creates no duplicate notice and emits no repeated stall log.
|
||||
- **GIVEN** a persisted notice belongs to an earlier prompt generation,
|
||||
**WHEN** a later prompt is `RUNNING`, **THEN** the earlier notice is not
|
||||
rendered and a delayed stall event cannot create a new notice for it.
|
||||
- **GIVEN** a stall notice is visible, **WHEN** the user activates
|
||||
**Cancel turn**, **THEN** the existing cancellation path settles the turn and
|
||||
the session becomes available for new input without a backend restart.
|
||||
- **GIVEN** a stall notice is visible on a phone viewport, **WHEN** the user taps
|
||||
**Cancel turn**, **THEN** the same cancellation outcome is reachable through
|
||||
an inline, content-width touch target of at least 44px.
|
||||
- **GIVEN** a quiet but legitimate long-running turn, **WHEN** the inactivity
|
||||
threshold passes and the user does not cancel, **THEN** Kandev leaves the
|
||||
turn and process running.
|
||||
|
||||
## Out of scope
|
||||
|
||||
- Automatically timing out, failing, cancelling, or killing quiet turns.
|
||||
- Making the inactivity threshold user-configurable.
|
||||
- Provider-specific repair of shell commands that hold child-process streams
|
||||
open.
|
||||
- Treating event silence as proof that an agent process crashed.
|
||||
- Allowing Resume or direct prompt admission while the session remains
|
||||
`RUNNING`.
|
||||
Reference in New Issue
Block a user