From 1614720ceb72a9384f57736d697a299a6fcedf57 Mon Sep 17 00:00:00 2001 From: Benjamin Date: Mon, 21 Sep 2026 16:39:42 +0800 Subject: [PATCH] fix(task): terminalize sessions orphaned by backend restart via reconciliation sweep (#3833) Co-authored-by: Carlos Florencio --- .../runtime/lifecycle/manager_execution.go | 40 ++ .../manager_interaction_accessors_test.go | 23 + apps/backend/internal/backendapp/adapters.go | 13 +- .../internal/backendapp/orchestrator.go | 3 + .../internal/task/models/resume_safety.go | 4 +- .../task/repository/sqlite/session.go | 110 ++++- .../archived_session_reconciliation.go | 173 +++++++- .../orphaned_session_reconciliation_test.go | 399 ++++++++++++++++++ apps/backend/internal/task/service/service.go | 20 + .../internal/task/service/service_tasks.go | 4 +- .../plan.md | 106 +++++ ...1-terminalize-restart-orphaned-sessions.md | 117 +++++ docs/specs/tasks/README.md | 1 + ...estart-orphaned-session-terminalization.md | 105 +++++ ...estart-orphaned-session-terminalization.md | 130 ++++++ 15 files changed, 1215 insertions(+), 33 deletions(-) create mode 100644 apps/backend/internal/task/service/orphaned_session_reconciliation_test.go create mode 100644 docs/plans/restart-orphaned-session-terminalization/plan.md create mode 100644 docs/plans/restart-orphaned-session-terminalization/task-01-terminalize-restart-orphaned-sessions.md create mode 100644 docs/specs/tasks/requirements/restart-orphaned-session-terminalization.md create mode 100644 docs/specs/tasks/system-design/restart-orphaned-session-terminalization.md diff --git a/apps/backend/internal/agent/runtime/lifecycle/manager_execution.go b/apps/backend/internal/agent/runtime/lifecycle/manager_execution.go index 317b99d64d..6d868a6d01 100644 --- a/apps/backend/internal/agent/runtime/lifecycle/manager_execution.go +++ b/apps/backend/internal/agent/runtime/lifecycle/manager_execution.go @@ -21,6 +21,7 @@ import ( "github.com/kandev/kandev/internal/secrets" "github.com/kandev/kandev/internal/task/models" "github.com/kandev/kandev/internal/worktree" + v1 "github.com/kandev/kandev/pkg/api/v1" ) // ErrSessionWorkspaceNotReady indicates the task session exists but does not yet @@ -496,6 +497,45 @@ func (m *Manager) IsAgentCommandConfigured(executionID string) bool { return configured } +// HasLiveAgentExecution reports whether a session owns an agent execution that +// can still protect its durable lifecycle row from orphan reconciliation. +// Workspace-only executions have no agent command and do not count. Terminal +// executions do not count even when they remain in the store briefly. A live +// passthrough process protects its session through the interactive runner. +func (m *Manager) HasLiveAgentExecution(sessionID string) bool { + execution, exists := m.executionStore.GetBySessionID(sessionID) + if !exists || execution == nil { + return false + } + + var agentCommand, passthroughProcessID string + var status v1.AgentStatus + if err := m.executionStore.WithRLock(execution.ID, func(current *AgentExecution) { + agentCommand = current.AgentCommand + passthroughProcessID = current.PassthroughProcessID + status = current.Status + }); err != nil { + return false + } + if isTerminalStatus(status) { + return false + } + if agentCommand != "" { + return true + } + if passthroughProcessID == "" { + return false + } + runner := m.GetInteractiveRunner() + if runner == nil { + // The process handle is evidence of agent ownership, but an unavailable + // runner cannot prove that the process is gone. Keep the session safe + // from destructive orphan cleanup until the runtime can answer. + return true + } + return runner.IsProcessReadyOrPending(passthroughProcessID) +} + // EnsurePassthroughExecution ensures an execution exists for a passthrough session // and starts the passthrough process if needed. This is called when the terminal // handler receives a connection for a session that might need recovery after backend restart. diff --git a/apps/backend/internal/agent/runtime/lifecycle/manager_interaction_accessors_test.go b/apps/backend/internal/agent/runtime/lifecycle/manager_interaction_accessors_test.go index ba4889c753..31e07aedcb 100644 --- a/apps/backend/internal/agent/runtime/lifecycle/manager_interaction_accessors_test.go +++ b/apps/backend/internal/agent/runtime/lifecycle/manager_interaction_accessors_test.go @@ -179,6 +179,29 @@ func TestIsAgentCommandConfigured(t *testing.T) { require.False(t, mgr.IsAgentCommandConfigured("exec-absent")) } +func TestHasLiveAgentExecutionIgnoresWorkspaceInfrastructureAndTerminalAgents(t *testing.T) { + mgr := newTestManager(t) + require.NoError(t, mgr.executionStore.Add(&AgentExecution{ + ID: "exec-workspace", SessionID: "session-workspace", Status: v1.AgentStatusRunning, + })) + require.NoError(t, mgr.executionStore.Add(&AgentExecution{ + ID: "exec-agent", SessionID: "session-agent", AgentCommand: "claude", + Status: v1.AgentStatusStarting, + })) + require.NoError(t, mgr.executionStore.Add(&AgentExecution{ + ID: "exec-terminal", SessionID: "session-terminal", AgentCommand: "claude", + Status: v1.AgentStatusFailed, + })) + + require.False(t, mgr.HasLiveAgentExecution("session-workspace"), + "workspace-only execution must not shield an orphaned session") + require.True(t, mgr.HasLiveAgentExecution("session-agent"), + "a configured non-terminal agent execution must shield its session") + require.False(t, mgr.HasLiveAgentExecution("session-terminal"), + "terminal executions must not shield a session") + require.False(t, mgr.HasLiveAgentExecution("session-absent")) +} + func TestResolveTaskEnvironmentIDPrefersInMemoryExecution(t *testing.T) { mgr := newTestManager(t) mgr.workspaceInfoProvider = &mockWorkspaceInfoProvider{infos: map[string]*WorkspaceInfo{ diff --git a/apps/backend/internal/backendapp/adapters.go b/apps/backend/internal/backendapp/adapters.go index b79b8bfc11..53e3d506ca 100644 --- a/apps/backend/internal/backendapp/adapters.go +++ b/apps/backend/internal/backendapp/adapters.go @@ -971,7 +971,10 @@ func (a *lifecycleAdapter) LiveSessionIDsForTask(taskID string) []string { references := a.mgr.ListExecutionsForTask(taskID) sessionIDs := make([]string, 0, len(references)) for _, reference := range references { - if reference.SessionID != "" { + // ListExecutionsForTask includes workspace-only infrastructure created + // by file/shell access. Only an execution with an agent command (or a + // live passthrough process) owns the agent session lifecycle. + if reference.SessionID != "" && a.mgr.HasLiveAgentExecution(reference.SessionID) { sessionIDs = append(sessionIDs, reference.SessionID) } } @@ -1018,6 +1021,14 @@ func (a *lifecycleAdapter) ResolveAgentProfile(ctx context.Context, profileID st }, nil } +// HasLiveExecution reports whether the session still has an agent execution +// owned by the runtime. It backs the task service's orphan-session +// reconciliation sweep; workspace-only infrastructure is not a live agent, +// and this lookup must never lazily create an execution. +func (a *lifecycleAdapter) HasLiveExecution(sessionID string) bool { + return a.mgr.HasLiveAgentExecution(sessionID) +} + // GetGitLog retrieves the git log for a session from baseCommit to HEAD. // If targetBranch is provided, uses dynamic merge-base calculation for accurate filtering. func (a *lifecycleAdapter) GetGitLog(ctx context.Context, sessionID, baseCommit string, limit int, targetBranch string) (*client.GitLogResult, error) { diff --git a/apps/backend/internal/backendapp/orchestrator.go b/apps/backend/internal/backendapp/orchestrator.go index e1e17effba..34cf37cdb2 100644 --- a/apps/backend/internal/backendapp/orchestrator.go +++ b/apps/backend/internal/backendapp/orchestrator.go @@ -170,6 +170,9 @@ func provideOrchestrator( // Runtime-aware liveness lets durable cleanup treat a not-found stop for a // confirmed-dead local runtime as already stopped instead of retrying forever. taskSvc.SetRowLivenessProber(agentManagerClient) + // The orphan-session sweep terminalizes stale STARTING/RUNNING sessions no + // live in-memory execution backs (backend-restart residue, #3711). + taskSvc.SetExecutionLivenessChecker(agentManagerClient) // The session reconciliation sweep's active-task pass (stall detection and // orphaned-session healing) verifies "no live execution" against the agent // runtime's in-memory execution store through this registry. diff --git a/apps/backend/internal/task/models/resume_safety.go b/apps/backend/internal/task/models/resume_safety.go index 3e06290755..5efc7da3d8 100644 --- a/apps/backend/internal/task/models/resume_safety.go +++ b/apps/backend/internal/task/models/resume_safety.go @@ -72,9 +72,7 @@ func RowMustBePreserved(running *ExecutorRunning, sessionState TaskSessionState) // SessionArchiveCancelReason and SessionArchiveTreeCancelReason are the // TaskSession.ErrorMessage values written when a session is auto-cancelled -// by archiving — Service.ArchiveTask's CancelActiveTaskSessionsByTaskID call -// (single-task archive) and HandoffService.cancelActiveRuns (cascade archive) -// respectively. Distinct reasons keep the cancellation source visible in +// by archiving. Distinct reasons keep the cancellation source visible in // session history and diagnostics. const ( SessionArchiveCancelReason = "task archived" diff --git a/apps/backend/internal/task/repository/sqlite/session.go b/apps/backend/internal/task/repository/sqlite/session.go index 15726f9f5e..b0ba83d03e 100644 --- a/apps/backend/internal/task/repository/sqlite/session.go +++ b/apps/backend/internal/task/repository/sqlite/session.go @@ -2112,7 +2112,7 @@ func completedAtForTaskSessionState(status models.TaskSessionState, now time.Tim // timed out. Returned sessions therefore carry only the fields the // RETURNING clause selects (ID, TaskID, AgentProfileID, // AgentProfileSnapshot, IsPassthrough, Name, ReviewStatus, Metadata, -// TaskEnvironmentID, State, UpdatedAt) — every other models.TaskSession +// TaskEnvironmentID, State, UpdatedAt, IsPrimary) — every other models.TaskSession // field is left at its zero value, and callers must not rely on fields // outside this list being populated. func (r *Repository) CancelActiveTaskSessionsByTaskID(ctx context.Context, taskID, reason string) ([]*models.TaskSession, error) { @@ -2131,7 +2131,7 @@ func (r *Repository) CancelActiveTaskSessionsByTaskID(ctx context.Context, taskI WHERE task_id = ? AND state IN ('CREATED', 'STARTING', 'RUNNING', 'WAITING_FOR_INPUT') RETURNING id, agent_profile_id, agent_profile_snapshot, is_passthrough, name, - review_status, metadata, task_environment_id, state, updated_at + review_status, metadata, task_environment_id, state, updated_at, is_primary `), string(models.TaskSessionStateCancelled), reason, now, now, taskID) if err != nil { return nil, err @@ -2149,6 +2149,97 @@ func (r *Repository) CancelActiveTaskSessionsByTaskID(ctx context.Context, taskI return sessions, rows.Err() } +// ListStaleRunningSessionsOnUnarchivedTasks returns every STARTING/RUNNING +// session of an unarchived task whose updated_at is older than staleBefore. +// It is the candidate list for the orphan-session reconciliation sweep (see +// service.runOrphanedSessionReconciliation): a session still holding one of +// those states past the launch grace window, with no live in-memory execution +// backing it, can never reach a terminal state on its own — its actor died +// with the process. The staleBefore cutoff is applied in SQL so the sweep +// never even loads fresh rows that may belong to an in-flight launch. +// +// Not part of the SessionRepository interface: it exists for that one sweep, +// which reaches it through the narrow orphanedSessionRepository capability — +// widening the interface would force every test double in the tree to grow +// methods this sweep never exercises through them. +func (r *Repository) ListStaleRunningSessionsOnUnarchivedTasks(ctx context.Context, staleBefore time.Time) ([]*models.TaskSession, error) { + rows, err := r.ro.QueryContext(ctx, r.ro.Rebind(` + SELECT `+taskSessionSelectCols+` `+taskSessionFromClause+` + JOIN tasks t ON t.id = ts.task_id + WHERE ts.state IN ('STARTING', 'RUNNING') + AND ts.updated_at < ? + AND t.archived_at IS NULL + ORDER BY ts.updated_at ASC + `), staleBefore) + if err != nil { + return nil, err + } + defer func() { _ = rows.Close() }() + + sessions, err := r.scanTaskSessions(ctx, rows) + if err != nil { + return nil, err + } + return r.loadWorktreesBatch(ctx, sessions) +} + +// CancelRunningTaskSessionByID transitions a single stale STARTING/RUNNING +// session to CANCELLED, returning the transitioned row, or nil when the +// session no longer matches (already terminal, raced to another state, or +// refreshed since the sweep read it). Like CancelActiveTaskSessionsByTaskID +// it is a pure DB state change that requires no live agent execution, and it +// is session-scoped so the orphan sweep can terminalize one session without +// cancelling healthy sibling sessions of the same task. +// +// staleBefore re-asserts the sweep's staleness cutoff at write time, closing +// the read-then-write race against an in-flight launch: a launch CAS-writes +// its session to STARTING (bumping updated_at) before it registers an +// execution in the in-memory store, so between the sweep's liveness check and +// this UPDATE the row can pass from "no live execution" to "launch in +// progress". A row refreshed since the candidate read no longer satisfies +// updated_at < staleBefore, the UPDATE matches nothing, and the next tick +// re-evaluates a fresh row that the grace window protects until its +// execution registers. +// +// The UPDATE and the row selection happen in one atomic RETURNING statement; +// the returned row carries only the fields that clause selects (same set as +// CancelActiveTaskSessionsByTaskID, minus TaskID — callers pass the task ID +// they already know). The write detaches from ctx like +// CancelActiveTaskSessionsByTaskID: once the sweep has decided this session +// is orphaned, the terminal transition must not be lost to a caller-context +// cancellation, and the 10s bound keeps a locked SQLite writer from stalling +// the sweep pass. A failed write simply retries on the next sweep tick. +func (r *Repository) CancelRunningTaskSessionByID(ctx context.Context, sessionID, reason string, staleBefore time.Time) (*models.TaskSession, error) { + now := time.Now().UTC() + writeCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second) + defer cancel() + rows, err := r.db.QueryContext(writeCtx, r.db.Rebind(` + UPDATE task_sessions + SET state = ?, error_message = ?, completed_at = ?, updated_at = ? + WHERE id = ? + AND state IN ('STARTING', 'RUNNING') + AND updated_at < ? + RETURNING id, agent_profile_id, agent_profile_snapshot, is_passthrough, name, + review_status, metadata, task_environment_id, state, updated_at, is_primary + `), string(models.TaskSessionStateCancelled), reason, now, now, sessionID, staleBefore) + if err != nil { + return nil, err + } + defer func() { _ = rows.Close() }() + + if !rows.Next() { + return nil, rows.Err() + } + // The scanner backfills TaskID from the parameter because RETURNING + // does not carry it; sessionID is not the task ID, so set it from the + // candidate row the sweep already read instead. + session, err := scanCancelledTaskSessionRow(rows, "") + if err != nil { + return nil, err + } + return session, rows.Err() +} + // CancelActiveTaskSessionsByIDs is documented on the SessionRepository // interface. It shares CancelActiveTaskSessionsByTaskID's atomic // UPDATE ... RETURNING shape and detached-but-bounded write context, so the @@ -2175,7 +2266,7 @@ func (r *Repository) CancelActiveTaskSessionsByIDs(ctx context.Context, taskID s AND id IN (`+placeholders+`) AND state IN ('CREATED', 'STARTING', 'RUNNING', 'WAITING_FOR_INPUT') RETURNING id, agent_profile_id, agent_profile_snapshot, is_passthrough, name, - review_status, metadata, task_environment_id, state, updated_at + review_status, metadata, task_environment_id, state, updated_at, is_primary `), args...) if err != nil { return nil, err @@ -2242,9 +2333,9 @@ func (r *Repository) CancelActiveTaskSessionsByCandidates( SET state = ?, error_message = ?, completed_at = ?, updated_at = ? WHERE task_id = ? AND state IN ('CREATED', 'STARTING', 'RUNNING', 'WAITING_FOR_INPUT') - AND (`+strings.Join(predicates, " OR ")+`) + AND (`+strings.Join(predicates, " OR ")+`) RETURNING id, agent_profile_id, agent_profile_snapshot, is_passthrough, name, - review_status, metadata, task_environment_id, state, updated_at + review_status, metadata, task_environment_id, state, updated_at, is_primary `), args...) if err != nil { return nil, err @@ -2305,15 +2396,16 @@ func activeSessionCancellationCandidatePredicate( // *models.TaskSession, mirroring scanTaskSessionRow's JSON-unmarshal and // int-to-bool/nullable-string conventions but for the narrower RETURNING // column set (id, agent_profile_id, agent_profile_snapshot, is_passthrough, -// name, review_status, metadata, task_environment_id, state, updated_at). -// taskID backfills TaskID, which RETURNING cannot supply since it's a query -// parameter, not a returned column. +// name, review_status, metadata, task_environment_id, state, updated_at, +// is_primary). taskID backfills TaskID, which RETURNING cannot supply since +// it's a query parameter, not a returned column. func scanCancelledTaskSessionRow(rows *sql.Rows, taskID string) (*models.TaskSession, error) { session := &models.TaskSession{TaskID: taskID} var state string var metadataJSON string var agentProfileSnapshotJSON string var isPassthrough int + var isPrimary int var reviewStatus sql.NullString var agentProfileID sql.NullString var name sql.NullString @@ -2321,12 +2413,14 @@ func scanCancelledTaskSessionRow(rows *sql.Rows, taskID string) (*models.TaskSes if err := rows.Scan( &session.ID, &agentProfileID, &agentProfileSnapshotJSON, &isPassthrough, &name, &reviewStatus, &metadataJSON, &session.TaskEnvironmentID, &state, &session.UpdatedAt, + &isPrimary, ); err != nil { return nil, err } session.State = models.TaskSessionState(state) session.IsPassthrough = isPassthrough == 1 + session.IsPrimary = isPrimary == 1 if reviewStatus.Valid { session.ReviewStatus = models.ReviewStatus(reviewStatus.String) } diff --git a/apps/backend/internal/task/service/archived_session_reconciliation.go b/apps/backend/internal/task/service/archived_session_reconciliation.go index 5c6784112a..cf1a708441 100644 --- a/apps/backend/internal/task/service/archived_session_reconciliation.go +++ b/apps/backend/internal/task/service/archived_session_reconciliation.go @@ -4,6 +4,7 @@ import ( "context" "time" + "github.com/kandev/kandev/internal/common/constants" "github.com/kandev/kandev/internal/task/archivecascade" "github.com/kandev/kandev/internal/task/models" "go.uber.org/zap" @@ -19,29 +20,28 @@ import ( // sessions whose backing execution is gone (see runActiveSessionSweep in // active_session_stall.go). // -// finalizeCancelledSessions (see service_tasks.go) already bounds its -// session-cancellation retry to a handful of fixed attempts inside -// ArchiveTask's own request: enough to ride out a brief SQLite writer-lock -// blip, but not unbounded. If SQLite's single writer stays occupied for -// longer than every attempt combined, that bounded retry gets exhausted -// after archived_at has already committed, and nothing else in the request -// path ever retries the transition. The archived task is then left with -// sessions stuck in an active DB state (CREATED/STARTING/RUNNING/ +// Pass 1 — archived tasks. finalizeCancelledSessions (see service_tasks.go) +// already bounds its session-cancellation retry to a handful of fixed attempts +// inside ArchiveTask's own request: enough to ride out a brief SQLite +// writer-lock blip, but not unbounded. If SQLite's single writer stays +// occupied for longer than every attempt combined, that bounded retry gets +// exhausted after archived_at has already committed, and nothing else in the +// request path ever retries the transition. The archived task is then left +// with sessions stuck in an active DB state (CREATED/STARTING/RUNNING/ // WAITING_FOR_INPUT) forever, and event-driven clients that key their // "is running" indicators off session.state_changed never learn the // sessions actually stopped. // -// The archived pass is the eventual-consistency backstop for that residual -// gap, following the exact periodic-sweep shape StartAutoArchiveLoop already -// uses: on a fixed interval, list every archived task that still has an -// active session, and re-invoke finalizeCancelledSessions for each. Calling -// finalizeCancelledSessions again is always safe — its underlying -// CancelActiveTaskSessionsByTaskID UPDATE only matches sessions still in an -// active state, so a task that was already fully reconciled (by ArchiveTask -// itself or by an earlier sweep pass) simply yields an empty cancelledSessions -// slice and is a no-op. As long as the process keeps running, a later pass -// retries once the SQLite contention that exhausted the in-line retry has -// cleared — unlike the fixed 3-attempt budget, this sweep never gives up. +// Pass 2 — orphaned sessions of unarchived tasks. A backend restart while an +// ACP prompt turn is open kills the actor that would have transitioned the +// session (the turn-complete path dies with the process), leaving +// task_sessions stuck in STARTING/RUNNING forever on tasks that were never +// archived. Every tick, stale STARTING/RUNNING sessions with no live +// in-memory execution backing them are terminalized. +// +// Both passes follow the exact periodic-sweep shape StartAutoArchiveLoop +// already uses and re-invoke the same finalize transitions; as long as the +// process keeps running, a later pass retries what an earlier pass missed. func (s *Service) StartSessionReconciliationLoop(ctx context.Context) { ticker := time.NewTicker(1 * time.Minute) go func() { @@ -52,6 +52,7 @@ func (s *Service) StartSessionReconciliationLoop(ctx context.Context) { return case now := <-ticker.C: s.runArchivedSessionReconciliation(ctx) + s.runOrphanedSessionReconciliation(ctx) s.runActiveSessionSweep(ctx, now) } } @@ -59,6 +60,14 @@ func (s *Service) StartSessionReconciliationLoop(ctx context.Context) { s.logger.Info("session reconciliation loop started (every 1 minute)") } +// StartArchivedSessionReconciliationLoop keeps the pre-stall-sweep entry +// point available for integrations that started the archived pass directly. +// New startup wiring should use StartSessionReconciliationLoop so all passes +// run from one ticker. +func (s *Service) StartArchivedSessionReconciliationLoop(ctx context.Context) { + s.StartSessionReconciliationLoop(ctx) +} + func (s *Service) runArchivedSessionReconciliation(ctx context.Context) { deadline := archivecascade.ArchiveDeadline(ctx) reconcileCtx, cancel := archivecascade.ContinuationContextUntil(ctx, deadline) @@ -89,3 +98,129 @@ func (s *Service) runArchivedSessionReconciliation(ctx context.Context) { s.finalizeCancelledSessions(reconcileCtx, taskID, activeSessions, deadline, models.SessionArchiveCancelReason) } } + +// orphanedSessionGraceAllowance is the slice added to the launch budget when +// deriving the orphan sweep's staleness cutoff, mirroring the session-ceiling +// reservation expiry: preparation is bounded by constants.AgentLaunchTimeout +// (read, not copied, so an operator-raised preparation budget is honored), +// and the allowance covers the start deadline inside the launch goroutine. +const orphanedSessionGraceAllowance = 5 * time.Minute + +// orphanedSessionGracePeriod is how stale a STARTING/RUNNING session's last +// write must be before the sweep may consider it unbacked. A session in those +// states whose row is fresher than one full launch budget can still belong to +// an in-flight launch whose execution has not reached the in-memory store +// yet, so the sweep leaves it alone. +func orphanedSessionGracePeriod() time.Duration { + return constants.AgentLaunchTimeout + orphanedSessionGraceAllowance +} + +// orphanedSessionRepository is the narrow capability the orphan sweep needs +// off its sessions repository. Kept as an optional interface — type-asserted +// off s.sessions like taskWorkspaceMetadataCASSetter — rather than widening +// repository.SessionRepository, whose many test doubles would each have to +// grow methods this sweep never exercises through them. The concrete sqlite +// repository is the only production implementer. +type orphanedSessionRepository interface { + ListStaleRunningSessionsOnUnarchivedTasks(ctx context.Context, staleBefore time.Time) ([]*models.TaskSession, error) + CancelRunningTaskSessionByID(ctx context.Context, sessionID, reason string, staleBefore time.Time) (*models.TaskSession, error) +} + +// runOrphanedSessionReconciliation is pass 2 of the reconciliation loop: it +// terminalizes stale STARTING/RUNNING sessions of unarchived tasks that no +// live in-memory execution backs. Without it, a backend restart mid-turn +// leaves those sessions RUNNING forever (#3711): the actor that would have +// transitioned them died with the process, and no other path ever retries. +// Sessions backed by a live execution — including survivors re-tracked by +// startup recovery — are never touched, and the per-session cancel keeps +// healthy sibling sessions of the same task (e.g. WAITING_FOR_INPUT) intact. +// A session whose cancellation write fails is simply retried on the next +// tick; the sweep never gives up for as long as the process runs. +func (s *Service) runOrphanedSessionReconciliation(ctx context.Context) { + if s.executionLivenessChecker == nil { + // Absence-from-store is the sweep's only dead signal; without the + // checker it can never prove a session unbacked, so the pass is inert. + return + } + repo, ok := s.sessions.(orphanedSessionRepository) + if !ok { + return + } + staleBefore := time.Now().UTC().Add(-orphanedSessionGracePeriod()) + candidates, err := repo.ListStaleRunningSessionsOnUnarchivedTasks(ctx, staleBefore) + if err != nil { + s.logger.Error("orphaned-session reconciliation: failed to list candidates", zap.Error(err)) + return + } + if len(candidates) == 0 { + return + } + s.reconcileOrphanedSessions(ctx, candidates, staleBefore) +} + +func (s *Service) reconcileOrphanedSessions(ctx context.Context, candidates []*models.TaskSession, staleBefore time.Time) { + deadline := archivecascade.ArchiveDeadline(ctx) + reconcileCtx, cancel := archivecascade.ContinuationContextUntil(ctx, deadline) + defer cancel() + s.reconcileOrphanedSessionsUntil(reconcileCtx, candidates, staleBefore, deadline) +} + +// reconcileOrphanedSessionsUntil applies one orphan sweep within the supplied +// admission deadline. A cancellation write may finish after that deadline +// because the repository owns a detached write context; its post-commit +// effects therefore receive a fresh bounded context instead of the expired +// sweep context. +func (s *Service) reconcileOrphanedSessionsUntil( + ctx context.Context, + candidates []*models.TaskSession, + staleBefore time.Time, + deadline time.Time, +) { + repo, ok := s.sessions.(orphanedSessionRepository) + if !ok { + return + } + for _, session := range candidates { + if session == nil || session.ID == "" { + continue + } + if !time.Now().Before(deadline) { + s.logger.Debug("orphaned-session reconciliation: admission deadline reached", + zap.Time("deadline", deadline)) + return + } + // Re-check liveness at the moment of the write, not just at the + // candidate read: a launch that raced the grace window since the read + // must not be reaped. The staleBefore predicate inside the cancel + // UPDATE is the second half of that guard — it re-asserts the + // staleness cutoff so a row refreshed by an in-flight launch between + // this check and the write no longer matches. + if s.executionLivenessChecker.HasLiveExecution(session.ID) { + continue + } + cancelled, err := repo.CancelRunningTaskSessionByID(ctx, session.ID, models.SessionOrphanedCancelReason, staleBefore) + if err != nil { + s.logger.Warn("orphaned-session reconciliation: failed to cancel session", + zap.String("session_id", session.ID), + zap.Error(err)) + continue + } + if cancelled == nil { + // Raced to another state between the candidate read and the + // write; its new owner is responsible for it now. + continue + } + s.logger.Info("orphaned-session reconciliation: terminalized unbacked session", + zap.String("task_id", session.TaskID), + zap.String("session_id", session.ID), + zap.String("previous_state", string(session.State))) + completionDeadline := time.Now().Add(taskPublicationTimeout) + completionCtx, cancelCompletion := context.WithDeadline( + context.WithoutCancel(ctx), completionDeadline, + ) + s.runCancelledSessionEffects(completionCtx, session.TaskID, + []*models.TaskSession{session}, []*models.TaskSession{cancelled}, + completionDeadline, models.SessionOrphanedCancelReason) + cancelCompletion() + } +} diff --git a/apps/backend/internal/task/service/orphaned_session_reconciliation_test.go b/apps/backend/internal/task/service/orphaned_session_reconciliation_test.go new file mode 100644 index 0000000000..81064d8e0f --- /dev/null +++ b/apps/backend/internal/task/service/orphaned_session_reconciliation_test.go @@ -0,0 +1,399 @@ +package service + +import ( + "context" + "sync" + "testing" + "time" + + sqliterepo "github.com/kandev/kandev/internal/task/repository/sqlite" + + "github.com/kandev/kandev/internal/events" + "github.com/kandev/kandev/internal/task/models" + "github.com/kandev/kandev/internal/task/repository" +) + +// fakeExecutionLiveness is the test double for the orphan sweep's +// execution-liveness seam: only session IDs in live map as backed by a live +// in-memory execution. +type fakeExecutionLiveness struct { + live map[string]bool +} + +func (f *fakeExecutionLiveness) HasLiveExecution(sessionID string) bool { + return f.live[sessionID] +} + +// seedOrphanSweepFixtures creates the one workspace and workflow every +// orphan-sweep test task hangs off. +func seedOrphanSweepFixtures(t *testing.T, repo *sqliterepo.Repository) { + t.Helper() + ctx := context.Background() + if err := repo.CreateWorkspace(ctx, &models.Workspace{ID: "ws-orphan", Name: "Workspace"}); err != nil { + t.Fatalf("CreateWorkspace: %v", err) + } + if err := repo.CreateWorkflow(ctx, &models.Workflow{ID: "wf-orphan", WorkspaceID: "ws-orphan", Name: "Workflow"}); err != nil { + t.Fatalf("CreateWorkflow: %v", err) + } +} + +func seedOrphanSweepTask(t *testing.T, repo *sqliterepo.Repository, taskID string, archived bool) { + t.Helper() + ctx := context.Background() + if err := repo.CreateTask(ctx, &models.Task{ + ID: taskID, WorkspaceID: "ws-orphan", WorkflowID: "wf-orphan", WorkflowStepID: "step-1", + Title: "Test " + taskID, Priority: "medium", + }); err != nil { + t.Fatalf("CreateTask(%s): %v", taskID, err) + } + if archived { + if _, err := repo.DB().ExecContext(ctx, + `UPDATE tasks SET archived_at = ? WHERE id = ?`, time.Now().UTC(), taskID); err != nil { + t.Fatalf("archive %s via SQL: %v", taskID, err) + } + } +} + +func TestService_OrphanedSessionReconciliationTerminalizesUnbackedSessions(t *testing.T) { + svc, eventBus, repo := createTestService(t) + ctx := context.Background() + seedOrphanSweepFixtures(t, repo) + + stale := time.Now().UTC().Add(-30 * time.Minute) + fresh := time.Now().UTC().Add(-1 * time.Minute) + + seedOrphanSweepTask(t, repo, "task-orphaned", false) + seedOrphanSweepTask(t, repo, "task-live-exec", false) + seedOrphanSweepTask(t, repo, "task-inflight-launch", false) + seedOrphanSweepTask(t, repo, "task-primary", false) + seedOrphanSweepTask(t, repo, "task-archived", true) + + seedSession := func(id, taskID string, state models.TaskSessionState, updatedAt time.Time) { + t.Helper() + if err := repo.CreateTaskSession(ctx, &models.TaskSession{ + ID: id, TaskID: taskID, State: state, + AgentProfileID: "agent-1", IsPrimary: true, UpdatedAt: updatedAt, StartedAt: updatedAt, + }); err != nil { + t.Fatalf("CreateTaskSession(%s): %v", id, err) + } + } + // The orphan this sweep exists for: unarchived task, stale RUNNING row, + // nothing in the execution store backs it (backend restart mid-turn). + seedSession("session-orphaned-running", "task-orphaned", models.TaskSessionStateRunning, stale) + // Same orphan class in STARTING: the launch died with the process. + seedSession("session-orphaned-starting", "task-orphaned", models.TaskSessionStateStarting, stale) + // Healthy sibling of the orphaned task: waiting for user input, must + // survive the sweep even though the task lost its RUNNING session. + seedSession("session-healthy-sibling", "task-orphaned", models.TaskSessionStateWaitingForInput, stale) + // Stale RUNNING row that a live execution still backs (e.g. re-tracked by + // startup recovery): must never be terminalized. + seedSession("session-live-execution", "task-live-exec", models.TaskSessionStateRunning, stale) + // Fresh RUNNING row: an in-flight launch may not have reached the + // in-memory store yet, so the grace window must protect it. + seedSession("session-inflight-launch", "task-inflight-launch", models.TaskSessionStateRunning, fresh) + // Archived task with a stale RUNNING session: the archived pass owns it, + // the orphan pass must not touch it. + seedSession("session-archived-task", "task-archived", models.TaskSessionStateRunning, stale) + // Primary session on the orphaned task: the CANCELLED event must carry + // is_primary=true so the status-summary projection keeps the durable + // primary assignment. + seedSession("session-primary", "task-primary", models.TaskSessionStateRunning, stale) + + svc.SetExecutionLivenessChecker(&fakeExecutionLiveness{ + live: map[string]bool{"session-live-execution": true}, + }) + svc.runOrphanedSessionReconciliation(ctx) + + assertState := func(id string, want models.TaskSessionState) { + t.Helper() + session, err := repo.GetTaskSession(ctx, id) + if err != nil { + t.Fatalf("GetTaskSession(%s): %v", id, err) + } + if session.State != want { + t.Errorf("session %s state = %q, want %q", id, session.State, want) + } + } + assertState("session-orphaned-running", models.TaskSessionStateCancelled) + assertState("session-orphaned-starting", models.TaskSessionStateCancelled) + assertState("session-healthy-sibling", models.TaskSessionStateWaitingForInput) + assertState("session-live-execution", models.TaskSessionStateRunning) + assertState("session-inflight-launch", models.TaskSessionStateRunning) + assertState("session-archived-task", models.TaskSessionStateRunning) + assertState("session-primary", models.TaskSessionStateCancelled) + + orphaned, err := repo.GetTaskSession(ctx, "session-orphaned-running") + if err != nil { + t.Fatalf("GetTaskSession(orphaned): %v", err) + } + if orphaned.ErrorMessage != models.SessionOrphanedCancelReason { + t.Errorf("orphaned session error_message = %q, want %q", + orphaned.ErrorMessage, models.SessionOrphanedCancelReason) + } + + // The event must carry the durable is_primary flag: the primary session's + // cancellation reports is_primary=true rather than the zero value. + if data := sessionCancelledEvent(eventBus, "session-primary"); data != nil { + if isPrimary, ok := data["is_primary"].(bool); !ok || !isPrimary { + t.Errorf("primary session cancellation event is_primary = %v, want true", data["is_primary"]) + } + } else { + t.Error("expected a cancellation event for session-primary") + } + + for _, id := range []string{"session-orphaned-running", "session-orphaned-starting", "session-primary"} { + if !sessionCancelledEventPublished(eventBus, id) { + t.Errorf("expected a session.state_changed event for %s from the orphan sweep, got none", id) + } + } + for _, id := range []string{"session-healthy-sibling", "session-live-execution", "session-inflight-launch", "session-archived-task"} { + if sessionCancelledEventPublished(eventBus, id) { + t.Errorf("unexpected session.state_changed event for %s", id) + } + } +} + +func sessionCancelledEventPublished(eventBus *MockEventBus, sessionID string) bool { + for _, evt := range eventBus.GetPublishedEvents() { + if evt.Type != events.TaskSessionStateChanged { + continue + } + data, ok := evt.Data.(map[string]interface{}) + if ok && data["session_id"] == sessionID && data["new_state"] == string(models.TaskSessionStateCancelled) { + return true + } + } + return false +} + +// sessionCancelledEvent returns the last session.state_changed payload for +// sessionID, or nil when none published. The orphan sweep's event must carry +// the durable is_primary flag — the status-summary projector demotes the +// session when a cancellation event reports is_primary: false. +func sessionCancelledEvent(eventBus *MockEventBus, sessionID string) map[string]interface{} { + var found map[string]interface{} + for _, evt := range eventBus.GetPublishedEvents() { + if evt.Type != events.TaskSessionStateChanged { + continue + } + data, ok := evt.Data.(map[string]interface{}) + if ok && data["session_id"] == sessionID && data["new_state"] == string(models.TaskSessionStateCancelled) { + found = data + } + } + return found +} + +// TestService_OrphanedSessionReconciliationRequiresLivenessChecker pins the +// fail-safe: with no execution-liveness seam wired, the sweep must be inert — +// absence-from-store is its only dead signal, and a nil checker can never +// prove a session unbacked. +func TestService_OrphanedSessionReconciliationRequiresLivenessChecker(t *testing.T) { + svc, _, repo := createTestService(t) + ctx := context.Background() + seedOrphanSweepFixtures(t, repo) + seedOrphanSweepTask(t, repo, "task-orphaned", false) + if err := repo.CreateTaskSession(ctx, &models.TaskSession{ + ID: "session-orphaned", TaskID: "task-orphaned", State: models.TaskSessionStateRunning, + AgentProfileID: "agent-1", IsPrimary: true, + UpdatedAt: time.Now().UTC().Add(-30 * time.Minute), StartedAt: time.Now().UTC().Add(-30 * time.Minute), + }); err != nil { + t.Fatalf("CreateTaskSession: %v", err) + } + + // No SetExecutionLivenessChecker call. + svc.runOrphanedSessionReconciliation(ctx) + + session, err := repo.GetTaskSession(ctx, "session-orphaned") + if err != nil { + t.Fatalf("GetTaskSession: %v", err) + } + if session.State != models.TaskSessionStateRunning { + t.Fatalf("session state = %q, want RUNNING (sweep must stay inert without the liveness seam)", session.State) + } +} + +func TestService_OrphanedSessionReconciliationStopsAtAdmissionDeadline(t *testing.T) { + svc, eventBus, repo := createTestService(t) + setupCtx := context.Background() + seedOrphanSweepFixtures(t, repo) + seedOrphanSweepTask(t, repo, "task-expired-pass", false) + stale := time.Now().UTC().Add(-30 * time.Minute) + if err := repo.CreateTaskSession(setupCtx, &models.TaskSession{ + ID: "session-expired-pass", TaskID: "task-expired-pass", State: models.TaskSessionStateRunning, + AgentProfileID: "agent-1", IsPrimary: true, UpdatedAt: stale, StartedAt: stale, + }); err != nil { + t.Fatalf("CreateTaskSession: %v", err) + } + candidate, err := repo.GetTaskSession(setupCtx, "session-expired-pass") + if err != nil { + t.Fatalf("GetTaskSession: %v", err) + } + svc.SetExecutionLivenessChecker(&fakeExecutionLiveness{}) + + expiredCtx, cancel := context.WithDeadline(context.Background(), time.Now().Add(-time.Second)) + defer cancel() + svc.reconcileOrphanedSessions(expiredCtx, []*models.TaskSession{candidate}, stale) + + session, err := repo.GetTaskSession(setupCtx, "session-expired-pass") + if err != nil { + t.Fatalf("GetTaskSession after expired pass: %v", err) + } + if session.State != models.TaskSessionStateRunning { + t.Fatalf("session state = %q, want RUNNING after an expired pass", session.State) + } + if sessionCancelledEventPublished(eventBus, "session-expired-pass") { + t.Fatal("expired pass must not publish a cancellation event") + } +} + +type delayedOrphanSessionRepository struct { + *sqliterepo.Repository + entered chan struct{} + release chan struct{} + once sync.Once +} + +func (r *delayedOrphanSessionRepository) CancelRunningTaskSessionByID( + ctx context.Context, + sessionID, reason string, + staleBefore time.Time, +) (*models.TaskSession, error) { + r.once.Do(func() { close(r.entered) }) + <-r.release + return r.Repository.CancelRunningTaskSessionByID(ctx, sessionID, reason, staleBefore) +} + +func TestService_OrphanedSessionReconciliationUsesFreshEffectsContextAfterDeadline(t *testing.T) { + delayed := &delayedOrphanSessionRepository{ + entered: make(chan struct{}), + release: make(chan struct{}), + } + svc, eventBus, repo := createTestServiceWithSessionsRepo(t, func(repo *sqliterepo.Repository) repository.SessionRepository { + delayed.Repository = repo + return delayed + }) + setupCtx := context.Background() + seedOrphanSweepFixtures(t, repo) + seedOrphanSweepTask(t, repo, "task-late-effects", false) + stale := time.Now().UTC().Add(-30 * time.Minute) + if err := repo.CreateTaskSession(setupCtx, &models.TaskSession{ + ID: "session-late-effects", TaskID: "task-late-effects", State: models.TaskSessionStateRunning, + AgentProfileID: "agent-1", IsPrimary: true, UpdatedAt: stale, StartedAt: stale, + }); err != nil { + t.Fatalf("CreateTaskSession: %v", err) + } + candidate, err := repo.GetTaskSession(setupCtx, "session-late-effects") + if err != nil { + t.Fatalf("GetTaskSession: %v", err) + } + clarifications := &recordingTaskClarificationCanceller{} + svc.SetClarificationCanceller(clarifications) + svc.SetExecutionLivenessChecker(&fakeExecutionLiveness{}) + + passCtx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + done := make(chan struct{}) + go func() { + svc.reconcileOrphanedSessions(passCtx, []*models.TaskSession{candidate}, stale.Add(time.Second)) + close(done) + }() + select { + case <-delayed.entered: + case <-time.After(time.Second): + t.Fatal("orphan cancellation did not start") + } + select { + case <-passCtx.Done(): + case <-time.After(time.Second): + t.Fatal("reconciliation pass did not reach its deadline") + } + close(delayed.release) + select { + case <-done: + case <-time.After(time.Second): + t.Fatal("reconciliation did not finish after the delayed write was released") + } + + if len(clarifications.contextErrs) != 1 || clarifications.contextErrs[0] != nil { + t.Fatalf("clarification cleanup context errors = %v, want one nil error after a late commit", clarifications.contextErrs) + } + if !sessionCancelledEventPublished(eventBus, "session-late-effects") { + t.Fatal("expected cancellation event after the delayed write committed") + } +} + +// TestService_OrphanedSessionReconciliationSparesRowRefreshedSinceCandidateRead +// pins the read-then-write guard: a launch CAS-writes its session to STARTING +// (bumping updated_at) before it registers an execution in the in-memory +// store, so the row can be refreshed between the sweep's candidate read and +// its cancellation write. The cancel's staleBefore predicate must fail to +// match that refreshed row instead of cancelling a launch in progress. +func TestService_OrphanedSessionReconciliationSparesRowRefreshedSinceCandidateRead(t *testing.T) { + svc, eventBus, repo := createTestService(t) + ctx := context.Background() + seedOrphanSweepFixtures(t, repo) + seedOrphanSweepTask(t, repo, "task-relaunch", false) + if err := repo.CreateTaskSession(ctx, &models.TaskSession{ + ID: "session-relaunch", TaskID: "task-relaunch", State: models.TaskSessionStateStarting, + AgentProfileID: "agent-1", IsPrimary: true, + UpdatedAt: time.Now().UTC().Add(-30 * time.Minute), StartedAt: time.Now().UTC().Add(-30 * time.Minute), + }); err != nil { + t.Fatalf("CreateTaskSession: %v", err) + } + + // Simulate the race: the candidate read sees the stale row, then an + // in-flight launch refreshes it before the sweep's write lands. + intercept := &livenessInterceptChecker{delegate: &fakeExecutionLiveness{}} + svc.SetExecutionLivenessChecker(intercept) + intercept.onCheck = func(sessionID string) { + if sessionID != "session-relaunch" || intercept.refreshed { + return + } + intercept.refreshed = true + now := time.Now().UTC() + if _, err := repo.DB().ExecContext(ctx, + `UPDATE task_sessions SET state = ?, updated_at = ? WHERE id = ?`, + string(models.TaskSessionStateStarting), now, sessionID); err != nil { + t.Errorf("refresh session during liveness check: %v", err) + } + } + + svc.runOrphanedSessionReconciliation(ctx) + + session, err := repo.GetTaskSession(ctx, "session-relaunch") + if err != nil { + t.Fatalf("GetTaskSession: %v", err) + } + if session.State != models.TaskSessionStateStarting { + t.Fatalf("session state = %q, want STARTING (a row refreshed since the candidate read must not be reaped)", session.State) + } + if sessionCancelledEventPublished(eventBus, "session-relaunch") { + t.Error("unexpected session.state_changed event for the refreshed session") + } +} + +// livenessInterceptChecker delegates to a fake liveness checker and lets a +// test mutate the DB between the sweep's liveness check and its write, +// reproducing the in-flight-launch race window. +type livenessInterceptChecker struct { + delegate *fakeExecutionLiveness + refreshed bool + onCheck func(sessionID string) +} + +func (l *livenessInterceptChecker) HasLiveExecution(sessionID string) bool { + if l.onCheck != nil { + l.onCheck(sessionID) + } + return l.delegate.HasLiveExecution(sessionID) +} + +// TestSessionOrphanedCancelReasonIsNotArchiveReason pins the reason taxonomy: +// resume paths branch on IsArchiveCancelReason, so the orphan reason must +// stay distinct from the archive reasons. +func TestSessionOrphanedCancelReasonIsNotArchiveReason(t *testing.T) { + if models.IsArchiveCancelReason(models.SessionOrphanedCancelReason) { + t.Fatal("SessionOrphanedCancelReason must not be an archive cancel reason") + } +} diff --git a/apps/backend/internal/task/service/service.go b/apps/backend/internal/task/service/service.go index 6a2f7e351d..c857344299 100644 --- a/apps/backend/internal/task/service/service.go +++ b/apps/backend/internal/task/service/service.go @@ -179,6 +179,16 @@ type TaskRowLivenessProber interface { RowLiveness(row *models.ExecutorRunning) models.ProcessLiveness } +// TaskExecutionLivenessChecker reports whether a task session still has a +// live agent execution backing it in the agent runtime's in-memory store. +// Implementers must distinguish agent-owned executions from workspace-only +// infrastructure and must answer from the store only — never lazily +// (re)create an execution the way the GetOrEnsureExecution recovery +// chokepoint does. +type TaskExecutionLivenessChecker interface { + HasLiveExecution(sessionID string) bool +} + // TaskResourceCleanupActivityGate serializes durable cleanup with install-wide maintenance. type TaskResourceCleanupActivityGate interface { AcquireTaskResourceCleanup(context.Context) (TaskResourceCleanupActivityLease, error) @@ -481,6 +491,7 @@ type Service struct { parkedProjectionCanceller ParkedProjectionCanceller sessionCeilingReleaser SessionCeilingReleaser rowLivenessProber TaskRowLivenessProber + executionLivenessChecker TaskExecutionLivenessChecker contextWindowResetter func(context.Context, string) error cleanupActivity TaskResourceCleanupActivityGate branchMaterializer BranchMaterializer @@ -882,6 +893,15 @@ func (s *Service) SetRowLivenessProber(prober TaskRowLivenessProber) { s.rowLivenessProber = prober } +// SetExecutionLivenessChecker wires the in-memory execution-store lookup +// (satisfied by the lifecycle adapter) used by the orphan-session +// reconciliation sweep. It is optional; when unwired the sweep is inert, +// because absent-from-store is its only dead signal and a nil checker can +// never prove a session unbacked. +func (s *Service) SetExecutionLivenessChecker(checker TaskExecutionLivenessChecker) { + s.executionLivenessChecker = checker +} + // SetContextWindowResetter wires the guarded context-window reset callback // owned by the orchestrator. It is optional for isolated task-service users; // those callers fall back to clearing the session metadata directly. diff --git a/apps/backend/internal/task/service/service_tasks.go b/apps/backend/internal/task/service/service_tasks.go index 3a00c2c572..4c805559b7 100644 --- a/apps/backend/internal/task/service/service_tasks.go +++ b/apps/backend/internal/task/service/service_tasks.go @@ -2855,7 +2855,7 @@ func (s *Service) runCancelledSessionEffects( _, err := s.clarificationCanceller.ExpireSessionAndNotify(expireCtx, session.ID) cancelExpire() if err != nil { - s.logger.Error("failed to expire clarification after archive cancellation; response claims remain quarantined", + s.logger.Error("failed to expire clarification after session cancellation; response claims remain quarantined", zap.String("task_id", taskID), zap.String("session_id", session.ID), zap.Error(err)) @@ -2872,7 +2872,7 @@ func (s *Service) runCancelledSessionEffects( cancelParked() } } - // CancelActiveTaskSessionsByTaskID is a bulk writer: RETURNING reports every + // The cancellation writers are bulk writers: RETURNING reports every // row's post-update CANCELLED state, not which were in an AC-1 state before // the update, so every returned id is released unconditionally rather than // branched on state (AC-51e). Releasing an id that held no reservation is a diff --git a/docs/plans/restart-orphaned-session-terminalization/plan.md b/docs/plans/restart-orphaned-session-terminalization/plan.md new file mode 100644 index 0000000000..e87c07e521 --- /dev/null +++ b/docs/plans/restart-orphaned-session-terminalization/plan.md @@ -0,0 +1,106 @@ +--- +created: 2026-09-20 +status: implemented +requirements: + - REQ-TASKS-RESTART-ORPHAN-SESSIONS-001 +system_design: + - ../../specs/tasks/system-design/restart-orphaned-session-terminalization.md +legacy_specs: [] +--- + +# Implementation Plan: Restart-Orphaned Session Terminalization + +## Overview + +A backend restart while an ACP prompt turn was open left `task_sessions` +stuck in `STARTING`/`RUNNING` forever on unarchived tasks: the actor that +would have transitioned them died with the process, and the existing 1-minute +reconciliation sweep only covered archived tasks. Extend that same sweep with +a second pass that terminalizes stale `STARTING`/`RUNNING` sessions of +unarchived tasks when no live in-memory execution backs them, after a grace +window of one launch budget so an in-flight launch is never reaped. + +## Scope + +### In scope + +- A second pass in the existing reconciliation loop covering unarchived + tasks' stale `STARTING`/`RUNNING` sessions. +- Candidate cutoff derived from `constants.AgentLaunchTimeout` plus a 5-minute + allowance. +- An execution-liveness seam (`TaskExecutionLivenessChecker` / + `HasLiveExecution`) answered from the in-memory store only. +- Per-session cancellation (`CancelRunningTaskSessionByID`) with a dedicated + `"orphaned by backend restart"` reason, distinct from archive reasons. +- Shared post-cancellation effects extracted from the archive path + (`notifyCancelledSessions`). +- Service-level regression tests: unbacked sessions terminalized, live-backed + and fresh rows untouched, healthy siblings survive, inert without the + liveness seam. + +### Out of scope + +- Turn resume or transcript replay; drain-on-shutdown; agent-side heartbeats; + manual SQL recovery; archived tasks (existing pass owns them). + +## Technical approach + +Reuse the archived pass's shape verbatim: same ticker, same continuation +context, same retry-forever semantics. The candidate query filters in SQL so +fresh rows are never loaded; the liveness re-check happens at write time so a +launch racing the grace window is never reaped. Cancellation is session-scoped +(`UPDATE ... RETURNING` on rows still in `STARTING`/`RUNNING`) so healthy +sibling sessions survive. The `orphanedSessionRepository` capability keeps the +two new repository methods off the `SessionRepository` interface and its test +doubles. + +## Tests + +- `TestService_OrphanedSessionReconciliationTerminalizesUnbackedSessions` + (`apps/backend/internal/task/service/orphaned_session_reconciliation_test.go`): + covers AC-TASKS-RESTART-ORPHAN-SESSIONS-001.1, .2, .3, .4, .5, .8, and .10 — + stale unbacked `RUNNING`/`STARTING` sessions reach `CANCELLED` with the + orphan reason and a `session.state_changed` publish whose `is_primary` + matches the durable flag; live-backed, fresh, archived-task, and + `WAITING_FOR_INPUT` sibling rows stay untouched. +- `TestService_OrphanedSessionReconciliationRequiresLivenessChecker`: covers + AC-TASKS-RESTART-ORPHAN-SESSIONS-001.6 — without the liveness seam the sweep + is inert. +- `TestService_OrphanedSessionReconciliationSparesRowRefreshedSinceCandidateRead`: + covers AC-TASKS-RESTART-ORPHAN-SESSIONS-001.9 — a row refreshed between the + candidate read and the cancel write (the in-flight-launch race) is spared. +- `TestSessionOrphanedCancelReasonIsNotArchiveReason`: covers + AC-TASKS-RESTART-ORPHAN-SESSIONS-001.4's distinctness invariant. + +## E2E tests + +None: the sweep is a periodic backend behavior with a one-minute tick and a +launch-budget grace window; the service-level test pins the full transition. +A repro (start session, kill backend mid-turn, restart) is recorded in +issue #3711's verification plan. + +## Work orders + +- [x] [Task 01: Terminalize restart-orphaned sessions via the reconciliation sweep](task-01-terminalize-restart-orphaned-sessions.md) + +## Verification results + +- `(cd apps/backend && go test ./internal/task/service -run 'TestService_OrphanedSessionReconciliation|TestService_ArchivedSessionReconciliation' -count=1)` — pass. +- `(cd apps/backend && go test ./internal/task/repository/... -count=1)` — pass. +- `(cd apps/backend && go run ./cmd/sqlguard ./internal)` — clean after rebinding the candidate query's placeholder. +- `(cd apps/backend && gofmt -l internal/task internal/backendapp)` — no output; `go vet` clean. +- `python3 scripts/lint-spec-files.py --all` and `python3 scripts/list-docs.py validate` — pass. +- Full `go test ./internal/task/...`: three pre-existing failures + (`TestDesktopDiscoveryRootPersists…`, `TestArchiveTaskCleanup…`, + `TestTaskLifecycleCleanup_MissingWorktree`) reproduce identically on the + base commit in a scratch worktree, so none is attributable to this change. + +## Risks + +- Reaping a live session whose execution legitimately left the store would + lose work; the liveness re-check at write time and the fresh-row grace + window bound that risk to zero in the steady state. +- The sweep cannot distinguish "execution gone because restart" from + "execution gone because of a runtime bookkeeping bug"; both are terminal + sessions by the same argument — nothing will transition them — so + terminalizing is correct in either case. diff --git a/docs/plans/restart-orphaned-session-terminalization/task-01-terminalize-restart-orphaned-sessions.md b/docs/plans/restart-orphaned-session-terminalization/task-01-terminalize-restart-orphaned-sessions.md new file mode 100644 index 0000000000..412082eecd --- /dev/null +++ b/docs/plans/restart-orphaned-session-terminalization/task-01-terminalize-restart-orphaned-sessions.md @@ -0,0 +1,117 @@ +--- +id: "01-terminalize-restart-orphaned-sessions" +title: "Terminalize restart-orphaned sessions via the reconciliation sweep" +status: done +wave: 1 +depends_on: [] +plan: "plan.md" +requirements: + - REQ-TASKS-RESTART-ORPHAN-SESSIONS-001 +acceptance_criteria: + - AC-TASKS-RESTART-ORPHAN-SESSIONS-001.1 + - AC-TASKS-RESTART-ORPHAN-SESSIONS-001.2 + - AC-TASKS-RESTART-ORPHAN-SESSIONS-001.3 + - AC-TASKS-RESTART-ORPHAN-SESSIONS-001.4 + - AC-TASKS-RESTART-ORPHAN-SESSIONS-001.5 + - AC-TASKS-RESTART-ORPHAN-SESSIONS-001.6 + - AC-TASKS-RESTART-ORPHAN-SESSIONS-001.7 + - AC-TASKS-RESTART-ORPHAN-SESSIONS-001.8 + - AC-TASKS-RESTART-ORPHAN-SESSIONS-001.9 + - AC-TASKS-RESTART-ORPHAN-SESSIONS-001.10 +system_design: + - ../../specs/tasks/system-design/restart-orphaned-session-terminalization.md +--- + +# Task 01: Terminalize Restart-Orphaned Sessions via the Reconciliation Sweep + +## Summary + +Extend the existing 1-minute task-session reconciliation sweep with a second +pass: terminalize stale `STARTING`/`RUNNING` sessions of unarchived tasks that +no live in-memory execution backs, so a backend restart mid-turn no longer +leaves sessions stuck RUNNING forever (#3711). + +## In scope + +- `runOrphanedSessionReconciliation` / `reconcileOrphanedSessions` in the + reconciliation loop file, gated on a wired `TaskExecutionLivenessChecker`. +- Repository methods `ListStaleRunningSessionsOnUnarchivedTasks` and + `CancelRunningTaskSessionByID` behind the `orphanedSessionRepository` + capability. +- `SessionOrphanedCancelReason` distinct from archive cancel reasons. +- Extract `notifyCancelledSessions` from `finalizeCancelledSessions` so the + orphan pass reuses the archive path's post-cancellation effects. +- Wire `HasLiveExecution` in the lifecycle adapter and the orchestrator. + +## Out of scope + +- Archived tasks' sessions (the archived pass owns them). +- Turn resume, drain-on-shutdown, heartbeats, adapter changes, schema + changes. + +## Acceptance + +- After a backend restart mid-turn, an unbacked stale `STARTING`/`RUNNING` + session reaches `CANCELLED` on the first sweep after the grace window, with + the orphan reason and a `session.state_changed` publish, while live-backed, + fresh, and archived-task rows stay untouched and healthy siblings survive. +- Without the liveness seam the sweep does nothing. + +## Verification + +```bash +(cd apps/backend && go test ./internal/task/service -run 'TestService_OrphanedSessionReconciliation|TestService_ArchivedSessionReconciliation' -count=1 -v) +(cd apps/backend && go test ./internal/task/repository/... -count=1) +(cd apps/backend && go run ./cmd/sqlguard ./internal) +(cd apps/backend && gofmt -l internal/task internal/backendapp) +(cd apps/backend && go vet ./internal/task/... ./internal/backendapp) +python3 scripts/lint-spec-files.py --all +python3 scripts/list-docs.py validate +``` + +## Files likely touched + +- `apps/backend/internal/task/service/archived_session_reconciliation.go` +- `apps/backend/internal/task/service/orphaned_session_reconciliation_test.go` +- `apps/backend/internal/task/service/service.go` +- `apps/backend/internal/task/service/service_tasks.go` +- `apps/backend/internal/task/models/resume_safety.go` +- `apps/backend/internal/task/repository/sqlite/session.go` +- `apps/backend/internal/backendapp/adapters.go` +- `apps/backend/internal/backendapp/orchestrator.go` + +## Dependencies + +None. + +## Risks + +- The liveness checker must answer from the in-memory store only — a lazy + creator would mask the sweep's dead signal. + +## Parallelism + +`sequential` + +## Inputs + +- `docs/specs/tasks/requirements/restart-orphaned-session-terminalization.md` +- `docs/specs/tasks/system-design/restart-orphaned-session-terminalization.md` +- `apps/backend/internal/task/service/archived_session_reconciliation.go` + +## Results + +Implemented in commit `aa7dc293b` (8 files, +458/−32): the second sweep pass, +the two repository methods behind the `orphanedSessionRepository` capability, +the orphan reason constant, the shared `notifyCancelledSessions` extraction, +and the lifecycle-adapter `HasLiveExecution` wiring. Follow-up commits added +the `Rebind` the SQL-portability gate requires, this specification package, +and two review-fix hardenings: the cancel statement re-asserts the staleness +cutoff (`updated_at < staleBefore`) so an in-flight launch refreshing its row +between the sweep's liveness check and the write is never reaped +(AC-.9, `TestService_OrphanedSessionReconciliationSparesRowRefreshedSinceCandidateRead`), +and both cancellation RETURNING clauses now select `is_primary` so the +published event carries the durable primary flag (AC-.10; this also repairs +the pre-existing archive path, which had the same gap). Verification commands +in the plan all pass; the failing packages in the full suite reproduce +identically on the base commit and are environmental. diff --git a/docs/specs/tasks/README.md b/docs/specs/tasks/README.md index 82bc6a2170..96460010a1 100644 --- a/docs/specs/tasks/README.md +++ b/docs/specs/tasks/README.md @@ -93,6 +93,7 @@ signals, and task-scoped scheduling contracts. - [Task Terminal Persistence](requirements/task-terminal-persistence.md) - [Runtime Task-State Publication Order](requirements/runtime-state-publication-order.md) - [Session Delete Preserves Task Workspaces](requirements/session-delete-resource-cleanup.md) +- [Restart-Orphaned Session Terminalization](requirements/restart-orphaned-session-terminalization.md) - [Sidebar Task Editing](requirements/sidebar-task-edit.md) - [Task Actions Menu on Preview and Detail Surfaces](requirements/task-actions-menu.md) - [Task menu grouping](requirements/task-menu-grouping.md) diff --git a/docs/specs/tasks/requirements/restart-orphaned-session-terminalization.md b/docs/specs/tasks/requirements/restart-orphaned-session-terminalization.md new file mode 100644 index 0000000000..fd23e45a1a --- /dev/null +++ b/docs/specs/tasks/requirements/restart-orphaned-session-terminalization.md @@ -0,0 +1,105 @@ +--- +status: draft +system: tasks +created: 2026-09-20 +owners: + - kandev +--- + +# Restart-Orphaned Session Terminalization Requirements + +## Overview + +A task session records the lifecycle of one agent conversation turn. When the +backend process dies while a session's turn is open — a graceful restart, a +crash, or a kill — the component that would have transitioned the session to a +terminal state dies with it. No other actor retries that transition, so the +session row stays in an active state forever and the task appears to wait for +an agent that already finished. + +The tasks system owns the task-session lifecycle, so it owns this +reconciliation: the system itself must detect and terminalize sessions that +lost their backing actor, without depending on the dead process, the agent, or +a human SQL edit. + +## Terminology + +- **Active session state:** A `task_sessions.state` value that implies work is + in flight: `CREATED`, `STARTING`, `RUNNING`, or `WAITING_FOR_INPUT`. +- **Terminal session state:** A state a session never leaves: `COMPLETED`, + `FAILED`, or `CANCELLED`. +- **Orphaned session:** An active-state session whose transition actor is gone: + the backend that would complete or fail its turn terminated, and no live + agent execution backs the session anymore. +- **Live execution:** An agent execution the current backend process tracks in + its in-memory execution store. A session backed by a live execution is not + orphaned even when its row is stale. +- **Launch grace window:** The interval between a session row entering + `STARTING` and its execution necessarily being visible in the in-memory + store. A session inside this window is not eligible for orphan detection. +- **Reconciliation sweep:** The periodic pass that lists candidate sessions + and terminalizes the orphaned ones. + +## Requirements + +### REQ-TASKS-RESTART-ORPHAN-SESSIONS-001: Sessions orphaned by a backend restart reach a terminal state + +**Intent:** A backend restart must not leave task sessions stuck in an active +state forever. The system must self-heal: detect sessions whose backing actor +died and terminalize them within a bounded time. + +**User story:** As a Kandev user, I want a session that lost its agent process +to a backend restart to show as stopped rather than stay "running" forever, so +that the task board stays truthful and the task can move on. + +#### Acceptance criteria + +- **AC-TASKS-RESTART-ORPHAN-SESSIONS-001.1:** When a backend process + terminates while a task session is in `STARTING` or `RUNNING`, and no live + execution backs that session, the system shall transition that session to + `CANCELLED` within one reconciliation sweep after the launch grace window + elapses, without any user action. +- **AC-TASKS-RESTART-ORPHAN-SESSIONS-001.2:** When a session in `STARTING` or + `RUNNING` is backed by a live execution in the in-memory store, the system + shall leave its state untouched, regardless of how stale its row is. +- **AC-TASKS-RESTART-ORPHAN-SESSIONS-001.3:** When a session's last write is + more recent than the launch grace window, the system shall not consider it + orphaned, so an in-flight launch whose execution has not yet reached the + in-memory store is never reaped. +- **AC-TASKS-RESTART-ORPHAN-SESSIONS-001.4:** When the sweep terminalizes a + session, the system shall record a cancellation reason that names the + backend restart as the source, distinct from archive cancellations and user + stops, so session history keeps the source visible. +- **AC-TASKS-RESTART-ORPHAN-SESSIONS-001.5:** When the sweep terminalizes a + session, the system shall run the same post-cancellation effects as an + archive cancellation: expire terminal clarifications, clear parked + projections, release session-ceiling reservations, and publish a + `session.state_changed` event for each session actually cancelled. +- **AC-TASKS-RESTART-ORPHAN-SESSIONS-001.6:** When the sweep cannot consult + the in-memory execution store, it shall remain inert rather than + terminalize sessions it cannot prove unbacked. +- **AC-TASKS-RESTART-ORPHAN-SESSIONS-001.7:** When a cancellation write fails, + the system shall retry it on the next sweep tick, and shall never give up on + a session for as long as the process runs. +- **AC-TASKS-RESTART-ORPHAN-SESSIONS-001.8:** When the sweep terminalizes one + session of a task, the system shall leave the task's other sessions + untouched, so healthy siblings (for example `WAITING_FOR_INPUT`) survive. +- **AC-TASKS-RESTART-ORPHAN-SESSIONS-001.9:** When a session row is refreshed + between the sweep's candidate read and its cancellation write (an in-flight + launch CAS-writing `STARTING` before it registers an execution), the system + shall not terminalize that session; the staleness cutoff is re-asserted at + write time inside the same statement as the transition. +- **AC-TASKS-RESTART-ORPHAN-SESSIONS-001.10:** When the sweep publishes a + session cancellation event, the event shall carry the session's durable + `is_primary` flag, so the status-summary projection keeps the durable + primary-session assignment. + +## Out of scope + +- Resuming or replaying a turn that was interrupted mid-flight; the orphaned + session is terminalized, not resumed. +- Detecting orphans of already-archived tasks; the archived-task + reconciliation pass owns those. +- Agent-side heartbeats or adapter-side turn-state persistence. +- Stopping a live agent process; the sweep only terminalizes DB rows of + sessions with no live execution. diff --git a/docs/specs/tasks/system-design/restart-orphaned-session-terminalization.md b/docs/specs/tasks/system-design/restart-orphaned-session-terminalization.md new file mode 100644 index 0000000000..5a21b7e14f --- /dev/null +++ b/docs/specs/tasks/system-design/restart-orphaned-session-terminalization.md @@ -0,0 +1,130 @@ +--- +status: current +system: tasks +requirements: + - REQ-TASKS-RESTART-ORPHAN-SESSIONS-001 +--- + +# Restart-Orphaned Session Terminalization System Design + +## Purpose and boundaries + +The tasks system owns the task-session lifecycle, including the transition of +a session into a terminal state. This design covers the periodic sweep that +terminalizes `STARTING`/`RUNNING` sessions of unarchived tasks that no live +in-memory agent execution backs — the residue of a backend process that died +mid-turn. + +It extends the existing reconciliation loop +(`internal/task/service/archived_session_reconciliation.go`) with a second +pass; it does not introduce a new loop, new persistence, or adapter changes. +Adjacent contracts this design uses but does not own: the agent runtime's +in-memory execution store (executors system), session-ceiling reservations +(tasks system, reservation contract), and the archived-task reconciliation +pass that owns archived tasks' stranded sessions. + +## Requirement mapping + +| Requirement | Design section | +| --- | --- | +| `REQ-TASKS-RESTART-ORPHAN-SESSIONS-001` | [Control flow](#control-flow), [Failure and recovery](#failure-and-recovery) | + +## Components and responsibilities + +- **Reconciliation loop** (`Service.StartArchivedSessionReconciliationLoop`): + ticks every minute and runs both passes — the archived pass first, then the + orphan pass. +- **Orphan pass** (`Service.runOrphanedSessionReconciliation`): lists stale + candidates, filters by live-execution absence, cancels each, runs + post-cancellation effects. +- **Session repository** (`sqlite.Repository`): candidate query + `ListStaleRunningSessionsOnUnarchivedTasks` and per-session cancel + `CancelRunningTaskSessionByID`. Not part of the `SessionRepository` + interface; reached through the narrow `orphanedSessionRepository` + capability so test doubles stay unchanged. +- **Execution-liveness seam** (`TaskExecutionLivenessChecker`): reports + whether a session has a live in-memory execution. Satisfied by the + lifecycle adapter (`HasLiveExecution`), which answers from the in-memory + store only and never lazily creates an execution. +- **Shared post-cancellation effects** (`Service.notifyCancelledSessions`): + extracted from the archive cancellation path and reused verbatim: + clarification expiry, parked-projection clearing, ceiling release, and the + `session.state_changed` publish. + +## Data and contracts + +- Candidate query joins `task_sessions` to `tasks` and selects rows where + `ts.state IN ('STARTING','RUNNING')`, `ts.updated_at < staleBefore`, and + `t.archived_at IS NULL`, ordered by `updated_at` ascending. +- `staleBefore` is `now - (constants.AgentLaunchTimeout + 5m grace + allowance)`: one full launch budget plus the start-deadline allowance, read + (not copied) from `constants.AgentLaunchTimeout` so an operator-raised + preparation budget is honored. The same cutoff is passed into the cancel + statement and re-asserted as `updated_at < staleBefore` there, so a row + refreshed between the candidate read and the write no longer matches. +- `CancelRunningTaskSessionByID(sessionID, reason, staleBefore)` is an + `UPDATE ... RETURNING` over rows still in `STARTING`/`RUNNING` whose + `updated_at` still predates the cutoff; it returns `nil` when the session + raced to another state or was refreshed, and the write detaches from the + caller context with a 10-second timeout. The RETURNING clause selects + `is_primary` alongside the other event-payload fields, so the published + cancellation event carries the durable primary flag. +- Cancellation reason constant: `models.SessionOrphanedCancelReason` + (`"orphaned by backend restart"`), distinct from the archive reasons; + `IsArchiveCancelReason` returns false for it. + +## Control flow + +1. Every tick, pass 2 runs after the archived pass. +2. If no execution-liveness checker is wired, the pass is inert: absence + from the store is the only dead signal, and a nil checker can never prove + a session unbacked. +3. List candidates with the stale-before cutoff computed from the launch + budget. +4. For each candidate, re-check `HasLiveExecution(session.ID)` at the moment + of the write — a launch that raced the grace window since the read must + not be reaped. +5. Cancel via `CancelRunningTaskSessionByID`; the statement re-asserts both + the state set (`STARTING`/`RUNNING`) and the staleness cutoff + (`updated_at < staleBefore`), so a row refreshed by an in-flight launch + between the liveness check and this write matches nothing; on `nil` + (raced or refreshed) skip; on error warn and let the next tick retry. +6. Run the shared post-cancellation effects with the orphan reason. + +## Failure and recovery + +- A failed candidate read logs an error and returns; the next tick retries. +- A failed cancel logs a warn and continues to the next candidate; the next + tick retries that session. The sweep never gives up for as long as the + process runs. +- The cancel write is context-detached (`context.WithoutCancel` + 10s bound), + so the terminal transition survives a client disconnect and cannot block a + sweep pass indefinitely on a locked writer. +- Pass 2 is a no-op on clean trees: fresh rows are filtered by the grace + window, live-backed rows by the liveness check, archived tasks' rows by the + archived-pass filter, and terminal rows by the candidate query. + +## Persistence + +No schema change. The sweep writes only `task_sessions.state`, +`error_message`, `completed_at`, and `updated_at`. Survivors re-tracked by +startup recovery are in the in-memory store and stay untouched, so the sweep +composes with agent-survival re-tracking rather than racing it. + +## Security + +No new trust boundary. The sweep runs in-process with the task service's own +credentials and touches only `task_sessions` rows of unarchived tasks. + +## Observability + +- `task-session reconciliation loop started (every 1 minute)` at loop start. +- `orphaned-session reconciliation: found candidates` / `terminalized + unbacked session` (Info, with task and session IDs, previous state). +- `failed to list candidates` / `failed to cancel session` (Error/Warn with + the underlying error). + +## Related decisions + +- Issue #3711 and its approved fix plan: extend the existing sweep rather + than drain-on-shutdown, turn-state persistence, heartbeats, or manual SQL.