fix(task): terminalize sessions orphaned by backend restart via reconciliation sweep (#3833)

Co-authored-by: Carlos Florencio <carlosmflorencio@gmail.com>
This commit is contained in:
Benjamin
2026-09-21 16:39:42 +08:00
committed by GitHub
parent 00c7469d81
commit 1614720ceb
15 changed files with 1215 additions and 33 deletions
@@ -21,6 +21,7 @@ import (
"github.com/kandev/kandev/internal/secrets"
"github.com/kandev/kandev/internal/task/models"
"github.com/kandev/kandev/internal/worktree"
v1 "github.com/kandev/kandev/pkg/api/v1"
)
// ErrSessionWorkspaceNotReady indicates the task session exists but does not yet
@@ -496,6 +497,45 @@ func (m *Manager) IsAgentCommandConfigured(executionID string) bool {
return configured
}
// HasLiveAgentExecution reports whether a session owns an agent execution that
// can still protect its durable lifecycle row from orphan reconciliation.
// Workspace-only executions have no agent command and do not count. Terminal
// executions do not count even when they remain in the store briefly. A live
// passthrough process protects its session through the interactive runner.
func (m *Manager) HasLiveAgentExecution(sessionID string) bool {
execution, exists := m.executionStore.GetBySessionID(sessionID)
if !exists || execution == nil {
return false
}
var agentCommand, passthroughProcessID string
var status v1.AgentStatus
if err := m.executionStore.WithRLock(execution.ID, func(current *AgentExecution) {
agentCommand = current.AgentCommand
passthroughProcessID = current.PassthroughProcessID
status = current.Status
}); err != nil {
return false
}
if isTerminalStatus(status) {
return false
}
if agentCommand != "" {
return true
}
if passthroughProcessID == "" {
return false
}
runner := m.GetInteractiveRunner()
if runner == nil {
// The process handle is evidence of agent ownership, but an unavailable
// runner cannot prove that the process is gone. Keep the session safe
// from destructive orphan cleanup until the runtime can answer.
return true
}
return runner.IsProcessReadyOrPending(passthroughProcessID)
}
// EnsurePassthroughExecution ensures an execution exists for a passthrough session
// and starts the passthrough process if needed. This is called when the terminal
// handler receives a connection for a session that might need recovery after backend restart.
@@ -179,6 +179,29 @@ func TestIsAgentCommandConfigured(t *testing.T) {
require.False(t, mgr.IsAgentCommandConfigured("exec-absent"))
}
func TestHasLiveAgentExecutionIgnoresWorkspaceInfrastructureAndTerminalAgents(t *testing.T) {
mgr := newTestManager(t)
require.NoError(t, mgr.executionStore.Add(&AgentExecution{
ID: "exec-workspace", SessionID: "session-workspace", Status: v1.AgentStatusRunning,
}))
require.NoError(t, mgr.executionStore.Add(&AgentExecution{
ID: "exec-agent", SessionID: "session-agent", AgentCommand: "claude",
Status: v1.AgentStatusStarting,
}))
require.NoError(t, mgr.executionStore.Add(&AgentExecution{
ID: "exec-terminal", SessionID: "session-terminal", AgentCommand: "claude",
Status: v1.AgentStatusFailed,
}))
require.False(t, mgr.HasLiveAgentExecution("session-workspace"),
"workspace-only execution must not shield an orphaned session")
require.True(t, mgr.HasLiveAgentExecution("session-agent"),
"a configured non-terminal agent execution must shield its session")
require.False(t, mgr.HasLiveAgentExecution("session-terminal"),
"terminal executions must not shield a session")
require.False(t, mgr.HasLiveAgentExecution("session-absent"))
}
func TestResolveTaskEnvironmentIDPrefersInMemoryExecution(t *testing.T) {
mgr := newTestManager(t)
mgr.workspaceInfoProvider = &mockWorkspaceInfoProvider{infos: map[string]*WorkspaceInfo{
+12 -1
View File
@@ -971,7 +971,10 @@ func (a *lifecycleAdapter) LiveSessionIDsForTask(taskID string) []string {
references := a.mgr.ListExecutionsForTask(taskID)
sessionIDs := make([]string, 0, len(references))
for _, reference := range references {
if reference.SessionID != "" {
// ListExecutionsForTask includes workspace-only infrastructure created
// by file/shell access. Only an execution with an agent command (or a
// live passthrough process) owns the agent session lifecycle.
if reference.SessionID != "" && a.mgr.HasLiveAgentExecution(reference.SessionID) {
sessionIDs = append(sessionIDs, reference.SessionID)
}
}
@@ -1018,6 +1021,14 @@ func (a *lifecycleAdapter) ResolveAgentProfile(ctx context.Context, profileID st
}, nil
}
// HasLiveExecution reports whether the session still has an agent execution
// owned by the runtime. It backs the task service's orphan-session
// reconciliation sweep; workspace-only infrastructure is not a live agent,
// and this lookup must never lazily create an execution.
func (a *lifecycleAdapter) HasLiveExecution(sessionID string) bool {
return a.mgr.HasLiveAgentExecution(sessionID)
}
// GetGitLog retrieves the git log for a session from baseCommit to HEAD.
// If targetBranch is provided, uses dynamic merge-base calculation for accurate filtering.
func (a *lifecycleAdapter) GetGitLog(ctx context.Context, sessionID, baseCommit string, limit int, targetBranch string) (*client.GitLogResult, error) {
@@ -170,6 +170,9 @@ func provideOrchestrator(
// Runtime-aware liveness lets durable cleanup treat a not-found stop for a
// confirmed-dead local runtime as already stopped instead of retrying forever.
taskSvc.SetRowLivenessProber(agentManagerClient)
// The orphan-session sweep terminalizes stale STARTING/RUNNING sessions no
// live in-memory execution backs (backend-restart residue, #3711).
taskSvc.SetExecutionLivenessChecker(agentManagerClient)
// The session reconciliation sweep's active-task pass (stall detection and
// orphaned-session healing) verifies "no live execution" against the agent
// runtime's in-memory execution store through this registry.
@@ -72,9 +72,7 @@ func RowMustBePreserved(running *ExecutorRunning, sessionState TaskSessionState)
// SessionArchiveCancelReason and SessionArchiveTreeCancelReason are the
// TaskSession.ErrorMessage values written when a session is auto-cancelled
// by archiving — Service.ArchiveTask's CancelActiveTaskSessionsByTaskID call
// (single-task archive) and HandoffService.cancelActiveRuns (cascade archive)
// respectively. Distinct reasons keep the cancellation source visible in
// by archiving. Distinct reasons keep the cancellation source visible in
// session history and diagnostics.
const (
SessionArchiveCancelReason = "task archived"
@@ -2112,7 +2112,7 @@ func completedAtForTaskSessionState(status models.TaskSessionState, now time.Tim
// timed out. Returned sessions therefore carry only the fields the
// RETURNING clause selects (ID, TaskID, AgentProfileID,
// AgentProfileSnapshot, IsPassthrough, Name, ReviewStatus, Metadata,
// TaskEnvironmentID, State, UpdatedAt) — every other models.TaskSession
// TaskEnvironmentID, State, UpdatedAt, IsPrimary) — every other models.TaskSession
// field is left at its zero value, and callers must not rely on fields
// outside this list being populated.
func (r *Repository) CancelActiveTaskSessionsByTaskID(ctx context.Context, taskID, reason string) ([]*models.TaskSession, error) {
@@ -2131,7 +2131,7 @@ func (r *Repository) CancelActiveTaskSessionsByTaskID(ctx context.Context, taskI
WHERE task_id = ?
AND state IN ('CREATED', 'STARTING', 'RUNNING', 'WAITING_FOR_INPUT')
RETURNING id, agent_profile_id, agent_profile_snapshot, is_passthrough, name,
review_status, metadata, task_environment_id, state, updated_at
review_status, metadata, task_environment_id, state, updated_at, is_primary
`), string(models.TaskSessionStateCancelled), reason, now, now, taskID)
if err != nil {
return nil, err
@@ -2149,6 +2149,97 @@ func (r *Repository) CancelActiveTaskSessionsByTaskID(ctx context.Context, taskI
return sessions, rows.Err()
}
// ListStaleRunningSessionsOnUnarchivedTasks returns every STARTING/RUNNING
// session of an unarchived task whose updated_at is older than staleBefore.
// It is the candidate list for the orphan-session reconciliation sweep (see
// service.runOrphanedSessionReconciliation): a session still holding one of
// those states past the launch grace window, with no live in-memory execution
// backing it, can never reach a terminal state on its own — its actor died
// with the process. The staleBefore cutoff is applied in SQL so the sweep
// never even loads fresh rows that may belong to an in-flight launch.
//
// Not part of the SessionRepository interface: it exists for that one sweep,
// which reaches it through the narrow orphanedSessionRepository capability —
// widening the interface would force every test double in the tree to grow
// methods this sweep never exercises through them.
func (r *Repository) ListStaleRunningSessionsOnUnarchivedTasks(ctx context.Context, staleBefore time.Time) ([]*models.TaskSession, error) {
rows, err := r.ro.QueryContext(ctx, r.ro.Rebind(`
SELECT `+taskSessionSelectCols+` `+taskSessionFromClause+`
JOIN tasks t ON t.id = ts.task_id
WHERE ts.state IN ('STARTING', 'RUNNING')
AND ts.updated_at < ?
AND t.archived_at IS NULL
ORDER BY ts.updated_at ASC
`), staleBefore)
if err != nil {
return nil, err
}
defer func() { _ = rows.Close() }()
sessions, err := r.scanTaskSessions(ctx, rows)
if err != nil {
return nil, err
}
return r.loadWorktreesBatch(ctx, sessions)
}
// CancelRunningTaskSessionByID transitions a single stale STARTING/RUNNING
// session to CANCELLED, returning the transitioned row, or nil when the
// session no longer matches (already terminal, raced to another state, or
// refreshed since the sweep read it). Like CancelActiveTaskSessionsByTaskID
// it is a pure DB state change that requires no live agent execution, and it
// is session-scoped so the orphan sweep can terminalize one session without
// cancelling healthy sibling sessions of the same task.
//
// staleBefore re-asserts the sweep's staleness cutoff at write time, closing
// the read-then-write race against an in-flight launch: a launch CAS-writes
// its session to STARTING (bumping updated_at) before it registers an
// execution in the in-memory store, so between the sweep's liveness check and
// this UPDATE the row can pass from "no live execution" to "launch in
// progress". A row refreshed since the candidate read no longer satisfies
// updated_at < staleBefore, the UPDATE matches nothing, and the next tick
// re-evaluates a fresh row that the grace window protects until its
// execution registers.
//
// The UPDATE and the row selection happen in one atomic RETURNING statement;
// the returned row carries only the fields that clause selects (same set as
// CancelActiveTaskSessionsByTaskID, minus TaskID — callers pass the task ID
// they already know). The write detaches from ctx like
// CancelActiveTaskSessionsByTaskID: once the sweep has decided this session
// is orphaned, the terminal transition must not be lost to a caller-context
// cancellation, and the 10s bound keeps a locked SQLite writer from stalling
// the sweep pass. A failed write simply retries on the next sweep tick.
func (r *Repository) CancelRunningTaskSessionByID(ctx context.Context, sessionID, reason string, staleBefore time.Time) (*models.TaskSession, error) {
now := time.Now().UTC()
writeCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
defer cancel()
rows, err := r.db.QueryContext(writeCtx, r.db.Rebind(`
UPDATE task_sessions
SET state = ?, error_message = ?, completed_at = ?, updated_at = ?
WHERE id = ?
AND state IN ('STARTING', 'RUNNING')
AND updated_at < ?
RETURNING id, agent_profile_id, agent_profile_snapshot, is_passthrough, name,
review_status, metadata, task_environment_id, state, updated_at, is_primary
`), string(models.TaskSessionStateCancelled), reason, now, now, sessionID, staleBefore)
if err != nil {
return nil, err
}
defer func() { _ = rows.Close() }()
if !rows.Next() {
return nil, rows.Err()
}
// The scanner backfills TaskID from the parameter because RETURNING
// does not carry it; sessionID is not the task ID, so set it from the
// candidate row the sweep already read instead.
session, err := scanCancelledTaskSessionRow(rows, "")
if err != nil {
return nil, err
}
return session, rows.Err()
}
// CancelActiveTaskSessionsByIDs is documented on the SessionRepository
// interface. It shares CancelActiveTaskSessionsByTaskID's atomic
// UPDATE ... RETURNING shape and detached-but-bounded write context, so the
@@ -2175,7 +2266,7 @@ func (r *Repository) CancelActiveTaskSessionsByIDs(ctx context.Context, taskID s
AND id IN (`+placeholders+`)
AND state IN ('CREATED', 'STARTING', 'RUNNING', 'WAITING_FOR_INPUT')
RETURNING id, agent_profile_id, agent_profile_snapshot, is_passthrough, name,
review_status, metadata, task_environment_id, state, updated_at
review_status, metadata, task_environment_id, state, updated_at, is_primary
`), args...)
if err != nil {
return nil, err
@@ -2242,9 +2333,9 @@ func (r *Repository) CancelActiveTaskSessionsByCandidates(
SET state = ?, error_message = ?, completed_at = ?, updated_at = ?
WHERE task_id = ?
AND state IN ('CREATED', 'STARTING', 'RUNNING', 'WAITING_FOR_INPUT')
AND (`+strings.Join(predicates, " OR ")+`)
AND (`+strings.Join(predicates, " OR ")+`)
RETURNING id, agent_profile_id, agent_profile_snapshot, is_passthrough, name,
review_status, metadata, task_environment_id, state, updated_at
review_status, metadata, task_environment_id, state, updated_at, is_primary
`), args...)
if err != nil {
return nil, err
@@ -2305,15 +2396,16 @@ func activeSessionCancellationCandidatePredicate(
// *models.TaskSession, mirroring scanTaskSessionRow's JSON-unmarshal and
// int-to-bool/nullable-string conventions but for the narrower RETURNING
// column set (id, agent_profile_id, agent_profile_snapshot, is_passthrough,
// name, review_status, metadata, task_environment_id, state, updated_at).
// taskID backfills TaskID, which RETURNING cannot supply since it's a query
// parameter, not a returned column.
// name, review_status, metadata, task_environment_id, state, updated_at,
// is_primary). taskID backfills TaskID, which RETURNING cannot supply since
// it's a query parameter, not a returned column.
func scanCancelledTaskSessionRow(rows *sql.Rows, taskID string) (*models.TaskSession, error) {
session := &models.TaskSession{TaskID: taskID}
var state string
var metadataJSON string
var agentProfileSnapshotJSON string
var isPassthrough int
var isPrimary int
var reviewStatus sql.NullString
var agentProfileID sql.NullString
var name sql.NullString
@@ -2321,12 +2413,14 @@ func scanCancelledTaskSessionRow(rows *sql.Rows, taskID string) (*models.TaskSes
if err := rows.Scan(
&session.ID, &agentProfileID, &agentProfileSnapshotJSON, &isPassthrough, &name,
&reviewStatus, &metadataJSON, &session.TaskEnvironmentID, &state, &session.UpdatedAt,
&isPrimary,
); err != nil {
return nil, err
}
session.State = models.TaskSessionState(state)
session.IsPassthrough = isPassthrough == 1
session.IsPrimary = isPrimary == 1
if reviewStatus.Valid {
session.ReviewStatus = models.ReviewStatus(reviewStatus.String)
}
@@ -4,6 +4,7 @@ import (
"context"
"time"
"github.com/kandev/kandev/internal/common/constants"
"github.com/kandev/kandev/internal/task/archivecascade"
"github.com/kandev/kandev/internal/task/models"
"go.uber.org/zap"
@@ -19,29 +20,28 @@ import (
// sessions whose backing execution is gone (see runActiveSessionSweep in
// active_session_stall.go).
//
// finalizeCancelledSessions (see service_tasks.go) already bounds its
// session-cancellation retry to a handful of fixed attempts inside
// ArchiveTask's own request: enough to ride out a brief SQLite writer-lock
// blip, but not unbounded. If SQLite's single writer stays occupied for
// longer than every attempt combined, that bounded retry gets exhausted
// after archived_at has already committed, and nothing else in the request
// path ever retries the transition. The archived task is then left with
// sessions stuck in an active DB state (CREATED/STARTING/RUNNING/
// Pass 1 — archived tasks. finalizeCancelledSessions (see service_tasks.go)
// already bounds its session-cancellation retry to a handful of fixed attempts
// inside ArchiveTask's own request: enough to ride out a brief SQLite
// writer-lock blip, but not unbounded. If SQLite's single writer stays
// occupied for longer than every attempt combined, that bounded retry gets
// exhausted after archived_at has already committed, and nothing else in the
// request path ever retries the transition. The archived task is then left
// with sessions stuck in an active DB state (CREATED/STARTING/RUNNING/
// WAITING_FOR_INPUT) forever, and event-driven clients that key their
// "is running" indicators off session.state_changed never learn the
// sessions actually stopped.
//
// The archived pass is the eventual-consistency backstop for that residual
// gap, following the exact periodic-sweep shape StartAutoArchiveLoop already
// uses: on a fixed interval, list every archived task that still has an
// active session, and re-invoke finalizeCancelledSessions for each. Calling
// finalizeCancelledSessions again is always safe — its underlying
// CancelActiveTaskSessionsByTaskID UPDATE only matches sessions still in an
// active state, so a task that was already fully reconciled (by ArchiveTask
// itself or by an earlier sweep pass) simply yields an empty cancelledSessions
// slice and is a no-op. As long as the process keeps running, a later pass
// retries once the SQLite contention that exhausted the in-line retry has
// cleared — unlike the fixed 3-attempt budget, this sweep never gives up.
// Pass 2 — orphaned sessions of unarchived tasks. A backend restart while an
// ACP prompt turn is open kills the actor that would have transitioned the
// session (the turn-complete path dies with the process), leaving
// task_sessions stuck in STARTING/RUNNING forever on tasks that were never
// archived. Every tick, stale STARTING/RUNNING sessions with no live
// in-memory execution backing them are terminalized.
//
// Both passes follow the exact periodic-sweep shape StartAutoArchiveLoop
// already uses and re-invoke the same finalize transitions; as long as the
// process keeps running, a later pass retries what an earlier pass missed.
func (s *Service) StartSessionReconciliationLoop(ctx context.Context) {
ticker := time.NewTicker(1 * time.Minute)
go func() {
@@ -52,6 +52,7 @@ func (s *Service) StartSessionReconciliationLoop(ctx context.Context) {
return
case now := <-ticker.C:
s.runArchivedSessionReconciliation(ctx)
s.runOrphanedSessionReconciliation(ctx)
s.runActiveSessionSweep(ctx, now)
}
}
@@ -59,6 +60,14 @@ func (s *Service) StartSessionReconciliationLoop(ctx context.Context) {
s.logger.Info("session reconciliation loop started (every 1 minute)")
}
// StartArchivedSessionReconciliationLoop keeps the pre-stall-sweep entry
// point available for integrations that started the archived pass directly.
// New startup wiring should use StartSessionReconciliationLoop so all passes
// run from one ticker.
func (s *Service) StartArchivedSessionReconciliationLoop(ctx context.Context) {
s.StartSessionReconciliationLoop(ctx)
}
func (s *Service) runArchivedSessionReconciliation(ctx context.Context) {
deadline := archivecascade.ArchiveDeadline(ctx)
reconcileCtx, cancel := archivecascade.ContinuationContextUntil(ctx, deadline)
@@ -89,3 +98,129 @@ func (s *Service) runArchivedSessionReconciliation(ctx context.Context) {
s.finalizeCancelledSessions(reconcileCtx, taskID, activeSessions, deadline, models.SessionArchiveCancelReason)
}
}
// orphanedSessionGraceAllowance is the slice added to the launch budget when
// deriving the orphan sweep's staleness cutoff, mirroring the session-ceiling
// reservation expiry: preparation is bounded by constants.AgentLaunchTimeout
// (read, not copied, so an operator-raised preparation budget is honored),
// and the allowance covers the start deadline inside the launch goroutine.
const orphanedSessionGraceAllowance = 5 * time.Minute
// orphanedSessionGracePeriod is how stale a STARTING/RUNNING session's last
// write must be before the sweep may consider it unbacked. A session in those
// states whose row is fresher than one full launch budget can still belong to
// an in-flight launch whose execution has not reached the in-memory store
// yet, so the sweep leaves it alone.
func orphanedSessionGracePeriod() time.Duration {
return constants.AgentLaunchTimeout + orphanedSessionGraceAllowance
}
// orphanedSessionRepository is the narrow capability the orphan sweep needs
// off its sessions repository. Kept as an optional interface — type-asserted
// off s.sessions like taskWorkspaceMetadataCASSetter — rather than widening
// repository.SessionRepository, whose many test doubles would each have to
// grow methods this sweep never exercises through them. The concrete sqlite
// repository is the only production implementer.
type orphanedSessionRepository interface {
ListStaleRunningSessionsOnUnarchivedTasks(ctx context.Context, staleBefore time.Time) ([]*models.TaskSession, error)
CancelRunningTaskSessionByID(ctx context.Context, sessionID, reason string, staleBefore time.Time) (*models.TaskSession, error)
}
// runOrphanedSessionReconciliation is pass 2 of the reconciliation loop: it
// terminalizes stale STARTING/RUNNING sessions of unarchived tasks that no
// live in-memory execution backs. Without it, a backend restart mid-turn
// leaves those sessions RUNNING forever (#3711): the actor that would have
// transitioned them died with the process, and no other path ever retries.
// Sessions backed by a live execution — including survivors re-tracked by
// startup recovery — are never touched, and the per-session cancel keeps
// healthy sibling sessions of the same task (e.g. WAITING_FOR_INPUT) intact.
// A session whose cancellation write fails is simply retried on the next
// tick; the sweep never gives up for as long as the process runs.
func (s *Service) runOrphanedSessionReconciliation(ctx context.Context) {
if s.executionLivenessChecker == nil {
// Absence-from-store is the sweep's only dead signal; without the
// checker it can never prove a session unbacked, so the pass is inert.
return
}
repo, ok := s.sessions.(orphanedSessionRepository)
if !ok {
return
}
staleBefore := time.Now().UTC().Add(-orphanedSessionGracePeriod())
candidates, err := repo.ListStaleRunningSessionsOnUnarchivedTasks(ctx, staleBefore)
if err != nil {
s.logger.Error("orphaned-session reconciliation: failed to list candidates", zap.Error(err))
return
}
if len(candidates) == 0 {
return
}
s.reconcileOrphanedSessions(ctx, candidates, staleBefore)
}
func (s *Service) reconcileOrphanedSessions(ctx context.Context, candidates []*models.TaskSession, staleBefore time.Time) {
deadline := archivecascade.ArchiveDeadline(ctx)
reconcileCtx, cancel := archivecascade.ContinuationContextUntil(ctx, deadline)
defer cancel()
s.reconcileOrphanedSessionsUntil(reconcileCtx, candidates, staleBefore, deadline)
}
// reconcileOrphanedSessionsUntil applies one orphan sweep within the supplied
// admission deadline. A cancellation write may finish after that deadline
// because the repository owns a detached write context; its post-commit
// effects therefore receive a fresh bounded context instead of the expired
// sweep context.
func (s *Service) reconcileOrphanedSessionsUntil(
ctx context.Context,
candidates []*models.TaskSession,
staleBefore time.Time,
deadline time.Time,
) {
repo, ok := s.sessions.(orphanedSessionRepository)
if !ok {
return
}
for _, session := range candidates {
if session == nil || session.ID == "" {
continue
}
if !time.Now().Before(deadline) {
s.logger.Debug("orphaned-session reconciliation: admission deadline reached",
zap.Time("deadline", deadline))
return
}
// Re-check liveness at the moment of the write, not just at the
// candidate read: a launch that raced the grace window since the read
// must not be reaped. The staleBefore predicate inside the cancel
// UPDATE is the second half of that guard — it re-asserts the
// staleness cutoff so a row refreshed by an in-flight launch between
// this check and the write no longer matches.
if s.executionLivenessChecker.HasLiveExecution(session.ID) {
continue
}
cancelled, err := repo.CancelRunningTaskSessionByID(ctx, session.ID, models.SessionOrphanedCancelReason, staleBefore)
if err != nil {
s.logger.Warn("orphaned-session reconciliation: failed to cancel session",
zap.String("session_id", session.ID),
zap.Error(err))
continue
}
if cancelled == nil {
// Raced to another state between the candidate read and the
// write; its new owner is responsible for it now.
continue
}
s.logger.Info("orphaned-session reconciliation: terminalized unbacked session",
zap.String("task_id", session.TaskID),
zap.String("session_id", session.ID),
zap.String("previous_state", string(session.State)))
completionDeadline := time.Now().Add(taskPublicationTimeout)
completionCtx, cancelCompletion := context.WithDeadline(
context.WithoutCancel(ctx), completionDeadline,
)
s.runCancelledSessionEffects(completionCtx, session.TaskID,
[]*models.TaskSession{session}, []*models.TaskSession{cancelled},
completionDeadline, models.SessionOrphanedCancelReason)
cancelCompletion()
}
}
@@ -0,0 +1,399 @@
package service
import (
"context"
"sync"
"testing"
"time"
sqliterepo "github.com/kandev/kandev/internal/task/repository/sqlite"
"github.com/kandev/kandev/internal/events"
"github.com/kandev/kandev/internal/task/models"
"github.com/kandev/kandev/internal/task/repository"
)
// fakeExecutionLiveness is the test double for the orphan sweep's
// execution-liveness seam: only session IDs in live map as backed by a live
// in-memory execution.
type fakeExecutionLiveness struct {
live map[string]bool
}
func (f *fakeExecutionLiveness) HasLiveExecution(sessionID string) bool {
return f.live[sessionID]
}
// seedOrphanSweepFixtures creates the one workspace and workflow every
// orphan-sweep test task hangs off.
func seedOrphanSweepFixtures(t *testing.T, repo *sqliterepo.Repository) {
t.Helper()
ctx := context.Background()
if err := repo.CreateWorkspace(ctx, &models.Workspace{ID: "ws-orphan", Name: "Workspace"}); err != nil {
t.Fatalf("CreateWorkspace: %v", err)
}
if err := repo.CreateWorkflow(ctx, &models.Workflow{ID: "wf-orphan", WorkspaceID: "ws-orphan", Name: "Workflow"}); err != nil {
t.Fatalf("CreateWorkflow: %v", err)
}
}
func seedOrphanSweepTask(t *testing.T, repo *sqliterepo.Repository, taskID string, archived bool) {
t.Helper()
ctx := context.Background()
if err := repo.CreateTask(ctx, &models.Task{
ID: taskID, WorkspaceID: "ws-orphan", WorkflowID: "wf-orphan", WorkflowStepID: "step-1",
Title: "Test " + taskID, Priority: "medium",
}); err != nil {
t.Fatalf("CreateTask(%s): %v", taskID, err)
}
if archived {
if _, err := repo.DB().ExecContext(ctx,
`UPDATE tasks SET archived_at = ? WHERE id = ?`, time.Now().UTC(), taskID); err != nil {
t.Fatalf("archive %s via SQL: %v", taskID, err)
}
}
}
func TestService_OrphanedSessionReconciliationTerminalizesUnbackedSessions(t *testing.T) {
svc, eventBus, repo := createTestService(t)
ctx := context.Background()
seedOrphanSweepFixtures(t, repo)
stale := time.Now().UTC().Add(-30 * time.Minute)
fresh := time.Now().UTC().Add(-1 * time.Minute)
seedOrphanSweepTask(t, repo, "task-orphaned", false)
seedOrphanSweepTask(t, repo, "task-live-exec", false)
seedOrphanSweepTask(t, repo, "task-inflight-launch", false)
seedOrphanSweepTask(t, repo, "task-primary", false)
seedOrphanSweepTask(t, repo, "task-archived", true)
seedSession := func(id, taskID string, state models.TaskSessionState, updatedAt time.Time) {
t.Helper()
if err := repo.CreateTaskSession(ctx, &models.TaskSession{
ID: id, TaskID: taskID, State: state,
AgentProfileID: "agent-1", IsPrimary: true, UpdatedAt: updatedAt, StartedAt: updatedAt,
}); err != nil {
t.Fatalf("CreateTaskSession(%s): %v", id, err)
}
}
// The orphan this sweep exists for: unarchived task, stale RUNNING row,
// nothing in the execution store backs it (backend restart mid-turn).
seedSession("session-orphaned-running", "task-orphaned", models.TaskSessionStateRunning, stale)
// Same orphan class in STARTING: the launch died with the process.
seedSession("session-orphaned-starting", "task-orphaned", models.TaskSessionStateStarting, stale)
// Healthy sibling of the orphaned task: waiting for user input, must
// survive the sweep even though the task lost its RUNNING session.
seedSession("session-healthy-sibling", "task-orphaned", models.TaskSessionStateWaitingForInput, stale)
// Stale RUNNING row that a live execution still backs (e.g. re-tracked by
// startup recovery): must never be terminalized.
seedSession("session-live-execution", "task-live-exec", models.TaskSessionStateRunning, stale)
// Fresh RUNNING row: an in-flight launch may not have reached the
// in-memory store yet, so the grace window must protect it.
seedSession("session-inflight-launch", "task-inflight-launch", models.TaskSessionStateRunning, fresh)
// Archived task with a stale RUNNING session: the archived pass owns it,
// the orphan pass must not touch it.
seedSession("session-archived-task", "task-archived", models.TaskSessionStateRunning, stale)
// Primary session on the orphaned task: the CANCELLED event must carry
// is_primary=true so the status-summary projection keeps the durable
// primary assignment.
seedSession("session-primary", "task-primary", models.TaskSessionStateRunning, stale)
svc.SetExecutionLivenessChecker(&fakeExecutionLiveness{
live: map[string]bool{"session-live-execution": true},
})
svc.runOrphanedSessionReconciliation(ctx)
assertState := func(id string, want models.TaskSessionState) {
t.Helper()
session, err := repo.GetTaskSession(ctx, id)
if err != nil {
t.Fatalf("GetTaskSession(%s): %v", id, err)
}
if session.State != want {
t.Errorf("session %s state = %q, want %q", id, session.State, want)
}
}
assertState("session-orphaned-running", models.TaskSessionStateCancelled)
assertState("session-orphaned-starting", models.TaskSessionStateCancelled)
assertState("session-healthy-sibling", models.TaskSessionStateWaitingForInput)
assertState("session-live-execution", models.TaskSessionStateRunning)
assertState("session-inflight-launch", models.TaskSessionStateRunning)
assertState("session-archived-task", models.TaskSessionStateRunning)
assertState("session-primary", models.TaskSessionStateCancelled)
orphaned, err := repo.GetTaskSession(ctx, "session-orphaned-running")
if err != nil {
t.Fatalf("GetTaskSession(orphaned): %v", err)
}
if orphaned.ErrorMessage != models.SessionOrphanedCancelReason {
t.Errorf("orphaned session error_message = %q, want %q",
orphaned.ErrorMessage, models.SessionOrphanedCancelReason)
}
// The event must carry the durable is_primary flag: the primary session's
// cancellation reports is_primary=true rather than the zero value.
if data := sessionCancelledEvent(eventBus, "session-primary"); data != nil {
if isPrimary, ok := data["is_primary"].(bool); !ok || !isPrimary {
t.Errorf("primary session cancellation event is_primary = %v, want true", data["is_primary"])
}
} else {
t.Error("expected a cancellation event for session-primary")
}
for _, id := range []string{"session-orphaned-running", "session-orphaned-starting", "session-primary"} {
if !sessionCancelledEventPublished(eventBus, id) {
t.Errorf("expected a session.state_changed event for %s from the orphan sweep, got none", id)
}
}
for _, id := range []string{"session-healthy-sibling", "session-live-execution", "session-inflight-launch", "session-archived-task"} {
if sessionCancelledEventPublished(eventBus, id) {
t.Errorf("unexpected session.state_changed event for %s", id)
}
}
}
func sessionCancelledEventPublished(eventBus *MockEventBus, sessionID string) bool {
for _, evt := range eventBus.GetPublishedEvents() {
if evt.Type != events.TaskSessionStateChanged {
continue
}
data, ok := evt.Data.(map[string]interface{})
if ok && data["session_id"] == sessionID && data["new_state"] == string(models.TaskSessionStateCancelled) {
return true
}
}
return false
}
// sessionCancelledEvent returns the last session.state_changed payload for
// sessionID, or nil when none published. The orphan sweep's event must carry
// the durable is_primary flag — the status-summary projector demotes the
// session when a cancellation event reports is_primary: false.
func sessionCancelledEvent(eventBus *MockEventBus, sessionID string) map[string]interface{} {
var found map[string]interface{}
for _, evt := range eventBus.GetPublishedEvents() {
if evt.Type != events.TaskSessionStateChanged {
continue
}
data, ok := evt.Data.(map[string]interface{})
if ok && data["session_id"] == sessionID && data["new_state"] == string(models.TaskSessionStateCancelled) {
found = data
}
}
return found
}
// TestService_OrphanedSessionReconciliationRequiresLivenessChecker pins the
// fail-safe: with no execution-liveness seam wired, the sweep must be inert —
// absence-from-store is its only dead signal, and a nil checker can never
// prove a session unbacked.
func TestService_OrphanedSessionReconciliationRequiresLivenessChecker(t *testing.T) {
svc, _, repo := createTestService(t)
ctx := context.Background()
seedOrphanSweepFixtures(t, repo)
seedOrphanSweepTask(t, repo, "task-orphaned", false)
if err := repo.CreateTaskSession(ctx, &models.TaskSession{
ID: "session-orphaned", TaskID: "task-orphaned", State: models.TaskSessionStateRunning,
AgentProfileID: "agent-1", IsPrimary: true,
UpdatedAt: time.Now().UTC().Add(-30 * time.Minute), StartedAt: time.Now().UTC().Add(-30 * time.Minute),
}); err != nil {
t.Fatalf("CreateTaskSession: %v", err)
}
// No SetExecutionLivenessChecker call.
svc.runOrphanedSessionReconciliation(ctx)
session, err := repo.GetTaskSession(ctx, "session-orphaned")
if err != nil {
t.Fatalf("GetTaskSession: %v", err)
}
if session.State != models.TaskSessionStateRunning {
t.Fatalf("session state = %q, want RUNNING (sweep must stay inert without the liveness seam)", session.State)
}
}
func TestService_OrphanedSessionReconciliationStopsAtAdmissionDeadline(t *testing.T) {
svc, eventBus, repo := createTestService(t)
setupCtx := context.Background()
seedOrphanSweepFixtures(t, repo)
seedOrphanSweepTask(t, repo, "task-expired-pass", false)
stale := time.Now().UTC().Add(-30 * time.Minute)
if err := repo.CreateTaskSession(setupCtx, &models.TaskSession{
ID: "session-expired-pass", TaskID: "task-expired-pass", State: models.TaskSessionStateRunning,
AgentProfileID: "agent-1", IsPrimary: true, UpdatedAt: stale, StartedAt: stale,
}); err != nil {
t.Fatalf("CreateTaskSession: %v", err)
}
candidate, err := repo.GetTaskSession(setupCtx, "session-expired-pass")
if err != nil {
t.Fatalf("GetTaskSession: %v", err)
}
svc.SetExecutionLivenessChecker(&fakeExecutionLiveness{})
expiredCtx, cancel := context.WithDeadline(context.Background(), time.Now().Add(-time.Second))
defer cancel()
svc.reconcileOrphanedSessions(expiredCtx, []*models.TaskSession{candidate}, stale)
session, err := repo.GetTaskSession(setupCtx, "session-expired-pass")
if err != nil {
t.Fatalf("GetTaskSession after expired pass: %v", err)
}
if session.State != models.TaskSessionStateRunning {
t.Fatalf("session state = %q, want RUNNING after an expired pass", session.State)
}
if sessionCancelledEventPublished(eventBus, "session-expired-pass") {
t.Fatal("expired pass must not publish a cancellation event")
}
}
type delayedOrphanSessionRepository struct {
*sqliterepo.Repository
entered chan struct{}
release chan struct{}
once sync.Once
}
func (r *delayedOrphanSessionRepository) CancelRunningTaskSessionByID(
ctx context.Context,
sessionID, reason string,
staleBefore time.Time,
) (*models.TaskSession, error) {
r.once.Do(func() { close(r.entered) })
<-r.release
return r.Repository.CancelRunningTaskSessionByID(ctx, sessionID, reason, staleBefore)
}
func TestService_OrphanedSessionReconciliationUsesFreshEffectsContextAfterDeadline(t *testing.T) {
delayed := &delayedOrphanSessionRepository{
entered: make(chan struct{}),
release: make(chan struct{}),
}
svc, eventBus, repo := createTestServiceWithSessionsRepo(t, func(repo *sqliterepo.Repository) repository.SessionRepository {
delayed.Repository = repo
return delayed
})
setupCtx := context.Background()
seedOrphanSweepFixtures(t, repo)
seedOrphanSweepTask(t, repo, "task-late-effects", false)
stale := time.Now().UTC().Add(-30 * time.Minute)
if err := repo.CreateTaskSession(setupCtx, &models.TaskSession{
ID: "session-late-effects", TaskID: "task-late-effects", State: models.TaskSessionStateRunning,
AgentProfileID: "agent-1", IsPrimary: true, UpdatedAt: stale, StartedAt: stale,
}); err != nil {
t.Fatalf("CreateTaskSession: %v", err)
}
candidate, err := repo.GetTaskSession(setupCtx, "session-late-effects")
if err != nil {
t.Fatalf("GetTaskSession: %v", err)
}
clarifications := &recordingTaskClarificationCanceller{}
svc.SetClarificationCanceller(clarifications)
svc.SetExecutionLivenessChecker(&fakeExecutionLiveness{})
passCtx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond)
defer cancel()
done := make(chan struct{})
go func() {
svc.reconcileOrphanedSessions(passCtx, []*models.TaskSession{candidate}, stale.Add(time.Second))
close(done)
}()
select {
case <-delayed.entered:
case <-time.After(time.Second):
t.Fatal("orphan cancellation did not start")
}
select {
case <-passCtx.Done():
case <-time.After(time.Second):
t.Fatal("reconciliation pass did not reach its deadline")
}
close(delayed.release)
select {
case <-done:
case <-time.After(time.Second):
t.Fatal("reconciliation did not finish after the delayed write was released")
}
if len(clarifications.contextErrs) != 1 || clarifications.contextErrs[0] != nil {
t.Fatalf("clarification cleanup context errors = %v, want one nil error after a late commit", clarifications.contextErrs)
}
if !sessionCancelledEventPublished(eventBus, "session-late-effects") {
t.Fatal("expected cancellation event after the delayed write committed")
}
}
// TestService_OrphanedSessionReconciliationSparesRowRefreshedSinceCandidateRead
// pins the read-then-write guard: a launch CAS-writes its session to STARTING
// (bumping updated_at) before it registers an execution in the in-memory
// store, so the row can be refreshed between the sweep's candidate read and
// its cancellation write. The cancel's staleBefore predicate must fail to
// match that refreshed row instead of cancelling a launch in progress.
func TestService_OrphanedSessionReconciliationSparesRowRefreshedSinceCandidateRead(t *testing.T) {
svc, eventBus, repo := createTestService(t)
ctx := context.Background()
seedOrphanSweepFixtures(t, repo)
seedOrphanSweepTask(t, repo, "task-relaunch", false)
if err := repo.CreateTaskSession(ctx, &models.TaskSession{
ID: "session-relaunch", TaskID: "task-relaunch", State: models.TaskSessionStateStarting,
AgentProfileID: "agent-1", IsPrimary: true,
UpdatedAt: time.Now().UTC().Add(-30 * time.Minute), StartedAt: time.Now().UTC().Add(-30 * time.Minute),
}); err != nil {
t.Fatalf("CreateTaskSession: %v", err)
}
// Simulate the race: the candidate read sees the stale row, then an
// in-flight launch refreshes it before the sweep's write lands.
intercept := &livenessInterceptChecker{delegate: &fakeExecutionLiveness{}}
svc.SetExecutionLivenessChecker(intercept)
intercept.onCheck = func(sessionID string) {
if sessionID != "session-relaunch" || intercept.refreshed {
return
}
intercept.refreshed = true
now := time.Now().UTC()
if _, err := repo.DB().ExecContext(ctx,
`UPDATE task_sessions SET state = ?, updated_at = ? WHERE id = ?`,
string(models.TaskSessionStateStarting), now, sessionID); err != nil {
t.Errorf("refresh session during liveness check: %v", err)
}
}
svc.runOrphanedSessionReconciliation(ctx)
session, err := repo.GetTaskSession(ctx, "session-relaunch")
if err != nil {
t.Fatalf("GetTaskSession: %v", err)
}
if session.State != models.TaskSessionStateStarting {
t.Fatalf("session state = %q, want STARTING (a row refreshed since the candidate read must not be reaped)", session.State)
}
if sessionCancelledEventPublished(eventBus, "session-relaunch") {
t.Error("unexpected session.state_changed event for the refreshed session")
}
}
// livenessInterceptChecker delegates to a fake liveness checker and lets a
// test mutate the DB between the sweep's liveness check and its write,
// reproducing the in-flight-launch race window.
type livenessInterceptChecker struct {
delegate *fakeExecutionLiveness
refreshed bool
onCheck func(sessionID string)
}
func (l *livenessInterceptChecker) HasLiveExecution(sessionID string) bool {
if l.onCheck != nil {
l.onCheck(sessionID)
}
return l.delegate.HasLiveExecution(sessionID)
}
// TestSessionOrphanedCancelReasonIsNotArchiveReason pins the reason taxonomy:
// resume paths branch on IsArchiveCancelReason, so the orphan reason must
// stay distinct from the archive reasons.
func TestSessionOrphanedCancelReasonIsNotArchiveReason(t *testing.T) {
if models.IsArchiveCancelReason(models.SessionOrphanedCancelReason) {
t.Fatal("SessionOrphanedCancelReason must not be an archive cancel reason")
}
}
@@ -179,6 +179,16 @@ type TaskRowLivenessProber interface {
RowLiveness(row *models.ExecutorRunning) models.ProcessLiveness
}
// TaskExecutionLivenessChecker reports whether a task session still has a
// live agent execution backing it in the agent runtime's in-memory store.
// Implementers must distinguish agent-owned executions from workspace-only
// infrastructure and must answer from the store only — never lazily
// (re)create an execution the way the GetOrEnsureExecution recovery
// chokepoint does.
type TaskExecutionLivenessChecker interface {
HasLiveExecution(sessionID string) bool
}
// TaskResourceCleanupActivityGate serializes durable cleanup with install-wide maintenance.
type TaskResourceCleanupActivityGate interface {
AcquireTaskResourceCleanup(context.Context) (TaskResourceCleanupActivityLease, error)
@@ -481,6 +491,7 @@ type Service struct {
parkedProjectionCanceller ParkedProjectionCanceller
sessionCeilingReleaser SessionCeilingReleaser
rowLivenessProber TaskRowLivenessProber
executionLivenessChecker TaskExecutionLivenessChecker
contextWindowResetter func(context.Context, string) error
cleanupActivity TaskResourceCleanupActivityGate
branchMaterializer BranchMaterializer
@@ -882,6 +893,15 @@ func (s *Service) SetRowLivenessProber(prober TaskRowLivenessProber) {
s.rowLivenessProber = prober
}
// SetExecutionLivenessChecker wires the in-memory execution-store lookup
// (satisfied by the lifecycle adapter) used by the orphan-session
// reconciliation sweep. It is optional; when unwired the sweep is inert,
// because absent-from-store is its only dead signal and a nil checker can
// never prove a session unbacked.
func (s *Service) SetExecutionLivenessChecker(checker TaskExecutionLivenessChecker) {
s.executionLivenessChecker = checker
}
// SetContextWindowResetter wires the guarded context-window reset callback
// owned by the orchestrator. It is optional for isolated task-service users;
// those callers fall back to clearing the session metadata directly.
@@ -2855,7 +2855,7 @@ func (s *Service) runCancelledSessionEffects(
_, err := s.clarificationCanceller.ExpireSessionAndNotify(expireCtx, session.ID)
cancelExpire()
if err != nil {
s.logger.Error("failed to expire clarification after archive cancellation; response claims remain quarantined",
s.logger.Error("failed to expire clarification after session cancellation; response claims remain quarantined",
zap.String("task_id", taskID),
zap.String("session_id", session.ID),
zap.Error(err))
@@ -2872,7 +2872,7 @@ func (s *Service) runCancelledSessionEffects(
cancelParked()
}
}
// CancelActiveTaskSessionsByTaskID is a bulk writer: RETURNING reports every
// The cancellation writers are bulk writers: RETURNING reports every
// row's post-update CANCELLED state, not which were in an AC-1 state before
// the update, so every returned id is released unconditionally rather than
// branched on state (AC-51e). Releasing an id that held no reservation is a
@@ -0,0 +1,106 @@
---
created: 2026-09-20
status: implemented
requirements:
- REQ-TASKS-RESTART-ORPHAN-SESSIONS-001
system_design:
- ../../specs/tasks/system-design/restart-orphaned-session-terminalization.md
legacy_specs: []
---
# Implementation Plan: Restart-Orphaned Session Terminalization
## Overview
A backend restart while an ACP prompt turn was open left `task_sessions`
stuck in `STARTING`/`RUNNING` forever on unarchived tasks: the actor that
would have transitioned them died with the process, and the existing 1-minute
reconciliation sweep only covered archived tasks. Extend that same sweep with
a second pass that terminalizes stale `STARTING`/`RUNNING` sessions of
unarchived tasks when no live in-memory execution backs them, after a grace
window of one launch budget so an in-flight launch is never reaped.
## Scope
### In scope
- A second pass in the existing reconciliation loop covering unarchived
tasks' stale `STARTING`/`RUNNING` sessions.
- Candidate cutoff derived from `constants.AgentLaunchTimeout` plus a 5-minute
allowance.
- An execution-liveness seam (`TaskExecutionLivenessChecker` /
`HasLiveExecution`) answered from the in-memory store only.
- Per-session cancellation (`CancelRunningTaskSessionByID`) with a dedicated
`"orphaned by backend restart"` reason, distinct from archive reasons.
- Shared post-cancellation effects extracted from the archive path
(`notifyCancelledSessions`).
- Service-level regression tests: unbacked sessions terminalized, live-backed
and fresh rows untouched, healthy siblings survive, inert without the
liveness seam.
### Out of scope
- Turn resume or transcript replay; drain-on-shutdown; agent-side heartbeats;
manual SQL recovery; archived tasks (existing pass owns them).
## Technical approach
Reuse the archived pass's shape verbatim: same ticker, same continuation
context, same retry-forever semantics. The candidate query filters in SQL so
fresh rows are never loaded; the liveness re-check happens at write time so a
launch racing the grace window is never reaped. Cancellation is session-scoped
(`UPDATE ... RETURNING` on rows still in `STARTING`/`RUNNING`) so healthy
sibling sessions survive. The `orphanedSessionRepository` capability keeps the
two new repository methods off the `SessionRepository` interface and its test
doubles.
## Tests
- `TestService_OrphanedSessionReconciliationTerminalizesUnbackedSessions`
(`apps/backend/internal/task/service/orphaned_session_reconciliation_test.go`):
covers AC-TASKS-RESTART-ORPHAN-SESSIONS-001.1, .2, .3, .4, .5, .8, and .10 —
stale unbacked `RUNNING`/`STARTING` sessions reach `CANCELLED` with the
orphan reason and a `session.state_changed` publish whose `is_primary`
matches the durable flag; live-backed, fresh, archived-task, and
`WAITING_FOR_INPUT` sibling rows stay untouched.
- `TestService_OrphanedSessionReconciliationRequiresLivenessChecker`: covers
AC-TASKS-RESTART-ORPHAN-SESSIONS-001.6 — without the liveness seam the sweep
is inert.
- `TestService_OrphanedSessionReconciliationSparesRowRefreshedSinceCandidateRead`:
covers AC-TASKS-RESTART-ORPHAN-SESSIONS-001.9 — a row refreshed between the
candidate read and the cancel write (the in-flight-launch race) is spared.
- `TestSessionOrphanedCancelReasonIsNotArchiveReason`: covers
AC-TASKS-RESTART-ORPHAN-SESSIONS-001.4's distinctness invariant.
## E2E tests
None: the sweep is a periodic backend behavior with a one-minute tick and a
launch-budget grace window; the service-level test pins the full transition.
A repro (start session, kill backend mid-turn, restart) is recorded in
issue #3711's verification plan.
## Work orders
- [x] [Task 01: Terminalize restart-orphaned sessions via the reconciliation sweep](task-01-terminalize-restart-orphaned-sessions.md)
## Verification results
- `(cd apps/backend && go test ./internal/task/service -run 'TestService_OrphanedSessionReconciliation|TestService_ArchivedSessionReconciliation' -count=1)` — pass.
- `(cd apps/backend && go test ./internal/task/repository/... -count=1)` — pass.
- `(cd apps/backend && go run ./cmd/sqlguard ./internal)` — clean after rebinding the candidate query's placeholder.
- `(cd apps/backend && gofmt -l internal/task internal/backendapp)` — no output; `go vet` clean.
- `python3 scripts/lint-spec-files.py --all` and `python3 scripts/list-docs.py validate` — pass.
- Full `go test ./internal/task/...`: three pre-existing failures
(`TestDesktopDiscoveryRootPersists…`, `TestArchiveTaskCleanup…`,
`TestTaskLifecycleCleanup_MissingWorktree`) reproduce identically on the
base commit in a scratch worktree, so none is attributable to this change.
## Risks
- Reaping a live session whose execution legitimately left the store would
lose work; the liveness re-check at write time and the fresh-row grace
window bound that risk to zero in the steady state.
- The sweep cannot distinguish "execution gone because restart" from
"execution gone because of a runtime bookkeeping bug"; both are terminal
sessions by the same argument — nothing will transition them — so
terminalizing is correct in either case.
@@ -0,0 +1,117 @@
---
id: "01-terminalize-restart-orphaned-sessions"
title: "Terminalize restart-orphaned sessions via the reconciliation sweep"
status: done
wave: 1
depends_on: []
plan: "plan.md"
requirements:
- REQ-TASKS-RESTART-ORPHAN-SESSIONS-001
acceptance_criteria:
- AC-TASKS-RESTART-ORPHAN-SESSIONS-001.1
- AC-TASKS-RESTART-ORPHAN-SESSIONS-001.2
- AC-TASKS-RESTART-ORPHAN-SESSIONS-001.3
- AC-TASKS-RESTART-ORPHAN-SESSIONS-001.4
- AC-TASKS-RESTART-ORPHAN-SESSIONS-001.5
- AC-TASKS-RESTART-ORPHAN-SESSIONS-001.6
- AC-TASKS-RESTART-ORPHAN-SESSIONS-001.7
- AC-TASKS-RESTART-ORPHAN-SESSIONS-001.8
- AC-TASKS-RESTART-ORPHAN-SESSIONS-001.9
- AC-TASKS-RESTART-ORPHAN-SESSIONS-001.10
system_design:
- ../../specs/tasks/system-design/restart-orphaned-session-terminalization.md
---
# Task 01: Terminalize Restart-Orphaned Sessions via the Reconciliation Sweep
## Summary
Extend the existing 1-minute task-session reconciliation sweep with a second
pass: terminalize stale `STARTING`/`RUNNING` sessions of unarchived tasks that
no live in-memory execution backs, so a backend restart mid-turn no longer
leaves sessions stuck RUNNING forever (#3711).
## In scope
- `runOrphanedSessionReconciliation` / `reconcileOrphanedSessions` in the
reconciliation loop file, gated on a wired `TaskExecutionLivenessChecker`.
- Repository methods `ListStaleRunningSessionsOnUnarchivedTasks` and
`CancelRunningTaskSessionByID` behind the `orphanedSessionRepository`
capability.
- `SessionOrphanedCancelReason` distinct from archive cancel reasons.
- Extract `notifyCancelledSessions` from `finalizeCancelledSessions` so the
orphan pass reuses the archive path's post-cancellation effects.
- Wire `HasLiveExecution` in the lifecycle adapter and the orchestrator.
## Out of scope
- Archived tasks' sessions (the archived pass owns them).
- Turn resume, drain-on-shutdown, heartbeats, adapter changes, schema
changes.
## Acceptance
- After a backend restart mid-turn, an unbacked stale `STARTING`/`RUNNING`
session reaches `CANCELLED` on the first sweep after the grace window, with
the orphan reason and a `session.state_changed` publish, while live-backed,
fresh, and archived-task rows stay untouched and healthy siblings survive.
- Without the liveness seam the sweep does nothing.
## Verification
```bash
(cd apps/backend && go test ./internal/task/service -run 'TestService_OrphanedSessionReconciliation|TestService_ArchivedSessionReconciliation' -count=1 -v)
(cd apps/backend && go test ./internal/task/repository/... -count=1)
(cd apps/backend && go run ./cmd/sqlguard ./internal)
(cd apps/backend && gofmt -l internal/task internal/backendapp)
(cd apps/backend && go vet ./internal/task/... ./internal/backendapp)
python3 scripts/lint-spec-files.py --all
python3 scripts/list-docs.py validate
```
## Files likely touched
- `apps/backend/internal/task/service/archived_session_reconciliation.go`
- `apps/backend/internal/task/service/orphaned_session_reconciliation_test.go`
- `apps/backend/internal/task/service/service.go`
- `apps/backend/internal/task/service/service_tasks.go`
- `apps/backend/internal/task/models/resume_safety.go`
- `apps/backend/internal/task/repository/sqlite/session.go`
- `apps/backend/internal/backendapp/adapters.go`
- `apps/backend/internal/backendapp/orchestrator.go`
## Dependencies
None.
## Risks
- The liveness checker must answer from the in-memory store only — a lazy
creator would mask the sweep's dead signal.
## Parallelism
`sequential`
## Inputs
- `docs/specs/tasks/requirements/restart-orphaned-session-terminalization.md`
- `docs/specs/tasks/system-design/restart-orphaned-session-terminalization.md`
- `apps/backend/internal/task/service/archived_session_reconciliation.go`
## Results
Implemented in commit `aa7dc293b` (8 files, +458/−32): the second sweep pass,
the two repository methods behind the `orphanedSessionRepository` capability,
the orphan reason constant, the shared `notifyCancelledSessions` extraction,
and the lifecycle-adapter `HasLiveExecution` wiring. Follow-up commits added
the `Rebind` the SQL-portability gate requires, this specification package,
and two review-fix hardenings: the cancel statement re-asserts the staleness
cutoff (`updated_at < staleBefore`) so an in-flight launch refreshing its row
between the sweep's liveness check and the write is never reaped
(AC-.9, `TestService_OrphanedSessionReconciliationSparesRowRefreshedSinceCandidateRead`),
and both cancellation RETURNING clauses now select `is_primary` so the
published event carries the durable primary flag (AC-.10; this also repairs
the pre-existing archive path, which had the same gap). Verification commands
in the plan all pass; the failing packages in the full suite reproduce
identically on the base commit and are environmental.
+1
View File
@@ -93,6 +93,7 @@ signals, and task-scoped scheduling contracts.
- [Task Terminal Persistence](requirements/task-terminal-persistence.md)
- [Runtime Task-State Publication Order](requirements/runtime-state-publication-order.md)
- [Session Delete Preserves Task Workspaces](requirements/session-delete-resource-cleanup.md)
- [Restart-Orphaned Session Terminalization](requirements/restart-orphaned-session-terminalization.md)
- [Sidebar Task Editing](requirements/sidebar-task-edit.md)
- [Task Actions Menu on Preview and Detail Surfaces](requirements/task-actions-menu.md)
- [Task menu grouping](requirements/task-menu-grouping.md)
@@ -0,0 +1,105 @@
---
status: draft
system: tasks
created: 2026-09-20
owners:
- kandev
---
# Restart-Orphaned Session Terminalization Requirements
## Overview
A task session records the lifecycle of one agent conversation turn. When the
backend process dies while a session's turn is open — a graceful restart, a
crash, or a kill — the component that would have transitioned the session to a
terminal state dies with it. No other actor retries that transition, so the
session row stays in an active state forever and the task appears to wait for
an agent that already finished.
The tasks system owns the task-session lifecycle, so it owns this
reconciliation: the system itself must detect and terminalize sessions that
lost their backing actor, without depending on the dead process, the agent, or
a human SQL edit.
## Terminology
- **Active session state:** A `task_sessions.state` value that implies work is
in flight: `CREATED`, `STARTING`, `RUNNING`, or `WAITING_FOR_INPUT`.
- **Terminal session state:** A state a session never leaves: `COMPLETED`,
`FAILED`, or `CANCELLED`.
- **Orphaned session:** An active-state session whose transition actor is gone:
the backend that would complete or fail its turn terminated, and no live
agent execution backs the session anymore.
- **Live execution:** An agent execution the current backend process tracks in
its in-memory execution store. A session backed by a live execution is not
orphaned even when its row is stale.
- **Launch grace window:** The interval between a session row entering
`STARTING` and its execution necessarily being visible in the in-memory
store. A session inside this window is not eligible for orphan detection.
- **Reconciliation sweep:** The periodic pass that lists candidate sessions
and terminalizes the orphaned ones.
## Requirements
### REQ-TASKS-RESTART-ORPHAN-SESSIONS-001: Sessions orphaned by a backend restart reach a terminal state
**Intent:** A backend restart must not leave task sessions stuck in an active
state forever. The system must self-heal: detect sessions whose backing actor
died and terminalize them within a bounded time.
**User story:** As a Kandev user, I want a session that lost its agent process
to a backend restart to show as stopped rather than stay "running" forever, so
that the task board stays truthful and the task can move on.
#### Acceptance criteria
- **AC-TASKS-RESTART-ORPHAN-SESSIONS-001.1:** When a backend process
terminates while a task session is in `STARTING` or `RUNNING`, and no live
execution backs that session, the system shall transition that session to
`CANCELLED` within one reconciliation sweep after the launch grace window
elapses, without any user action.
- **AC-TASKS-RESTART-ORPHAN-SESSIONS-001.2:** When a session in `STARTING` or
`RUNNING` is backed by a live execution in the in-memory store, the system
shall leave its state untouched, regardless of how stale its row is.
- **AC-TASKS-RESTART-ORPHAN-SESSIONS-001.3:** When a session's last write is
more recent than the launch grace window, the system shall not consider it
orphaned, so an in-flight launch whose execution has not yet reached the
in-memory store is never reaped.
- **AC-TASKS-RESTART-ORPHAN-SESSIONS-001.4:** When the sweep terminalizes a
session, the system shall record a cancellation reason that names the
backend restart as the source, distinct from archive cancellations and user
stops, so session history keeps the source visible.
- **AC-TASKS-RESTART-ORPHAN-SESSIONS-001.5:** When the sweep terminalizes a
session, the system shall run the same post-cancellation effects as an
archive cancellation: expire terminal clarifications, clear parked
projections, release session-ceiling reservations, and publish a
`session.state_changed` event for each session actually cancelled.
- **AC-TASKS-RESTART-ORPHAN-SESSIONS-001.6:** When the sweep cannot consult
the in-memory execution store, it shall remain inert rather than
terminalize sessions it cannot prove unbacked.
- **AC-TASKS-RESTART-ORPHAN-SESSIONS-001.7:** When a cancellation write fails,
the system shall retry it on the next sweep tick, and shall never give up on
a session for as long as the process runs.
- **AC-TASKS-RESTART-ORPHAN-SESSIONS-001.8:** When the sweep terminalizes one
session of a task, the system shall leave the task's other sessions
untouched, so healthy siblings (for example `WAITING_FOR_INPUT`) survive.
- **AC-TASKS-RESTART-ORPHAN-SESSIONS-001.9:** When a session row is refreshed
between the sweep's candidate read and its cancellation write (an in-flight
launch CAS-writing `STARTING` before it registers an execution), the system
shall not terminalize that session; the staleness cutoff is re-asserted at
write time inside the same statement as the transition.
- **AC-TASKS-RESTART-ORPHAN-SESSIONS-001.10:** When the sweep publishes a
session cancellation event, the event shall carry the session's durable
`is_primary` flag, so the status-summary projection keeps the durable
primary-session assignment.
## Out of scope
- Resuming or replaying a turn that was interrupted mid-flight; the orphaned
session is terminalized, not resumed.
- Detecting orphans of already-archived tasks; the archived-task
reconciliation pass owns those.
- Agent-side heartbeats or adapter-side turn-state persistence.
- Stopping a live agent process; the sweep only terminalizes DB rows of
sessions with no live execution.
@@ -0,0 +1,130 @@
---
status: current
system: tasks
requirements:
- REQ-TASKS-RESTART-ORPHAN-SESSIONS-001
---
# Restart-Orphaned Session Terminalization System Design
## Purpose and boundaries
The tasks system owns the task-session lifecycle, including the transition of
a session into a terminal state. This design covers the periodic sweep that
terminalizes `STARTING`/`RUNNING` sessions of unarchived tasks that no live
in-memory agent execution backs — the residue of a backend process that died
mid-turn.
It extends the existing reconciliation loop
(`internal/task/service/archived_session_reconciliation.go`) with a second
pass; it does not introduce a new loop, new persistence, or adapter changes.
Adjacent contracts this design uses but does not own: the agent runtime's
in-memory execution store (executors system), session-ceiling reservations
(tasks system, reservation contract), and the archived-task reconciliation
pass that owns archived tasks' stranded sessions.
## Requirement mapping
| Requirement | Design section |
| --- | --- |
| `REQ-TASKS-RESTART-ORPHAN-SESSIONS-001` | [Control flow](#control-flow), [Failure and recovery](#failure-and-recovery) |
## Components and responsibilities
- **Reconciliation loop** (`Service.StartArchivedSessionReconciliationLoop`):
ticks every minute and runs both passes — the archived pass first, then the
orphan pass.
- **Orphan pass** (`Service.runOrphanedSessionReconciliation`): lists stale
candidates, filters by live-execution absence, cancels each, runs
post-cancellation effects.
- **Session repository** (`sqlite.Repository`): candidate query
`ListStaleRunningSessionsOnUnarchivedTasks` and per-session cancel
`CancelRunningTaskSessionByID`. Not part of the `SessionRepository`
interface; reached through the narrow `orphanedSessionRepository`
capability so test doubles stay unchanged.
- **Execution-liveness seam** (`TaskExecutionLivenessChecker`): reports
whether a session has a live in-memory execution. Satisfied by the
lifecycle adapter (`HasLiveExecution`), which answers from the in-memory
store only and never lazily creates an execution.
- **Shared post-cancellation effects** (`Service.notifyCancelledSessions`):
extracted from the archive cancellation path and reused verbatim:
clarification expiry, parked-projection clearing, ceiling release, and the
`session.state_changed` publish.
## Data and contracts
- Candidate query joins `task_sessions` to `tasks` and selects rows where
`ts.state IN ('STARTING','RUNNING')`, `ts.updated_at < staleBefore`, and
`t.archived_at IS NULL`, ordered by `updated_at` ascending.
- `staleBefore` is `now - (constants.AgentLaunchTimeout + 5m grace
allowance)`: one full launch budget plus the start-deadline allowance, read
(not copied) from `constants.AgentLaunchTimeout` so an operator-raised
preparation budget is honored. The same cutoff is passed into the cancel
statement and re-asserted as `updated_at < staleBefore` there, so a row
refreshed between the candidate read and the write no longer matches.
- `CancelRunningTaskSessionByID(sessionID, reason, staleBefore)` is an
`UPDATE ... RETURNING` over rows still in `STARTING`/`RUNNING` whose
`updated_at` still predates the cutoff; it returns `nil` when the session
raced to another state or was refreshed, and the write detaches from the
caller context with a 10-second timeout. The RETURNING clause selects
`is_primary` alongside the other event-payload fields, so the published
cancellation event carries the durable primary flag.
- Cancellation reason constant: `models.SessionOrphanedCancelReason`
(`"orphaned by backend restart"`), distinct from the archive reasons;
`IsArchiveCancelReason` returns false for it.
## Control flow
1. Every tick, pass 2 runs after the archived pass.
2. If no execution-liveness checker is wired, the pass is inert: absence
from the store is the only dead signal, and a nil checker can never prove
a session unbacked.
3. List candidates with the stale-before cutoff computed from the launch
budget.
4. For each candidate, re-check `HasLiveExecution(session.ID)` at the moment
of the write — a launch that raced the grace window since the read must
not be reaped.
5. Cancel via `CancelRunningTaskSessionByID`; the statement re-asserts both
the state set (`STARTING`/`RUNNING`) and the staleness cutoff
(`updated_at < staleBefore`), so a row refreshed by an in-flight launch
between the liveness check and this write matches nothing; on `nil`
(raced or refreshed) skip; on error warn and let the next tick retry.
6. Run the shared post-cancellation effects with the orphan reason.
## Failure and recovery
- A failed candidate read logs an error and returns; the next tick retries.
- A failed cancel logs a warn and continues to the next candidate; the next
tick retries that session. The sweep never gives up for as long as the
process runs.
- The cancel write is context-detached (`context.WithoutCancel` + 10s bound),
so the terminal transition survives a client disconnect and cannot block a
sweep pass indefinitely on a locked writer.
- Pass 2 is a no-op on clean trees: fresh rows are filtered by the grace
window, live-backed rows by the liveness check, archived tasks' rows by the
archived-pass filter, and terminal rows by the candidate query.
## Persistence
No schema change. The sweep writes only `task_sessions.state`,
`error_message`, `completed_at`, and `updated_at`. Survivors re-tracked by
startup recovery are in the in-memory store and stay untouched, so the sweep
composes with agent-survival re-tracking rather than racing it.
## Security
No new trust boundary. The sweep runs in-process with the task service's own
credentials and touches only `task_sessions` rows of unarchived tasks.
## Observability
- `task-session reconciliation loop started (every 1 minute)` at loop start.
- `orphaned-session reconciliation: found candidates` / `terminalized
unbacked session` (Info, with task and session IDs, previous state).
- `failed to list candidates` / `failed to cancel session` (Error/Warn with
the underlying error).
## Related decisions
- Issue #3711 and its approved fix plan: extend the existing sweep rather
than drain-on-shutdown, turn-state persistence, heartbeats, or manual SQL.