fix(task): terminalize sessions orphaned by backend restart via reconciliation sweep (#3833)
Co-authored-by: Carlos Florencio <carlosmflorencio@gmail.com>
This commit is contained in:
@@ -21,6 +21,7 @@ import (
|
||||
"github.com/kandev/kandev/internal/secrets"
|
||||
"github.com/kandev/kandev/internal/task/models"
|
||||
"github.com/kandev/kandev/internal/worktree"
|
||||
v1 "github.com/kandev/kandev/pkg/api/v1"
|
||||
)
|
||||
|
||||
// ErrSessionWorkspaceNotReady indicates the task session exists but does not yet
|
||||
@@ -496,6 +497,45 @@ func (m *Manager) IsAgentCommandConfigured(executionID string) bool {
|
||||
return configured
|
||||
}
|
||||
|
||||
// HasLiveAgentExecution reports whether a session owns an agent execution that
|
||||
// can still protect its durable lifecycle row from orphan reconciliation.
|
||||
// Workspace-only executions have no agent command and do not count. Terminal
|
||||
// executions do not count even when they remain in the store briefly. A live
|
||||
// passthrough process protects its session through the interactive runner.
|
||||
func (m *Manager) HasLiveAgentExecution(sessionID string) bool {
|
||||
execution, exists := m.executionStore.GetBySessionID(sessionID)
|
||||
if !exists || execution == nil {
|
||||
return false
|
||||
}
|
||||
|
||||
var agentCommand, passthroughProcessID string
|
||||
var status v1.AgentStatus
|
||||
if err := m.executionStore.WithRLock(execution.ID, func(current *AgentExecution) {
|
||||
agentCommand = current.AgentCommand
|
||||
passthroughProcessID = current.PassthroughProcessID
|
||||
status = current.Status
|
||||
}); err != nil {
|
||||
return false
|
||||
}
|
||||
if isTerminalStatus(status) {
|
||||
return false
|
||||
}
|
||||
if agentCommand != "" {
|
||||
return true
|
||||
}
|
||||
if passthroughProcessID == "" {
|
||||
return false
|
||||
}
|
||||
runner := m.GetInteractiveRunner()
|
||||
if runner == nil {
|
||||
// The process handle is evidence of agent ownership, but an unavailable
|
||||
// runner cannot prove that the process is gone. Keep the session safe
|
||||
// from destructive orphan cleanup until the runtime can answer.
|
||||
return true
|
||||
}
|
||||
return runner.IsProcessReadyOrPending(passthroughProcessID)
|
||||
}
|
||||
|
||||
// EnsurePassthroughExecution ensures an execution exists for a passthrough session
|
||||
// and starts the passthrough process if needed. This is called when the terminal
|
||||
// handler receives a connection for a session that might need recovery after backend restart.
|
||||
|
||||
@@ -179,6 +179,29 @@ func TestIsAgentCommandConfigured(t *testing.T) {
|
||||
require.False(t, mgr.IsAgentCommandConfigured("exec-absent"))
|
||||
}
|
||||
|
||||
func TestHasLiveAgentExecutionIgnoresWorkspaceInfrastructureAndTerminalAgents(t *testing.T) {
|
||||
mgr := newTestManager(t)
|
||||
require.NoError(t, mgr.executionStore.Add(&AgentExecution{
|
||||
ID: "exec-workspace", SessionID: "session-workspace", Status: v1.AgentStatusRunning,
|
||||
}))
|
||||
require.NoError(t, mgr.executionStore.Add(&AgentExecution{
|
||||
ID: "exec-agent", SessionID: "session-agent", AgentCommand: "claude",
|
||||
Status: v1.AgentStatusStarting,
|
||||
}))
|
||||
require.NoError(t, mgr.executionStore.Add(&AgentExecution{
|
||||
ID: "exec-terminal", SessionID: "session-terminal", AgentCommand: "claude",
|
||||
Status: v1.AgentStatusFailed,
|
||||
}))
|
||||
|
||||
require.False(t, mgr.HasLiveAgentExecution("session-workspace"),
|
||||
"workspace-only execution must not shield an orphaned session")
|
||||
require.True(t, mgr.HasLiveAgentExecution("session-agent"),
|
||||
"a configured non-terminal agent execution must shield its session")
|
||||
require.False(t, mgr.HasLiveAgentExecution("session-terminal"),
|
||||
"terminal executions must not shield a session")
|
||||
require.False(t, mgr.HasLiveAgentExecution("session-absent"))
|
||||
}
|
||||
|
||||
func TestResolveTaskEnvironmentIDPrefersInMemoryExecution(t *testing.T) {
|
||||
mgr := newTestManager(t)
|
||||
mgr.workspaceInfoProvider = &mockWorkspaceInfoProvider{infos: map[string]*WorkspaceInfo{
|
||||
|
||||
@@ -971,7 +971,10 @@ func (a *lifecycleAdapter) LiveSessionIDsForTask(taskID string) []string {
|
||||
references := a.mgr.ListExecutionsForTask(taskID)
|
||||
sessionIDs := make([]string, 0, len(references))
|
||||
for _, reference := range references {
|
||||
if reference.SessionID != "" {
|
||||
// ListExecutionsForTask includes workspace-only infrastructure created
|
||||
// by file/shell access. Only an execution with an agent command (or a
|
||||
// live passthrough process) owns the agent session lifecycle.
|
||||
if reference.SessionID != "" && a.mgr.HasLiveAgentExecution(reference.SessionID) {
|
||||
sessionIDs = append(sessionIDs, reference.SessionID)
|
||||
}
|
||||
}
|
||||
@@ -1018,6 +1021,14 @@ func (a *lifecycleAdapter) ResolveAgentProfile(ctx context.Context, profileID st
|
||||
}, nil
|
||||
}
|
||||
|
||||
// HasLiveExecution reports whether the session still has an agent execution
|
||||
// owned by the runtime. It backs the task service's orphan-session
|
||||
// reconciliation sweep; workspace-only infrastructure is not a live agent,
|
||||
// and this lookup must never lazily create an execution.
|
||||
func (a *lifecycleAdapter) HasLiveExecution(sessionID string) bool {
|
||||
return a.mgr.HasLiveAgentExecution(sessionID)
|
||||
}
|
||||
|
||||
// GetGitLog retrieves the git log for a session from baseCommit to HEAD.
|
||||
// If targetBranch is provided, uses dynamic merge-base calculation for accurate filtering.
|
||||
func (a *lifecycleAdapter) GetGitLog(ctx context.Context, sessionID, baseCommit string, limit int, targetBranch string) (*client.GitLogResult, error) {
|
||||
|
||||
@@ -170,6 +170,9 @@ func provideOrchestrator(
|
||||
// Runtime-aware liveness lets durable cleanup treat a not-found stop for a
|
||||
// confirmed-dead local runtime as already stopped instead of retrying forever.
|
||||
taskSvc.SetRowLivenessProber(agentManagerClient)
|
||||
// The orphan-session sweep terminalizes stale STARTING/RUNNING sessions no
|
||||
// live in-memory execution backs (backend-restart residue, #3711).
|
||||
taskSvc.SetExecutionLivenessChecker(agentManagerClient)
|
||||
// The session reconciliation sweep's active-task pass (stall detection and
|
||||
// orphaned-session healing) verifies "no live execution" against the agent
|
||||
// runtime's in-memory execution store through this registry.
|
||||
|
||||
@@ -72,9 +72,7 @@ func RowMustBePreserved(running *ExecutorRunning, sessionState TaskSessionState)
|
||||
|
||||
// SessionArchiveCancelReason and SessionArchiveTreeCancelReason are the
|
||||
// TaskSession.ErrorMessage values written when a session is auto-cancelled
|
||||
// by archiving — Service.ArchiveTask's CancelActiveTaskSessionsByTaskID call
|
||||
// (single-task archive) and HandoffService.cancelActiveRuns (cascade archive)
|
||||
// respectively. Distinct reasons keep the cancellation source visible in
|
||||
// by archiving. Distinct reasons keep the cancellation source visible in
|
||||
// session history and diagnostics.
|
||||
const (
|
||||
SessionArchiveCancelReason = "task archived"
|
||||
|
||||
@@ -2112,7 +2112,7 @@ func completedAtForTaskSessionState(status models.TaskSessionState, now time.Tim
|
||||
// timed out. Returned sessions therefore carry only the fields the
|
||||
// RETURNING clause selects (ID, TaskID, AgentProfileID,
|
||||
// AgentProfileSnapshot, IsPassthrough, Name, ReviewStatus, Metadata,
|
||||
// TaskEnvironmentID, State, UpdatedAt) — every other models.TaskSession
|
||||
// TaskEnvironmentID, State, UpdatedAt, IsPrimary) — every other models.TaskSession
|
||||
// field is left at its zero value, and callers must not rely on fields
|
||||
// outside this list being populated.
|
||||
func (r *Repository) CancelActiveTaskSessionsByTaskID(ctx context.Context, taskID, reason string) ([]*models.TaskSession, error) {
|
||||
@@ -2131,7 +2131,7 @@ func (r *Repository) CancelActiveTaskSessionsByTaskID(ctx context.Context, taskI
|
||||
WHERE task_id = ?
|
||||
AND state IN ('CREATED', 'STARTING', 'RUNNING', 'WAITING_FOR_INPUT')
|
||||
RETURNING id, agent_profile_id, agent_profile_snapshot, is_passthrough, name,
|
||||
review_status, metadata, task_environment_id, state, updated_at
|
||||
review_status, metadata, task_environment_id, state, updated_at, is_primary
|
||||
`), string(models.TaskSessionStateCancelled), reason, now, now, taskID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -2149,6 +2149,97 @@ func (r *Repository) CancelActiveTaskSessionsByTaskID(ctx context.Context, taskI
|
||||
return sessions, rows.Err()
|
||||
}
|
||||
|
||||
// ListStaleRunningSessionsOnUnarchivedTasks returns every STARTING/RUNNING
|
||||
// session of an unarchived task whose updated_at is older than staleBefore.
|
||||
// It is the candidate list for the orphan-session reconciliation sweep (see
|
||||
// service.runOrphanedSessionReconciliation): a session still holding one of
|
||||
// those states past the launch grace window, with no live in-memory execution
|
||||
// backing it, can never reach a terminal state on its own — its actor died
|
||||
// with the process. The staleBefore cutoff is applied in SQL so the sweep
|
||||
// never even loads fresh rows that may belong to an in-flight launch.
|
||||
//
|
||||
// Not part of the SessionRepository interface: it exists for that one sweep,
|
||||
// which reaches it through the narrow orphanedSessionRepository capability —
|
||||
// widening the interface would force every test double in the tree to grow
|
||||
// methods this sweep never exercises through them.
|
||||
func (r *Repository) ListStaleRunningSessionsOnUnarchivedTasks(ctx context.Context, staleBefore time.Time) ([]*models.TaskSession, error) {
|
||||
rows, err := r.ro.QueryContext(ctx, r.ro.Rebind(`
|
||||
SELECT `+taskSessionSelectCols+` `+taskSessionFromClause+`
|
||||
JOIN tasks t ON t.id = ts.task_id
|
||||
WHERE ts.state IN ('STARTING', 'RUNNING')
|
||||
AND ts.updated_at < ?
|
||||
AND t.archived_at IS NULL
|
||||
ORDER BY ts.updated_at ASC
|
||||
`), staleBefore)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer func() { _ = rows.Close() }()
|
||||
|
||||
sessions, err := r.scanTaskSessions(ctx, rows)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return r.loadWorktreesBatch(ctx, sessions)
|
||||
}
|
||||
|
||||
// CancelRunningTaskSessionByID transitions a single stale STARTING/RUNNING
|
||||
// session to CANCELLED, returning the transitioned row, or nil when the
|
||||
// session no longer matches (already terminal, raced to another state, or
|
||||
// refreshed since the sweep read it). Like CancelActiveTaskSessionsByTaskID
|
||||
// it is a pure DB state change that requires no live agent execution, and it
|
||||
// is session-scoped so the orphan sweep can terminalize one session without
|
||||
// cancelling healthy sibling sessions of the same task.
|
||||
//
|
||||
// staleBefore re-asserts the sweep's staleness cutoff at write time, closing
|
||||
// the read-then-write race against an in-flight launch: a launch CAS-writes
|
||||
// its session to STARTING (bumping updated_at) before it registers an
|
||||
// execution in the in-memory store, so between the sweep's liveness check and
|
||||
// this UPDATE the row can pass from "no live execution" to "launch in
|
||||
// progress". A row refreshed since the candidate read no longer satisfies
|
||||
// updated_at < staleBefore, the UPDATE matches nothing, and the next tick
|
||||
// re-evaluates a fresh row that the grace window protects until its
|
||||
// execution registers.
|
||||
//
|
||||
// The UPDATE and the row selection happen in one atomic RETURNING statement;
|
||||
// the returned row carries only the fields that clause selects (same set as
|
||||
// CancelActiveTaskSessionsByTaskID, minus TaskID — callers pass the task ID
|
||||
// they already know). The write detaches from ctx like
|
||||
// CancelActiveTaskSessionsByTaskID: once the sweep has decided this session
|
||||
// is orphaned, the terminal transition must not be lost to a caller-context
|
||||
// cancellation, and the 10s bound keeps a locked SQLite writer from stalling
|
||||
// the sweep pass. A failed write simply retries on the next sweep tick.
|
||||
func (r *Repository) CancelRunningTaskSessionByID(ctx context.Context, sessionID, reason string, staleBefore time.Time) (*models.TaskSession, error) {
|
||||
now := time.Now().UTC()
|
||||
writeCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
|
||||
defer cancel()
|
||||
rows, err := r.db.QueryContext(writeCtx, r.db.Rebind(`
|
||||
UPDATE task_sessions
|
||||
SET state = ?, error_message = ?, completed_at = ?, updated_at = ?
|
||||
WHERE id = ?
|
||||
AND state IN ('STARTING', 'RUNNING')
|
||||
AND updated_at < ?
|
||||
RETURNING id, agent_profile_id, agent_profile_snapshot, is_passthrough, name,
|
||||
review_status, metadata, task_environment_id, state, updated_at, is_primary
|
||||
`), string(models.TaskSessionStateCancelled), reason, now, now, sessionID, staleBefore)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer func() { _ = rows.Close() }()
|
||||
|
||||
if !rows.Next() {
|
||||
return nil, rows.Err()
|
||||
}
|
||||
// The scanner backfills TaskID from the parameter because RETURNING
|
||||
// does not carry it; sessionID is not the task ID, so set it from the
|
||||
// candidate row the sweep already read instead.
|
||||
session, err := scanCancelledTaskSessionRow(rows, "")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return session, rows.Err()
|
||||
}
|
||||
|
||||
// CancelActiveTaskSessionsByIDs is documented on the SessionRepository
|
||||
// interface. It shares CancelActiveTaskSessionsByTaskID's atomic
|
||||
// UPDATE ... RETURNING shape and detached-but-bounded write context, so the
|
||||
@@ -2175,7 +2266,7 @@ func (r *Repository) CancelActiveTaskSessionsByIDs(ctx context.Context, taskID s
|
||||
AND id IN (`+placeholders+`)
|
||||
AND state IN ('CREATED', 'STARTING', 'RUNNING', 'WAITING_FOR_INPUT')
|
||||
RETURNING id, agent_profile_id, agent_profile_snapshot, is_passthrough, name,
|
||||
review_status, metadata, task_environment_id, state, updated_at
|
||||
review_status, metadata, task_environment_id, state, updated_at, is_primary
|
||||
`), args...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -2242,9 +2333,9 @@ func (r *Repository) CancelActiveTaskSessionsByCandidates(
|
||||
SET state = ?, error_message = ?, completed_at = ?, updated_at = ?
|
||||
WHERE task_id = ?
|
||||
AND state IN ('CREATED', 'STARTING', 'RUNNING', 'WAITING_FOR_INPUT')
|
||||
AND (`+strings.Join(predicates, " OR ")+`)
|
||||
AND (`+strings.Join(predicates, " OR ")+`)
|
||||
RETURNING id, agent_profile_id, agent_profile_snapshot, is_passthrough, name,
|
||||
review_status, metadata, task_environment_id, state, updated_at
|
||||
review_status, metadata, task_environment_id, state, updated_at, is_primary
|
||||
`), args...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -2305,15 +2396,16 @@ func activeSessionCancellationCandidatePredicate(
|
||||
// *models.TaskSession, mirroring scanTaskSessionRow's JSON-unmarshal and
|
||||
// int-to-bool/nullable-string conventions but for the narrower RETURNING
|
||||
// column set (id, agent_profile_id, agent_profile_snapshot, is_passthrough,
|
||||
// name, review_status, metadata, task_environment_id, state, updated_at).
|
||||
// taskID backfills TaskID, which RETURNING cannot supply since it's a query
|
||||
// parameter, not a returned column.
|
||||
// name, review_status, metadata, task_environment_id, state, updated_at,
|
||||
// is_primary). taskID backfills TaskID, which RETURNING cannot supply since
|
||||
// it's a query parameter, not a returned column.
|
||||
func scanCancelledTaskSessionRow(rows *sql.Rows, taskID string) (*models.TaskSession, error) {
|
||||
session := &models.TaskSession{TaskID: taskID}
|
||||
var state string
|
||||
var metadataJSON string
|
||||
var agentProfileSnapshotJSON string
|
||||
var isPassthrough int
|
||||
var isPrimary int
|
||||
var reviewStatus sql.NullString
|
||||
var agentProfileID sql.NullString
|
||||
var name sql.NullString
|
||||
@@ -2321,12 +2413,14 @@ func scanCancelledTaskSessionRow(rows *sql.Rows, taskID string) (*models.TaskSes
|
||||
if err := rows.Scan(
|
||||
&session.ID, &agentProfileID, &agentProfileSnapshotJSON, &isPassthrough, &name,
|
||||
&reviewStatus, &metadataJSON, &session.TaskEnvironmentID, &state, &session.UpdatedAt,
|
||||
&isPrimary,
|
||||
); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
session.State = models.TaskSessionState(state)
|
||||
session.IsPassthrough = isPassthrough == 1
|
||||
session.IsPrimary = isPrimary == 1
|
||||
if reviewStatus.Valid {
|
||||
session.ReviewStatus = models.ReviewStatus(reviewStatus.String)
|
||||
}
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"github.com/kandev/kandev/internal/common/constants"
|
||||
"github.com/kandev/kandev/internal/task/archivecascade"
|
||||
"github.com/kandev/kandev/internal/task/models"
|
||||
"go.uber.org/zap"
|
||||
@@ -19,29 +20,28 @@ import (
|
||||
// sessions whose backing execution is gone (see runActiveSessionSweep in
|
||||
// active_session_stall.go).
|
||||
//
|
||||
// finalizeCancelledSessions (see service_tasks.go) already bounds its
|
||||
// session-cancellation retry to a handful of fixed attempts inside
|
||||
// ArchiveTask's own request: enough to ride out a brief SQLite writer-lock
|
||||
// blip, but not unbounded. If SQLite's single writer stays occupied for
|
||||
// longer than every attempt combined, that bounded retry gets exhausted
|
||||
// after archived_at has already committed, and nothing else in the request
|
||||
// path ever retries the transition. The archived task is then left with
|
||||
// sessions stuck in an active DB state (CREATED/STARTING/RUNNING/
|
||||
// Pass 1 — archived tasks. finalizeCancelledSessions (see service_tasks.go)
|
||||
// already bounds its session-cancellation retry to a handful of fixed attempts
|
||||
// inside ArchiveTask's own request: enough to ride out a brief SQLite
|
||||
// writer-lock blip, but not unbounded. If SQLite's single writer stays
|
||||
// occupied for longer than every attempt combined, that bounded retry gets
|
||||
// exhausted after archived_at has already committed, and nothing else in the
|
||||
// request path ever retries the transition. The archived task is then left
|
||||
// with sessions stuck in an active DB state (CREATED/STARTING/RUNNING/
|
||||
// WAITING_FOR_INPUT) forever, and event-driven clients that key their
|
||||
// "is running" indicators off session.state_changed never learn the
|
||||
// sessions actually stopped.
|
||||
//
|
||||
// The archived pass is the eventual-consistency backstop for that residual
|
||||
// gap, following the exact periodic-sweep shape StartAutoArchiveLoop already
|
||||
// uses: on a fixed interval, list every archived task that still has an
|
||||
// active session, and re-invoke finalizeCancelledSessions for each. Calling
|
||||
// finalizeCancelledSessions again is always safe — its underlying
|
||||
// CancelActiveTaskSessionsByTaskID UPDATE only matches sessions still in an
|
||||
// active state, so a task that was already fully reconciled (by ArchiveTask
|
||||
// itself or by an earlier sweep pass) simply yields an empty cancelledSessions
|
||||
// slice and is a no-op. As long as the process keeps running, a later pass
|
||||
// retries once the SQLite contention that exhausted the in-line retry has
|
||||
// cleared — unlike the fixed 3-attempt budget, this sweep never gives up.
|
||||
// Pass 2 — orphaned sessions of unarchived tasks. A backend restart while an
|
||||
// ACP prompt turn is open kills the actor that would have transitioned the
|
||||
// session (the turn-complete path dies with the process), leaving
|
||||
// task_sessions stuck in STARTING/RUNNING forever on tasks that were never
|
||||
// archived. Every tick, stale STARTING/RUNNING sessions with no live
|
||||
// in-memory execution backing them are terminalized.
|
||||
//
|
||||
// Both passes follow the exact periodic-sweep shape StartAutoArchiveLoop
|
||||
// already uses and re-invoke the same finalize transitions; as long as the
|
||||
// process keeps running, a later pass retries what an earlier pass missed.
|
||||
func (s *Service) StartSessionReconciliationLoop(ctx context.Context) {
|
||||
ticker := time.NewTicker(1 * time.Minute)
|
||||
go func() {
|
||||
@@ -52,6 +52,7 @@ func (s *Service) StartSessionReconciliationLoop(ctx context.Context) {
|
||||
return
|
||||
case now := <-ticker.C:
|
||||
s.runArchivedSessionReconciliation(ctx)
|
||||
s.runOrphanedSessionReconciliation(ctx)
|
||||
s.runActiveSessionSweep(ctx, now)
|
||||
}
|
||||
}
|
||||
@@ -59,6 +60,14 @@ func (s *Service) StartSessionReconciliationLoop(ctx context.Context) {
|
||||
s.logger.Info("session reconciliation loop started (every 1 minute)")
|
||||
}
|
||||
|
||||
// StartArchivedSessionReconciliationLoop keeps the pre-stall-sweep entry
|
||||
// point available for integrations that started the archived pass directly.
|
||||
// New startup wiring should use StartSessionReconciliationLoop so all passes
|
||||
// run from one ticker.
|
||||
func (s *Service) StartArchivedSessionReconciliationLoop(ctx context.Context) {
|
||||
s.StartSessionReconciliationLoop(ctx)
|
||||
}
|
||||
|
||||
func (s *Service) runArchivedSessionReconciliation(ctx context.Context) {
|
||||
deadline := archivecascade.ArchiveDeadline(ctx)
|
||||
reconcileCtx, cancel := archivecascade.ContinuationContextUntil(ctx, deadline)
|
||||
@@ -89,3 +98,129 @@ func (s *Service) runArchivedSessionReconciliation(ctx context.Context) {
|
||||
s.finalizeCancelledSessions(reconcileCtx, taskID, activeSessions, deadline, models.SessionArchiveCancelReason)
|
||||
}
|
||||
}
|
||||
|
||||
// orphanedSessionGraceAllowance is the slice added to the launch budget when
|
||||
// deriving the orphan sweep's staleness cutoff, mirroring the session-ceiling
|
||||
// reservation expiry: preparation is bounded by constants.AgentLaunchTimeout
|
||||
// (read, not copied, so an operator-raised preparation budget is honored),
|
||||
// and the allowance covers the start deadline inside the launch goroutine.
|
||||
const orphanedSessionGraceAllowance = 5 * time.Minute
|
||||
|
||||
// orphanedSessionGracePeriod is how stale a STARTING/RUNNING session's last
|
||||
// write must be before the sweep may consider it unbacked. A session in those
|
||||
// states whose row is fresher than one full launch budget can still belong to
|
||||
// an in-flight launch whose execution has not reached the in-memory store
|
||||
// yet, so the sweep leaves it alone.
|
||||
func orphanedSessionGracePeriod() time.Duration {
|
||||
return constants.AgentLaunchTimeout + orphanedSessionGraceAllowance
|
||||
}
|
||||
|
||||
// orphanedSessionRepository is the narrow capability the orphan sweep needs
|
||||
// off its sessions repository. Kept as an optional interface — type-asserted
|
||||
// off s.sessions like taskWorkspaceMetadataCASSetter — rather than widening
|
||||
// repository.SessionRepository, whose many test doubles would each have to
|
||||
// grow methods this sweep never exercises through them. The concrete sqlite
|
||||
// repository is the only production implementer.
|
||||
type orphanedSessionRepository interface {
|
||||
ListStaleRunningSessionsOnUnarchivedTasks(ctx context.Context, staleBefore time.Time) ([]*models.TaskSession, error)
|
||||
CancelRunningTaskSessionByID(ctx context.Context, sessionID, reason string, staleBefore time.Time) (*models.TaskSession, error)
|
||||
}
|
||||
|
||||
// runOrphanedSessionReconciliation is pass 2 of the reconciliation loop: it
|
||||
// terminalizes stale STARTING/RUNNING sessions of unarchived tasks that no
|
||||
// live in-memory execution backs. Without it, a backend restart mid-turn
|
||||
// leaves those sessions RUNNING forever (#3711): the actor that would have
|
||||
// transitioned them died with the process, and no other path ever retries.
|
||||
// Sessions backed by a live execution — including survivors re-tracked by
|
||||
// startup recovery — are never touched, and the per-session cancel keeps
|
||||
// healthy sibling sessions of the same task (e.g. WAITING_FOR_INPUT) intact.
|
||||
// A session whose cancellation write fails is simply retried on the next
|
||||
// tick; the sweep never gives up for as long as the process runs.
|
||||
func (s *Service) runOrphanedSessionReconciliation(ctx context.Context) {
|
||||
if s.executionLivenessChecker == nil {
|
||||
// Absence-from-store is the sweep's only dead signal; without the
|
||||
// checker it can never prove a session unbacked, so the pass is inert.
|
||||
return
|
||||
}
|
||||
repo, ok := s.sessions.(orphanedSessionRepository)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
staleBefore := time.Now().UTC().Add(-orphanedSessionGracePeriod())
|
||||
candidates, err := repo.ListStaleRunningSessionsOnUnarchivedTasks(ctx, staleBefore)
|
||||
if err != nil {
|
||||
s.logger.Error("orphaned-session reconciliation: failed to list candidates", zap.Error(err))
|
||||
return
|
||||
}
|
||||
if len(candidates) == 0 {
|
||||
return
|
||||
}
|
||||
s.reconcileOrphanedSessions(ctx, candidates, staleBefore)
|
||||
}
|
||||
|
||||
func (s *Service) reconcileOrphanedSessions(ctx context.Context, candidates []*models.TaskSession, staleBefore time.Time) {
|
||||
deadline := archivecascade.ArchiveDeadline(ctx)
|
||||
reconcileCtx, cancel := archivecascade.ContinuationContextUntil(ctx, deadline)
|
||||
defer cancel()
|
||||
s.reconcileOrphanedSessionsUntil(reconcileCtx, candidates, staleBefore, deadline)
|
||||
}
|
||||
|
||||
// reconcileOrphanedSessionsUntil applies one orphan sweep within the supplied
|
||||
// admission deadline. A cancellation write may finish after that deadline
|
||||
// because the repository owns a detached write context; its post-commit
|
||||
// effects therefore receive a fresh bounded context instead of the expired
|
||||
// sweep context.
|
||||
func (s *Service) reconcileOrphanedSessionsUntil(
|
||||
ctx context.Context,
|
||||
candidates []*models.TaskSession,
|
||||
staleBefore time.Time,
|
||||
deadline time.Time,
|
||||
) {
|
||||
repo, ok := s.sessions.(orphanedSessionRepository)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
for _, session := range candidates {
|
||||
if session == nil || session.ID == "" {
|
||||
continue
|
||||
}
|
||||
if !time.Now().Before(deadline) {
|
||||
s.logger.Debug("orphaned-session reconciliation: admission deadline reached",
|
||||
zap.Time("deadline", deadline))
|
||||
return
|
||||
}
|
||||
// Re-check liveness at the moment of the write, not just at the
|
||||
// candidate read: a launch that raced the grace window since the read
|
||||
// must not be reaped. The staleBefore predicate inside the cancel
|
||||
// UPDATE is the second half of that guard — it re-asserts the
|
||||
// staleness cutoff so a row refreshed by an in-flight launch between
|
||||
// this check and the write no longer matches.
|
||||
if s.executionLivenessChecker.HasLiveExecution(session.ID) {
|
||||
continue
|
||||
}
|
||||
cancelled, err := repo.CancelRunningTaskSessionByID(ctx, session.ID, models.SessionOrphanedCancelReason, staleBefore)
|
||||
if err != nil {
|
||||
s.logger.Warn("orphaned-session reconciliation: failed to cancel session",
|
||||
zap.String("session_id", session.ID),
|
||||
zap.Error(err))
|
||||
continue
|
||||
}
|
||||
if cancelled == nil {
|
||||
// Raced to another state between the candidate read and the
|
||||
// write; its new owner is responsible for it now.
|
||||
continue
|
||||
}
|
||||
s.logger.Info("orphaned-session reconciliation: terminalized unbacked session",
|
||||
zap.String("task_id", session.TaskID),
|
||||
zap.String("session_id", session.ID),
|
||||
zap.String("previous_state", string(session.State)))
|
||||
completionDeadline := time.Now().Add(taskPublicationTimeout)
|
||||
completionCtx, cancelCompletion := context.WithDeadline(
|
||||
context.WithoutCancel(ctx), completionDeadline,
|
||||
)
|
||||
s.runCancelledSessionEffects(completionCtx, session.TaskID,
|
||||
[]*models.TaskSession{session}, []*models.TaskSession{cancelled},
|
||||
completionDeadline, models.SessionOrphanedCancelReason)
|
||||
cancelCompletion()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,399 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
sqliterepo "github.com/kandev/kandev/internal/task/repository/sqlite"
|
||||
|
||||
"github.com/kandev/kandev/internal/events"
|
||||
"github.com/kandev/kandev/internal/task/models"
|
||||
"github.com/kandev/kandev/internal/task/repository"
|
||||
)
|
||||
|
||||
// fakeExecutionLiveness is the test double for the orphan sweep's
|
||||
// execution-liveness seam: only session IDs in live map as backed by a live
|
||||
// in-memory execution.
|
||||
type fakeExecutionLiveness struct {
|
||||
live map[string]bool
|
||||
}
|
||||
|
||||
func (f *fakeExecutionLiveness) HasLiveExecution(sessionID string) bool {
|
||||
return f.live[sessionID]
|
||||
}
|
||||
|
||||
// seedOrphanSweepFixtures creates the one workspace and workflow every
|
||||
// orphan-sweep test task hangs off.
|
||||
func seedOrphanSweepFixtures(t *testing.T, repo *sqliterepo.Repository) {
|
||||
t.Helper()
|
||||
ctx := context.Background()
|
||||
if err := repo.CreateWorkspace(ctx, &models.Workspace{ID: "ws-orphan", Name: "Workspace"}); err != nil {
|
||||
t.Fatalf("CreateWorkspace: %v", err)
|
||||
}
|
||||
if err := repo.CreateWorkflow(ctx, &models.Workflow{ID: "wf-orphan", WorkspaceID: "ws-orphan", Name: "Workflow"}); err != nil {
|
||||
t.Fatalf("CreateWorkflow: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func seedOrphanSweepTask(t *testing.T, repo *sqliterepo.Repository, taskID string, archived bool) {
|
||||
t.Helper()
|
||||
ctx := context.Background()
|
||||
if err := repo.CreateTask(ctx, &models.Task{
|
||||
ID: taskID, WorkspaceID: "ws-orphan", WorkflowID: "wf-orphan", WorkflowStepID: "step-1",
|
||||
Title: "Test " + taskID, Priority: "medium",
|
||||
}); err != nil {
|
||||
t.Fatalf("CreateTask(%s): %v", taskID, err)
|
||||
}
|
||||
if archived {
|
||||
if _, err := repo.DB().ExecContext(ctx,
|
||||
`UPDATE tasks SET archived_at = ? WHERE id = ?`, time.Now().UTC(), taskID); err != nil {
|
||||
t.Fatalf("archive %s via SQL: %v", taskID, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestService_OrphanedSessionReconciliationTerminalizesUnbackedSessions(t *testing.T) {
|
||||
svc, eventBus, repo := createTestService(t)
|
||||
ctx := context.Background()
|
||||
seedOrphanSweepFixtures(t, repo)
|
||||
|
||||
stale := time.Now().UTC().Add(-30 * time.Minute)
|
||||
fresh := time.Now().UTC().Add(-1 * time.Minute)
|
||||
|
||||
seedOrphanSweepTask(t, repo, "task-orphaned", false)
|
||||
seedOrphanSweepTask(t, repo, "task-live-exec", false)
|
||||
seedOrphanSweepTask(t, repo, "task-inflight-launch", false)
|
||||
seedOrphanSweepTask(t, repo, "task-primary", false)
|
||||
seedOrphanSweepTask(t, repo, "task-archived", true)
|
||||
|
||||
seedSession := func(id, taskID string, state models.TaskSessionState, updatedAt time.Time) {
|
||||
t.Helper()
|
||||
if err := repo.CreateTaskSession(ctx, &models.TaskSession{
|
||||
ID: id, TaskID: taskID, State: state,
|
||||
AgentProfileID: "agent-1", IsPrimary: true, UpdatedAt: updatedAt, StartedAt: updatedAt,
|
||||
}); err != nil {
|
||||
t.Fatalf("CreateTaskSession(%s): %v", id, err)
|
||||
}
|
||||
}
|
||||
// The orphan this sweep exists for: unarchived task, stale RUNNING row,
|
||||
// nothing in the execution store backs it (backend restart mid-turn).
|
||||
seedSession("session-orphaned-running", "task-orphaned", models.TaskSessionStateRunning, stale)
|
||||
// Same orphan class in STARTING: the launch died with the process.
|
||||
seedSession("session-orphaned-starting", "task-orphaned", models.TaskSessionStateStarting, stale)
|
||||
// Healthy sibling of the orphaned task: waiting for user input, must
|
||||
// survive the sweep even though the task lost its RUNNING session.
|
||||
seedSession("session-healthy-sibling", "task-orphaned", models.TaskSessionStateWaitingForInput, stale)
|
||||
// Stale RUNNING row that a live execution still backs (e.g. re-tracked by
|
||||
// startup recovery): must never be terminalized.
|
||||
seedSession("session-live-execution", "task-live-exec", models.TaskSessionStateRunning, stale)
|
||||
// Fresh RUNNING row: an in-flight launch may not have reached the
|
||||
// in-memory store yet, so the grace window must protect it.
|
||||
seedSession("session-inflight-launch", "task-inflight-launch", models.TaskSessionStateRunning, fresh)
|
||||
// Archived task with a stale RUNNING session: the archived pass owns it,
|
||||
// the orphan pass must not touch it.
|
||||
seedSession("session-archived-task", "task-archived", models.TaskSessionStateRunning, stale)
|
||||
// Primary session on the orphaned task: the CANCELLED event must carry
|
||||
// is_primary=true so the status-summary projection keeps the durable
|
||||
// primary assignment.
|
||||
seedSession("session-primary", "task-primary", models.TaskSessionStateRunning, stale)
|
||||
|
||||
svc.SetExecutionLivenessChecker(&fakeExecutionLiveness{
|
||||
live: map[string]bool{"session-live-execution": true},
|
||||
})
|
||||
svc.runOrphanedSessionReconciliation(ctx)
|
||||
|
||||
assertState := func(id string, want models.TaskSessionState) {
|
||||
t.Helper()
|
||||
session, err := repo.GetTaskSession(ctx, id)
|
||||
if err != nil {
|
||||
t.Fatalf("GetTaskSession(%s): %v", id, err)
|
||||
}
|
||||
if session.State != want {
|
||||
t.Errorf("session %s state = %q, want %q", id, session.State, want)
|
||||
}
|
||||
}
|
||||
assertState("session-orphaned-running", models.TaskSessionStateCancelled)
|
||||
assertState("session-orphaned-starting", models.TaskSessionStateCancelled)
|
||||
assertState("session-healthy-sibling", models.TaskSessionStateWaitingForInput)
|
||||
assertState("session-live-execution", models.TaskSessionStateRunning)
|
||||
assertState("session-inflight-launch", models.TaskSessionStateRunning)
|
||||
assertState("session-archived-task", models.TaskSessionStateRunning)
|
||||
assertState("session-primary", models.TaskSessionStateCancelled)
|
||||
|
||||
orphaned, err := repo.GetTaskSession(ctx, "session-orphaned-running")
|
||||
if err != nil {
|
||||
t.Fatalf("GetTaskSession(orphaned): %v", err)
|
||||
}
|
||||
if orphaned.ErrorMessage != models.SessionOrphanedCancelReason {
|
||||
t.Errorf("orphaned session error_message = %q, want %q",
|
||||
orphaned.ErrorMessage, models.SessionOrphanedCancelReason)
|
||||
}
|
||||
|
||||
// The event must carry the durable is_primary flag: the primary session's
|
||||
// cancellation reports is_primary=true rather than the zero value.
|
||||
if data := sessionCancelledEvent(eventBus, "session-primary"); data != nil {
|
||||
if isPrimary, ok := data["is_primary"].(bool); !ok || !isPrimary {
|
||||
t.Errorf("primary session cancellation event is_primary = %v, want true", data["is_primary"])
|
||||
}
|
||||
} else {
|
||||
t.Error("expected a cancellation event for session-primary")
|
||||
}
|
||||
|
||||
for _, id := range []string{"session-orphaned-running", "session-orphaned-starting", "session-primary"} {
|
||||
if !sessionCancelledEventPublished(eventBus, id) {
|
||||
t.Errorf("expected a session.state_changed event for %s from the orphan sweep, got none", id)
|
||||
}
|
||||
}
|
||||
for _, id := range []string{"session-healthy-sibling", "session-live-execution", "session-inflight-launch", "session-archived-task"} {
|
||||
if sessionCancelledEventPublished(eventBus, id) {
|
||||
t.Errorf("unexpected session.state_changed event for %s", id)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func sessionCancelledEventPublished(eventBus *MockEventBus, sessionID string) bool {
|
||||
for _, evt := range eventBus.GetPublishedEvents() {
|
||||
if evt.Type != events.TaskSessionStateChanged {
|
||||
continue
|
||||
}
|
||||
data, ok := evt.Data.(map[string]interface{})
|
||||
if ok && data["session_id"] == sessionID && data["new_state"] == string(models.TaskSessionStateCancelled) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// sessionCancelledEvent returns the last session.state_changed payload for
|
||||
// sessionID, or nil when none published. The orphan sweep's event must carry
|
||||
// the durable is_primary flag — the status-summary projector demotes the
|
||||
// session when a cancellation event reports is_primary: false.
|
||||
func sessionCancelledEvent(eventBus *MockEventBus, sessionID string) map[string]interface{} {
|
||||
var found map[string]interface{}
|
||||
for _, evt := range eventBus.GetPublishedEvents() {
|
||||
if evt.Type != events.TaskSessionStateChanged {
|
||||
continue
|
||||
}
|
||||
data, ok := evt.Data.(map[string]interface{})
|
||||
if ok && data["session_id"] == sessionID && data["new_state"] == string(models.TaskSessionStateCancelled) {
|
||||
found = data
|
||||
}
|
||||
}
|
||||
return found
|
||||
}
|
||||
|
||||
// TestService_OrphanedSessionReconciliationRequiresLivenessChecker pins the
|
||||
// fail-safe: with no execution-liveness seam wired, the sweep must be inert —
|
||||
// absence-from-store is its only dead signal, and a nil checker can never
|
||||
// prove a session unbacked.
|
||||
func TestService_OrphanedSessionReconciliationRequiresLivenessChecker(t *testing.T) {
|
||||
svc, _, repo := createTestService(t)
|
||||
ctx := context.Background()
|
||||
seedOrphanSweepFixtures(t, repo)
|
||||
seedOrphanSweepTask(t, repo, "task-orphaned", false)
|
||||
if err := repo.CreateTaskSession(ctx, &models.TaskSession{
|
||||
ID: "session-orphaned", TaskID: "task-orphaned", State: models.TaskSessionStateRunning,
|
||||
AgentProfileID: "agent-1", IsPrimary: true,
|
||||
UpdatedAt: time.Now().UTC().Add(-30 * time.Minute), StartedAt: time.Now().UTC().Add(-30 * time.Minute),
|
||||
}); err != nil {
|
||||
t.Fatalf("CreateTaskSession: %v", err)
|
||||
}
|
||||
|
||||
// No SetExecutionLivenessChecker call.
|
||||
svc.runOrphanedSessionReconciliation(ctx)
|
||||
|
||||
session, err := repo.GetTaskSession(ctx, "session-orphaned")
|
||||
if err != nil {
|
||||
t.Fatalf("GetTaskSession: %v", err)
|
||||
}
|
||||
if session.State != models.TaskSessionStateRunning {
|
||||
t.Fatalf("session state = %q, want RUNNING (sweep must stay inert without the liveness seam)", session.State)
|
||||
}
|
||||
}
|
||||
|
||||
func TestService_OrphanedSessionReconciliationStopsAtAdmissionDeadline(t *testing.T) {
|
||||
svc, eventBus, repo := createTestService(t)
|
||||
setupCtx := context.Background()
|
||||
seedOrphanSweepFixtures(t, repo)
|
||||
seedOrphanSweepTask(t, repo, "task-expired-pass", false)
|
||||
stale := time.Now().UTC().Add(-30 * time.Minute)
|
||||
if err := repo.CreateTaskSession(setupCtx, &models.TaskSession{
|
||||
ID: "session-expired-pass", TaskID: "task-expired-pass", State: models.TaskSessionStateRunning,
|
||||
AgentProfileID: "agent-1", IsPrimary: true, UpdatedAt: stale, StartedAt: stale,
|
||||
}); err != nil {
|
||||
t.Fatalf("CreateTaskSession: %v", err)
|
||||
}
|
||||
candidate, err := repo.GetTaskSession(setupCtx, "session-expired-pass")
|
||||
if err != nil {
|
||||
t.Fatalf("GetTaskSession: %v", err)
|
||||
}
|
||||
svc.SetExecutionLivenessChecker(&fakeExecutionLiveness{})
|
||||
|
||||
expiredCtx, cancel := context.WithDeadline(context.Background(), time.Now().Add(-time.Second))
|
||||
defer cancel()
|
||||
svc.reconcileOrphanedSessions(expiredCtx, []*models.TaskSession{candidate}, stale)
|
||||
|
||||
session, err := repo.GetTaskSession(setupCtx, "session-expired-pass")
|
||||
if err != nil {
|
||||
t.Fatalf("GetTaskSession after expired pass: %v", err)
|
||||
}
|
||||
if session.State != models.TaskSessionStateRunning {
|
||||
t.Fatalf("session state = %q, want RUNNING after an expired pass", session.State)
|
||||
}
|
||||
if sessionCancelledEventPublished(eventBus, "session-expired-pass") {
|
||||
t.Fatal("expired pass must not publish a cancellation event")
|
||||
}
|
||||
}
|
||||
|
||||
type delayedOrphanSessionRepository struct {
|
||||
*sqliterepo.Repository
|
||||
entered chan struct{}
|
||||
release chan struct{}
|
||||
once sync.Once
|
||||
}
|
||||
|
||||
func (r *delayedOrphanSessionRepository) CancelRunningTaskSessionByID(
|
||||
ctx context.Context,
|
||||
sessionID, reason string,
|
||||
staleBefore time.Time,
|
||||
) (*models.TaskSession, error) {
|
||||
r.once.Do(func() { close(r.entered) })
|
||||
<-r.release
|
||||
return r.Repository.CancelRunningTaskSessionByID(ctx, sessionID, reason, staleBefore)
|
||||
}
|
||||
|
||||
func TestService_OrphanedSessionReconciliationUsesFreshEffectsContextAfterDeadline(t *testing.T) {
|
||||
delayed := &delayedOrphanSessionRepository{
|
||||
entered: make(chan struct{}),
|
||||
release: make(chan struct{}),
|
||||
}
|
||||
svc, eventBus, repo := createTestServiceWithSessionsRepo(t, func(repo *sqliterepo.Repository) repository.SessionRepository {
|
||||
delayed.Repository = repo
|
||||
return delayed
|
||||
})
|
||||
setupCtx := context.Background()
|
||||
seedOrphanSweepFixtures(t, repo)
|
||||
seedOrphanSweepTask(t, repo, "task-late-effects", false)
|
||||
stale := time.Now().UTC().Add(-30 * time.Minute)
|
||||
if err := repo.CreateTaskSession(setupCtx, &models.TaskSession{
|
||||
ID: "session-late-effects", TaskID: "task-late-effects", State: models.TaskSessionStateRunning,
|
||||
AgentProfileID: "agent-1", IsPrimary: true, UpdatedAt: stale, StartedAt: stale,
|
||||
}); err != nil {
|
||||
t.Fatalf("CreateTaskSession: %v", err)
|
||||
}
|
||||
candidate, err := repo.GetTaskSession(setupCtx, "session-late-effects")
|
||||
if err != nil {
|
||||
t.Fatalf("GetTaskSession: %v", err)
|
||||
}
|
||||
clarifications := &recordingTaskClarificationCanceller{}
|
||||
svc.SetClarificationCanceller(clarifications)
|
||||
svc.SetExecutionLivenessChecker(&fakeExecutionLiveness{})
|
||||
|
||||
passCtx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond)
|
||||
defer cancel()
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
svc.reconcileOrphanedSessions(passCtx, []*models.TaskSession{candidate}, stale.Add(time.Second))
|
||||
close(done)
|
||||
}()
|
||||
select {
|
||||
case <-delayed.entered:
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("orphan cancellation did not start")
|
||||
}
|
||||
select {
|
||||
case <-passCtx.Done():
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("reconciliation pass did not reach its deadline")
|
||||
}
|
||||
close(delayed.release)
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("reconciliation did not finish after the delayed write was released")
|
||||
}
|
||||
|
||||
if len(clarifications.contextErrs) != 1 || clarifications.contextErrs[0] != nil {
|
||||
t.Fatalf("clarification cleanup context errors = %v, want one nil error after a late commit", clarifications.contextErrs)
|
||||
}
|
||||
if !sessionCancelledEventPublished(eventBus, "session-late-effects") {
|
||||
t.Fatal("expected cancellation event after the delayed write committed")
|
||||
}
|
||||
}
|
||||
|
||||
// TestService_OrphanedSessionReconciliationSparesRowRefreshedSinceCandidateRead
|
||||
// pins the read-then-write guard: a launch CAS-writes its session to STARTING
|
||||
// (bumping updated_at) before it registers an execution in the in-memory
|
||||
// store, so the row can be refreshed between the sweep's candidate read and
|
||||
// its cancellation write. The cancel's staleBefore predicate must fail to
|
||||
// match that refreshed row instead of cancelling a launch in progress.
|
||||
func TestService_OrphanedSessionReconciliationSparesRowRefreshedSinceCandidateRead(t *testing.T) {
|
||||
svc, eventBus, repo := createTestService(t)
|
||||
ctx := context.Background()
|
||||
seedOrphanSweepFixtures(t, repo)
|
||||
seedOrphanSweepTask(t, repo, "task-relaunch", false)
|
||||
if err := repo.CreateTaskSession(ctx, &models.TaskSession{
|
||||
ID: "session-relaunch", TaskID: "task-relaunch", State: models.TaskSessionStateStarting,
|
||||
AgentProfileID: "agent-1", IsPrimary: true,
|
||||
UpdatedAt: time.Now().UTC().Add(-30 * time.Minute), StartedAt: time.Now().UTC().Add(-30 * time.Minute),
|
||||
}); err != nil {
|
||||
t.Fatalf("CreateTaskSession: %v", err)
|
||||
}
|
||||
|
||||
// Simulate the race: the candidate read sees the stale row, then an
|
||||
// in-flight launch refreshes it before the sweep's write lands.
|
||||
intercept := &livenessInterceptChecker{delegate: &fakeExecutionLiveness{}}
|
||||
svc.SetExecutionLivenessChecker(intercept)
|
||||
intercept.onCheck = func(sessionID string) {
|
||||
if sessionID != "session-relaunch" || intercept.refreshed {
|
||||
return
|
||||
}
|
||||
intercept.refreshed = true
|
||||
now := time.Now().UTC()
|
||||
if _, err := repo.DB().ExecContext(ctx,
|
||||
`UPDATE task_sessions SET state = ?, updated_at = ? WHERE id = ?`,
|
||||
string(models.TaskSessionStateStarting), now, sessionID); err != nil {
|
||||
t.Errorf("refresh session during liveness check: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
svc.runOrphanedSessionReconciliation(ctx)
|
||||
|
||||
session, err := repo.GetTaskSession(ctx, "session-relaunch")
|
||||
if err != nil {
|
||||
t.Fatalf("GetTaskSession: %v", err)
|
||||
}
|
||||
if session.State != models.TaskSessionStateStarting {
|
||||
t.Fatalf("session state = %q, want STARTING (a row refreshed since the candidate read must not be reaped)", session.State)
|
||||
}
|
||||
if sessionCancelledEventPublished(eventBus, "session-relaunch") {
|
||||
t.Error("unexpected session.state_changed event for the refreshed session")
|
||||
}
|
||||
}
|
||||
|
||||
// livenessInterceptChecker delegates to a fake liveness checker and lets a
|
||||
// test mutate the DB between the sweep's liveness check and its write,
|
||||
// reproducing the in-flight-launch race window.
|
||||
type livenessInterceptChecker struct {
|
||||
delegate *fakeExecutionLiveness
|
||||
refreshed bool
|
||||
onCheck func(sessionID string)
|
||||
}
|
||||
|
||||
func (l *livenessInterceptChecker) HasLiveExecution(sessionID string) bool {
|
||||
if l.onCheck != nil {
|
||||
l.onCheck(sessionID)
|
||||
}
|
||||
return l.delegate.HasLiveExecution(sessionID)
|
||||
}
|
||||
|
||||
// TestSessionOrphanedCancelReasonIsNotArchiveReason pins the reason taxonomy:
|
||||
// resume paths branch on IsArchiveCancelReason, so the orphan reason must
|
||||
// stay distinct from the archive reasons.
|
||||
func TestSessionOrphanedCancelReasonIsNotArchiveReason(t *testing.T) {
|
||||
if models.IsArchiveCancelReason(models.SessionOrphanedCancelReason) {
|
||||
t.Fatal("SessionOrphanedCancelReason must not be an archive cancel reason")
|
||||
}
|
||||
}
|
||||
@@ -179,6 +179,16 @@ type TaskRowLivenessProber interface {
|
||||
RowLiveness(row *models.ExecutorRunning) models.ProcessLiveness
|
||||
}
|
||||
|
||||
// TaskExecutionLivenessChecker reports whether a task session still has a
|
||||
// live agent execution backing it in the agent runtime's in-memory store.
|
||||
// Implementers must distinguish agent-owned executions from workspace-only
|
||||
// infrastructure and must answer from the store only — never lazily
|
||||
// (re)create an execution the way the GetOrEnsureExecution recovery
|
||||
// chokepoint does.
|
||||
type TaskExecutionLivenessChecker interface {
|
||||
HasLiveExecution(sessionID string) bool
|
||||
}
|
||||
|
||||
// TaskResourceCleanupActivityGate serializes durable cleanup with install-wide maintenance.
|
||||
type TaskResourceCleanupActivityGate interface {
|
||||
AcquireTaskResourceCleanup(context.Context) (TaskResourceCleanupActivityLease, error)
|
||||
@@ -481,6 +491,7 @@ type Service struct {
|
||||
parkedProjectionCanceller ParkedProjectionCanceller
|
||||
sessionCeilingReleaser SessionCeilingReleaser
|
||||
rowLivenessProber TaskRowLivenessProber
|
||||
executionLivenessChecker TaskExecutionLivenessChecker
|
||||
contextWindowResetter func(context.Context, string) error
|
||||
cleanupActivity TaskResourceCleanupActivityGate
|
||||
branchMaterializer BranchMaterializer
|
||||
@@ -882,6 +893,15 @@ func (s *Service) SetRowLivenessProber(prober TaskRowLivenessProber) {
|
||||
s.rowLivenessProber = prober
|
||||
}
|
||||
|
||||
// SetExecutionLivenessChecker wires the in-memory execution-store lookup
|
||||
// (satisfied by the lifecycle adapter) used by the orphan-session
|
||||
// reconciliation sweep. It is optional; when unwired the sweep is inert,
|
||||
// because absent-from-store is its only dead signal and a nil checker can
|
||||
// never prove a session unbacked.
|
||||
func (s *Service) SetExecutionLivenessChecker(checker TaskExecutionLivenessChecker) {
|
||||
s.executionLivenessChecker = checker
|
||||
}
|
||||
|
||||
// SetContextWindowResetter wires the guarded context-window reset callback
|
||||
// owned by the orchestrator. It is optional for isolated task-service users;
|
||||
// those callers fall back to clearing the session metadata directly.
|
||||
|
||||
@@ -2855,7 +2855,7 @@ func (s *Service) runCancelledSessionEffects(
|
||||
_, err := s.clarificationCanceller.ExpireSessionAndNotify(expireCtx, session.ID)
|
||||
cancelExpire()
|
||||
if err != nil {
|
||||
s.logger.Error("failed to expire clarification after archive cancellation; response claims remain quarantined",
|
||||
s.logger.Error("failed to expire clarification after session cancellation; response claims remain quarantined",
|
||||
zap.String("task_id", taskID),
|
||||
zap.String("session_id", session.ID),
|
||||
zap.Error(err))
|
||||
@@ -2872,7 +2872,7 @@ func (s *Service) runCancelledSessionEffects(
|
||||
cancelParked()
|
||||
}
|
||||
}
|
||||
// CancelActiveTaskSessionsByTaskID is a bulk writer: RETURNING reports every
|
||||
// The cancellation writers are bulk writers: RETURNING reports every
|
||||
// row's post-update CANCELLED state, not which were in an AC-1 state before
|
||||
// the update, so every returned id is released unconditionally rather than
|
||||
// branched on state (AC-51e). Releasing an id that held no reservation is a
|
||||
|
||||
@@ -0,0 +1,106 @@
|
||||
---
|
||||
created: 2026-09-20
|
||||
status: implemented
|
||||
requirements:
|
||||
- REQ-TASKS-RESTART-ORPHAN-SESSIONS-001
|
||||
system_design:
|
||||
- ../../specs/tasks/system-design/restart-orphaned-session-terminalization.md
|
||||
legacy_specs: []
|
||||
---
|
||||
|
||||
# Implementation Plan: Restart-Orphaned Session Terminalization
|
||||
|
||||
## Overview
|
||||
|
||||
A backend restart while an ACP prompt turn was open left `task_sessions`
|
||||
stuck in `STARTING`/`RUNNING` forever on unarchived tasks: the actor that
|
||||
would have transitioned them died with the process, and the existing 1-minute
|
||||
reconciliation sweep only covered archived tasks. Extend that same sweep with
|
||||
a second pass that terminalizes stale `STARTING`/`RUNNING` sessions of
|
||||
unarchived tasks when no live in-memory execution backs them, after a grace
|
||||
window of one launch budget so an in-flight launch is never reaped.
|
||||
|
||||
## Scope
|
||||
|
||||
### In scope
|
||||
|
||||
- A second pass in the existing reconciliation loop covering unarchived
|
||||
tasks' stale `STARTING`/`RUNNING` sessions.
|
||||
- Candidate cutoff derived from `constants.AgentLaunchTimeout` plus a 5-minute
|
||||
allowance.
|
||||
- An execution-liveness seam (`TaskExecutionLivenessChecker` /
|
||||
`HasLiveExecution`) answered from the in-memory store only.
|
||||
- Per-session cancellation (`CancelRunningTaskSessionByID`) with a dedicated
|
||||
`"orphaned by backend restart"` reason, distinct from archive reasons.
|
||||
- Shared post-cancellation effects extracted from the archive path
|
||||
(`notifyCancelledSessions`).
|
||||
- Service-level regression tests: unbacked sessions terminalized, live-backed
|
||||
and fresh rows untouched, healthy siblings survive, inert without the
|
||||
liveness seam.
|
||||
|
||||
### Out of scope
|
||||
|
||||
- Turn resume or transcript replay; drain-on-shutdown; agent-side heartbeats;
|
||||
manual SQL recovery; archived tasks (existing pass owns them).
|
||||
|
||||
## Technical approach
|
||||
|
||||
Reuse the archived pass's shape verbatim: same ticker, same continuation
|
||||
context, same retry-forever semantics. The candidate query filters in SQL so
|
||||
fresh rows are never loaded; the liveness re-check happens at write time so a
|
||||
launch racing the grace window is never reaped. Cancellation is session-scoped
|
||||
(`UPDATE ... RETURNING` on rows still in `STARTING`/`RUNNING`) so healthy
|
||||
sibling sessions survive. The `orphanedSessionRepository` capability keeps the
|
||||
two new repository methods off the `SessionRepository` interface and its test
|
||||
doubles.
|
||||
|
||||
## Tests
|
||||
|
||||
- `TestService_OrphanedSessionReconciliationTerminalizesUnbackedSessions`
|
||||
(`apps/backend/internal/task/service/orphaned_session_reconciliation_test.go`):
|
||||
covers AC-TASKS-RESTART-ORPHAN-SESSIONS-001.1, .2, .3, .4, .5, .8, and .10 —
|
||||
stale unbacked `RUNNING`/`STARTING` sessions reach `CANCELLED` with the
|
||||
orphan reason and a `session.state_changed` publish whose `is_primary`
|
||||
matches the durable flag; live-backed, fresh, archived-task, and
|
||||
`WAITING_FOR_INPUT` sibling rows stay untouched.
|
||||
- `TestService_OrphanedSessionReconciliationRequiresLivenessChecker`: covers
|
||||
AC-TASKS-RESTART-ORPHAN-SESSIONS-001.6 — without the liveness seam the sweep
|
||||
is inert.
|
||||
- `TestService_OrphanedSessionReconciliationSparesRowRefreshedSinceCandidateRead`:
|
||||
covers AC-TASKS-RESTART-ORPHAN-SESSIONS-001.9 — a row refreshed between the
|
||||
candidate read and the cancel write (the in-flight-launch race) is spared.
|
||||
- `TestSessionOrphanedCancelReasonIsNotArchiveReason`: covers
|
||||
AC-TASKS-RESTART-ORPHAN-SESSIONS-001.4's distinctness invariant.
|
||||
|
||||
## E2E tests
|
||||
|
||||
None: the sweep is a periodic backend behavior with a one-minute tick and a
|
||||
launch-budget grace window; the service-level test pins the full transition.
|
||||
A repro (start session, kill backend mid-turn, restart) is recorded in
|
||||
issue #3711's verification plan.
|
||||
|
||||
## Work orders
|
||||
|
||||
- [x] [Task 01: Terminalize restart-orphaned sessions via the reconciliation sweep](task-01-terminalize-restart-orphaned-sessions.md)
|
||||
|
||||
## Verification results
|
||||
|
||||
- `(cd apps/backend && go test ./internal/task/service -run 'TestService_OrphanedSessionReconciliation|TestService_ArchivedSessionReconciliation' -count=1)` — pass.
|
||||
- `(cd apps/backend && go test ./internal/task/repository/... -count=1)` — pass.
|
||||
- `(cd apps/backend && go run ./cmd/sqlguard ./internal)` — clean after rebinding the candidate query's placeholder.
|
||||
- `(cd apps/backend && gofmt -l internal/task internal/backendapp)` — no output; `go vet` clean.
|
||||
- `python3 scripts/lint-spec-files.py --all` and `python3 scripts/list-docs.py validate` — pass.
|
||||
- Full `go test ./internal/task/...`: three pre-existing failures
|
||||
(`TestDesktopDiscoveryRootPersists…`, `TestArchiveTaskCleanup…`,
|
||||
`TestTaskLifecycleCleanup_MissingWorktree`) reproduce identically on the
|
||||
base commit in a scratch worktree, so none is attributable to this change.
|
||||
|
||||
## Risks
|
||||
|
||||
- Reaping a live session whose execution legitimately left the store would
|
||||
lose work; the liveness re-check at write time and the fresh-row grace
|
||||
window bound that risk to zero in the steady state.
|
||||
- The sweep cannot distinguish "execution gone because restart" from
|
||||
"execution gone because of a runtime bookkeeping bug"; both are terminal
|
||||
sessions by the same argument — nothing will transition them — so
|
||||
terminalizing is correct in either case.
|
||||
+117
@@ -0,0 +1,117 @@
|
||||
---
|
||||
id: "01-terminalize-restart-orphaned-sessions"
|
||||
title: "Terminalize restart-orphaned sessions via the reconciliation sweep"
|
||||
status: done
|
||||
wave: 1
|
||||
depends_on: []
|
||||
plan: "plan.md"
|
||||
requirements:
|
||||
- REQ-TASKS-RESTART-ORPHAN-SESSIONS-001
|
||||
acceptance_criteria:
|
||||
- AC-TASKS-RESTART-ORPHAN-SESSIONS-001.1
|
||||
- AC-TASKS-RESTART-ORPHAN-SESSIONS-001.2
|
||||
- AC-TASKS-RESTART-ORPHAN-SESSIONS-001.3
|
||||
- AC-TASKS-RESTART-ORPHAN-SESSIONS-001.4
|
||||
- AC-TASKS-RESTART-ORPHAN-SESSIONS-001.5
|
||||
- AC-TASKS-RESTART-ORPHAN-SESSIONS-001.6
|
||||
- AC-TASKS-RESTART-ORPHAN-SESSIONS-001.7
|
||||
- AC-TASKS-RESTART-ORPHAN-SESSIONS-001.8
|
||||
- AC-TASKS-RESTART-ORPHAN-SESSIONS-001.9
|
||||
- AC-TASKS-RESTART-ORPHAN-SESSIONS-001.10
|
||||
system_design:
|
||||
- ../../specs/tasks/system-design/restart-orphaned-session-terminalization.md
|
||||
---
|
||||
|
||||
# Task 01: Terminalize Restart-Orphaned Sessions via the Reconciliation Sweep
|
||||
|
||||
## Summary
|
||||
|
||||
Extend the existing 1-minute task-session reconciliation sweep with a second
|
||||
pass: terminalize stale `STARTING`/`RUNNING` sessions of unarchived tasks that
|
||||
no live in-memory execution backs, so a backend restart mid-turn no longer
|
||||
leaves sessions stuck RUNNING forever (#3711).
|
||||
|
||||
## In scope
|
||||
|
||||
- `runOrphanedSessionReconciliation` / `reconcileOrphanedSessions` in the
|
||||
reconciliation loop file, gated on a wired `TaskExecutionLivenessChecker`.
|
||||
- Repository methods `ListStaleRunningSessionsOnUnarchivedTasks` and
|
||||
`CancelRunningTaskSessionByID` behind the `orphanedSessionRepository`
|
||||
capability.
|
||||
- `SessionOrphanedCancelReason` distinct from archive cancel reasons.
|
||||
- Extract `notifyCancelledSessions` from `finalizeCancelledSessions` so the
|
||||
orphan pass reuses the archive path's post-cancellation effects.
|
||||
- Wire `HasLiveExecution` in the lifecycle adapter and the orchestrator.
|
||||
|
||||
## Out of scope
|
||||
|
||||
- Archived tasks' sessions (the archived pass owns them).
|
||||
- Turn resume, drain-on-shutdown, heartbeats, adapter changes, schema
|
||||
changes.
|
||||
|
||||
## Acceptance
|
||||
|
||||
- After a backend restart mid-turn, an unbacked stale `STARTING`/`RUNNING`
|
||||
session reaches `CANCELLED` on the first sweep after the grace window, with
|
||||
the orphan reason and a `session.state_changed` publish, while live-backed,
|
||||
fresh, and archived-task rows stay untouched and healthy siblings survive.
|
||||
- Without the liveness seam the sweep does nothing.
|
||||
|
||||
## Verification
|
||||
|
||||
```bash
|
||||
(cd apps/backend && go test ./internal/task/service -run 'TestService_OrphanedSessionReconciliation|TestService_ArchivedSessionReconciliation' -count=1 -v)
|
||||
(cd apps/backend && go test ./internal/task/repository/... -count=1)
|
||||
(cd apps/backend && go run ./cmd/sqlguard ./internal)
|
||||
(cd apps/backend && gofmt -l internal/task internal/backendapp)
|
||||
(cd apps/backend && go vet ./internal/task/... ./internal/backendapp)
|
||||
python3 scripts/lint-spec-files.py --all
|
||||
python3 scripts/list-docs.py validate
|
||||
```
|
||||
|
||||
## Files likely touched
|
||||
|
||||
- `apps/backend/internal/task/service/archived_session_reconciliation.go`
|
||||
- `apps/backend/internal/task/service/orphaned_session_reconciliation_test.go`
|
||||
- `apps/backend/internal/task/service/service.go`
|
||||
- `apps/backend/internal/task/service/service_tasks.go`
|
||||
- `apps/backend/internal/task/models/resume_safety.go`
|
||||
- `apps/backend/internal/task/repository/sqlite/session.go`
|
||||
- `apps/backend/internal/backendapp/adapters.go`
|
||||
- `apps/backend/internal/backendapp/orchestrator.go`
|
||||
|
||||
## Dependencies
|
||||
|
||||
None.
|
||||
|
||||
## Risks
|
||||
|
||||
- The liveness checker must answer from the in-memory store only — a lazy
|
||||
creator would mask the sweep's dead signal.
|
||||
|
||||
## Parallelism
|
||||
|
||||
`sequential`
|
||||
|
||||
## Inputs
|
||||
|
||||
- `docs/specs/tasks/requirements/restart-orphaned-session-terminalization.md`
|
||||
- `docs/specs/tasks/system-design/restart-orphaned-session-terminalization.md`
|
||||
- `apps/backend/internal/task/service/archived_session_reconciliation.go`
|
||||
|
||||
## Results
|
||||
|
||||
Implemented in commit `aa7dc293b` (8 files, +458/−32): the second sweep pass,
|
||||
the two repository methods behind the `orphanedSessionRepository` capability,
|
||||
the orphan reason constant, the shared `notifyCancelledSessions` extraction,
|
||||
and the lifecycle-adapter `HasLiveExecution` wiring. Follow-up commits added
|
||||
the `Rebind` the SQL-portability gate requires, this specification package,
|
||||
and two review-fix hardenings: the cancel statement re-asserts the staleness
|
||||
cutoff (`updated_at < staleBefore`) so an in-flight launch refreshing its row
|
||||
between the sweep's liveness check and the write is never reaped
|
||||
(AC-.9, `TestService_OrphanedSessionReconciliationSparesRowRefreshedSinceCandidateRead`),
|
||||
and both cancellation RETURNING clauses now select `is_primary` so the
|
||||
published event carries the durable primary flag (AC-.10; this also repairs
|
||||
the pre-existing archive path, which had the same gap). Verification commands
|
||||
in the plan all pass; the failing packages in the full suite reproduce
|
||||
identically on the base commit and are environmental.
|
||||
@@ -93,6 +93,7 @@ signals, and task-scoped scheduling contracts.
|
||||
- [Task Terminal Persistence](requirements/task-terminal-persistence.md)
|
||||
- [Runtime Task-State Publication Order](requirements/runtime-state-publication-order.md)
|
||||
- [Session Delete Preserves Task Workspaces](requirements/session-delete-resource-cleanup.md)
|
||||
- [Restart-Orphaned Session Terminalization](requirements/restart-orphaned-session-terminalization.md)
|
||||
- [Sidebar Task Editing](requirements/sidebar-task-edit.md)
|
||||
- [Task Actions Menu on Preview and Detail Surfaces](requirements/task-actions-menu.md)
|
||||
- [Task menu grouping](requirements/task-menu-grouping.md)
|
||||
|
||||
@@ -0,0 +1,105 @@
|
||||
---
|
||||
status: draft
|
||||
system: tasks
|
||||
created: 2026-09-20
|
||||
owners:
|
||||
- kandev
|
||||
---
|
||||
|
||||
# Restart-Orphaned Session Terminalization Requirements
|
||||
|
||||
## Overview
|
||||
|
||||
A task session records the lifecycle of one agent conversation turn. When the
|
||||
backend process dies while a session's turn is open — a graceful restart, a
|
||||
crash, or a kill — the component that would have transitioned the session to a
|
||||
terminal state dies with it. No other actor retries that transition, so the
|
||||
session row stays in an active state forever and the task appears to wait for
|
||||
an agent that already finished.
|
||||
|
||||
The tasks system owns the task-session lifecycle, so it owns this
|
||||
reconciliation: the system itself must detect and terminalize sessions that
|
||||
lost their backing actor, without depending on the dead process, the agent, or
|
||||
a human SQL edit.
|
||||
|
||||
## Terminology
|
||||
|
||||
- **Active session state:** A `task_sessions.state` value that implies work is
|
||||
in flight: `CREATED`, `STARTING`, `RUNNING`, or `WAITING_FOR_INPUT`.
|
||||
- **Terminal session state:** A state a session never leaves: `COMPLETED`,
|
||||
`FAILED`, or `CANCELLED`.
|
||||
- **Orphaned session:** An active-state session whose transition actor is gone:
|
||||
the backend that would complete or fail its turn terminated, and no live
|
||||
agent execution backs the session anymore.
|
||||
- **Live execution:** An agent execution the current backend process tracks in
|
||||
its in-memory execution store. A session backed by a live execution is not
|
||||
orphaned even when its row is stale.
|
||||
- **Launch grace window:** The interval between a session row entering
|
||||
`STARTING` and its execution necessarily being visible in the in-memory
|
||||
store. A session inside this window is not eligible for orphan detection.
|
||||
- **Reconciliation sweep:** The periodic pass that lists candidate sessions
|
||||
and terminalizes the orphaned ones.
|
||||
|
||||
## Requirements
|
||||
|
||||
### REQ-TASKS-RESTART-ORPHAN-SESSIONS-001: Sessions orphaned by a backend restart reach a terminal state
|
||||
|
||||
**Intent:** A backend restart must not leave task sessions stuck in an active
|
||||
state forever. The system must self-heal: detect sessions whose backing actor
|
||||
died and terminalize them within a bounded time.
|
||||
|
||||
**User story:** As a Kandev user, I want a session that lost its agent process
|
||||
to a backend restart to show as stopped rather than stay "running" forever, so
|
||||
that the task board stays truthful and the task can move on.
|
||||
|
||||
#### Acceptance criteria
|
||||
|
||||
- **AC-TASKS-RESTART-ORPHAN-SESSIONS-001.1:** When a backend process
|
||||
terminates while a task session is in `STARTING` or `RUNNING`, and no live
|
||||
execution backs that session, the system shall transition that session to
|
||||
`CANCELLED` within one reconciliation sweep after the launch grace window
|
||||
elapses, without any user action.
|
||||
- **AC-TASKS-RESTART-ORPHAN-SESSIONS-001.2:** When a session in `STARTING` or
|
||||
`RUNNING` is backed by a live execution in the in-memory store, the system
|
||||
shall leave its state untouched, regardless of how stale its row is.
|
||||
- **AC-TASKS-RESTART-ORPHAN-SESSIONS-001.3:** When a session's last write is
|
||||
more recent than the launch grace window, the system shall not consider it
|
||||
orphaned, so an in-flight launch whose execution has not yet reached the
|
||||
in-memory store is never reaped.
|
||||
- **AC-TASKS-RESTART-ORPHAN-SESSIONS-001.4:** When the sweep terminalizes a
|
||||
session, the system shall record a cancellation reason that names the
|
||||
backend restart as the source, distinct from archive cancellations and user
|
||||
stops, so session history keeps the source visible.
|
||||
- **AC-TASKS-RESTART-ORPHAN-SESSIONS-001.5:** When the sweep terminalizes a
|
||||
session, the system shall run the same post-cancellation effects as an
|
||||
archive cancellation: expire terminal clarifications, clear parked
|
||||
projections, release session-ceiling reservations, and publish a
|
||||
`session.state_changed` event for each session actually cancelled.
|
||||
- **AC-TASKS-RESTART-ORPHAN-SESSIONS-001.6:** When the sweep cannot consult
|
||||
the in-memory execution store, it shall remain inert rather than
|
||||
terminalize sessions it cannot prove unbacked.
|
||||
- **AC-TASKS-RESTART-ORPHAN-SESSIONS-001.7:** When a cancellation write fails,
|
||||
the system shall retry it on the next sweep tick, and shall never give up on
|
||||
a session for as long as the process runs.
|
||||
- **AC-TASKS-RESTART-ORPHAN-SESSIONS-001.8:** When the sweep terminalizes one
|
||||
session of a task, the system shall leave the task's other sessions
|
||||
untouched, so healthy siblings (for example `WAITING_FOR_INPUT`) survive.
|
||||
- **AC-TASKS-RESTART-ORPHAN-SESSIONS-001.9:** When a session row is refreshed
|
||||
between the sweep's candidate read and its cancellation write (an in-flight
|
||||
launch CAS-writing `STARTING` before it registers an execution), the system
|
||||
shall not terminalize that session; the staleness cutoff is re-asserted at
|
||||
write time inside the same statement as the transition.
|
||||
- **AC-TASKS-RESTART-ORPHAN-SESSIONS-001.10:** When the sweep publishes a
|
||||
session cancellation event, the event shall carry the session's durable
|
||||
`is_primary` flag, so the status-summary projection keeps the durable
|
||||
primary-session assignment.
|
||||
|
||||
## Out of scope
|
||||
|
||||
- Resuming or replaying a turn that was interrupted mid-flight; the orphaned
|
||||
session is terminalized, not resumed.
|
||||
- Detecting orphans of already-archived tasks; the archived-task
|
||||
reconciliation pass owns those.
|
||||
- Agent-side heartbeats or adapter-side turn-state persistence.
|
||||
- Stopping a live agent process; the sweep only terminalizes DB rows of
|
||||
sessions with no live execution.
|
||||
@@ -0,0 +1,130 @@
|
||||
---
|
||||
status: current
|
||||
system: tasks
|
||||
requirements:
|
||||
- REQ-TASKS-RESTART-ORPHAN-SESSIONS-001
|
||||
---
|
||||
|
||||
# Restart-Orphaned Session Terminalization System Design
|
||||
|
||||
## Purpose and boundaries
|
||||
|
||||
The tasks system owns the task-session lifecycle, including the transition of
|
||||
a session into a terminal state. This design covers the periodic sweep that
|
||||
terminalizes `STARTING`/`RUNNING` sessions of unarchived tasks that no live
|
||||
in-memory agent execution backs — the residue of a backend process that died
|
||||
mid-turn.
|
||||
|
||||
It extends the existing reconciliation loop
|
||||
(`internal/task/service/archived_session_reconciliation.go`) with a second
|
||||
pass; it does not introduce a new loop, new persistence, or adapter changes.
|
||||
Adjacent contracts this design uses but does not own: the agent runtime's
|
||||
in-memory execution store (executors system), session-ceiling reservations
|
||||
(tasks system, reservation contract), and the archived-task reconciliation
|
||||
pass that owns archived tasks' stranded sessions.
|
||||
|
||||
## Requirement mapping
|
||||
|
||||
| Requirement | Design section |
|
||||
| --- | --- |
|
||||
| `REQ-TASKS-RESTART-ORPHAN-SESSIONS-001` | [Control flow](#control-flow), [Failure and recovery](#failure-and-recovery) |
|
||||
|
||||
## Components and responsibilities
|
||||
|
||||
- **Reconciliation loop** (`Service.StartArchivedSessionReconciliationLoop`):
|
||||
ticks every minute and runs both passes — the archived pass first, then the
|
||||
orphan pass.
|
||||
- **Orphan pass** (`Service.runOrphanedSessionReconciliation`): lists stale
|
||||
candidates, filters by live-execution absence, cancels each, runs
|
||||
post-cancellation effects.
|
||||
- **Session repository** (`sqlite.Repository`): candidate query
|
||||
`ListStaleRunningSessionsOnUnarchivedTasks` and per-session cancel
|
||||
`CancelRunningTaskSessionByID`. Not part of the `SessionRepository`
|
||||
interface; reached through the narrow `orphanedSessionRepository`
|
||||
capability so test doubles stay unchanged.
|
||||
- **Execution-liveness seam** (`TaskExecutionLivenessChecker`): reports
|
||||
whether a session has a live in-memory execution. Satisfied by the
|
||||
lifecycle adapter (`HasLiveExecution`), which answers from the in-memory
|
||||
store only and never lazily creates an execution.
|
||||
- **Shared post-cancellation effects** (`Service.notifyCancelledSessions`):
|
||||
extracted from the archive cancellation path and reused verbatim:
|
||||
clarification expiry, parked-projection clearing, ceiling release, and the
|
||||
`session.state_changed` publish.
|
||||
|
||||
## Data and contracts
|
||||
|
||||
- Candidate query joins `task_sessions` to `tasks` and selects rows where
|
||||
`ts.state IN ('STARTING','RUNNING')`, `ts.updated_at < staleBefore`, and
|
||||
`t.archived_at IS NULL`, ordered by `updated_at` ascending.
|
||||
- `staleBefore` is `now - (constants.AgentLaunchTimeout + 5m grace
|
||||
allowance)`: one full launch budget plus the start-deadline allowance, read
|
||||
(not copied) from `constants.AgentLaunchTimeout` so an operator-raised
|
||||
preparation budget is honored. The same cutoff is passed into the cancel
|
||||
statement and re-asserted as `updated_at < staleBefore` there, so a row
|
||||
refreshed between the candidate read and the write no longer matches.
|
||||
- `CancelRunningTaskSessionByID(sessionID, reason, staleBefore)` is an
|
||||
`UPDATE ... RETURNING` over rows still in `STARTING`/`RUNNING` whose
|
||||
`updated_at` still predates the cutoff; it returns `nil` when the session
|
||||
raced to another state or was refreshed, and the write detaches from the
|
||||
caller context with a 10-second timeout. The RETURNING clause selects
|
||||
`is_primary` alongside the other event-payload fields, so the published
|
||||
cancellation event carries the durable primary flag.
|
||||
- Cancellation reason constant: `models.SessionOrphanedCancelReason`
|
||||
(`"orphaned by backend restart"`), distinct from the archive reasons;
|
||||
`IsArchiveCancelReason` returns false for it.
|
||||
|
||||
## Control flow
|
||||
|
||||
1. Every tick, pass 2 runs after the archived pass.
|
||||
2. If no execution-liveness checker is wired, the pass is inert: absence
|
||||
from the store is the only dead signal, and a nil checker can never prove
|
||||
a session unbacked.
|
||||
3. List candidates with the stale-before cutoff computed from the launch
|
||||
budget.
|
||||
4. For each candidate, re-check `HasLiveExecution(session.ID)` at the moment
|
||||
of the write — a launch that raced the grace window since the read must
|
||||
not be reaped.
|
||||
5. Cancel via `CancelRunningTaskSessionByID`; the statement re-asserts both
|
||||
the state set (`STARTING`/`RUNNING`) and the staleness cutoff
|
||||
(`updated_at < staleBefore`), so a row refreshed by an in-flight launch
|
||||
between the liveness check and this write matches nothing; on `nil`
|
||||
(raced or refreshed) skip; on error warn and let the next tick retry.
|
||||
6. Run the shared post-cancellation effects with the orphan reason.
|
||||
|
||||
## Failure and recovery
|
||||
|
||||
- A failed candidate read logs an error and returns; the next tick retries.
|
||||
- A failed cancel logs a warn and continues to the next candidate; the next
|
||||
tick retries that session. The sweep never gives up for as long as the
|
||||
process runs.
|
||||
- The cancel write is context-detached (`context.WithoutCancel` + 10s bound),
|
||||
so the terminal transition survives a client disconnect and cannot block a
|
||||
sweep pass indefinitely on a locked writer.
|
||||
- Pass 2 is a no-op on clean trees: fresh rows are filtered by the grace
|
||||
window, live-backed rows by the liveness check, archived tasks' rows by the
|
||||
archived-pass filter, and terminal rows by the candidate query.
|
||||
|
||||
## Persistence
|
||||
|
||||
No schema change. The sweep writes only `task_sessions.state`,
|
||||
`error_message`, `completed_at`, and `updated_at`. Survivors re-tracked by
|
||||
startup recovery are in the in-memory store and stay untouched, so the sweep
|
||||
composes with agent-survival re-tracking rather than racing it.
|
||||
|
||||
## Security
|
||||
|
||||
No new trust boundary. The sweep runs in-process with the task service's own
|
||||
credentials and touches only `task_sessions` rows of unarchived tasks.
|
||||
|
||||
## Observability
|
||||
|
||||
- `task-session reconciliation loop started (every 1 minute)` at loop start.
|
||||
- `orphaned-session reconciliation: found candidates` / `terminalized
|
||||
unbacked session` (Info, with task and session IDs, previous state).
|
||||
- `failed to list candidates` / `failed to cancel session` (Error/Warn with
|
||||
the underlying error).
|
||||
|
||||
## Related decisions
|
||||
|
||||
- Issue #3711 and its approved fix plan: extend the existing sweep rather
|
||||
than drain-on-shutdown, turn-state persistence, heartbeats, or manual SQL.
|
||||
Reference in New Issue
Block a user