diff --git a/apps/backend/internal/backendapp/main.go b/apps/backend/internal/backendapp/main.go index 697ad7e7b7..9cee05b032 100644 --- a/apps/backend/internal/backendapp/main.go +++ b/apps/backend/internal/backendapp/main.go @@ -751,6 +751,7 @@ func startGatewayAndServe( } services.Task.StartAutoArchiveLoop(ctx) + services.Task.StartArchivedSessionReconciliationLoop(ctx) services.Task.StartQuickChatExpirationLoop(ctx) // ============================================ diff --git a/apps/backend/internal/orchestrator/executor/executor_mocks_test.go b/apps/backend/internal/orchestrator/executor/executor_mocks_test.go index 50df9d5d76..5f1c900d8e 100644 --- a/apps/backend/internal/orchestrator/executor/executor_mocks_test.go +++ b/apps/backend/internal/orchestrator/executor/executor_mocks_test.go @@ -608,6 +608,9 @@ func (m *mockRepository) ArchiveTask(ctx context.Context, id string) error { ret func (m *mockRepository) ListTasksForAutoArchive(ctx context.Context) ([]*models.Task, error) { return nil, nil } +func (m *mockRepository) ListArchivedTasksWithActiveSessions(ctx context.Context) ([]string, error) { + return nil, nil +} func (m *mockRepository) GetWorkspace(ctx context.Context, id string) (*models.Workspace, error) { return nil, nil diff --git a/apps/backend/internal/task/handlers/process_handlers_test.go b/apps/backend/internal/task/handlers/process_handlers_test.go index 6b06b32570..4b595fd956 100644 --- a/apps/backend/internal/task/handlers/process_handlers_test.go +++ b/apps/backend/internal/task/handlers/process_handlers_test.go @@ -83,6 +83,9 @@ func (m *mockRepository) ArchiveTask(ctx context.Context, id string) error { func (m *mockRepository) ListTasksForAutoArchive(ctx context.Context) ([]*models.Task, error) { return nil, nil } +func (m *mockRepository) ListArchivedTasksWithActiveSessions(ctx context.Context) ([]string, error) { + return nil, nil +} func (m *mockRepository) ListExpiredQuickChatTasks(ctx context.Context, cutoff time.Time) ([]*models.Task, error) { return nil, nil } @@ -297,8 +300,8 @@ func (m *mockRepository) ListActiveTaskSessions(ctx context.Context) ([]*models. func (m *mockRepository) ListActiveTaskSessionsByTaskID(ctx context.Context, taskID string) ([]*models.TaskSession, error) { return nil, nil } -func (m *mockRepository) CancelActiveTaskSessionsByTaskID(ctx context.Context, taskID, reason string) (int64, error) { - return 0, nil +func (m *mockRepository) CancelActiveTaskSessionsByTaskID(ctx context.Context, taskID, reason string) ([]*models.TaskSession, error) { + return nil, nil } func (m *mockRepository) HasActiveTaskSessionsByAgentProfile(ctx context.Context, agentProfileID string) (bool, error) { return false, nil diff --git a/apps/backend/internal/task/repository/interface.go b/apps/backend/internal/task/repository/interface.go index 0081e1ec66..e0760f3d84 100644 --- a/apps/backend/internal/task/repository/interface.go +++ b/apps/backend/internal/task/repository/interface.go @@ -52,6 +52,13 @@ type TaskRepository interface { // Returns whether the row was updated. UnarchiveTask(ctx context.Context, id string) (bool, error) ListTasksForAutoArchive(ctx context.Context) ([]*models.Task, error) + // ListArchivedTasksWithActiveSessions returns the IDs of archived tasks + // (archived_at IS NOT NULL) that still have at least one task_sessions + // row in an active DB state (CREATED/STARTING/RUNNING/WAITING_FOR_INPUT). + // Candidate list for the periodic reconciliation sweep that recovers + // sessions left stranded when finalizeCancelledSessions's bounded + // in-line retry was exhausted by sustained SQLite writer contention. + ListArchivedTasksWithActiveSessions(ctx context.Context) ([]string, error) ListExpiredQuickChatTasks(ctx context.Context, cutoff time.Time) ([]*models.Task, error) DeleteExpiredQuickChatTask(ctx context.Context, id string, cutoff time.Time) (bool, error) // CountOpenWatcherCreatedTasks returns the number of open watcher-created @@ -183,7 +190,7 @@ type SessionRepository interface { ListTaskSessions(ctx context.Context, taskID string) ([]*models.TaskSession, error) ListActiveTaskSessions(ctx context.Context) ([]*models.TaskSession, error) ListActiveTaskSessionsByTaskID(ctx context.Context, taskID string) ([]*models.TaskSession, error) - CancelActiveTaskSessionsByTaskID(ctx context.Context, taskID, reason string) (int64, error) + CancelActiveTaskSessionsByTaskID(ctx context.Context, taskID, reason string) ([]*models.TaskSession, error) HasActiveTaskSessionsByAgentProfile(ctx context.Context, agentProfileID string) (bool, error) GetActiveTaskInfoByAgentProfile(ctx context.Context, agentProfileID string) ([]agentdto.ActiveTaskInfo, error) HasActiveTaskSessionsByExecutor(ctx context.Context, executorID string) (bool, error) diff --git a/apps/backend/internal/task/repository/sqlite/session.go b/apps/backend/internal/task/repository/sqlite/session.go index 4fefc9d3d6..97955326ef 100644 --- a/apps/backend/internal/task/repository/sqlite/session.go +++ b/apps/backend/internal/task/repository/sqlite/session.go @@ -754,22 +754,112 @@ func completedAtForTaskSessionState(status models.TaskSessionState, now time.Tim // CancelActiveTaskSessionsByTaskID transitions every active session of a task // (CREATED/STARTING/RUNNING/WAITING_FOR_INPUT) to CANCELLED, returning the -// number of rows changed. The transition is a pure DB state change and does not -// require a live agent execution, making it the authoritative way to finalize a -// task's sessions independent of agent-process teardown. -func (r *Repository) CancelActiveTaskSessionsByTaskID(ctx context.Context, taskID, reason string) (int64, error) { +// full row of each session actually transitioned. The transition is a pure +// DB state change and does not require a live agent execution, making it +// the authoritative way to finalize a task's sessions independent of +// agent-process teardown. +// +// The UPDATE and the row selection happen in a single atomic statement via +// RETURNING, so the returned rows are exactly what this call changed — a +// session created or transitioned to active state concurrently, after this +// statement starts, is simply outside its snapshot; the task_id + state +// predicate is evaluated once per row as it commits, so no session matching +// it at commit time is missed by a separate pre-update snapshot. Callers use +// the returned sessions to publish a matching session.state_changed event +// per session — without this, clients that cache session state +// independently of the task (e.g. an Office task list's "is running" +// indicator) never learn the session left its active state and spin +// forever after the owning task is archived. +// +// The RETURNING clause carries every field publishSessionsCancelled needs +// to build its event payload directly, so callers never fall back to a +// separate post-commit read (e.g. GetTaskSession) to assemble the event — +// closing the read-after-write gap where a session could commit CANCELLED +// but its event never gets published because that follow-up read failed or +// timed out. Returned sessions therefore carry only the fields the +// RETURNING clause selects (ID, TaskID, AgentProfileID, +// AgentProfileSnapshot, IsPassthrough, Name, ReviewStatus, Metadata, +// TaskEnvironmentID, State, UpdatedAt) — every other models.TaskSession +// field is left at its zero value, and callers must not rely on fields +// outside this list being populated. +func (r *Repository) CancelActiveTaskSessionsByTaskID(ctx context.Context, taskID, reason string) ([]*models.TaskSession, error) { now := time.Now().UTC() - writeCtx := context.WithoutCancel(ctx) - result, err := r.db.ExecContext(writeCtx, r.db.Rebind(` + // Detach from ctx via WithoutCancel: this write must survive a client + // disconnect (see the doc comment above) so the CANCELLED transition and + // its returned rows are never lost to a caller that hung up mid-request. + // Bound it with a timeout so a locked SQLite UPDATE can't block forever + // on a request-independent DB stall just because the deadline was + // dropped. + writeCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second) + defer cancel() + rows, err := r.db.QueryContext(writeCtx, r.db.Rebind(` UPDATE task_sessions SET state = ?, error_message = ?, completed_at = ?, updated_at = ? WHERE task_id = ? AND state IN ('CREATED', 'STARTING', 'RUNNING', 'WAITING_FOR_INPUT') + RETURNING id, agent_profile_id, agent_profile_snapshot, is_passthrough, name, + review_status, metadata, task_environment_id, state, updated_at `), string(models.TaskSessionStateCancelled), reason, now, now, taskID) if err != nil { - return 0, err + return nil, err } - return result.RowsAffected() + defer func() { _ = rows.Close() }() + + var sessions []*models.TaskSession + for rows.Next() { + session, err := scanCancelledTaskSessionRow(rows, taskID) + if err != nil { + return nil, err + } + sessions = append(sessions, session) + } + return sessions, rows.Err() +} + +// scanCancelledTaskSessionRow scans one row produced by +// CancelActiveTaskSessionsByTaskID's UPDATE ... RETURNING into a +// *models.TaskSession, mirroring scanTaskSessionRow's JSON-unmarshal and +// int-to-bool/nullable-string conventions but for the narrower RETURNING +// column set (id, agent_profile_id, agent_profile_snapshot, is_passthrough, +// name, review_status, metadata, task_environment_id, state, updated_at). +// taskID backfills TaskID, which RETURNING cannot supply since it's a query +// parameter, not a returned column. +func scanCancelledTaskSessionRow(rows *sql.Rows, taskID string) (*models.TaskSession, error) { + session := &models.TaskSession{TaskID: taskID} + var state string + var metadataJSON string + var agentProfileSnapshotJSON string + var isPassthrough int + var reviewStatus sql.NullString + var agentProfileID sql.NullString + var name sql.NullString + + if err := rows.Scan( + &session.ID, &agentProfileID, &agentProfileSnapshotJSON, &isPassthrough, &name, + &reviewStatus, &metadataJSON, &session.TaskEnvironmentID, &state, &session.UpdatedAt, + ); err != nil { + return nil, err + } + + session.State = models.TaskSessionState(state) + session.IsPassthrough = isPassthrough == 1 + if reviewStatus.Valid { + session.ReviewStatus = models.ReviewStatus(reviewStatus.String) + } + if agentProfileID.Valid { + session.AgentProfileID = agentProfileID.String + } + if name.Valid { + session.Name = name.String + } + if err := unmarshalSessionJSON(metadataJSON, &session.Metadata, "agent session metadata"); err != nil { + return nil, err + } + if err := unmarshalSessionJSON(agentProfileSnapshotJSON, &session.AgentProfileSnapshot, "agent profile snapshot"); err != nil { + return nil, err + } + + return session, nil } // UpdateSessionMetadata updates only the metadata column of a session, diff --git a/apps/backend/internal/task/repository/sqlite/session_test.go b/apps/backend/internal/task/repository/sqlite/session_test.go index 2beaece63f..0755fc7ab9 100644 --- a/apps/backend/internal/task/repository/sqlite/session_test.go +++ b/apps/backend/internal/task/repository/sqlite/session_test.go @@ -17,6 +17,9 @@ import ( "github.com/stretchr/testify/require" ) +// newRepoForSessionTests creates a fresh SQLite-backed Repository in a +// temporary directory for use by session tests, closing the connection via +// t.Cleanup. func newRepoForSessionTests(t *testing.T) *Repository { t.Helper() dbPath := filepath.Join(t.TempDir(), "session-test.db") @@ -77,6 +80,10 @@ func insertAgentMsg(t *testing.T, repo *Repository, id, sessionID, turnID, autho } } +// TestRenameTaskSession verifies RenameTaskSession rejects a missing +// session, updates and clears the session name (with ToAPI omitting an +// empty name), and that a name set via CreateTaskSession round-trips +// through ListTaskSessions. func TestRenameTaskSession(t *testing.T) { repo := newRepoForSessionTests(t) ctx := context.Background() @@ -139,6 +146,12 @@ func TestRenameTaskSession(t *testing.T) { } } +// TestTaskSessionNotFoundErrorsAreTyped verifies that GetTaskSession, +// UpdateTaskSession, UpdateTaskSessionState, and UpdateTaskSessionBaseCommit +// all return ErrTaskSessionNotFound for a missing session, that +// GetTaskSessionByTaskAndAgent translates not-found to (nil, nil), and that +// GetPrimarySessionByTaskID returns ErrNoPrimarySession, while operations on +// an existing session succeed. func TestTaskSessionNotFoundErrorsAreTyped(t *testing.T) { repo := newRepoForSessionTests(t) ctx := context.Background() @@ -176,6 +189,10 @@ func TestTaskSessionNotFoundErrorsAreTyped(t *testing.T) { } } +// TestSetSessionMetadataKeyIfAbsentSQLiteIsWriteOnce verifies that on SQLite +// SetSessionMetadataKeyIfAbsent stores the value on first call and reports +// no write (leaving the original value in place) on a subsequent call with +// the same key. func TestSetSessionMetadataKeyIfAbsentSQLiteIsWriteOnce(t *testing.T) { repo := newRepoForSessionTests(t) seedForMsgTest(t, repo, "task-baseline", "session-baseline", "turn-baseline") @@ -206,6 +223,9 @@ func TestSetSessionMetadataKeyIfAbsentSQLiteIsWriteOnce(t *testing.T) { } } +// TestSetSessionMetadataKeyIfAbsentQueryUsesPostgresJSONB verifies the +// Postgres dialect's write-once query uses jsonb_set/jsonb_extract_path +// rather than SQLite's json_set/json_type/json functions. func TestSetSessionMetadataKeyIfAbsentQueryUsesPostgresJSONB(t *testing.T) { query := setSessionMetadataKeyIfAbsentQuery(dialect.PGX) if strings.Contains(query, "json_set") || strings.Contains(query, "json_type") || strings.Contains(query, "json(?)") { @@ -216,6 +236,10 @@ func TestSetSessionMetadataKeyIfAbsentQueryUsesPostgresJSONB(t *testing.T) { } } +// TestListTaskSessionWorktreesFiltersInactiveRows verifies that +// ListTaskSessionWorktrees and ListWorktreesBySessionIDs exclude worktree +// rows marked deleted via either the status column or a set deleted_at +// timestamp, returning only the still-active worktree. func TestListTaskSessionWorktreesFiltersInactiveRows(t *testing.T) { repo := newRepoForSessionTests(t) ctx := context.Background() @@ -281,6 +305,10 @@ func TestListTaskSessionWorktreesFiltersInactiveRows(t *testing.T) { } } +// TestUpdateTaskSessionWorktreeBranchByRepositoryScopesUpdate verifies +// UpdateTaskSessionWorktreeBranchByRepository updates the branch only for +// the worktree matching the given repository, leaving other repositories' +// worktrees on the same session unchanged. func TestUpdateTaskSessionWorktreeBranchByRepositoryScopesUpdate(t *testing.T) { repo := newRepoForSessionTests(t) ctx := context.Background() @@ -727,6 +755,8 @@ func insertSession(t *testing.T, repo *Repository, sessionID, taskID, state stri } } +// sessionState reads and returns the state column of the task_sessions row +// for the given session ID, failing the test if the row cannot be read. func sessionState(t *testing.T, repo *Repository, sessionID string) string { t.Helper() var state string @@ -737,6 +767,11 @@ func sessionState(t *testing.T, repo *Repository, sessionID string) string { return state } +// TestCancelActiveTaskSessionIsTerminalSafe verifies CancelActiveTaskSession +// transitions a RUNNING session to CANCELLED and returns the stored +// updated_at as the cancellation timestamp, while a session already in a +// terminal state (COMPLETED) is left unchanged and reported as not +// cancelled. func TestCancelActiveTaskSessionIsTerminalSafe(t *testing.T) { repo := newRepoForSessionTests(t) ctx := context.Background() @@ -776,6 +811,10 @@ func TestCancelActiveTaskSessionIsTerminalSafe(t *testing.T) { } } +// TestUpdateTaskSessionStateIfCurrentRejectsStaleActiveWriter verifies that +// after a session is cancelled, a conditional UpdateTaskSessionStateIfCurrent +// call from a writer still expecting the prior RUNNING state is rejected and +// the session remains CANCELLED. func TestUpdateTaskSessionStateIfCurrentRejectsStaleActiveWriter(t *testing.T) { repo := newRepoForSessionTests(t) ctx := context.Background() @@ -805,6 +844,11 @@ func TestUpdateTaskSessionStateIfCurrentRejectsStaleActiveWriter(t *testing.T) { } } +// TestUpdateTaskSessionIfCurrentStateRejectsStaleFullRowWriter verifies that +// after a session is cancelled out from under a full-row writer, a +// subsequent UpdateTaskSessionIfCurrentState call using the writer's stale +// in-memory state is rejected, leaving the session CANCELLED with its +// executor ID unset. func TestUpdateTaskSessionIfCurrentStateRejectsStaleFullRowWriter(t *testing.T) { repo := newRepoForSessionTests(t) ctx := context.Background() @@ -832,6 +876,10 @@ func TestUpdateTaskSessionIfCurrentStateRejectsStaleFullRowWriter(t *testing.T) require.Empty(t, stored.ExecutorID) } +// TestUpdateTaskSessionWithMetadataRejectsInvalidMetadataBeforeStateWrite +// verifies that UpdateTaskSessionWithMetadata returns an error and leaves +// both the session state and existing metadata untouched when the supplied +// metadata cannot be marshalled. func TestUpdateTaskSessionWithMetadataRejectsInvalidMetadataBeforeStateWrite(t *testing.T) { repo := newRepoForSessionTests(t) ctx := context.Background() @@ -862,6 +910,10 @@ func TestUpdateTaskSessionWithMetadataRejectsInvalidMetadataBeforeStateWrite(t * } } +// TestUpdateTaskSessionIfCurrentStateRemovingMetadataKeys verifies that +// UpdateTaskSessionIfCurrentStateRemovingMetadataKeys removes the requested +// metadata key while preserving other keys when the session's current state +// matches the expected state. func TestUpdateTaskSessionIfCurrentStateRemovingMetadataKeys(t *testing.T) { repo := newRepoForSessionTests(t) ctx := context.Background() @@ -886,6 +938,10 @@ func TestUpdateTaskSessionIfCurrentStateRemovingMetadataKeys(t *testing.T) { require.Equal(t, "newer", stored.Metadata["keep"]) } +// TestUpdateTaskSessionIfCurrentStateRemovingMetadataKeysStateMismatch +// verifies that UpdateTaskSessionIfCurrentStateRemovingMetadataKeys is a +// no-op, leaving all metadata untouched, when the expected state does not +// match the session's current state. func TestUpdateTaskSessionIfCurrentStateRemovingMetadataKeysStateMismatch(t *testing.T) { repo := newRepoForSessionTests(t) ctx := context.Background() @@ -910,6 +966,9 @@ func TestUpdateTaskSessionIfCurrentStateRemovingMetadataKeysStateMismatch(t *tes require.Equal(t, "untouched", stored.Metadata["keep"]) } +// TestDismissLastAgentErrorDoesNotOverwriteNewerError verifies that +// DismissLastAgentError ignores a dismiss request for a stale error once a +// newer error has been recorded, leaving the newer error undismissed. func TestDismissLastAgentErrorDoesNotOverwriteNewerError(t *testing.T) { repo := newRepoForSessionTests(t) ctx := context.Background() @@ -950,6 +1009,10 @@ func TestDismissLastAgentErrorDoesNotOverwriteNewerError(t *testing.T) { } } +// TestDismissLastAgentErrorMatchesEquivalentTimestampText verifies that +// DismissLastAgentError matches the stored error even when its occurred_at +// timestamp was persisted as RFC3339Nano text rather than the parsed +// time.Time value, and marks it dismissed. func TestDismissLastAgentErrorMatchesEquivalentTimestampText(t *testing.T) { repo := newRepoForSessionTests(t) ctx := context.Background() @@ -990,6 +1053,9 @@ func TestDismissLastAgentErrorMatchesEquivalentTimestampText(t *testing.T) { } } +// sessionCancellationMetadata reads and returns the error_message, +// completed_at, and updated_at columns of the task_sessions row for the +// given session ID. func sessionCancellationMetadata(t *testing.T, repo *Repository, sessionID string) (string, sql.NullTime, time.Time) { t.Helper() var errorMessage string @@ -1004,6 +1070,10 @@ func sessionCancellationMetadata(t *testing.T, repo *Repository, sessionID strin return errorMessage, completedAt, updatedAt } +// assertReapedSession asserts that the session was transitioned to +// CANCELLED with error_message "task archived", a completed_at timestamp +// set, and updated_at at or after reapedAfter, recording any mismatch as a +// test failure. func assertReapedSession(t *testing.T, repo *Repository, sessionID string, reapedAfter time.Time) { t.Helper() if got := sessionState(t, repo, sessionID); got != "CANCELLED" { @@ -1023,9 +1093,9 @@ func assertReapedSession(t *testing.T, repo *Repository, sessionID string, reape // TestCancelActiveTaskSessionsByTaskID verifies the archive reaper transitions // only the target task's still-active sessions to CANCELLED, leaves terminal -// sessions and other tasks untouched, and reports the rows changed. It also -// confirms the repo-delete guard reports the repository as free afterward — -// the end-to-end purpose of the reap. +// sessions and other tasks untouched, and reports exactly the sessions it +// changed. It also confirms the repo-delete guard reports the repository as +// free afterward — the end-to-end purpose of the reap. func TestCancelActiveTaskSessionsByTaskID(t *testing.T) { repo := newRepoForSessionTests(t) ctx := context.Background() @@ -1041,14 +1111,28 @@ func TestCancelActiveTaskSessionsByTaskID(t *testing.T) { seedRepoLink(t, repo, "ws-r", "repo-other", "task-other", "sess-o1", "RUNNING") reapedAfter := time.Now().UTC() - reaped, err := repo.CancelActiveTaskSessionsByTaskID(ctx, "task-r", "task archived") + sessions, err := repo.CancelActiveTaskSessionsByTaskID(ctx, "task-r", "task archived") if err != nil { t.Fatalf("unexpected error: %v", err) } - if reaped != 4 { - t.Errorf("expected 4 active sessions reaped, got %d", reaped) + wantIDs := []string{"sess-r1", "sess-r2", "sess-r3", "sess-r4"} + var gotIDs []string + for _, sess := range sessions { + gotIDs = append(gotIDs, sess.ID) + if sess.TaskID != "task-r" { + t.Errorf("session %s TaskID = %q, want task-r", sess.ID, sess.TaskID) + } + if sess.State != models.TaskSessionStateCancelled { + t.Errorf("session %s State = %q, want CANCELLED", sess.ID, sess.State) + } + if sess.UpdatedAt.Before(reapedAfter) { + t.Errorf("session %s UpdatedAt = %s, want >= %s", sess.ID, sess.UpdatedAt, reapedAfter) + } } - for _, sessionID := range []string{"sess-r1", "sess-r2", "sess-r3", "sess-r4"} { + if !sameStringSet(gotIDs, wantIDs) { + t.Errorf("cancelled ids = %v, want %v", gotIDs, wantIDs) + } + for _, sessionID := range wantIDs { assertReapedSession(t, repo, sessionID, reapedAfter) } if got := sessionState(t, repo, "sess-r5"); got != "COMPLETED" { @@ -1069,11 +1153,33 @@ func TestCancelActiveTaskSessionsByTaskID(t *testing.T) { } // Idempotent: a second call changes nothing. - reaped, err = repo.CancelActiveTaskSessionsByTaskID(ctx, "task-r", "task archived") + sessions, err = repo.CancelActiveTaskSessionsByTaskID(ctx, "task-r", "task archived") if err != nil { t.Fatalf("unexpected error on second call: %v", err) } - if reaped != 0 { - t.Errorf("expected 0 rows on idempotent re-run, got %d", reaped) + if len(sessions) != 0 { + t.Errorf("expected 0 sessions on idempotent re-run, got %v", sessions) } } + +// sameStringSet reports whether got and want contain the same strings, +// ignoring order and duplicates count-for-count — used to compare the set of +// cancelled session IDs against an expected set regardless of return order. +func sameStringSet(got, want []string) bool { + if len(got) != len(want) { + return false + } + set := make(map[string]int, len(want)) + for _, s := range want { + set[s]++ + } + for _, s := range got { + set[s]-- + } + for _, count := range set { + if count != 0 { + return false + } + } + return true +} diff --git a/apps/backend/internal/task/repository/sqlite/task.go b/apps/backend/internal/task/repository/sqlite/task.go index ac1d0f6e74..fa8c5afe41 100644 --- a/apps/backend/internal/task/repository/sqlite/task.go +++ b/apps/backend/internal/task/repository/sqlite/task.go @@ -1069,6 +1069,31 @@ func (r *Repository) ListTasksForAutoArchive(ctx context.Context) ([]*models.Tas return r.scanTasks(rows) } +// ListArchivedTasksWithActiveSessions returns the IDs of archived tasks that +// still have at least one task_sessions row in an active DB state +// (CREATED/STARTING/RUNNING/WAITING_FOR_INPUT). This is the candidate list +// for the periodic reconciliation sweep (see +// service.StartArchivedSessionReconciliationLoop): finalizeCancelledSessions +// bounds its session-cancellation retry to a handful of attempts, so +// sustained SQLite writer contention can exhaust it and leave an archived +// task's sessions stuck active with no session.state_changed event ever +// delivered. The sweep re-derives this list every pass, so a later attempt +// recovers the task once the contention clears. +func (r *Repository) ListArchivedTasksWithActiveSessions(ctx context.Context) ([]string, error) { + var ids []string + err := r.ro.SelectContext(ctx, &ids, ` + SELECT DISTINCT t.id + FROM tasks t + JOIN task_sessions ts ON ts.task_id = t.id + WHERE t.archived_at IS NOT NULL + AND ts.state IN ('CREATED', 'STARTING', 'RUNNING', 'WAITING_FOR_INPUT') + `) + if err != nil { + return nil, err + } + return ids, nil +} + // ListExpiredQuickChatTasks returns quick-chat tasks whose last task/session // activity is older than cutoff. Active sessions are excluded so in-use chats // are never deleted by the idle sweeper. diff --git a/apps/backend/internal/task/service/archived_session_reconciliation.go b/apps/backend/internal/task/service/archived_session_reconciliation.go new file mode 100644 index 0000000000..9f53cfb29f --- /dev/null +++ b/apps/backend/internal/task/service/archived_session_reconciliation.go @@ -0,0 +1,79 @@ +package service + +import ( + "context" + "time" + + "go.uber.org/zap" +) + +// StartArchivedSessionReconciliationLoop starts a background goroutine that +// periodically re-finalizes archived tasks whose sessions never made it to a +// terminal DB state. +// +// finalizeCancelledSessions (see service_tasks.go) already bounds its +// session-cancellation retry to a handful of fixed attempts inside +// ArchiveTask's own request: enough to ride out a brief SQLite writer-lock +// blip, but not unbounded. If SQLite's single writer stays occupied for +// longer than every attempt combined, that bounded retry gets exhausted +// after archived_at has already committed, and nothing else in the request +// path ever retries the transition. The archived task is then left with +// sessions stuck in an active DB state (CREATED/STARTING/RUNNING/ +// WAITING_FOR_INPUT) forever, and event-driven clients that key their +// "is running" indicators off session.state_changed never learn the +// sessions actually stopped. +// +// This sweep is the eventual-consistency backstop for that residual gap, +// following the exact periodic-sweep shape StartAutoArchiveLoop already +// uses: on a fixed interval, list every archived task that still has an +// active session, and re-invoke finalizeCancelledSessions for each. Calling +// finalizeCancelledSessions again is always safe — its underlying +// CancelActiveTaskSessionsByTaskID UPDATE only matches sessions still in an +// active state, so a task that was already fully reconciled (by ArchiveTask +// itself or by an earlier sweep pass) simply yields an empty cancelledSessions +// slice and is a no-op. As long as the process keeps running, a later pass +// retries once the SQLite contention that exhausted the in-line retry has +// cleared — unlike the fixed 3-attempt budget, this sweep never gives up. +func (s *Service) StartArchivedSessionReconciliationLoop(ctx context.Context) { + ticker := time.NewTicker(1 * time.Minute) + go func() { + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + s.runArchivedSessionReconciliation(ctx) + } + } + }() + s.logger.Info("archived-session reconciliation loop started (every 1 minute)") +} + +func (s *Service) runArchivedSessionReconciliation(ctx context.Context) { + taskIDs, err := s.tasks.ListArchivedTasksWithActiveSessions(ctx) + if err != nil { + s.logger.Error("archived-session reconciliation: failed to list candidates", zap.Error(err)) + return + } + if len(taskIDs) == 0 { + return + } + + s.logger.Info("archived-session reconciliation: found candidates", zap.Int("count", len(taskIDs))) + for _, taskID := range taskIDs { + activeSessions, err := s.sessions.ListActiveTaskSessionsByTaskID(ctx, taskID) + if err != nil { + s.logger.Warn("archived-session reconciliation: failed to list active sessions", + zap.String("task_id", taskID), + zap.Error(err)) + continue + } + if len(activeSessions) == 0 { + // Already reconciled by a concurrent pass or by ArchiveTask itself + // between the candidate list query and this read. + continue + } + s.finalizeCancelledSessions(ctx, taskID, activeSessions) + } +} diff --git a/apps/backend/internal/task/service/archived_session_reconciliation_test.go b/apps/backend/internal/task/service/archived_session_reconciliation_test.go new file mode 100644 index 0000000000..e75e98eea8 --- /dev/null +++ b/apps/backend/internal/task/service/archived_session_reconciliation_test.go @@ -0,0 +1,104 @@ +package service + +import ( + "context" + "testing" + + "github.com/kandev/kandev/internal/events" + "github.com/kandev/kandev/internal/events/bus" + "github.com/kandev/kandev/internal/task/models" + "github.com/kandev/kandev/internal/task/repository" + sqliterepo "github.com/kandev/kandev/internal/task/repository/sqlite" +) + +// TestService_ArchivedSessionReconciliationRecoversStrandedSessions is the +// regression test for the residual gap Greptile flagged on finalizeCancelledSessions: +// its bounded in-line retry (3 attempts x 10s) can still be exhausted by +// sustained SQLite writer contention, leaving an archived task's session +// stuck active in the DB with no session.state_changed event ever +// delivered. It reuses flakyCancelSessionRepository (defined in +// service_test.go) to fail all 3 in-line attempts during ArchiveTask, +// simulating exhausted retries, then calls runArchivedSessionReconciliation +// directly (the sweep's single run, not the ticker loop) once the +// underlying repository would succeed again, and asserts the stranded +// session is recovered: CANCELLED in the DB and carrying a published +// session.state_changed event. +func TestService_ArchivedSessionReconciliationRecoversStrandedSessions(t *testing.T) { + flaky := &flakyCancelSessionRepository{failuresLeft: 3} + svc, eventBus, repo := createTestServiceWithSessionsRepo(t, func(repo *sqliterepo.Repository) repository.SessionRepository { + flaky.Repository = repo + return flaky + }) + + ctx := context.Background() + if err := repo.CreateWorkspace(ctx, &models.Workspace{ID: "ws-1", Name: "Workspace"}); err != nil { + t.Fatalf("CreateWorkspace: %v", err) + } + if err := repo.CreateWorkflow(ctx, &models.Workflow{ID: "wf-1", WorkspaceID: "ws-1", Name: "Workflow"}); err != nil { + t.Fatalf("CreateWorkflow: %v", err) + } + if err := repo.CreateTask(ctx, &models.Task{ + ID: "task-stranded", WorkspaceID: "ws-1", WorkflowID: "wf-1", WorkflowStepID: "step-1", + Title: "Test", Priority: "medium", + }); err != nil { + t.Fatalf("CreateTask: %v", err) + } + if err := repo.CreateTaskSession(ctx, &models.TaskSession{ + ID: "session-stranded", TaskID: "task-stranded", State: models.TaskSessionStateRunning, + AgentProfileID: "agent-1", IsPrimary: true, + }); err != nil { + t.Fatalf("CreateTaskSession: %v", err) + } + + // Exhaust finalizeCancelledSessions's bounded in-line retry during + // ArchiveTask. ArchiveTask must still succeed: the session-cancellation + // failure is best-effort and never fails the archive itself. + if err := svc.ArchiveTask(ctx, "task-stranded"); err != nil { + t.Fatalf("ArchiveTask: %v", err) + } + if got, want := flaky.callCount(), 3; got != want { + t.Fatalf("CancelActiveTaskSessionsByTaskID call count = %d, want %d (all 3 in-line attempts exhausted)", got, want) + } + + session, err := repo.GetTaskSession(ctx, "session-stranded") + if err != nil { + t.Fatalf("GetTaskSession: %v", err) + } + if session.State != models.TaskSessionStateRunning { + t.Fatalf("session state after ArchiveTask = %q, want RUNNING (retries should be exhausted, leaving the session stranded)", session.State) + } + + // flaky.failuresLeft is now exhausted (0 remaining), so the sweep's call + // to CancelActiveTaskSessionsByTaskID succeeds. Call the sweep's + // single-run method directly, not the ticker loop. + svc.runArchivedSessionReconciliation(ctx) + + session, err = repo.GetTaskSession(ctx, "session-stranded") + if err != nil { + t.Fatalf("GetTaskSession after reconciliation: %v", err) + } + if session.State != models.TaskSessionStateCancelled { + t.Fatalf("session state after reconciliation = %q, want CANCELLED", session.State) + } + + var found *bus.Event + for _, evt := range eventBus.GetPublishedEvents() { + if evt.Type != events.TaskSessionStateChanged { + continue + } + data, ok := evt.Data.(map[string]interface{}) + if ok && data["session_id"] == "session-stranded" { + found = evt + } + } + if found == nil { + t.Fatal("expected a session.state_changed event for session-stranded from the reconciliation sweep, got none") + } + data := found.Data.(map[string]interface{}) + if got := data["new_state"]; got != string(models.TaskSessionStateCancelled) { + t.Errorf("new_state = %v, want CANCELLED", got) + } + if got := data["task_id"]; got != "task-stranded" { + t.Errorf("task_id = %v, want task-stranded", got) + } +} diff --git a/apps/backend/internal/task/service/handoff_workspace_test.go b/apps/backend/internal/task/service/handoff_workspace_test.go index 1b108a35bb..3c39672500 100644 --- a/apps/backend/internal/task/service/handoff_workspace_test.go +++ b/apps/backend/internal/task/service/handoff_workspace_test.go @@ -379,6 +379,10 @@ func (r *phase4TaskRepo) ListTasksForAutoArchive(context.Context) ([]*models.Tas r.panicNotUsed("ListTasksForAutoArchive") return nil, nil } +func (r *phase4TaskRepo) ListArchivedTasksWithActiveSessions(context.Context) ([]string, error) { + r.panicNotUsed("ListArchivedTasksWithActiveSessions") + return nil, nil +} func (r *phase4TaskRepo) ListExpiredQuickChatTasks(context.Context, time.Time) ([]*models.Task, error) { r.panicNotUsed("ListExpiredQuickChatTasks") return nil, nil diff --git a/apps/backend/internal/task/service/service_events.go b/apps/backend/internal/task/service/service_events.go index 015f3ec5c6..acf919eef0 100644 --- a/apps/backend/internal/task/service/service_events.go +++ b/apps/backend/internal/task/service/service_events.go @@ -39,6 +39,92 @@ func (s *Service) PublishTaskDeleted(ctx context.Context, task *models.Task) { s.publishTaskEvent(ctx, events.TaskDeleted, task, nil) } +// Field names shared by every session.state_changed publish in this file — +// extracted to satisfy goconst without borrowing unrelated constants. +const ( + sessionEventFieldTaskID = "task_id" + sessionEventFieldSessionID = "session_id" + sessionEventFieldUpdatedAt = "updated_at" + sessionEventFieldName = "name" +) + +// publishSessionsCancelled publishes a session.state_changed event for each +// session in cancelledSessions — the full rows CancelActiveTaskSessionsByTaskID's +// UPDATE ... RETURNING already produced. The event's old_state is a +// best-effort hint from snapshot (the pre-cancellation session list a +// caller already had in hand) — used only for logging/diagnostics — while +// every other field, including new_state, comes directly from the returned +// row. +// +// This used to re-read each session via GetTaskSession after the UPDATE +// committed, which raced: if that lookup failed or timed out after commit, +// the CANCELLED transition was permanent but its event was silently lost +// (see the now-closed "Cancelled events escape reconciliation" review +// thread, PR #1891 comment 3638052588). Building the payload straight from +// cancelledSessions closes that gap structurally — there is no longer a +// separate post-commit read to race, so a session in cancelledSessions can +// no longer "vanish" between the write and the publish. +// +// ctx is expected to be detached-but-unbounded (callers typically pass a +// context.WithoutCancel derivative with no deadline of its own): each +// session in cancelledSessions gets its own independent 10-second timeout +// around the Publish call below, rather than the whole batch sharing one +// deadline. That way a single slow synchronous event subscriber can only +// ever starve its own session's publish, not the events for every session +// that comes after it in cancelledSessions. +// +// CancelActiveTaskSessionsByTaskID is a repository-level DB write with no +// event of its own; without this, any client cache kept fresh exclusively by +// session.state_changed (e.g. an Office task list's "is running" indicator) +// never learns the session left RUNNING/WAITING_FOR_INPUT and spins forever. +func (s *Service) publishSessionsCancelled( + ctx context.Context, + taskID string, + snapshot []*models.TaskSession, + cancelledSessions []*models.TaskSession, + reason string, +) { + if s.eventBus == nil { + return + } + oldStateByID := make(map[string]models.TaskSessionState, len(snapshot)) + for _, sess := range snapshot { + if sess != nil { + oldStateByID[sess.ID] = sess.State + } + } + for _, sess := range cancelledSessions { + sessCtx, cancel := context.WithTimeout(ctx, 10*time.Second) + data := map[string]interface{}{ + sessionEventFieldTaskID: taskID, + sessionEventFieldSessionID: sess.ID, + "old_state": string(oldStateByID[sess.ID]), + "new_state": string(sess.State), + "error_message": reason, + "agent_profile_id": sess.AgentProfileID, + "agent_profile_snapshot": sess.AgentProfileSnapshot, + "is_passthrough": sess.IsPassthrough, + sessionEventFieldUpdatedAt: sess.UpdatedAt.Format(time.RFC3339Nano), + sessionEventFieldName: sess.Name, + } + if sess.ReviewStatus != models.ReviewStatusNone { + data["review_status"] = string(sess.ReviewStatus) + } + if len(sess.Metadata) > 0 { + data["session_metadata"] = sess.Metadata + } + if sess.TaskEnvironmentID != "" { + data["task_environment_id"] = sess.TaskEnvironmentID + } + event := bus.NewEvent(events.TaskSessionStateChanged, "task-service", data) + if err := s.eventBus.Publish(sessCtx, events.TaskSessionStateChanged, event); err != nil { + s.logger.Error("failed to publish session cancellation event", + zap.String(sessionEventFieldTaskID, taskID), zap.String(sessionEventFieldSessionID, sess.ID), zap.Error(err)) + } + cancel() + } +} + // publishTaskEvent publishes task events to the event bus func (s *Service) publishTaskEvent(ctx context.Context, eventType string, task *models.Task, oldState *v1.TaskState, oldWorkflowIDs ...string) { s.publishTaskEventWithExtra(ctx, eventType, task, oldState, nil, oldWorkflowIDs...) diff --git a/apps/backend/internal/task/service/service_tasks.go b/apps/backend/internal/task/service/service_tasks.go index 6bc6521590..f24eb8b719 100644 --- a/apps/backend/internal/task/service/service_tasks.go +++ b/apps/backend/internal/task/service/service_tasks.go @@ -1176,34 +1176,34 @@ func (s *Service) ArchiveTask(ctx context.Context, id string) error { // that raced in after the snapshot and must clean itself up. s.registerTaskRuntimeStopOwners(stopTargets, true) - // 3b. Finalize active sessions in the DB. The async cleanup below tears down - // the agent processes; this records the terminal session state, which - // process teardown does not persist on its own. - if reaped, rerr := s.sessions.CancelActiveTaskSessionsByTaskID(ctx, id, "task archived"); rerr != nil { - s.logger.Warn("failed to reap active sessions on archive", - zap.String("task_id", id), - zap.Error(rerr)) - } else if reaped > 0 { - s.logger.Info("reaped active sessions on archive", - zap.String("task_id", id), - zap.Int64("count", reaped)) - } + // archived_at has committed above: the rest of this function only + // reports on that already-durable state (cancelling sessions, re-reading + // the task, publishing task.updated) or kicks off cleanup that already + // runs detached from ctx. A client disconnecting right after the write + // above must not stop any of that from finishing and reaching + // event-driven clients that don't share the archiving caller's + // connection. + finalizeCtx := context.WithoutCancel(ctx) + + // 3b. Finalize active sessions in the DB and publish their cancellation + // events. See finalizeCancelledSessions for the detailed rationale. + s.finalizeCancelledSessions(finalizeCtx, id, activeSessions) // 4. Re-read task for updated archived_at field - task, err = s.tasks.GetTask(ctx, id) + task, err = s.tasks.GetTask(finalizeCtx, id) if err != nil { return err } // 5. Publish task.updated event so frontend removes from board - s.publishTaskEvent(ctx, events.TaskUpdated, task, nil) + s.publishTaskEvent(finalizeCtx, events.TaskUpdated, task, nil) s.logger.Info("task archived", zap.String("task_id", id), zap.Duration("duration", time.Since(start))) // 6. Background: Stop agents and cleanup worktrees if cleanupJob != nil { - if err := s.StartPreparedTaskResourceCleanup(ctx, cleanupJob.OperationID); err != nil { + if err := s.StartPreparedTaskResourceCleanup(finalizeCtx, cleanupJob.OperationID); err != nil { s.logger.Warn("start committed archive resource cleanup", zap.String("job_id", cleanupJob.ID), zap.String("task_id", id), zap.Error(err)) } @@ -1215,6 +1215,69 @@ func (s *Service) ArchiveTask(ctx context.Context, id string) error { return nil } +// finalizeCancelledSessions finalizes an archived task's active sessions in +// the DB and publishes a session.state_changed event for each one actually +// cancelled. The async cleanup that follows tears down the agent processes; +// this records the terminal session state, which process teardown does not +// persist on its own. The DB write alone is invisible to any client cache +// (e.g. an Office task list's "is running" indicator) that's kept fresh +// exclusively by that event, and would otherwise show a live spinner +// forever after archive. +// +// CancelActiveTaskSessionsByTaskID is bounded by its own internal 10s +// timeout, so a single attempt can time out under SQLite writer contention +// alone — with archived_at already committed by the caller, that would +// silently leave this task's sessions stuck in an active DB state forever, +// since nothing else ever re-attempts the cancellation and no +// session.state_changed event is delivered to clear a client's running +// indicator. Retrying a small, fixed number of times — each attempt getting +// its own fresh 10s budget from the repository — meaningfully shrinks that +// window without making it unbounded. This is deliberately NOT a background +// reconciliation/sweep system, just closing the immediate race; if every +// attempt still fails, ArchiveTask has already committed archived_at, so +// this remains best-effort cleanup and is not a reason to fail the archive. +func (s *Service) finalizeCancelledSessions(ctx context.Context, taskID string, activeSessions []*models.TaskSession) { + const maxCancelAttempts = 3 + const cancelRetryBackoff = 250 * time.Millisecond + + var cancelledSessions []*models.TaskSession + var cancelErr error + for attempt := 1; attempt <= maxCancelAttempts; attempt++ { + cancelledSessions, cancelErr = s.sessions.CancelActiveTaskSessionsByTaskID(ctx, taskID, "task archived") + if cancelErr == nil { + break + } + if attempt < maxCancelAttempts { + time.Sleep(cancelRetryBackoff) + } + } + if cancelErr != nil { + s.logger.Error("failed to reap active sessions on archive after retries", + zap.String("task_id", taskID), + zap.Int("attempts", maxCancelAttempts), + zap.Error(cancelErr)) + return + } + if len(cancelledSessions) == 0 { + return + } + s.logger.Info("reaped active sessions on archive", + zap.String("task_id", taskID), + zap.Int("count", len(cancelledSessions))) + // Detach from ctx via WithoutCancel: the DB write above already + // committed on a detached context, so a client disconnect here must + // not also suppress the event publish below — event-driven clients + // need session.state_changed regardless of whether the archiving + // caller is still connected. + // Deliberately left unbounded (no timeout) here: publishSessionsCancelled + // gives each session in cancelledSessions its own independent 10s + // deadline around its Publish call, so one slow synchronous subscriber + // can no longer consume a shared batch-wide budget and starve the + // events for sessions later in the loop. + detachedCtx := context.WithoutCancel(ctx) + s.publishSessionsCancelled(detachedCtx, taskID, activeSessions, cancelledSessions, "task archived") +} + func (s *Service) registerTaskRuntimeStopOwners(stopTargets []taskStopTarget, force bool) { if s.executionStopper == nil { return diff --git a/apps/backend/internal/task/service/service_test.go b/apps/backend/internal/task/service/service_test.go index a8f474d313..0c4ca906bb 100644 --- a/apps/backend/internal/task/service/service_test.go +++ b/apps/backend/internal/task/service/service_test.go @@ -15,6 +15,7 @@ import ( "github.com/kandev/kandev/internal/agentruntime" "github.com/kandev/kandev/internal/common/logger" "github.com/kandev/kandev/internal/db" + "github.com/kandev/kandev/internal/events" "github.com/kandev/kandev/internal/events/bus" officesqlite "github.com/kandev/kandev/internal/office/repository/sqlite" "github.com/kandev/kandev/internal/task/models" @@ -79,6 +80,20 @@ func (m *MockEventBus) ClearEvents() { } func createTestService(t *testing.T) (*Service, *MockEventBus, *sqliterepo.Repository) { + t.Helper() + return createTestServiceWithSessionsRepo(t, func(repo *sqliterepo.Repository) repository.SessionRepository { + return repo + }) +} + +// createTestServiceWithSessionsRepo mirrors createTestService but lets a +// caller substitute the Sessions repository (e.g. to wrap it with a test-only +// hook) while reusing the same DB setup, migrations, and cleanup-worker +// wiring for every other field. +func createTestServiceWithSessionsRepo( + t *testing.T, + wrapSessions func(*sqliterepo.Repository) repository.SessionRepository, +) (*Service, *MockEventBus, *sqliterepo.Repository) { t.Helper() tmpDir := t.TempDir() dbConn, err := db.OpenSQLite(filepath.Join(tmpDir, "test.db")) @@ -118,7 +133,7 @@ func createTestService(t *testing.T) (*Service, *MockEventBus, *sqliterepo.Repos Workflows: repo, Messages: repo, Turns: repo, - Sessions: repo, + Sessions: wrapSessions(repo), GitSnapshots: repo, RepoEntities: repo, Executors: repo, @@ -1868,6 +1883,413 @@ func TestService_ArchiveTaskStopsExecutorRunningForTerminalSession(t *testing.T) } } +// TestService_ArchiveTaskPublishesSessionStateChangedForActiveSessions is the +// regression test for the stuck-spinner bug: archiving a task with an active +// session cancels that session in the DB (verified elsewhere), but any client +// cache kept fresh exclusively by session.state_changed — e.g. an Office task +// list's "is running" indicator — never learns about it unless ArchiveTask +// also publishes the event. Without the fix, the archived task's session +// would sit at RUNNING in every client cache forever. +func TestService_ArchiveTaskPublishesSessionStateChangedForActiveSessions(t *testing.T) { + svc, eventBus, repo := createTestService(t) + ctx := context.Background() + + if err := repo.CreateWorkspace(ctx, &models.Workspace{ID: "ws-1", Name: "Workspace"}); err != nil { + t.Fatalf("CreateWorkspace: %v", err) + } + if err := repo.CreateWorkflow(ctx, &models.Workflow{ID: "wf-1", WorkspaceID: "ws-1", Name: "Workflow"}); err != nil { + t.Fatalf("CreateWorkflow: %v", err) + } + if err := repo.CreateTask(ctx, &models.Task{ + ID: "task-1", WorkspaceID: "ws-1", WorkflowID: "wf-1", WorkflowStepID: "step-1", + Title: "Test", Priority: "medium", + }); err != nil { + t.Fatalf("CreateTask: %v", err) + } + if err := repo.CreateTaskSession(ctx, &models.TaskSession{ + ID: "session-running", TaskID: "task-1", State: models.TaskSessionStateRunning, + AgentProfileID: "agent-1", IsPrimary: true, + }); err != nil { + t.Fatalf("CreateTaskSession: %v", err) + } + + if err := svc.ArchiveTask(ctx, "task-1"); err != nil { + t.Fatalf("ArchiveTask: %v", err) + } + + var found *bus.Event + for _, evt := range eventBus.GetPublishedEvents() { + if evt.Type != events.TaskSessionStateChanged { + continue + } + data, ok := evt.Data.(map[string]interface{}) + if !ok || data["session_id"] != "session-running" { + continue + } + found = evt + } + if found == nil { + t.Fatal("expected a session.state_changed event for session-running, got none") + } + data := found.Data.(map[string]interface{}) + if got := data["old_state"]; got != string(models.TaskSessionStateRunning) { + t.Errorf("old_state = %v, want RUNNING", got) + } + if got := data["new_state"]; got != string(models.TaskSessionStateCancelled) { + t.Errorf("new_state = %v, want CANCELLED", got) + } + if got := data["task_id"]; got != "task-1" { + t.Errorf("task_id = %v, want task-1", got) + } + if got := data["agent_profile_id"]; got != "agent-1" { + t.Errorf("agent_profile_id = %v, want agent-1", got) + } + updatedAtStr, _ := data["updated_at"].(string) + if _, err := time.Parse(time.RFC3339Nano, updatedAtStr); err != nil { + t.Errorf("updated_at %q not parseable as RFC3339Nano: %v", updatedAtStr, err) + } + + session, err := repo.GetTaskSession(ctx, "session-running") + if err != nil { + t.Fatalf("GetTaskSession: %v", err) + } + if session.State != models.TaskSessionStateCancelled { + t.Errorf("session state = %q, want CANCELLED", session.State) + } +} + +// TestService_PublishSessionsCancelledCoversSessionsMissingFromSnapshot is a +// defensive-coverage test: CancelActiveTaskSessionsByTaskID re-evaluates +// active sessions atomically and returns full session rows directly, so a +// returned session can have no matching entry in a caller-supplied snapshot +// taken moments earlier. publishSessionsCancelled must build the event +// entirely from that returned row — old_state is the only field allowed to +// fall back to an empty best-effort hint when snapshot has nothing for the +// session's ID; every other field, including agent_profile_id, must come +// from the row cancelledSessions already carries, not be silently dropped +// or fabricated. +func TestService_PublishSessionsCancelledCoversSessionsMissingFromSnapshot(t *testing.T) { + svc, eventBus, _ := createTestService(t) + ctx := context.Background() + + // The exact shape of the race: CancelActiveTaskSessionsByTaskID returned + // this session's row, but the caller's pre-cancel snapshot is empty. + cancelledSession := &models.TaskSession{ + ID: "session-not-in-snapshot", + TaskID: "task-race", + State: models.TaskSessionStateCancelled, + AgentProfileID: "agent-1", + UpdatedAt: time.Now().UTC(), + } + + svc.publishSessionsCancelled(ctx, "task-race", nil, []*models.TaskSession{cancelledSession}, "task archived") + + var found *bus.Event + for _, evt := range eventBus.GetPublishedEvents() { + if evt.Type != events.TaskSessionStateChanged { + continue + } + data, ok := evt.Data.(map[string]interface{}) + if ok && data["session_id"] == "session-not-in-snapshot" { + found = evt + } + } + if found == nil { + t.Fatal("expected a session.state_changed event for the snapshot-missing session, got none") + } + data := found.Data.(map[string]interface{}) + if got := data["new_state"]; got != string(models.TaskSessionStateCancelled) { + t.Errorf("new_state = %v, want CANCELLED", got) + } + if got := data["old_state"]; got != "" { + t.Errorf("old_state = %v, want empty (no snapshot hint available)", got) + } + if got := data["agent_profile_id"]; got != "agent-1" { + t.Errorf("agent_profile_id = %v, want agent-1 (must come from the returned row, not the snapshot)", got) + } +} + +// cancelHookSessionRepository wraps the real session repository so a test +// can deterministically simulate a client disconnecting the instant after +// CancelActiveTaskSessionsByTaskID's DB write commits — the exact race +// finalizeCancelledSessions guards against by publishing on a +// context.WithoutCancel derivative. Test-only: production code has no +// equivalent hook. +type cancelHookSessionRepository struct { + *sqliterepo.Repository + afterCancelSessions func() +} + +func (r *cancelHookSessionRepository) CancelActiveTaskSessionsByTaskID( + ctx context.Context, taskID, reason string, +) ([]*models.TaskSession, error) { + cancelledSessions, err := r.Repository.CancelActiveTaskSessionsByTaskID(ctx, taskID, reason) + if r.afterCancelSessions != nil { + r.afterCancelSessions() + } + return cancelledSessions, err +} + +// TestService_ArchiveTaskPublishesEventAfterClientDisconnect is the +// regression test for the client-disconnect race: finalizeCancelledSessions +// commits the session-cancellation DB write on the caller's ctx, then +// publishes session.state_changed on a context.WithoutCancel derivative +// specifically so that a client disconnecting (cancelling ctx) in the instant +// right after that DB write still gets the event. A +// cancelHookSessionRepository simulates that race deterministically by +// cancelling ctx from inside CancelActiveTaskSessionsByTaskID, right after +// the real DB write returns. +func TestService_ArchiveTaskPublishesEventAfterClientDisconnect(t *testing.T) { + hook := &cancelHookSessionRepository{} + svc, eventBus, repo := createTestServiceWithSessionsRepo(t, func(repo *sqliterepo.Repository) repository.SessionRepository { + hook.Repository = repo + return hook + }) + + setupCtx := context.Background() + if err := repo.CreateWorkspace(setupCtx, &models.Workspace{ID: "ws-1", Name: "Workspace"}); err != nil { + t.Fatalf("CreateWorkspace: %v", err) + } + if err := repo.CreateWorkflow(setupCtx, &models.Workflow{ID: "wf-1", WorkspaceID: "ws-1", Name: "Workflow"}); err != nil { + t.Fatalf("CreateWorkflow: %v", err) + } + if err := repo.CreateTask(setupCtx, &models.Task{ + ID: "task-1", WorkspaceID: "ws-1", WorkflowID: "wf-1", WorkflowStepID: "step-1", + Title: "Test", Priority: "medium", + }); err != nil { + t.Fatalf("CreateTask: %v", err) + } + if err := repo.CreateTaskSession(setupCtx, &models.TaskSession{ + ID: "session-running", TaskID: "task-1", State: models.TaskSessionStateRunning, + AgentProfileID: "agent-1", IsPrimary: true, + }); err != nil { + t.Fatalf("CreateTaskSession: %v", err) + } + + ctx, cancel := context.WithCancel(context.Background()) + hook.afterCancelSessions = cancel + + if err := svc.ArchiveTask(ctx, "task-1"); err != nil { + t.Fatalf("ArchiveTask returned an error even though ctx was only cancelled after the sessions DB write committed: %v", err) + } + + var found *bus.Event + for _, evt := range eventBus.GetPublishedEvents() { + if evt.Type != events.TaskSessionStateChanged { + continue + } + data, ok := evt.Data.(map[string]interface{}) + if ok && data["session_id"] == "session-running" { + found = evt + } + } + if found == nil { + t.Fatal("expected a session.state_changed event for session-running despite the post-commit ctx cancellation, got none") + } + data := found.Data.(map[string]interface{}) + if got := data["session_id"]; got != "session-running" { + t.Errorf("session_id = %v, want session-running", got) + } + if got := data["new_state"]; got != string(models.TaskSessionStateCancelled) { + t.Errorf("new_state = %v, want CANCELLED", got) + } + updatedAtStr, _ := data["updated_at"].(string) + if _, err := time.Parse(time.RFC3339Nano, updatedAtStr); err != nil { + t.Errorf("updated_at %q not parseable as RFC3339Nano: %v", updatedAtStr, err) + } + + session, err := repo.GetTaskSession(context.Background(), "session-running") + if err != nil { + t.Fatalf("GetTaskSession: %v", err) + } + if session.State != models.TaskSessionStateCancelled { + t.Errorf("session state = %q, want CANCELLED", session.State) + } +} + +// flakyCancelSessionRepository wraps the real session repository so a test +// can deterministically simulate CancelActiveTaskSessionsByTaskID failing +// with a transient error (e.g. the kind of writer-contention timeout the +// repository's own internal 10s bound produces) on its first N-1 calls and +// succeeding on the Nth. Test-only: production code has no equivalent hook. +type flakyCancelSessionRepository struct { + *sqliterepo.Repository + mu sync.Mutex + failuresLeft int + calls int +} + +func (r *flakyCancelSessionRepository) CancelActiveTaskSessionsByTaskID( + ctx context.Context, taskID, reason string, +) ([]*models.TaskSession, error) { + r.mu.Lock() + r.calls++ + if r.failuresLeft > 0 { + r.failuresLeft-- + r.mu.Unlock() + return nil, errors.New("simulated transient sqlite writer contention timeout") + } + r.mu.Unlock() + return r.Repository.CancelActiveTaskSessionsByTaskID(ctx, taskID, reason) +} + +func (r *flakyCancelSessionRepository) callCount() int { + r.mu.Lock() + defer r.mu.Unlock() + return r.calls +} + +// TestService_ArchiveTaskRetriesTransientSessionCancellationFailure is the +// regression test for the retry loop in finalizeCancelledSessions: +// CancelActiveTaskSessionsByTaskID is bounded by its own internal timeout, +// so a lone attempt failing (e.g. to writer contention) used to be swallowed +// immediately, leaving the archived task's sessions stuck active with no +// cancellation event ever published. Failing the first two calls and +// succeeding on the third proves the retry loop keeps trying within its +// bounded attempt budget and still delivers the session.state_changed event +// once the underlying write finally succeeds. +func TestService_ArchiveTaskRetriesTransientSessionCancellationFailure(t *testing.T) { + flaky := &flakyCancelSessionRepository{failuresLeft: 2} + svc, eventBus, repo := createTestServiceWithSessionsRepo(t, func(repo *sqliterepo.Repository) repository.SessionRepository { + flaky.Repository = repo + return flaky + }) + + ctx := context.Background() + if err := repo.CreateWorkspace(ctx, &models.Workspace{ID: "ws-1", Name: "Workspace"}); err != nil { + t.Fatalf("CreateWorkspace: %v", err) + } + if err := repo.CreateWorkflow(ctx, &models.Workflow{ID: "wf-1", WorkspaceID: "ws-1", Name: "Workflow"}); err != nil { + t.Fatalf("CreateWorkflow: %v", err) + } + if err := repo.CreateTask(ctx, &models.Task{ + ID: "task-flaky", WorkspaceID: "ws-1", WorkflowID: "wf-1", WorkflowStepID: "step-1", + Title: "Test", Priority: "medium", + }); err != nil { + t.Fatalf("CreateTask: %v", err) + } + if err := repo.CreateTaskSession(ctx, &models.TaskSession{ + ID: "session-flaky", TaskID: "task-flaky", State: models.TaskSessionStateRunning, + AgentProfileID: "agent-1", IsPrimary: true, + }); err != nil { + t.Fatalf("CreateTaskSession: %v", err) + } + + if err := svc.ArchiveTask(ctx, "task-flaky"); err != nil { + t.Fatalf("ArchiveTask returned an error even though the retry loop should have recovered: %v", err) + } + + if got, want := flaky.callCount(), 3; got != want { + t.Fatalf("CancelActiveTaskSessionsByTaskID call count = %d, want %d (2 failures + 1 success)", got, want) + } + + var found *bus.Event + for _, evt := range eventBus.GetPublishedEvents() { + if evt.Type != events.TaskSessionStateChanged { + continue + } + data, ok := evt.Data.(map[string]interface{}) + if ok && data["session_id"] == "session-flaky" { + found = evt + } + } + if found == nil { + t.Fatal("expected a session.state_changed event for session-flaky after the retry loop recovered, got none") + } + data := found.Data.(map[string]interface{}) + if got := data["new_state"]; got != string(models.TaskSessionStateCancelled) { + t.Errorf("new_state = %v, want CANCELLED", got) + } + + session, err := repo.GetTaskSession(context.Background(), "session-flaky") + if err != nil { + t.Fatalf("GetTaskSession: %v", err) + } + if session.State != models.TaskSessionStateCancelled { + t.Errorf("session state = %q, want CANCELLED", session.State) + } +} + +// TestService_ArchiveTaskPublishesEventsForAllCancelledSessions is the +// regression test for the shared-deadline bug: publishSessionsCancelled used +// to run its per-session Publish call under one deadline shared across the +// whole cancelledSessions batch, so a slow synchronous subscriber for one +// session could starve the event for every session queued after it. +// Archiving a task with two active sessions and asserting both get their own +// session.state_changed event proves the fix (each session now gets an +// independent 10s timeout) without relying on a real timing race. +func TestService_ArchiveTaskPublishesEventsForAllCancelledSessions(t *testing.T) { + svc, eventBus, repo := createTestService(t) + ctx := context.Background() + + if err := repo.CreateWorkspace(ctx, &models.Workspace{ID: "ws-1", Name: "Workspace"}); err != nil { + t.Fatalf("CreateWorkspace: %v", err) + } + if err := repo.CreateWorkflow(ctx, &models.Workflow{ID: "wf-1", WorkspaceID: "ws-1", Name: "Workflow"}); err != nil { + t.Fatalf("CreateWorkflow: %v", err) + } + if err := repo.CreateTask(ctx, &models.Task{ + ID: "task-multi", WorkspaceID: "ws-1", WorkflowID: "wf-1", WorkflowStepID: "step-1", + Title: "Test", Priority: "medium", + }); err != nil { + t.Fatalf("CreateTask: %v", err) + } + if err := repo.CreateTaskSession(ctx, &models.TaskSession{ + ID: "session-running-a", TaskID: "task-multi", State: models.TaskSessionStateRunning, + AgentProfileID: "agent-1", IsPrimary: true, + }); err != nil { + t.Fatalf("CreateTaskSession(session-running-a): %v", err) + } + if err := repo.CreateTaskSession(ctx, &models.TaskSession{ + ID: "session-running-b", TaskID: "task-multi", State: models.TaskSessionStateWaitingForInput, + AgentProfileID: "agent-2", + }); err != nil { + t.Fatalf("CreateTaskSession(session-running-b): %v", err) + } + + if err := svc.ArchiveTask(ctx, "task-multi"); err != nil { + t.Fatalf("ArchiveTask: %v", err) + } + + foundBySessionID := make(map[string]*bus.Event) + for _, evt := range eventBus.GetPublishedEvents() { + if evt.Type != events.TaskSessionStateChanged { + continue + } + data, ok := evt.Data.(map[string]interface{}) + if !ok { + continue + } + sessionID, _ := data["session_id"].(string) + if sessionID == "session-running-a" || sessionID == "session-running-b" { + foundBySessionID[sessionID] = evt + } + } + for _, sessionID := range []string{"session-running-a", "session-running-b"} { + evt, ok := foundBySessionID[sessionID] + if !ok { + t.Fatalf("expected a session.state_changed event for %s, got none (events: %#v)", sessionID, eventBus.GetPublishedEvents()) + } + data := evt.Data.(map[string]interface{}) + if got := data["new_state"]; got != string(models.TaskSessionStateCancelled) { + t.Errorf("%s: new_state = %v, want CANCELLED", sessionID, got) + } + if got := data["task_id"]; got != "task-multi" { + t.Errorf("%s: task_id = %v, want task-multi", sessionID, got) + } + } + + for _, sessionID := range []string{"session-running-a", "session-running-b"} { + session, err := repo.GetTaskSession(ctx, sessionID) + if err != nil { + t.Fatalf("GetTaskSession(%s): %v", sessionID, err) + } + if session.State != models.TaskSessionStateCancelled { + t.Errorf("%s: session state = %q, want CANCELLED", sessionID, session.State) + } + } +} + func TestService_ArchiveTaskClaimsExactExecutionBeforeCancellingSession(t *testing.T) { svc, _, repo := createTestService(t) ctx := context.Background()