fix(backend): publish session state_changed when archiving cancels sessions (#1891)

This commit is contained in:
Clément Fiere
2026-07-23 15:08:41 +02:00
committed by GitHub
parent e1008737c4
commit 26c38b02e1
13 changed files with 1030 additions and 37 deletions
+1
View File
@@ -751,6 +751,7 @@ func startGatewayAndServe(
}
services.Task.StartAutoArchiveLoop(ctx)
services.Task.StartArchivedSessionReconciliationLoop(ctx)
services.Task.StartQuickChatExpirationLoop(ctx)
// ============================================
@@ -608,6 +608,9 @@ func (m *mockRepository) ArchiveTask(ctx context.Context, id string) error { ret
func (m *mockRepository) ListTasksForAutoArchive(ctx context.Context) ([]*models.Task, error) {
return nil, nil
}
func (m *mockRepository) ListArchivedTasksWithActiveSessions(ctx context.Context) ([]string, error) {
return nil, nil
}
func (m *mockRepository) GetWorkspace(ctx context.Context, id string) (*models.Workspace, error) {
return nil, nil
@@ -83,6 +83,9 @@ func (m *mockRepository) ArchiveTask(ctx context.Context, id string) error {
func (m *mockRepository) ListTasksForAutoArchive(ctx context.Context) ([]*models.Task, error) {
return nil, nil
}
func (m *mockRepository) ListArchivedTasksWithActiveSessions(ctx context.Context) ([]string, error) {
return nil, nil
}
func (m *mockRepository) ListExpiredQuickChatTasks(ctx context.Context, cutoff time.Time) ([]*models.Task, error) {
return nil, nil
}
@@ -297,8 +300,8 @@ func (m *mockRepository) ListActiveTaskSessions(ctx context.Context) ([]*models.
func (m *mockRepository) ListActiveTaskSessionsByTaskID(ctx context.Context, taskID string) ([]*models.TaskSession, error) {
return nil, nil
}
func (m *mockRepository) CancelActiveTaskSessionsByTaskID(ctx context.Context, taskID, reason string) (int64, error) {
return 0, nil
func (m *mockRepository) CancelActiveTaskSessionsByTaskID(ctx context.Context, taskID, reason string) ([]*models.TaskSession, error) {
return nil, nil
}
func (m *mockRepository) HasActiveTaskSessionsByAgentProfile(ctx context.Context, agentProfileID string) (bool, error) {
return false, nil
@@ -52,6 +52,13 @@ type TaskRepository interface {
// Returns whether the row was updated.
UnarchiveTask(ctx context.Context, id string) (bool, error)
ListTasksForAutoArchive(ctx context.Context) ([]*models.Task, error)
// ListArchivedTasksWithActiveSessions returns the IDs of archived tasks
// (archived_at IS NOT NULL) that still have at least one task_sessions
// row in an active DB state (CREATED/STARTING/RUNNING/WAITING_FOR_INPUT).
// Candidate list for the periodic reconciliation sweep that recovers
// sessions left stranded when finalizeCancelledSessions's bounded
// in-line retry was exhausted by sustained SQLite writer contention.
ListArchivedTasksWithActiveSessions(ctx context.Context) ([]string, error)
ListExpiredQuickChatTasks(ctx context.Context, cutoff time.Time) ([]*models.Task, error)
DeleteExpiredQuickChatTask(ctx context.Context, id string, cutoff time.Time) (bool, error)
// CountOpenWatcherCreatedTasks returns the number of open watcher-created
@@ -183,7 +190,7 @@ type SessionRepository interface {
ListTaskSessions(ctx context.Context, taskID string) ([]*models.TaskSession, error)
ListActiveTaskSessions(ctx context.Context) ([]*models.TaskSession, error)
ListActiveTaskSessionsByTaskID(ctx context.Context, taskID string) ([]*models.TaskSession, error)
CancelActiveTaskSessionsByTaskID(ctx context.Context, taskID, reason string) (int64, error)
CancelActiveTaskSessionsByTaskID(ctx context.Context, taskID, reason string) ([]*models.TaskSession, error)
HasActiveTaskSessionsByAgentProfile(ctx context.Context, agentProfileID string) (bool, error)
GetActiveTaskInfoByAgentProfile(ctx context.Context, agentProfileID string) ([]agentdto.ActiveTaskInfo, error)
HasActiveTaskSessionsByExecutor(ctx context.Context, executorID string) (bool, error)
@@ -754,22 +754,112 @@ func completedAtForTaskSessionState(status models.TaskSessionState, now time.Tim
// CancelActiveTaskSessionsByTaskID transitions every active session of a task
// (CREATED/STARTING/RUNNING/WAITING_FOR_INPUT) to CANCELLED, returning the
// number of rows changed. The transition is a pure DB state change and does not
// require a live agent execution, making it the authoritative way to finalize a
// task's sessions independent of agent-process teardown.
func (r *Repository) CancelActiveTaskSessionsByTaskID(ctx context.Context, taskID, reason string) (int64, error) {
// full row of each session actually transitioned. The transition is a pure
// DB state change and does not require a live agent execution, making it
// the authoritative way to finalize a task's sessions independent of
// agent-process teardown.
//
// The UPDATE and the row selection happen in a single atomic statement via
// RETURNING, so the returned rows are exactly what this call changed — a
// session created or transitioned to active state concurrently, after this
// statement starts, is simply outside its snapshot; the task_id + state
// predicate is evaluated once per row as it commits, so no session matching
// it at commit time is missed by a separate pre-update snapshot. Callers use
// the returned sessions to publish a matching session.state_changed event
// per session — without this, clients that cache session state
// independently of the task (e.g. an Office task list's "is running"
// indicator) never learn the session left its active state and spin
// forever after the owning task is archived.
//
// The RETURNING clause carries every field publishSessionsCancelled needs
// to build its event payload directly, so callers never fall back to a
// separate post-commit read (e.g. GetTaskSession) to assemble the event —
// closing the read-after-write gap where a session could commit CANCELLED
// but its event never gets published because that follow-up read failed or
// timed out. Returned sessions therefore carry only the fields the
// RETURNING clause selects (ID, TaskID, AgentProfileID,
// AgentProfileSnapshot, IsPassthrough, Name, ReviewStatus, Metadata,
// TaskEnvironmentID, State, UpdatedAt) — every other models.TaskSession
// field is left at its zero value, and callers must not rely on fields
// outside this list being populated.
func (r *Repository) CancelActiveTaskSessionsByTaskID(ctx context.Context, taskID, reason string) ([]*models.TaskSession, error) {
now := time.Now().UTC()
writeCtx := context.WithoutCancel(ctx)
result, err := r.db.ExecContext(writeCtx, r.db.Rebind(`
// Detach from ctx via WithoutCancel: this write must survive a client
// disconnect (see the doc comment above) so the CANCELLED transition and
// its returned rows are never lost to a caller that hung up mid-request.
// Bound it with a timeout so a locked SQLite UPDATE can't block forever
// on a request-independent DB stall just because the deadline was
// dropped.
writeCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
defer cancel()
rows, err := r.db.QueryContext(writeCtx, r.db.Rebind(`
UPDATE task_sessions
SET state = ?, error_message = ?, completed_at = ?, updated_at = ?
WHERE task_id = ?
AND state IN ('CREATED', 'STARTING', 'RUNNING', 'WAITING_FOR_INPUT')
RETURNING id, agent_profile_id, agent_profile_snapshot, is_passthrough, name,
review_status, metadata, task_environment_id, state, updated_at
`), string(models.TaskSessionStateCancelled), reason, now, now, taskID)
if err != nil {
return 0, err
return nil, err
}
return result.RowsAffected()
defer func() { _ = rows.Close() }()
var sessions []*models.TaskSession
for rows.Next() {
session, err := scanCancelledTaskSessionRow(rows, taskID)
if err != nil {
return nil, err
}
sessions = append(sessions, session)
}
return sessions, rows.Err()
}
// scanCancelledTaskSessionRow scans one row produced by
// CancelActiveTaskSessionsByTaskID's UPDATE ... RETURNING into a
// *models.TaskSession, mirroring scanTaskSessionRow's JSON-unmarshal and
// int-to-bool/nullable-string conventions but for the narrower RETURNING
// column set (id, agent_profile_id, agent_profile_snapshot, is_passthrough,
// name, review_status, metadata, task_environment_id, state, updated_at).
// taskID backfills TaskID, which RETURNING cannot supply since it's a query
// parameter, not a returned column.
func scanCancelledTaskSessionRow(rows *sql.Rows, taskID string) (*models.TaskSession, error) {
session := &models.TaskSession{TaskID: taskID}
var state string
var metadataJSON string
var agentProfileSnapshotJSON string
var isPassthrough int
var reviewStatus sql.NullString
var agentProfileID sql.NullString
var name sql.NullString
if err := rows.Scan(
&session.ID, &agentProfileID, &agentProfileSnapshotJSON, &isPassthrough, &name,
&reviewStatus, &metadataJSON, &session.TaskEnvironmentID, &state, &session.UpdatedAt,
); err != nil {
return nil, err
}
session.State = models.TaskSessionState(state)
session.IsPassthrough = isPassthrough == 1
if reviewStatus.Valid {
session.ReviewStatus = models.ReviewStatus(reviewStatus.String)
}
if agentProfileID.Valid {
session.AgentProfileID = agentProfileID.String
}
if name.Valid {
session.Name = name.String
}
if err := unmarshalSessionJSON(metadataJSON, &session.Metadata, "agent session metadata"); err != nil {
return nil, err
}
if err := unmarshalSessionJSON(agentProfileSnapshotJSON, &session.AgentProfileSnapshot, "agent profile snapshot"); err != nil {
return nil, err
}
return session, nil
}
// UpdateSessionMetadata updates only the metadata column of a session,
@@ -17,6 +17,9 @@ import (
"github.com/stretchr/testify/require"
)
// newRepoForSessionTests creates a fresh SQLite-backed Repository in a
// temporary directory for use by session tests, closing the connection via
// t.Cleanup.
func newRepoForSessionTests(t *testing.T) *Repository {
t.Helper()
dbPath := filepath.Join(t.TempDir(), "session-test.db")
@@ -77,6 +80,10 @@ func insertAgentMsg(t *testing.T, repo *Repository, id, sessionID, turnID, autho
}
}
// TestRenameTaskSession verifies RenameTaskSession rejects a missing
// session, updates and clears the session name (with ToAPI omitting an
// empty name), and that a name set via CreateTaskSession round-trips
// through ListTaskSessions.
func TestRenameTaskSession(t *testing.T) {
repo := newRepoForSessionTests(t)
ctx := context.Background()
@@ -139,6 +146,12 @@ func TestRenameTaskSession(t *testing.T) {
}
}
// TestTaskSessionNotFoundErrorsAreTyped verifies that GetTaskSession,
// UpdateTaskSession, UpdateTaskSessionState, and UpdateTaskSessionBaseCommit
// all return ErrTaskSessionNotFound for a missing session, that
// GetTaskSessionByTaskAndAgent translates not-found to (nil, nil), and that
// GetPrimarySessionByTaskID returns ErrNoPrimarySession, while operations on
// an existing session succeed.
func TestTaskSessionNotFoundErrorsAreTyped(t *testing.T) {
repo := newRepoForSessionTests(t)
ctx := context.Background()
@@ -176,6 +189,10 @@ func TestTaskSessionNotFoundErrorsAreTyped(t *testing.T) {
}
}
// TestSetSessionMetadataKeyIfAbsentSQLiteIsWriteOnce verifies that on SQLite
// SetSessionMetadataKeyIfAbsent stores the value on first call and reports
// no write (leaving the original value in place) on a subsequent call with
// the same key.
func TestSetSessionMetadataKeyIfAbsentSQLiteIsWriteOnce(t *testing.T) {
repo := newRepoForSessionTests(t)
seedForMsgTest(t, repo, "task-baseline", "session-baseline", "turn-baseline")
@@ -206,6 +223,9 @@ func TestSetSessionMetadataKeyIfAbsentSQLiteIsWriteOnce(t *testing.T) {
}
}
// TestSetSessionMetadataKeyIfAbsentQueryUsesPostgresJSONB verifies the
// Postgres dialect's write-once query uses jsonb_set/jsonb_extract_path
// rather than SQLite's json_set/json_type/json functions.
func TestSetSessionMetadataKeyIfAbsentQueryUsesPostgresJSONB(t *testing.T) {
query := setSessionMetadataKeyIfAbsentQuery(dialect.PGX)
if strings.Contains(query, "json_set") || strings.Contains(query, "json_type") || strings.Contains(query, "json(?)") {
@@ -216,6 +236,10 @@ func TestSetSessionMetadataKeyIfAbsentQueryUsesPostgresJSONB(t *testing.T) {
}
}
// TestListTaskSessionWorktreesFiltersInactiveRows verifies that
// ListTaskSessionWorktrees and ListWorktreesBySessionIDs exclude worktree
// rows marked deleted via either the status column or a set deleted_at
// timestamp, returning only the still-active worktree.
func TestListTaskSessionWorktreesFiltersInactiveRows(t *testing.T) {
repo := newRepoForSessionTests(t)
ctx := context.Background()
@@ -281,6 +305,10 @@ func TestListTaskSessionWorktreesFiltersInactiveRows(t *testing.T) {
}
}
// TestUpdateTaskSessionWorktreeBranchByRepositoryScopesUpdate verifies
// UpdateTaskSessionWorktreeBranchByRepository updates the branch only for
// the worktree matching the given repository, leaving other repositories'
// worktrees on the same session unchanged.
func TestUpdateTaskSessionWorktreeBranchByRepositoryScopesUpdate(t *testing.T) {
repo := newRepoForSessionTests(t)
ctx := context.Background()
@@ -727,6 +755,8 @@ func insertSession(t *testing.T, repo *Repository, sessionID, taskID, state stri
}
}
// sessionState reads and returns the state column of the task_sessions row
// for the given session ID, failing the test if the row cannot be read.
func sessionState(t *testing.T, repo *Repository, sessionID string) string {
t.Helper()
var state string
@@ -737,6 +767,11 @@ func sessionState(t *testing.T, repo *Repository, sessionID string) string {
return state
}
// TestCancelActiveTaskSessionIsTerminalSafe verifies CancelActiveTaskSession
// transitions a RUNNING session to CANCELLED and returns the stored
// updated_at as the cancellation timestamp, while a session already in a
// terminal state (COMPLETED) is left unchanged and reported as not
// cancelled.
func TestCancelActiveTaskSessionIsTerminalSafe(t *testing.T) {
repo := newRepoForSessionTests(t)
ctx := context.Background()
@@ -776,6 +811,10 @@ func TestCancelActiveTaskSessionIsTerminalSafe(t *testing.T) {
}
}
// TestUpdateTaskSessionStateIfCurrentRejectsStaleActiveWriter verifies that
// after a session is cancelled, a conditional UpdateTaskSessionStateIfCurrent
// call from a writer still expecting the prior RUNNING state is rejected and
// the session remains CANCELLED.
func TestUpdateTaskSessionStateIfCurrentRejectsStaleActiveWriter(t *testing.T) {
repo := newRepoForSessionTests(t)
ctx := context.Background()
@@ -805,6 +844,11 @@ func TestUpdateTaskSessionStateIfCurrentRejectsStaleActiveWriter(t *testing.T) {
}
}
// TestUpdateTaskSessionIfCurrentStateRejectsStaleFullRowWriter verifies that
// after a session is cancelled out from under a full-row writer, a
// subsequent UpdateTaskSessionIfCurrentState call using the writer's stale
// in-memory state is rejected, leaving the session CANCELLED with its
// executor ID unset.
func TestUpdateTaskSessionIfCurrentStateRejectsStaleFullRowWriter(t *testing.T) {
repo := newRepoForSessionTests(t)
ctx := context.Background()
@@ -832,6 +876,10 @@ func TestUpdateTaskSessionIfCurrentStateRejectsStaleFullRowWriter(t *testing.T)
require.Empty(t, stored.ExecutorID)
}
// TestUpdateTaskSessionWithMetadataRejectsInvalidMetadataBeforeStateWrite
// verifies that UpdateTaskSessionWithMetadata returns an error and leaves
// both the session state and existing metadata untouched when the supplied
// metadata cannot be marshalled.
func TestUpdateTaskSessionWithMetadataRejectsInvalidMetadataBeforeStateWrite(t *testing.T) {
repo := newRepoForSessionTests(t)
ctx := context.Background()
@@ -862,6 +910,10 @@ func TestUpdateTaskSessionWithMetadataRejectsInvalidMetadataBeforeStateWrite(t *
}
}
// TestUpdateTaskSessionIfCurrentStateRemovingMetadataKeys verifies that
// UpdateTaskSessionIfCurrentStateRemovingMetadataKeys removes the requested
// metadata key while preserving other keys when the session's current state
// matches the expected state.
func TestUpdateTaskSessionIfCurrentStateRemovingMetadataKeys(t *testing.T) {
repo := newRepoForSessionTests(t)
ctx := context.Background()
@@ -886,6 +938,10 @@ func TestUpdateTaskSessionIfCurrentStateRemovingMetadataKeys(t *testing.T) {
require.Equal(t, "newer", stored.Metadata["keep"])
}
// TestUpdateTaskSessionIfCurrentStateRemovingMetadataKeysStateMismatch
// verifies that UpdateTaskSessionIfCurrentStateRemovingMetadataKeys is a
// no-op, leaving all metadata untouched, when the expected state does not
// match the session's current state.
func TestUpdateTaskSessionIfCurrentStateRemovingMetadataKeysStateMismatch(t *testing.T) {
repo := newRepoForSessionTests(t)
ctx := context.Background()
@@ -910,6 +966,9 @@ func TestUpdateTaskSessionIfCurrentStateRemovingMetadataKeysStateMismatch(t *tes
require.Equal(t, "untouched", stored.Metadata["keep"])
}
// TestDismissLastAgentErrorDoesNotOverwriteNewerError verifies that
// DismissLastAgentError ignores a dismiss request for a stale error once a
// newer error has been recorded, leaving the newer error undismissed.
func TestDismissLastAgentErrorDoesNotOverwriteNewerError(t *testing.T) {
repo := newRepoForSessionTests(t)
ctx := context.Background()
@@ -950,6 +1009,10 @@ func TestDismissLastAgentErrorDoesNotOverwriteNewerError(t *testing.T) {
}
}
// TestDismissLastAgentErrorMatchesEquivalentTimestampText verifies that
// DismissLastAgentError matches the stored error even when its occurred_at
// timestamp was persisted as RFC3339Nano text rather than the parsed
// time.Time value, and marks it dismissed.
func TestDismissLastAgentErrorMatchesEquivalentTimestampText(t *testing.T) {
repo := newRepoForSessionTests(t)
ctx := context.Background()
@@ -990,6 +1053,9 @@ func TestDismissLastAgentErrorMatchesEquivalentTimestampText(t *testing.T) {
}
}
// sessionCancellationMetadata reads and returns the error_message,
// completed_at, and updated_at columns of the task_sessions row for the
// given session ID.
func sessionCancellationMetadata(t *testing.T, repo *Repository, sessionID string) (string, sql.NullTime, time.Time) {
t.Helper()
var errorMessage string
@@ -1004,6 +1070,10 @@ func sessionCancellationMetadata(t *testing.T, repo *Repository, sessionID strin
return errorMessage, completedAt, updatedAt
}
// assertReapedSession asserts that the session was transitioned to
// CANCELLED with error_message "task archived", a completed_at timestamp
// set, and updated_at at or after reapedAfter, recording any mismatch as a
// test failure.
func assertReapedSession(t *testing.T, repo *Repository, sessionID string, reapedAfter time.Time) {
t.Helper()
if got := sessionState(t, repo, sessionID); got != "CANCELLED" {
@@ -1023,9 +1093,9 @@ func assertReapedSession(t *testing.T, repo *Repository, sessionID string, reape
// TestCancelActiveTaskSessionsByTaskID verifies the archive reaper transitions
// only the target task's still-active sessions to CANCELLED, leaves terminal
// sessions and other tasks untouched, and reports the rows changed. It also
// confirms the repo-delete guard reports the repository as free afterward —
// the end-to-end purpose of the reap.
// sessions and other tasks untouched, and reports exactly the sessions it
// changed. It also confirms the repo-delete guard reports the repository as
// free afterward — the end-to-end purpose of the reap.
func TestCancelActiveTaskSessionsByTaskID(t *testing.T) {
repo := newRepoForSessionTests(t)
ctx := context.Background()
@@ -1041,14 +1111,28 @@ func TestCancelActiveTaskSessionsByTaskID(t *testing.T) {
seedRepoLink(t, repo, "ws-r", "repo-other", "task-other", "sess-o1", "RUNNING")
reapedAfter := time.Now().UTC()
reaped, err := repo.CancelActiveTaskSessionsByTaskID(ctx, "task-r", "task archived")
sessions, err := repo.CancelActiveTaskSessionsByTaskID(ctx, "task-r", "task archived")
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if reaped != 4 {
t.Errorf("expected 4 active sessions reaped, got %d", reaped)
wantIDs := []string{"sess-r1", "sess-r2", "sess-r3", "sess-r4"}
var gotIDs []string
for _, sess := range sessions {
gotIDs = append(gotIDs, sess.ID)
if sess.TaskID != "task-r" {
t.Errorf("session %s TaskID = %q, want task-r", sess.ID, sess.TaskID)
}
if sess.State != models.TaskSessionStateCancelled {
t.Errorf("session %s State = %q, want CANCELLED", sess.ID, sess.State)
}
if sess.UpdatedAt.Before(reapedAfter) {
t.Errorf("session %s UpdatedAt = %s, want >= %s", sess.ID, sess.UpdatedAt, reapedAfter)
}
}
for _, sessionID := range []string{"sess-r1", "sess-r2", "sess-r3", "sess-r4"} {
if !sameStringSet(gotIDs, wantIDs) {
t.Errorf("cancelled ids = %v, want %v", gotIDs, wantIDs)
}
for _, sessionID := range wantIDs {
assertReapedSession(t, repo, sessionID, reapedAfter)
}
if got := sessionState(t, repo, "sess-r5"); got != "COMPLETED" {
@@ -1069,11 +1153,33 @@ func TestCancelActiveTaskSessionsByTaskID(t *testing.T) {
}
// Idempotent: a second call changes nothing.
reaped, err = repo.CancelActiveTaskSessionsByTaskID(ctx, "task-r", "task archived")
sessions, err = repo.CancelActiveTaskSessionsByTaskID(ctx, "task-r", "task archived")
if err != nil {
t.Fatalf("unexpected error on second call: %v", err)
}
if reaped != 0 {
t.Errorf("expected 0 rows on idempotent re-run, got %d", reaped)
if len(sessions) != 0 {
t.Errorf("expected 0 sessions on idempotent re-run, got %v", sessions)
}
}
// sameStringSet reports whether got and want contain the same strings,
// ignoring order and duplicates count-for-count — used to compare the set of
// cancelled session IDs against an expected set regardless of return order.
func sameStringSet(got, want []string) bool {
if len(got) != len(want) {
return false
}
set := make(map[string]int, len(want))
for _, s := range want {
set[s]++
}
for _, s := range got {
set[s]--
}
for _, count := range set {
if count != 0 {
return false
}
}
return true
}
@@ -1069,6 +1069,31 @@ func (r *Repository) ListTasksForAutoArchive(ctx context.Context) ([]*models.Tas
return r.scanTasks(rows)
}
// ListArchivedTasksWithActiveSessions returns the IDs of archived tasks that
// still have at least one task_sessions row in an active DB state
// (CREATED/STARTING/RUNNING/WAITING_FOR_INPUT). This is the candidate list
// for the periodic reconciliation sweep (see
// service.StartArchivedSessionReconciliationLoop): finalizeCancelledSessions
// bounds its session-cancellation retry to a handful of attempts, so
// sustained SQLite writer contention can exhaust it and leave an archived
// task's sessions stuck active with no session.state_changed event ever
// delivered. The sweep re-derives this list every pass, so a later attempt
// recovers the task once the contention clears.
func (r *Repository) ListArchivedTasksWithActiveSessions(ctx context.Context) ([]string, error) {
var ids []string
err := r.ro.SelectContext(ctx, &ids, `
SELECT DISTINCT t.id
FROM tasks t
JOIN task_sessions ts ON ts.task_id = t.id
WHERE t.archived_at IS NOT NULL
AND ts.state IN ('CREATED', 'STARTING', 'RUNNING', 'WAITING_FOR_INPUT')
`)
if err != nil {
return nil, err
}
return ids, nil
}
// ListExpiredQuickChatTasks returns quick-chat tasks whose last task/session
// activity is older than cutoff. Active sessions are excluded so in-use chats
// are never deleted by the idle sweeper.
@@ -0,0 +1,79 @@
package service
import (
"context"
"time"
"go.uber.org/zap"
)
// StartArchivedSessionReconciliationLoop starts a background goroutine that
// periodically re-finalizes archived tasks whose sessions never made it to a
// terminal DB state.
//
// finalizeCancelledSessions (see service_tasks.go) already bounds its
// session-cancellation retry to a handful of fixed attempts inside
// ArchiveTask's own request: enough to ride out a brief SQLite writer-lock
// blip, but not unbounded. If SQLite's single writer stays occupied for
// longer than every attempt combined, that bounded retry gets exhausted
// after archived_at has already committed, and nothing else in the request
// path ever retries the transition. The archived task is then left with
// sessions stuck in an active DB state (CREATED/STARTING/RUNNING/
// WAITING_FOR_INPUT) forever, and event-driven clients that key their
// "is running" indicators off session.state_changed never learn the
// sessions actually stopped.
//
// This sweep is the eventual-consistency backstop for that residual gap,
// following the exact periodic-sweep shape StartAutoArchiveLoop already
// uses: on a fixed interval, list every archived task that still has an
// active session, and re-invoke finalizeCancelledSessions for each. Calling
// finalizeCancelledSessions again is always safe — its underlying
// CancelActiveTaskSessionsByTaskID UPDATE only matches sessions still in an
// active state, so a task that was already fully reconciled (by ArchiveTask
// itself or by an earlier sweep pass) simply yields an empty cancelledSessions
// slice and is a no-op. As long as the process keeps running, a later pass
// retries once the SQLite contention that exhausted the in-line retry has
// cleared — unlike the fixed 3-attempt budget, this sweep never gives up.
func (s *Service) StartArchivedSessionReconciliationLoop(ctx context.Context) {
ticker := time.NewTicker(1 * time.Minute)
go func() {
defer ticker.Stop()
for {
select {
case <-ctx.Done():
return
case <-ticker.C:
s.runArchivedSessionReconciliation(ctx)
}
}
}()
s.logger.Info("archived-session reconciliation loop started (every 1 minute)")
}
func (s *Service) runArchivedSessionReconciliation(ctx context.Context) {
taskIDs, err := s.tasks.ListArchivedTasksWithActiveSessions(ctx)
if err != nil {
s.logger.Error("archived-session reconciliation: failed to list candidates", zap.Error(err))
return
}
if len(taskIDs) == 0 {
return
}
s.logger.Info("archived-session reconciliation: found candidates", zap.Int("count", len(taskIDs)))
for _, taskID := range taskIDs {
activeSessions, err := s.sessions.ListActiveTaskSessionsByTaskID(ctx, taskID)
if err != nil {
s.logger.Warn("archived-session reconciliation: failed to list active sessions",
zap.String("task_id", taskID),
zap.Error(err))
continue
}
if len(activeSessions) == 0 {
// Already reconciled by a concurrent pass or by ArchiveTask itself
// between the candidate list query and this read.
continue
}
s.finalizeCancelledSessions(ctx, taskID, activeSessions)
}
}
@@ -0,0 +1,104 @@
package service
import (
"context"
"testing"
"github.com/kandev/kandev/internal/events"
"github.com/kandev/kandev/internal/events/bus"
"github.com/kandev/kandev/internal/task/models"
"github.com/kandev/kandev/internal/task/repository"
sqliterepo "github.com/kandev/kandev/internal/task/repository/sqlite"
)
// TestService_ArchivedSessionReconciliationRecoversStrandedSessions is the
// regression test for the residual gap Greptile flagged on finalizeCancelledSessions:
// its bounded in-line retry (3 attempts x 10s) can still be exhausted by
// sustained SQLite writer contention, leaving an archived task's session
// stuck active in the DB with no session.state_changed event ever
// delivered. It reuses flakyCancelSessionRepository (defined in
// service_test.go) to fail all 3 in-line attempts during ArchiveTask,
// simulating exhausted retries, then calls runArchivedSessionReconciliation
// directly (the sweep's single run, not the ticker loop) once the
// underlying repository would succeed again, and asserts the stranded
// session is recovered: CANCELLED in the DB and carrying a published
// session.state_changed event.
func TestService_ArchivedSessionReconciliationRecoversStrandedSessions(t *testing.T) {
flaky := &flakyCancelSessionRepository{failuresLeft: 3}
svc, eventBus, repo := createTestServiceWithSessionsRepo(t, func(repo *sqliterepo.Repository) repository.SessionRepository {
flaky.Repository = repo
return flaky
})
ctx := context.Background()
if err := repo.CreateWorkspace(ctx, &models.Workspace{ID: "ws-1", Name: "Workspace"}); err != nil {
t.Fatalf("CreateWorkspace: %v", err)
}
if err := repo.CreateWorkflow(ctx, &models.Workflow{ID: "wf-1", WorkspaceID: "ws-1", Name: "Workflow"}); err != nil {
t.Fatalf("CreateWorkflow: %v", err)
}
if err := repo.CreateTask(ctx, &models.Task{
ID: "task-stranded", WorkspaceID: "ws-1", WorkflowID: "wf-1", WorkflowStepID: "step-1",
Title: "Test", Priority: "medium",
}); err != nil {
t.Fatalf("CreateTask: %v", err)
}
if err := repo.CreateTaskSession(ctx, &models.TaskSession{
ID: "session-stranded", TaskID: "task-stranded", State: models.TaskSessionStateRunning,
AgentProfileID: "agent-1", IsPrimary: true,
}); err != nil {
t.Fatalf("CreateTaskSession: %v", err)
}
// Exhaust finalizeCancelledSessions's bounded in-line retry during
// ArchiveTask. ArchiveTask must still succeed: the session-cancellation
// failure is best-effort and never fails the archive itself.
if err := svc.ArchiveTask(ctx, "task-stranded"); err != nil {
t.Fatalf("ArchiveTask: %v", err)
}
if got, want := flaky.callCount(), 3; got != want {
t.Fatalf("CancelActiveTaskSessionsByTaskID call count = %d, want %d (all 3 in-line attempts exhausted)", got, want)
}
session, err := repo.GetTaskSession(ctx, "session-stranded")
if err != nil {
t.Fatalf("GetTaskSession: %v", err)
}
if session.State != models.TaskSessionStateRunning {
t.Fatalf("session state after ArchiveTask = %q, want RUNNING (retries should be exhausted, leaving the session stranded)", session.State)
}
// flaky.failuresLeft is now exhausted (0 remaining), so the sweep's call
// to CancelActiveTaskSessionsByTaskID succeeds. Call the sweep's
// single-run method directly, not the ticker loop.
svc.runArchivedSessionReconciliation(ctx)
session, err = repo.GetTaskSession(ctx, "session-stranded")
if err != nil {
t.Fatalf("GetTaskSession after reconciliation: %v", err)
}
if session.State != models.TaskSessionStateCancelled {
t.Fatalf("session state after reconciliation = %q, want CANCELLED", session.State)
}
var found *bus.Event
for _, evt := range eventBus.GetPublishedEvents() {
if evt.Type != events.TaskSessionStateChanged {
continue
}
data, ok := evt.Data.(map[string]interface{})
if ok && data["session_id"] == "session-stranded" {
found = evt
}
}
if found == nil {
t.Fatal("expected a session.state_changed event for session-stranded from the reconciliation sweep, got none")
}
data := found.Data.(map[string]interface{})
if got := data["new_state"]; got != string(models.TaskSessionStateCancelled) {
t.Errorf("new_state = %v, want CANCELLED", got)
}
if got := data["task_id"]; got != "task-stranded" {
t.Errorf("task_id = %v, want task-stranded", got)
}
}
@@ -379,6 +379,10 @@ func (r *phase4TaskRepo) ListTasksForAutoArchive(context.Context) ([]*models.Tas
r.panicNotUsed("ListTasksForAutoArchive")
return nil, nil
}
func (r *phase4TaskRepo) ListArchivedTasksWithActiveSessions(context.Context) ([]string, error) {
r.panicNotUsed("ListArchivedTasksWithActiveSessions")
return nil, nil
}
func (r *phase4TaskRepo) ListExpiredQuickChatTasks(context.Context, time.Time) ([]*models.Task, error) {
r.panicNotUsed("ListExpiredQuickChatTasks")
return nil, nil
@@ -39,6 +39,92 @@ func (s *Service) PublishTaskDeleted(ctx context.Context, task *models.Task) {
s.publishTaskEvent(ctx, events.TaskDeleted, task, nil)
}
// Field names shared by every session.state_changed publish in this file —
// extracted to satisfy goconst without borrowing unrelated constants.
const (
sessionEventFieldTaskID = "task_id"
sessionEventFieldSessionID = "session_id"
sessionEventFieldUpdatedAt = "updated_at"
sessionEventFieldName = "name"
)
// publishSessionsCancelled publishes a session.state_changed event for each
// session in cancelledSessions — the full rows CancelActiveTaskSessionsByTaskID's
// UPDATE ... RETURNING already produced. The event's old_state is a
// best-effort hint from snapshot (the pre-cancellation session list a
// caller already had in hand) — used only for logging/diagnostics — while
// every other field, including new_state, comes directly from the returned
// row.
//
// This used to re-read each session via GetTaskSession after the UPDATE
// committed, which raced: if that lookup failed or timed out after commit,
// the CANCELLED transition was permanent but its event was silently lost
// (see the now-closed "Cancelled events escape reconciliation" review
// thread, PR #1891 comment 3638052588). Building the payload straight from
// cancelledSessions closes that gap structurally — there is no longer a
// separate post-commit read to race, so a session in cancelledSessions can
// no longer "vanish" between the write and the publish.
//
// ctx is expected to be detached-but-unbounded (callers typically pass a
// context.WithoutCancel derivative with no deadline of its own): each
// session in cancelledSessions gets its own independent 10-second timeout
// around the Publish call below, rather than the whole batch sharing one
// deadline. That way a single slow synchronous event subscriber can only
// ever starve its own session's publish, not the events for every session
// that comes after it in cancelledSessions.
//
// CancelActiveTaskSessionsByTaskID is a repository-level DB write with no
// event of its own; without this, any client cache kept fresh exclusively by
// session.state_changed (e.g. an Office task list's "is running" indicator)
// never learns the session left RUNNING/WAITING_FOR_INPUT and spins forever.
func (s *Service) publishSessionsCancelled(
ctx context.Context,
taskID string,
snapshot []*models.TaskSession,
cancelledSessions []*models.TaskSession,
reason string,
) {
if s.eventBus == nil {
return
}
oldStateByID := make(map[string]models.TaskSessionState, len(snapshot))
for _, sess := range snapshot {
if sess != nil {
oldStateByID[sess.ID] = sess.State
}
}
for _, sess := range cancelledSessions {
sessCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
data := map[string]interface{}{
sessionEventFieldTaskID: taskID,
sessionEventFieldSessionID: sess.ID,
"old_state": string(oldStateByID[sess.ID]),
"new_state": string(sess.State),
"error_message": reason,
"agent_profile_id": sess.AgentProfileID,
"agent_profile_snapshot": sess.AgentProfileSnapshot,
"is_passthrough": sess.IsPassthrough,
sessionEventFieldUpdatedAt: sess.UpdatedAt.Format(time.RFC3339Nano),
sessionEventFieldName: sess.Name,
}
if sess.ReviewStatus != models.ReviewStatusNone {
data["review_status"] = string(sess.ReviewStatus)
}
if len(sess.Metadata) > 0 {
data["session_metadata"] = sess.Metadata
}
if sess.TaskEnvironmentID != "" {
data["task_environment_id"] = sess.TaskEnvironmentID
}
event := bus.NewEvent(events.TaskSessionStateChanged, "task-service", data)
if err := s.eventBus.Publish(sessCtx, events.TaskSessionStateChanged, event); err != nil {
s.logger.Error("failed to publish session cancellation event",
zap.String(sessionEventFieldTaskID, taskID), zap.String(sessionEventFieldSessionID, sess.ID), zap.Error(err))
}
cancel()
}
}
// publishTaskEvent publishes task events to the event bus
func (s *Service) publishTaskEvent(ctx context.Context, eventType string, task *models.Task, oldState *v1.TaskState, oldWorkflowIDs ...string) {
s.publishTaskEventWithExtra(ctx, eventType, task, oldState, nil, oldWorkflowIDs...)
@@ -1176,34 +1176,34 @@ func (s *Service) ArchiveTask(ctx context.Context, id string) error {
// that raced in after the snapshot and must clean itself up.
s.registerTaskRuntimeStopOwners(stopTargets, true)
// 3b. Finalize active sessions in the DB. The async cleanup below tears down
// the agent processes; this records the terminal session state, which
// process teardown does not persist on its own.
if reaped, rerr := s.sessions.CancelActiveTaskSessionsByTaskID(ctx, id, "task archived"); rerr != nil {
s.logger.Warn("failed to reap active sessions on archive",
zap.String("task_id", id),
zap.Error(rerr))
} else if reaped > 0 {
s.logger.Info("reaped active sessions on archive",
zap.String("task_id", id),
zap.Int64("count", reaped))
}
// archived_at has committed above: the rest of this function only
// reports on that already-durable state (cancelling sessions, re-reading
// the task, publishing task.updated) or kicks off cleanup that already
// runs detached from ctx. A client disconnecting right after the write
// above must not stop any of that from finishing and reaching
// event-driven clients that don't share the archiving caller's
// connection.
finalizeCtx := context.WithoutCancel(ctx)
// 3b. Finalize active sessions in the DB and publish their cancellation
// events. See finalizeCancelledSessions for the detailed rationale.
s.finalizeCancelledSessions(finalizeCtx, id, activeSessions)
// 4. Re-read task for updated archived_at field
task, err = s.tasks.GetTask(ctx, id)
task, err = s.tasks.GetTask(finalizeCtx, id)
if err != nil {
return err
}
// 5. Publish task.updated event so frontend removes from board
s.publishTaskEvent(ctx, events.TaskUpdated, task, nil)
s.publishTaskEvent(finalizeCtx, events.TaskUpdated, task, nil)
s.logger.Info("task archived",
zap.String("task_id", id),
zap.Duration("duration", time.Since(start)))
// 6. Background: Stop agents and cleanup worktrees
if cleanupJob != nil {
if err := s.StartPreparedTaskResourceCleanup(ctx, cleanupJob.OperationID); err != nil {
if err := s.StartPreparedTaskResourceCleanup(finalizeCtx, cleanupJob.OperationID); err != nil {
s.logger.Warn("start committed archive resource cleanup",
zap.String("job_id", cleanupJob.ID), zap.String("task_id", id), zap.Error(err))
}
@@ -1215,6 +1215,69 @@ func (s *Service) ArchiveTask(ctx context.Context, id string) error {
return nil
}
// finalizeCancelledSessions finalizes an archived task's active sessions in
// the DB and publishes a session.state_changed event for each one actually
// cancelled. The async cleanup that follows tears down the agent processes;
// this records the terminal session state, which process teardown does not
// persist on its own. The DB write alone is invisible to any client cache
// (e.g. an Office task list's "is running" indicator) that's kept fresh
// exclusively by that event, and would otherwise show a live spinner
// forever after archive.
//
// CancelActiveTaskSessionsByTaskID is bounded by its own internal 10s
// timeout, so a single attempt can time out under SQLite writer contention
// alone — with archived_at already committed by the caller, that would
// silently leave this task's sessions stuck in an active DB state forever,
// since nothing else ever re-attempts the cancellation and no
// session.state_changed event is delivered to clear a client's running
// indicator. Retrying a small, fixed number of times — each attempt getting
// its own fresh 10s budget from the repository — meaningfully shrinks that
// window without making it unbounded. This is deliberately NOT a background
// reconciliation/sweep system, just closing the immediate race; if every
// attempt still fails, ArchiveTask has already committed archived_at, so
// this remains best-effort cleanup and is not a reason to fail the archive.
func (s *Service) finalizeCancelledSessions(ctx context.Context, taskID string, activeSessions []*models.TaskSession) {
const maxCancelAttempts = 3
const cancelRetryBackoff = 250 * time.Millisecond
var cancelledSessions []*models.TaskSession
var cancelErr error
for attempt := 1; attempt <= maxCancelAttempts; attempt++ {
cancelledSessions, cancelErr = s.sessions.CancelActiveTaskSessionsByTaskID(ctx, taskID, "task archived")
if cancelErr == nil {
break
}
if attempt < maxCancelAttempts {
time.Sleep(cancelRetryBackoff)
}
}
if cancelErr != nil {
s.logger.Error("failed to reap active sessions on archive after retries",
zap.String("task_id", taskID),
zap.Int("attempts", maxCancelAttempts),
zap.Error(cancelErr))
return
}
if len(cancelledSessions) == 0 {
return
}
s.logger.Info("reaped active sessions on archive",
zap.String("task_id", taskID),
zap.Int("count", len(cancelledSessions)))
// Detach from ctx via WithoutCancel: the DB write above already
// committed on a detached context, so a client disconnect here must
// not also suppress the event publish below — event-driven clients
// need session.state_changed regardless of whether the archiving
// caller is still connected.
// Deliberately left unbounded (no timeout) here: publishSessionsCancelled
// gives each session in cancelledSessions its own independent 10s
// deadline around its Publish call, so one slow synchronous subscriber
// can no longer consume a shared batch-wide budget and starve the
// events for sessions later in the loop.
detachedCtx := context.WithoutCancel(ctx)
s.publishSessionsCancelled(detachedCtx, taskID, activeSessions, cancelledSessions, "task archived")
}
func (s *Service) registerTaskRuntimeStopOwners(stopTargets []taskStopTarget, force bool) {
if s.executionStopper == nil {
return
@@ -15,6 +15,7 @@ import (
"github.com/kandev/kandev/internal/agentruntime"
"github.com/kandev/kandev/internal/common/logger"
"github.com/kandev/kandev/internal/db"
"github.com/kandev/kandev/internal/events"
"github.com/kandev/kandev/internal/events/bus"
officesqlite "github.com/kandev/kandev/internal/office/repository/sqlite"
"github.com/kandev/kandev/internal/task/models"
@@ -79,6 +80,20 @@ func (m *MockEventBus) ClearEvents() {
}
func createTestService(t *testing.T) (*Service, *MockEventBus, *sqliterepo.Repository) {
t.Helper()
return createTestServiceWithSessionsRepo(t, func(repo *sqliterepo.Repository) repository.SessionRepository {
return repo
})
}
// createTestServiceWithSessionsRepo mirrors createTestService but lets a
// caller substitute the Sessions repository (e.g. to wrap it with a test-only
// hook) while reusing the same DB setup, migrations, and cleanup-worker
// wiring for every other field.
func createTestServiceWithSessionsRepo(
t *testing.T,
wrapSessions func(*sqliterepo.Repository) repository.SessionRepository,
) (*Service, *MockEventBus, *sqliterepo.Repository) {
t.Helper()
tmpDir := t.TempDir()
dbConn, err := db.OpenSQLite(filepath.Join(tmpDir, "test.db"))
@@ -118,7 +133,7 @@ func createTestService(t *testing.T) (*Service, *MockEventBus, *sqliterepo.Repos
Workflows: repo,
Messages: repo,
Turns: repo,
Sessions: repo,
Sessions: wrapSessions(repo),
GitSnapshots: repo,
RepoEntities: repo,
Executors: repo,
@@ -1868,6 +1883,413 @@ func TestService_ArchiveTaskStopsExecutorRunningForTerminalSession(t *testing.T)
}
}
// TestService_ArchiveTaskPublishesSessionStateChangedForActiveSessions is the
// regression test for the stuck-spinner bug: archiving a task with an active
// session cancels that session in the DB (verified elsewhere), but any client
// cache kept fresh exclusively by session.state_changed — e.g. an Office task
// list's "is running" indicator — never learns about it unless ArchiveTask
// also publishes the event. Without the fix, the archived task's session
// would sit at RUNNING in every client cache forever.
func TestService_ArchiveTaskPublishesSessionStateChangedForActiveSessions(t *testing.T) {
svc, eventBus, repo := createTestService(t)
ctx := context.Background()
if err := repo.CreateWorkspace(ctx, &models.Workspace{ID: "ws-1", Name: "Workspace"}); err != nil {
t.Fatalf("CreateWorkspace: %v", err)
}
if err := repo.CreateWorkflow(ctx, &models.Workflow{ID: "wf-1", WorkspaceID: "ws-1", Name: "Workflow"}); err != nil {
t.Fatalf("CreateWorkflow: %v", err)
}
if err := repo.CreateTask(ctx, &models.Task{
ID: "task-1", WorkspaceID: "ws-1", WorkflowID: "wf-1", WorkflowStepID: "step-1",
Title: "Test", Priority: "medium",
}); err != nil {
t.Fatalf("CreateTask: %v", err)
}
if err := repo.CreateTaskSession(ctx, &models.TaskSession{
ID: "session-running", TaskID: "task-1", State: models.TaskSessionStateRunning,
AgentProfileID: "agent-1", IsPrimary: true,
}); err != nil {
t.Fatalf("CreateTaskSession: %v", err)
}
if err := svc.ArchiveTask(ctx, "task-1"); err != nil {
t.Fatalf("ArchiveTask: %v", err)
}
var found *bus.Event
for _, evt := range eventBus.GetPublishedEvents() {
if evt.Type != events.TaskSessionStateChanged {
continue
}
data, ok := evt.Data.(map[string]interface{})
if !ok || data["session_id"] != "session-running" {
continue
}
found = evt
}
if found == nil {
t.Fatal("expected a session.state_changed event for session-running, got none")
}
data := found.Data.(map[string]interface{})
if got := data["old_state"]; got != string(models.TaskSessionStateRunning) {
t.Errorf("old_state = %v, want RUNNING", got)
}
if got := data["new_state"]; got != string(models.TaskSessionStateCancelled) {
t.Errorf("new_state = %v, want CANCELLED", got)
}
if got := data["task_id"]; got != "task-1" {
t.Errorf("task_id = %v, want task-1", got)
}
if got := data["agent_profile_id"]; got != "agent-1" {
t.Errorf("agent_profile_id = %v, want agent-1", got)
}
updatedAtStr, _ := data["updated_at"].(string)
if _, err := time.Parse(time.RFC3339Nano, updatedAtStr); err != nil {
t.Errorf("updated_at %q not parseable as RFC3339Nano: %v", updatedAtStr, err)
}
session, err := repo.GetTaskSession(ctx, "session-running")
if err != nil {
t.Fatalf("GetTaskSession: %v", err)
}
if session.State != models.TaskSessionStateCancelled {
t.Errorf("session state = %q, want CANCELLED", session.State)
}
}
// TestService_PublishSessionsCancelledCoversSessionsMissingFromSnapshot is a
// defensive-coverage test: CancelActiveTaskSessionsByTaskID re-evaluates
// active sessions atomically and returns full session rows directly, so a
// returned session can have no matching entry in a caller-supplied snapshot
// taken moments earlier. publishSessionsCancelled must build the event
// entirely from that returned row — old_state is the only field allowed to
// fall back to an empty best-effort hint when snapshot has nothing for the
// session's ID; every other field, including agent_profile_id, must come
// from the row cancelledSessions already carries, not be silently dropped
// or fabricated.
func TestService_PublishSessionsCancelledCoversSessionsMissingFromSnapshot(t *testing.T) {
svc, eventBus, _ := createTestService(t)
ctx := context.Background()
// The exact shape of the race: CancelActiveTaskSessionsByTaskID returned
// this session's row, but the caller's pre-cancel snapshot is empty.
cancelledSession := &models.TaskSession{
ID: "session-not-in-snapshot",
TaskID: "task-race",
State: models.TaskSessionStateCancelled,
AgentProfileID: "agent-1",
UpdatedAt: time.Now().UTC(),
}
svc.publishSessionsCancelled(ctx, "task-race", nil, []*models.TaskSession{cancelledSession}, "task archived")
var found *bus.Event
for _, evt := range eventBus.GetPublishedEvents() {
if evt.Type != events.TaskSessionStateChanged {
continue
}
data, ok := evt.Data.(map[string]interface{})
if ok && data["session_id"] == "session-not-in-snapshot" {
found = evt
}
}
if found == nil {
t.Fatal("expected a session.state_changed event for the snapshot-missing session, got none")
}
data := found.Data.(map[string]interface{})
if got := data["new_state"]; got != string(models.TaskSessionStateCancelled) {
t.Errorf("new_state = %v, want CANCELLED", got)
}
if got := data["old_state"]; got != "" {
t.Errorf("old_state = %v, want empty (no snapshot hint available)", got)
}
if got := data["agent_profile_id"]; got != "agent-1" {
t.Errorf("agent_profile_id = %v, want agent-1 (must come from the returned row, not the snapshot)", got)
}
}
// cancelHookSessionRepository wraps the real session repository so a test
// can deterministically simulate a client disconnecting the instant after
// CancelActiveTaskSessionsByTaskID's DB write commits — the exact race
// finalizeCancelledSessions guards against by publishing on a
// context.WithoutCancel derivative. Test-only: production code has no
// equivalent hook.
type cancelHookSessionRepository struct {
*sqliterepo.Repository
afterCancelSessions func()
}
func (r *cancelHookSessionRepository) CancelActiveTaskSessionsByTaskID(
ctx context.Context, taskID, reason string,
) ([]*models.TaskSession, error) {
cancelledSessions, err := r.Repository.CancelActiveTaskSessionsByTaskID(ctx, taskID, reason)
if r.afterCancelSessions != nil {
r.afterCancelSessions()
}
return cancelledSessions, err
}
// TestService_ArchiveTaskPublishesEventAfterClientDisconnect is the
// regression test for the client-disconnect race: finalizeCancelledSessions
// commits the session-cancellation DB write on the caller's ctx, then
// publishes session.state_changed on a context.WithoutCancel derivative
// specifically so that a client disconnecting (cancelling ctx) in the instant
// right after that DB write still gets the event. A
// cancelHookSessionRepository simulates that race deterministically by
// cancelling ctx from inside CancelActiveTaskSessionsByTaskID, right after
// the real DB write returns.
func TestService_ArchiveTaskPublishesEventAfterClientDisconnect(t *testing.T) {
hook := &cancelHookSessionRepository{}
svc, eventBus, repo := createTestServiceWithSessionsRepo(t, func(repo *sqliterepo.Repository) repository.SessionRepository {
hook.Repository = repo
return hook
})
setupCtx := context.Background()
if err := repo.CreateWorkspace(setupCtx, &models.Workspace{ID: "ws-1", Name: "Workspace"}); err != nil {
t.Fatalf("CreateWorkspace: %v", err)
}
if err := repo.CreateWorkflow(setupCtx, &models.Workflow{ID: "wf-1", WorkspaceID: "ws-1", Name: "Workflow"}); err != nil {
t.Fatalf("CreateWorkflow: %v", err)
}
if err := repo.CreateTask(setupCtx, &models.Task{
ID: "task-1", WorkspaceID: "ws-1", WorkflowID: "wf-1", WorkflowStepID: "step-1",
Title: "Test", Priority: "medium",
}); err != nil {
t.Fatalf("CreateTask: %v", err)
}
if err := repo.CreateTaskSession(setupCtx, &models.TaskSession{
ID: "session-running", TaskID: "task-1", State: models.TaskSessionStateRunning,
AgentProfileID: "agent-1", IsPrimary: true,
}); err != nil {
t.Fatalf("CreateTaskSession: %v", err)
}
ctx, cancel := context.WithCancel(context.Background())
hook.afterCancelSessions = cancel
if err := svc.ArchiveTask(ctx, "task-1"); err != nil {
t.Fatalf("ArchiveTask returned an error even though ctx was only cancelled after the sessions DB write committed: %v", err)
}
var found *bus.Event
for _, evt := range eventBus.GetPublishedEvents() {
if evt.Type != events.TaskSessionStateChanged {
continue
}
data, ok := evt.Data.(map[string]interface{})
if ok && data["session_id"] == "session-running" {
found = evt
}
}
if found == nil {
t.Fatal("expected a session.state_changed event for session-running despite the post-commit ctx cancellation, got none")
}
data := found.Data.(map[string]interface{})
if got := data["session_id"]; got != "session-running" {
t.Errorf("session_id = %v, want session-running", got)
}
if got := data["new_state"]; got != string(models.TaskSessionStateCancelled) {
t.Errorf("new_state = %v, want CANCELLED", got)
}
updatedAtStr, _ := data["updated_at"].(string)
if _, err := time.Parse(time.RFC3339Nano, updatedAtStr); err != nil {
t.Errorf("updated_at %q not parseable as RFC3339Nano: %v", updatedAtStr, err)
}
session, err := repo.GetTaskSession(context.Background(), "session-running")
if err != nil {
t.Fatalf("GetTaskSession: %v", err)
}
if session.State != models.TaskSessionStateCancelled {
t.Errorf("session state = %q, want CANCELLED", session.State)
}
}
// flakyCancelSessionRepository wraps the real session repository so a test
// can deterministically simulate CancelActiveTaskSessionsByTaskID failing
// with a transient error (e.g. the kind of writer-contention timeout the
// repository's own internal 10s bound produces) on its first N-1 calls and
// succeeding on the Nth. Test-only: production code has no equivalent hook.
type flakyCancelSessionRepository struct {
*sqliterepo.Repository
mu sync.Mutex
failuresLeft int
calls int
}
func (r *flakyCancelSessionRepository) CancelActiveTaskSessionsByTaskID(
ctx context.Context, taskID, reason string,
) ([]*models.TaskSession, error) {
r.mu.Lock()
r.calls++
if r.failuresLeft > 0 {
r.failuresLeft--
r.mu.Unlock()
return nil, errors.New("simulated transient sqlite writer contention timeout")
}
r.mu.Unlock()
return r.Repository.CancelActiveTaskSessionsByTaskID(ctx, taskID, reason)
}
func (r *flakyCancelSessionRepository) callCount() int {
r.mu.Lock()
defer r.mu.Unlock()
return r.calls
}
// TestService_ArchiveTaskRetriesTransientSessionCancellationFailure is the
// regression test for the retry loop in finalizeCancelledSessions:
// CancelActiveTaskSessionsByTaskID is bounded by its own internal timeout,
// so a lone attempt failing (e.g. to writer contention) used to be swallowed
// immediately, leaving the archived task's sessions stuck active with no
// cancellation event ever published. Failing the first two calls and
// succeeding on the third proves the retry loop keeps trying within its
// bounded attempt budget and still delivers the session.state_changed event
// once the underlying write finally succeeds.
func TestService_ArchiveTaskRetriesTransientSessionCancellationFailure(t *testing.T) {
flaky := &flakyCancelSessionRepository{failuresLeft: 2}
svc, eventBus, repo := createTestServiceWithSessionsRepo(t, func(repo *sqliterepo.Repository) repository.SessionRepository {
flaky.Repository = repo
return flaky
})
ctx := context.Background()
if err := repo.CreateWorkspace(ctx, &models.Workspace{ID: "ws-1", Name: "Workspace"}); err != nil {
t.Fatalf("CreateWorkspace: %v", err)
}
if err := repo.CreateWorkflow(ctx, &models.Workflow{ID: "wf-1", WorkspaceID: "ws-1", Name: "Workflow"}); err != nil {
t.Fatalf("CreateWorkflow: %v", err)
}
if err := repo.CreateTask(ctx, &models.Task{
ID: "task-flaky", WorkspaceID: "ws-1", WorkflowID: "wf-1", WorkflowStepID: "step-1",
Title: "Test", Priority: "medium",
}); err != nil {
t.Fatalf("CreateTask: %v", err)
}
if err := repo.CreateTaskSession(ctx, &models.TaskSession{
ID: "session-flaky", TaskID: "task-flaky", State: models.TaskSessionStateRunning,
AgentProfileID: "agent-1", IsPrimary: true,
}); err != nil {
t.Fatalf("CreateTaskSession: %v", err)
}
if err := svc.ArchiveTask(ctx, "task-flaky"); err != nil {
t.Fatalf("ArchiveTask returned an error even though the retry loop should have recovered: %v", err)
}
if got, want := flaky.callCount(), 3; got != want {
t.Fatalf("CancelActiveTaskSessionsByTaskID call count = %d, want %d (2 failures + 1 success)", got, want)
}
var found *bus.Event
for _, evt := range eventBus.GetPublishedEvents() {
if evt.Type != events.TaskSessionStateChanged {
continue
}
data, ok := evt.Data.(map[string]interface{})
if ok && data["session_id"] == "session-flaky" {
found = evt
}
}
if found == nil {
t.Fatal("expected a session.state_changed event for session-flaky after the retry loop recovered, got none")
}
data := found.Data.(map[string]interface{})
if got := data["new_state"]; got != string(models.TaskSessionStateCancelled) {
t.Errorf("new_state = %v, want CANCELLED", got)
}
session, err := repo.GetTaskSession(context.Background(), "session-flaky")
if err != nil {
t.Fatalf("GetTaskSession: %v", err)
}
if session.State != models.TaskSessionStateCancelled {
t.Errorf("session state = %q, want CANCELLED", session.State)
}
}
// TestService_ArchiveTaskPublishesEventsForAllCancelledSessions is the
// regression test for the shared-deadline bug: publishSessionsCancelled used
// to run its per-session Publish call under one deadline shared across the
// whole cancelledSessions batch, so a slow synchronous subscriber for one
// session could starve the event for every session queued after it.
// Archiving a task with two active sessions and asserting both get their own
// session.state_changed event proves the fix (each session now gets an
// independent 10s timeout) without relying on a real timing race.
func TestService_ArchiveTaskPublishesEventsForAllCancelledSessions(t *testing.T) {
svc, eventBus, repo := createTestService(t)
ctx := context.Background()
if err := repo.CreateWorkspace(ctx, &models.Workspace{ID: "ws-1", Name: "Workspace"}); err != nil {
t.Fatalf("CreateWorkspace: %v", err)
}
if err := repo.CreateWorkflow(ctx, &models.Workflow{ID: "wf-1", WorkspaceID: "ws-1", Name: "Workflow"}); err != nil {
t.Fatalf("CreateWorkflow: %v", err)
}
if err := repo.CreateTask(ctx, &models.Task{
ID: "task-multi", WorkspaceID: "ws-1", WorkflowID: "wf-1", WorkflowStepID: "step-1",
Title: "Test", Priority: "medium",
}); err != nil {
t.Fatalf("CreateTask: %v", err)
}
if err := repo.CreateTaskSession(ctx, &models.TaskSession{
ID: "session-running-a", TaskID: "task-multi", State: models.TaskSessionStateRunning,
AgentProfileID: "agent-1", IsPrimary: true,
}); err != nil {
t.Fatalf("CreateTaskSession(session-running-a): %v", err)
}
if err := repo.CreateTaskSession(ctx, &models.TaskSession{
ID: "session-running-b", TaskID: "task-multi", State: models.TaskSessionStateWaitingForInput,
AgentProfileID: "agent-2",
}); err != nil {
t.Fatalf("CreateTaskSession(session-running-b): %v", err)
}
if err := svc.ArchiveTask(ctx, "task-multi"); err != nil {
t.Fatalf("ArchiveTask: %v", err)
}
foundBySessionID := make(map[string]*bus.Event)
for _, evt := range eventBus.GetPublishedEvents() {
if evt.Type != events.TaskSessionStateChanged {
continue
}
data, ok := evt.Data.(map[string]interface{})
if !ok {
continue
}
sessionID, _ := data["session_id"].(string)
if sessionID == "session-running-a" || sessionID == "session-running-b" {
foundBySessionID[sessionID] = evt
}
}
for _, sessionID := range []string{"session-running-a", "session-running-b"} {
evt, ok := foundBySessionID[sessionID]
if !ok {
t.Fatalf("expected a session.state_changed event for %s, got none (events: %#v)", sessionID, eventBus.GetPublishedEvents())
}
data := evt.Data.(map[string]interface{})
if got := data["new_state"]; got != string(models.TaskSessionStateCancelled) {
t.Errorf("%s: new_state = %v, want CANCELLED", sessionID, got)
}
if got := data["task_id"]; got != "task-multi" {
t.Errorf("%s: task_id = %v, want task-multi", sessionID, got)
}
}
for _, sessionID := range []string{"session-running-a", "session-running-b"} {
session, err := repo.GetTaskSession(ctx, sessionID)
if err != nil {
t.Fatalf("GetTaskSession(%s): %v", sessionID, err)
}
if session.State != models.TaskSessionStateCancelled {
t.Errorf("%s: session state = %q, want CANCELLED", sessionID, session.State)
}
}
}
func TestService_ArchiveTaskClaimsExactExecutionBeforeCancellingSession(t *testing.T) {
svc, _, repo := createTestService(t)
ctx := context.Background()