fix: freeze archived task state against late REVIEW writes (#1706)

This commit is contained in:
Clément Fiere
2026-07-18 18:30:24 +02:00
committed by GitHub
parent 1ba81a7cd0
commit 69972b4f79
20 changed files with 1338 additions and 37 deletions
@@ -59,6 +59,14 @@ func (a *taskRepositoryAdapter) UpdateTaskStateIfCurrentIn(
return a.svc.UpdateTaskStateIfCurrentIn(ctx, taskID, state, allowed)
}
// UpdateTaskStateIfNotArchived is UpdateTaskStateIfCurrentIn without the
// prior-state constraint (see scheduler.TaskRepository doc).
func (a *taskRepositoryAdapter) UpdateTaskStateIfNotArchived(
ctx context.Context, taskID string, state v1.TaskState,
) (bool, error) {
return a.svc.UpdateTaskStateIfNotArchived(ctx, taskID, state)
}
// lifecycleAdapter adapts the lifecycle manager as an AgentManagerClient
type lifecycleAdapter struct {
mgr *lifecycle.Manager
@@ -73,6 +73,12 @@ func (a *taskRepositoryAdapter) UpdateTaskStateIfCurrentIn(
return a.svc.UpdateTaskStateIfCurrentIn(ctx, taskID, state, allowed)
}
func (a *taskRepositoryAdapter) UpdateTaskStateIfNotArchived(
ctx context.Context, taskID string, state v1.TaskState,
) (bool, error) {
return a.svc.UpdateTaskStateIfNotArchived(ctx, taskID, state)
}
// testMessageCreatorAdapter adapts the task service to the orchestrator.MessageCreator interface for tests
type testMessageCreatorAdapter struct {
svc *taskservice.Service
@@ -819,6 +819,16 @@ func (s *Service) setSessionWaitingForInput(ctx context.Context, taskID, session
s.writeTaskReviewState(ctx, taskID, sessionID)
}
// taskArchived reports whether a task row has been archived. Runtime-state
// writes (IN_PROGRESS on session start, REVIEW on turn completion/cancel/
// startup reconciliation) must never resurrect an archived task's state —
// once archived_at is set the row is frozen from the kanban's perspective,
// so every write site here has to skip it instead of reviving stale runtime
// state that raced the archive.
func taskArchived(task *models.Task) bool {
return task != nil && task.ArchivedAt != nil
}
func (s *Service) writeTaskReviewState(ctx context.Context, taskID, completedSessionID string) {
// Task lookup errors fail closed so office/archived guards cannot be bypassed
// by a transient repository failure.
@@ -831,6 +841,10 @@ func (s *Service) writeTaskReviewState(ctx context.Context, taskID, completedSes
s.logger.Debug("skipping REVIEW transition for office task",
zap.String("task_id", taskID))
return
} else if taskArchived(dbTask) {
s.logger.Debug("skipping REVIEW transition for archived task",
zap.String("task_id", taskID))
return
}
s.taskRuntimeStateMu.Lock()
@@ -921,6 +935,11 @@ func (s *Service) writeTaskReviewStateOnCancel(ctx context.Context, taskID, sess
if dbTask.AssigneeAgentProfileID != "" {
return
}
if taskArchived(dbTask) {
s.logger.Debug("skipping REVIEW transition after cancel for archived task",
zap.String("task_id", taskID))
return
}
s.taskRuntimeStateMu.Lock()
defer s.taskRuntimeStateMu.Unlock()
@@ -1026,7 +1045,7 @@ func (s *Service) writeTaskInProgressForRuntime(ctx context.Context, taskID, ses
zap.Error(err))
return
}
if task != nil && task.ArchivedAt != nil {
if taskArchived(task) {
s.logger.Debug("skipping IN_PROGRESS transition for archived task",
zap.String("task_id", taskID))
return
@@ -1065,11 +1084,17 @@ func (s *Service) writeTaskInProgressForRuntime(ctx context.Context, taskID, ses
}
}
if err := s.taskRepo.UpdateTaskState(ctx, taskID, v1.TaskStateInProgress); err != nil {
// UpdateTaskStateIfNotArchived (not the unconditional UpdateTaskState) so
// the write is atomic against archived_at: the taskArchived guard above
// reads the row before this call, and ArchiveTask can commit in that
// window without changing task.State, so only an archive-aware
// conditional write closes the race (PR #1706 review).
updated, err := s.taskRepo.UpdateTaskStateIfNotArchived(ctx, taskID, v1.TaskStateInProgress)
if err != nil {
s.logger.Error("failed to update task state to IN_PROGRESS",
zap.String("task_id", taskID),
zap.Error(err))
} else {
} else if updated {
s.logger.Info("task moved to IN_PROGRESS state",
zap.String("task_id", taskID))
}
@@ -1204,6 +1204,71 @@ func TestWriteTaskInProgressForRuntimeSkipsArchivedTask(t *testing.T) {
"runtime reconciliation must not promote archived tasks to IN_PROGRESS")
}
// TestWriteTaskInProgressForRuntimeUsesArchiveAwareCAS is the TOCTOU
// companion to TestWriteTaskInProgressForRuntimeSkipsArchivedTask
// (carlosflorencio review on PR #1706): the earlier taskArchived() guard is
// a plain, non-transactional read, so an ArchiveTask commit landing in the
// window between that read and the write could still resurrect the task's
// state if the write itself were the unconditional UpdateTaskState. Asserts
// the write goes through UpdateTaskStateIfNotArchived (tracked separately
// from the unconditional path via unconditionalWrites) instead.
func TestWriteTaskInProgressForRuntimeUsesArchiveAwareCAS(t *testing.T) {
ctx := context.Background()
repo := setupTestRepo(t)
seedSession(t, repo, "t1", "s1", "")
taskRepo := newMockTaskRepo()
svc := createTestService(repo, newMockStepGetter(), taskRepo)
svc.writeTaskInProgressForRuntime(ctx, "t1", "s1")
require.Equal(t, 1, taskRepo.stateWrites["t1"],
"runtime reconciliation must still promote a non-archived task to IN_PROGRESS")
require.Equal(t, v1.TaskStateInProgress, taskRepo.updatedStates["t1"])
require.Zero(t, taskRepo.unconditionalWrites["t1"],
"IN_PROGRESS write must use UpdateTaskStateIfNotArchived, not the unconditional UpdateTaskState")
}
func TestWriteTaskReviewStateSkipsArchivedTask(t *testing.T) {
ctx := context.Background()
repo := setupTestRepo(t)
seedSession(t, repo, "t1", "s1", "")
session, err := repo.GetTaskSession(ctx, "s1")
require.NoError(t, err)
session.State = models.TaskSessionStateWaitingForInput
require.NoError(t, repo.UpdateTaskSession(ctx, session))
require.NoError(t, repo.ArchiveTask(ctx, "t1"))
taskRepo := newMockTaskRepo()
taskRepo.tasks["t1"] = &v1.Task{ID: "t1", State: v1.TaskStateInProgress}
svc := createTestService(repo, newMockStepGetter(), taskRepo)
svc.writeTaskReviewState(ctx, "t1", "s1")
require.Zero(t, taskRepo.stateWrites["t1"],
"REVIEW reconcile must not resurrect an archived task's state")
}
func TestWriteTaskReviewStateOnCancelSkipsArchivedTask(t *testing.T) {
ctx := context.Background()
repo := setupTestRepo(t)
seedSession(t, repo, "t1", "s1", "")
session, err := repo.GetTaskSession(ctx, "s1")
require.NoError(t, err)
session.State = models.TaskSessionStateWaitingForInput
require.NoError(t, repo.UpdateTaskSession(ctx, session))
require.NoError(t, repo.ArchiveTask(ctx, "t1"))
taskRepo := newMockTaskRepo()
taskRepo.tasks["t1"] = &v1.Task{ID: "t1", State: v1.TaskStateInProgress}
svc := createTestService(repo, newMockStepGetter(), taskRepo)
svc.writeTaskReviewStateOnCancel(ctx, "t1", "s1")
require.Zero(t, taskRepo.stateWrites["t1"],
"cancel reconcile must not resurrect an archived task's state")
}
func TestWriteTaskReviewStateOnCancelSkipsWhenSessionActiveAgain(t *testing.T) {
ctx := context.Background()
repo := setupTestRepo(t)
@@ -96,15 +96,23 @@ type mockTaskRepo struct {
// dispatch's IN_PROGRESS write) — tests asserting a state was NEVER
// written at any point must check stateHistory, not updatedStates.
stateHistory map[string][]v1.TaskState
getTaskErr error // if set, GetTask returns this error
// unconditionalWrites counts UpdateTaskState (the non-CAS write) calls per
// task, tracked separately from stateWrites (which both UpdateTaskState
// and UpdateTaskStateIfCurrentIn increment). Startup/crash reconciliation
// callers must route guarded REVIEW writes through the CAS method so an
// archive can't race a late write; tests assert this stays 0 for those
// paths instead of just checking the resulting state.
unconditionalWrites map[string]int
getTaskErr error // if set, GetTask returns this error
}
func newMockTaskRepo() *mockTaskRepo {
return &mockTaskRepo{
tasks: make(map[string]*v1.Task),
updatedStates: make(map[string]v1.TaskState),
stateWrites: make(map[string]int),
stateHistory: make(map[string][]v1.TaskState),
tasks: make(map[string]*v1.Task),
updatedStates: make(map[string]v1.TaskState),
stateWrites: make(map[string]int),
stateHistory: make(map[string][]v1.TaskState),
unconditionalWrites: make(map[string]int),
}
}
@@ -130,6 +138,7 @@ func (m *mockTaskRepo) UpdateTaskState(_ context.Context, taskID string, state v
m.updatedStates[taskID] = state
m.stateWrites[taskID]++
m.stateHistory[taskID] = append(m.stateHistory[taskID], state)
m.unconditionalWrites[taskID]++
if t, ok := m.tasks[taskID]; ok {
t.State = state
}
@@ -158,6 +167,30 @@ func (m *mockTaskRepo) UpdateTaskStateIfCurrentIn(
return false, nil
}
// UpdateTaskStateIfNotArchived has no "allowed" precondition to check, so
// (unlike UpdateTaskStateIfCurrentIn above) this mock cannot model the
// archived_at race itself — v1.Task carries no ArchivedAt field for it to
// consult. It behaves like UpdateTaskState above: unconditional and
// always tracked, mutating the seeded task's State only when present —
// existing callers seed via seedMockTaskState only when they need to read
// the state back, not merely to assert a write happened.
// Real archived-freeze coverage for this CAS lives at the sqlite layer
// (task_state_cas_test.go) and the executor-layer mock (models.Task, which
// does carry ArchivedAt).
func (m *mockTaskRepo) UpdateTaskStateIfNotArchived(
_ context.Context, taskID string, state v1.TaskState,
) (bool, error) {
m.mu.Lock()
defer m.mu.Unlock()
m.updatedStates[taskID] = state
m.stateWrites[taskID]++
m.stateHistory[taskID] = append(m.stateHistory[taskID], state)
if t, ok := m.tasks[taskID]; ok {
t.State = state
}
return true, nil
}
// mockAgentManager is a minimal mock of executor.AgentManagerClient for testing.
type mockAgentManager struct {
isPassthrough bool
@@ -23,7 +23,21 @@ import (
type executorStore interface {
// Task
GetTask(ctx context.Context, id string) (*models.Task, error)
UpdateTaskState(ctx context.Context, id string, state v1.TaskState) error
// UpdateTaskStateIfNotArchived atomically transitions state unless the
// task is archived (archived_at IS NULL). Used instead of a plain
// unconditional UpdateTaskState for every runtime-driven task-state write
// (IN_PROGRESS on start/resume, FAILED on launch error) so a late write
// can never race an archive that lands after an earlier (non-transactional)
// archived-state check. Returns the pre-update state and whether a row
// was modified.
UpdateTaskStateIfNotArchived(ctx context.Context, id string, state v1.TaskState) (v1.TaskState, bool, error)
// UpdateTaskStateIfCurrentIn atomically transitions state only when the
// current state is in allowed AND the task is not archived (archived_at
// IS NULL). Used instead of UpdateTaskState for guarded REVIEW writes so
// a late write can never race an archive that lands after an earlier
// (non-transactional) archived-state check. Returns the pre-update state
// and whether a row was modified.
UpdateTaskStateIfCurrentIn(ctx context.Context, id string, state v1.TaskState, allowed []v1.TaskState) (v1.TaskState, bool, error)
// Task↔repo junction
GetPrimaryTaskRepository(ctx context.Context, taskID string) (*models.TaskRepository, error)
ListTaskRepositories(ctx context.Context, taskID string) ([]*models.TaskRepository, error)
@@ -148,7 +148,23 @@ func (e *Executor) writeTaskReviewStateIfNoWorkingSessions(ctx context.Context,
if e.hasOtherWorkingSessions(ctx, taskID, failedSessionID) {
return
}
if updateErr := e.updateTaskState(ctx, taskID, v1.TaskStateReview); updateErr != nil {
// When onTaskStateChange is configured, it owns event publishing for
// this write (see its doc comment) — keep routing through it rather
// than bypassing it. Only when NEITHER callback is set (no orchestrator
// wiring at all — production always wires both, see service.go's
// exec.SetOnTaskStateChange/SetOnTaskReviewStateReconcile) do we fall
// back to the archive-aware UpdateTaskStateIfCurrentIn CAS directly on
// the repository, so even that raw path can't race an archive that
// commits between shouldSkipFailedStartReviewForTask's read and this write.
if e.onTaskStateChange != nil {
if updateErr := e.onTaskStateChange(ctx, taskID, v1.TaskStateReview); updateErr != nil {
e.logger.Warn("failed to update task state to REVIEW after start error",
zap.String("task_id", taskID),
zap.Error(updateErr))
}
return
}
if _, _, updateErr := e.repo.UpdateTaskStateIfCurrentIn(ctx, taskID, v1.TaskStateReview, []v1.TaskState{v1.TaskStateInProgress, v1.TaskStateScheduling}); updateErr != nil {
e.logger.Warn("failed to update task state to REVIEW after start error",
zap.String("task_id", taskID),
zap.Error(updateErr))
@@ -170,6 +186,12 @@ func (e *Executor) shouldSkipFailedStartReviewForTask(ctx context.Context, taskI
zap.String("session_id", failedSessionID))
return true
}
if task != nil && task.ArchivedAt != nil {
e.logger.Debug("skipping failed-start task REVIEW state for archived task",
zap.String("task_id", taskID),
zap.String("session_id", failedSessionID))
return true
}
return false
}
@@ -227,12 +249,15 @@ func isRuntimeWorkingSessionState(state models.TaskSessionState) bool {
}
// updateTaskState updates a task's state, using the callback if set for event publishing,
// or falling back to the raw repository.
// or falling back to the archive-aware CAS (UpdateTaskStateIfNotArchived) directly. Every
// call site here is a runtime-driven write (IN_PROGRESS on start/resume, FAILED on launch
// error) that must never resurrect an archived task's state (PR #1706 review).
func (e *Executor) updateTaskState(ctx context.Context, taskID string, state v1.TaskState) error {
if e.onTaskStateChange != nil {
return e.onTaskStateChange(ctx, taskID, state)
}
return e.repo.UpdateTaskState(ctx, taskID, state)
_, _, err := e.repo.UpdateTaskStateIfNotArchived(ctx, taskID, state)
return err
}
// updateSessionState updates a session's state, using the callback if set for event publishing,
@@ -258,14 +258,48 @@ type mockRepository struct {
// transient DB error); if nil, the default map lookup is used.
getTaskSessionFunc func(ctx context.Context, id string) (*models.TaskSession, error)
createTaskSessionFunc func(ctx context.Context, session *models.TaskSession) error
// Optional hook invoked at the top of UpdateTaskStateIfCurrentIn, before
// it reads task state/archived_at. Lets tests simulate the exact TOCTOU
// window this CAS closes: an earlier (non-transactional) archived-state
// guard already passed, then an archive commits in the gap before this
// call — the hook mutates the task here to model that gap. If nil, the
// CAS runs immediately against the task as currently seeded.
preCASHook func(taskID string)
// updateTaskStateIfNotArchivedCh, when non-nil, receives a non-blocking
// signal every time UpdateTaskStateIfNotArchived records a call — lets
// tests wait via select instead of polling with time.Sleep for a
// background goroutine's call to land. Buffered so a signal is never
// lost even if the test hasn't started waiting yet.
updateTaskStateIfNotArchivedCh chan struct{}
// Track calls for verification
createTaskSessionCalls []*models.TaskSession
updateTaskSessionCalls []*models.TaskSession
setSessionMetadataKeyCalls []setSessionMetadataKeyCall
setSessionPrimaryCalls []string
createTaskEnvironmentCalls []*models.TaskEnvironment
updateTaskEnvironmentCalls []*models.TaskEnvironment
createTaskSessionCalls []*models.TaskSession
updateTaskSessionCalls []*models.TaskSession
setSessionMetadataKeyCalls []setSessionMetadataKeyCall
setSessionPrimaryCalls []string
createTaskEnvironmentCalls []*models.TaskEnvironment
updateTaskEnvironmentCalls []*models.TaskEnvironment
updateTaskStateIfCurrentInCalls []updateTaskStateIfCurrentInCall
updateTaskStateIfNotArchivedCalls []updateTaskStateIfNotArchivedCall
}
// updateTaskStateIfCurrentInCall records one UpdateTaskStateIfCurrentIn
// invocation so tests can assert callers route guarded REVIEW writes
// through the archive-aware CAS instead of the unconditional
// UpdateTaskState.
type updateTaskStateIfCurrentInCall struct {
TaskID string
State v1.TaskState
Allowed []v1.TaskState
}
// updateTaskStateIfNotArchivedCall records one UpdateTaskStateIfNotArchived
// invocation — the IN_PROGRESS/FAILED-writer analog of
// updateTaskStateIfCurrentInCall, used to assert those callers route
// through the archive-aware CAS instead of the unconditional UpdateTaskState.
type updateTaskStateIfNotArchivedCall struct {
TaskID string
State v1.TaskState
}
type setSessionMetadataKeyCall struct {
@@ -276,15 +310,16 @@ type setSessionMetadataKeyCall struct {
func newMockRepository() *mockRepository {
return &mockRepository{
sessions: make(map[string]*models.TaskSession),
tasks: make(map[string]*models.Task),
taskRepositories: make(map[string]*models.TaskRepository),
repositories: make(map[string]*models.Repository),
executors: make(map[string]*models.Executor),
executorProfiles: make(map[string]*models.ExecutorProfile),
executorsRunning: make(map[string]*models.ExecutorRunning),
taskEnvironments: make(map[string]*models.TaskEnvironment),
taskEnvironmentRepos: make(map[string][]*models.TaskEnvironmentRepo),
sessions: make(map[string]*models.TaskSession),
tasks: make(map[string]*models.Task),
taskRepositories: make(map[string]*models.TaskRepository),
repositories: make(map[string]*models.Repository),
executors: make(map[string]*models.Executor),
executorProfiles: make(map[string]*models.ExecutorProfile),
executorsRunning: make(map[string]*models.ExecutorRunning),
taskEnvironments: make(map[string]*models.TaskEnvironment),
taskEnvironmentRepos: make(map[string][]*models.TaskEnvironmentRepo),
updateTaskStateIfNotArchivedCh: make(chan struct{}, 8),
}
}
@@ -388,6 +423,80 @@ func (m *mockRepository) CreateTaskSessionWorktree(_ context.Context, worktree *
func (m *mockRepository) UpdateTaskState(ctx context.Context, taskID string, state v1.TaskState) error {
return nil
}
// UpdateTaskStateIfCurrentIn mirrors the real repository's archive-aware CAS
// semantics (task.go's UpdateTaskStateIfCurrentIn): the write only lands when
// the task's current state is in allowed AND the task is not archived.
func (m *mockRepository) UpdateTaskStateIfCurrentIn(
ctx context.Context, taskID string, state v1.TaskState, allowed []v1.TaskState,
) (v1.TaskState, bool, error) {
if m.preCASHook != nil {
m.preCASHook(taskID)
}
m.mu.Lock()
defer m.mu.Unlock()
m.updateTaskStateIfCurrentInCalls = append(m.updateTaskStateIfCurrentInCalls, updateTaskStateIfCurrentInCall{
TaskID: taskID, State: state, Allowed: allowed,
})
task, ok := m.tasks[taskID]
if !ok {
return "", false, fmt.Errorf("task not found: %s", taskID)
}
currentState := task.State
if task.ArchivedAt != nil {
return currentState, false, nil
}
for _, candidate := range allowed {
if currentState == candidate {
task.State = state
return currentState, true, nil
}
}
return currentState, false, nil
}
// UpdateTaskStateIfNotArchived mirrors the real repository's archive-aware
// CAS semantics (task.go's UpdateTaskStateIfNotArchived): unlike
// UpdateTaskStateIfCurrentIn, there is no "allowed" prior-state set — the
// write lands whenever the task is not archived. Reuses preCASHook so tests
// can model the same TOCTOU window (archive commits between an earlier
// non-transactional guard read and this call) for IN_PROGRESS/FAILED writers.
func (m *mockRepository) UpdateTaskStateIfNotArchived(
ctx context.Context, taskID string, state v1.TaskState,
) (v1.TaskState, bool, error) {
if m.preCASHook != nil {
m.preCASHook(taskID)
}
defer m.notifyUpdateTaskStateIfNotArchived()
m.mu.Lock()
defer m.mu.Unlock()
m.updateTaskStateIfNotArchivedCalls = append(m.updateTaskStateIfNotArchivedCalls, updateTaskStateIfNotArchivedCall{
TaskID: taskID, State: state,
})
task, ok := m.tasks[taskID]
if !ok {
return "", false, fmt.Errorf("task not found: %s", taskID)
}
currentState := task.State
if task.ArchivedAt != nil {
return currentState, false, nil
}
task.State = state
return currentState, true, nil
}
// notifyUpdateTaskStateIfNotArchived signals updateTaskStateIfNotArchivedCh
// (if the test wired one) that a call just landed. Non-blocking so it never
// stalls the caller when nothing is listening.
func (m *mockRepository) notifyUpdateTaskStateIfNotArchived() {
if m.updateTaskStateIfNotArchivedCh == nil {
return
}
select {
case m.updateTaskStateIfNotArchivedCh <- struct{}{}:
default:
}
}
func (m *mockRepository) ArchiveTask(ctx context.Context, id string) error { return nil }
func (m *mockRepository) ListTasksForAutoArchive(ctx context.Context) ([]*models.Task, error) {
return nil, nil
@@ -1358,6 +1358,215 @@ func TestRunAgentProcessAsync_ResumeDoesNotEscalateTaskState(t *testing.T) {
}
}
// TestRunAgentProcessAsync_ResumeFailureUsesRawCASWithoutCallbacks covers the
// true raw-fallback branch: neither onTaskReviewStateReconcile nor
// onTaskStateChange is configured (a standalone Executor with no orchestrator
// wiring at all — never the case in production, see service.go's
// exec.SetOnTaskStateChange/SetOnTaskReviewStateReconcile). The REVIEW write
// must go straight through the archive-aware UpdateTaskStateIfCurrentIn CAS
// on the repository rather than the unconditional UpdateTaskState, so a late
// write here still can't race an archive.
func TestRunAgentProcessAsync_ResumeFailureUsesRawCASWithoutCallbacks(t *testing.T) {
repo := newMockRepository()
repo.sessions["session-123"] = &models.TaskSession{
ID: "session-123", TaskID: "task-123", State: models.TaskSessionStateStarting,
}
repo.tasks["task-123"] = &models.Task{ID: "task-123", State: v1.TaskStateInProgress}
stopCh := make(chan struct{})
agentManager := &mockAgentManager{
startAgentProcessFunc: func(ctx context.Context, agentExecutionID string) error {
return fmt.Errorf("ACP initialize handshake failed: context deadline exceeded")
},
stopAgentFunc: func(ctx context.Context, agentExecutionID string, force bool) error {
close(stopCh)
return nil
},
}
exec := newTestExecutor(t, agentManager, repo)
// Deliberately no SetOnTaskStateChange / SetOnTaskReviewStateReconcile.
exec.runAgentProcessAsync(context.Background(), "task-123", "session-123", "exec-456",
func(ctx context.Context) { t.Error("onSuccess should not run on failure") },
false, true) // resume path: no escalation, fromResume=true
select {
case <-stopCh:
case <-time.After(5 * time.Second):
t.Fatal("timed out waiting for StopAgent to be called")
}
if got := repo.tasks["task-123"].State; got != v1.TaskStateReview {
t.Errorf("expected resume failure to reconcile task state to REVIEW via the raw CAS, got %q", got)
}
if len(repo.updateTaskStateIfCurrentInCalls) != 1 {
t.Fatalf("expected exactly 1 UpdateTaskStateIfCurrentIn call, got %d: %+v", len(repo.updateTaskStateIfCurrentInCalls), repo.updateTaskStateIfCurrentInCalls)
}
call := repo.updateTaskStateIfCurrentInCalls[0]
if call.TaskID != "task-123" || call.State != v1.TaskStateReview {
t.Errorf("unexpected CAS call: %+v", call)
}
if len(call.Allowed) != 2 || call.Allowed[0] != v1.TaskStateInProgress || call.Allowed[1] != v1.TaskStateScheduling {
t.Errorf("expected CAS allowed=[IN_PROGRESS, SCHEDULING], got %v", call.Allowed)
}
}
// TestRunAgentProcessAsync_ResumeFailureSkipsArchivedTaskRacingCAS is the
// TOCTOU companion to TestRunAgentProcessAsync_ResumeFailureUsesRawCASWithoutCallbacks
// (cubic review finding on PR #1706): the task is NOT archived when
// shouldSkipFailedStartReviewForTask's earlier (non-transactional) guard
// runs — that guard passes — and only archives via preCASHook right as
// UpdateTaskStateIfCurrentIn is invoked, modeling ArchiveTask committing in
// the exact gap between the guard read and the atomic write. Without the
// archived_at check inside the CAS itself (not just the earlier guard),
// this write would still land and resurrect the task to REVIEW.
func TestRunAgentProcessAsync_ResumeFailureSkipsArchivedTaskRacingCAS(t *testing.T) {
repo := newMockRepository()
repo.sessions["session-123"] = &models.TaskSession{
ID: "session-123", TaskID: "task-123", State: models.TaskSessionStateStarting,
}
repo.tasks["task-123"] = &models.Task{ID: "task-123", State: v1.TaskStateInProgress}
repo.preCASHook = func(taskID string) {
if task, ok := repo.tasks[taskID]; ok {
archivedAt := time.Now().UTC()
task.ArchivedAt = &archivedAt
}
}
stopCh := make(chan struct{})
agentManager := &mockAgentManager{
startAgentProcessFunc: func(ctx context.Context, agentExecutionID string) error {
return fmt.Errorf("ACP initialize handshake failed: context deadline exceeded")
},
stopAgentFunc: func(ctx context.Context, agentExecutionID string, force bool) error {
close(stopCh)
return nil
},
}
exec := newTestExecutor(t, agentManager, repo)
// Deliberately no SetOnTaskStateChange / SetOnTaskReviewStateReconcile.
exec.runAgentProcessAsync(context.Background(), "task-123", "session-123", "exec-456",
func(ctx context.Context) { t.Error("onSuccess should not run on failure") },
false, true) // resume path: no escalation, fromResume=true
select {
case <-stopCh:
case <-time.After(5 * time.Second):
t.Fatal("timed out waiting for StopAgent to be called")
}
if got := repo.tasks["task-123"].State; got != v1.TaskStateInProgress {
t.Errorf("expected archive racing the CAS to leave state untouched, got %q", got)
}
if len(repo.updateTaskStateIfCurrentInCalls) != 1 {
t.Fatalf("expected the CAS to still be called (guard passed pre-archive), got %+v", repo.updateTaskStateIfCurrentInCalls)
}
}
// waitForUpdateTaskStateIfNotArchivedCall blocks (via a channel signal, not
// polling) until startAgentProcessAsync's background goroutine has recorded
// its UpdateTaskStateIfNotArchived call. Needed because — unlike the
// resume-failure tests above, where StopAgent (closing the sync channel)
// runs strictly after the state write — startAgentProcessAsync's onSuccess
// callback has nothing observable after the write itself.
func waitForUpdateTaskStateIfNotArchivedCall(t *testing.T, repo *mockRepository) {
t.Helper()
select {
case <-repo.updateTaskStateIfNotArchivedCh:
case <-time.After(5 * time.Second):
t.Fatal("timed out waiting for UpdateTaskStateIfNotArchived call")
}
}
// TestStartAgentProcessAsync_UsesRawCASWithoutCallbacks is the IN_PROGRESS
// analog of TestRunAgentProcessAsync_ResumeFailureUsesRawCASWithoutCallbacks:
// with no SetOnTaskStateChange callback wired, startAgentProcessAsync's
// post-launch IN_PROGRESS write must go through the archive-aware
// UpdateTaskStateIfNotArchived CAS on the repository rather than the
// unconditional UpdateTaskState (carlosflorencio review on PR #1706:
// writeTaskInProgressForRuntime's ArchivedAt guard was followed by an
// unconditional write, leaving the same TOCTOU window open as the REVIEW
// writers had before the CAS fix).
func TestStartAgentProcessAsync_UsesRawCASWithoutCallbacks(t *testing.T) {
repo := newMockRepository()
repo.sessions["session-123"] = &models.TaskSession{
ID: "session-123", TaskID: "task-123", State: models.TaskSessionStateStarting,
}
repo.tasks["task-123"] = &models.Task{ID: "task-123", State: v1.TaskStateWaitingForInput}
startedCh := make(chan struct{})
agentManager := &mockAgentManager{
startAgentProcessFunc: func(ctx context.Context, agentExecutionID string) error {
close(startedCh)
return nil
},
}
exec := newTestExecutor(t, agentManager, repo)
// Deliberately no SetOnTaskStateChange.
exec.startAgentProcessAsync(context.Background(), "task-123", "session-123", "exec-456")
select {
case <-startedCh:
case <-time.After(5 * time.Second):
t.Fatal("timed out waiting for StartAgentProcess to be called")
}
waitForUpdateTaskStateIfNotArchivedCall(t, repo)
if got := repo.tasks["task-123"].State; got != v1.TaskStateInProgress {
t.Errorf("expected agent start success to promote task state to IN_PROGRESS via the raw CAS, got %q", got)
}
call := repo.updateTaskStateIfNotArchivedCalls[0]
if call.TaskID != "task-123" || call.State != v1.TaskStateInProgress {
t.Errorf("unexpected CAS call: %+v", call)
}
}
// TestStartAgentProcessAsync_SkipsArchivedTaskRacingCAS is the TOCTOU
// companion to TestStartAgentProcessAsync_UsesRawCASWithoutCallbacks: the
// task is NOT archived when startAgentProcessAsync's earlier archived guard
// (inside its onSuccess callback, via updateTaskState → the repo fallback)
// would have run, and only archives via preCASHook right as
// UpdateTaskStateIfNotArchived is invoked, modeling ArchiveTask committing
// in the exact gap between an earlier archived-state read and the atomic
// write. Without the archived_at check inside the CAS itself, this write
// would still land and resurrect the task to IN_PROGRESS.
func TestStartAgentProcessAsync_SkipsArchivedTaskRacingCAS(t *testing.T) {
repo := newMockRepository()
repo.sessions["session-123"] = &models.TaskSession{
ID: "session-123", TaskID: "task-123", State: models.TaskSessionStateStarting,
}
repo.tasks["task-123"] = &models.Task{ID: "task-123", State: v1.TaskStateWaitingForInput}
repo.preCASHook = func(taskID string) {
if task, ok := repo.tasks[taskID]; ok {
archivedAt := time.Now().UTC()
task.ArchivedAt = &archivedAt
}
}
startedCh := make(chan struct{})
agentManager := &mockAgentManager{
startAgentProcessFunc: func(ctx context.Context, agentExecutionID string) error {
close(startedCh)
return nil
},
}
exec := newTestExecutor(t, agentManager, repo)
// Deliberately no SetOnTaskStateChange.
exec.startAgentProcessAsync(context.Background(), "task-123", "session-123", "exec-456")
select {
case <-startedCh:
case <-time.After(5 * time.Second):
t.Fatal("timed out waiting for StartAgentProcess to be called")
}
waitForUpdateTaskStateIfNotArchivedCall(t, repo)
if got := repo.tasks["task-123"].State; got != v1.TaskStateWaitingForInput {
t.Errorf("expected archive racing the CAS to leave state untouched, got %q", got)
}
}
func TestRunAgentProcessAsync_ResumeFailureDoesNotReviewWhileSiblingWorks(t *testing.T) {
f := newRunAgentProcessAsyncFailureFixture(t)
f.repo.sessions["session-456"] = &models.TaskSession{
@@ -1459,6 +1668,30 @@ func TestWriteTaskReviewStateIfNoWorkingSessionsSkipsOfficeTask(t *testing.T) {
}
}
func TestWriteTaskReviewStateIfNoWorkingSessionsSkipsArchivedTask(t *testing.T) {
repo := newMockRepository()
archivedAt := time.Now().UTC()
repo.tasks["task-123"] = &models.Task{
ID: "task-123",
ArchivedAt: &archivedAt,
}
repo.sessions["session-123"] = &models.TaskSession{
ID: "session-123", TaskID: "task-123", State: models.TaskSessionStateFailed,
}
exec := newTestExecutor(t, &mockAgentManager{}, repo)
var taskStateUpdates []v1.TaskState
exec.SetOnTaskStateChange(func(ctx context.Context, taskID string, state v1.TaskState) error {
taskStateUpdates = append(taskStateUpdates, state)
return nil
})
exec.writeTaskReviewStateIfNoWorkingSessions(context.Background(), "task-123", "session-123")
if len(taskStateUpdates) != 0 {
t.Errorf("expected archived task to keep its frozen state, got %v", taskStateUpdates)
}
}
func TestStartAgentProcessOnResumePromotesTaskAfterSuccess(t *testing.T) {
repo := newMockRepository()
session := &models.TaskSession{
@@ -42,7 +42,18 @@ func DefaultSchedulerConfig() SchedulerConfig {
type TaskRepository interface {
GetTask(ctx context.Context, taskID string) (*v1.Task, error)
UpdateTaskState(ctx context.Context, taskID string, state v1.TaskState) error
// UpdateTaskStateIfCurrentIn atomically transitions state only when the
// task's current state is in allowed AND the task is not archived
// (archived_at IS NULL, enforced inside the write's own transaction —
// not just by a caller's earlier, non-transactional archived-state
// check). Returns whether a row was modified.
UpdateTaskStateIfCurrentIn(ctx context.Context, taskID string, state v1.TaskState, allowed []v1.TaskState) (bool, error)
// UpdateTaskStateIfNotArchived is UpdateTaskStateIfCurrentIn without the
// prior-state constraint — for writers (e.g. IN_PROGRESS runtime
// reconciliation) that legitimately fire from many prior states and only
// need the archived_at IS NULL guarantee. Returns whether a row was
// modified.
UpdateTaskStateIfNotArchived(ctx context.Context, taskID string, state v1.TaskState) (bool, error)
}
// QueueStatus contains queue statistics
@@ -233,6 +233,19 @@ func (r *testTaskRepository) UpdateTaskStateIfCurrentIn(
return false, nil
}
func (r *testTaskRepository) UpdateTaskStateIfNotArchived(
_ context.Context, taskID string, state v1.TaskState,
) (bool, error) {
r.mu.Lock()
defer r.mu.Unlock()
task, exists := r.tasks[taskID]
if !exists {
return false, fmt.Errorf("%w: %s", taskrepo.ErrTaskNotFound, taskID)
}
task.State = state
return true, nil
}
func createTestLogger() *logger.Logger {
log, _ := logger.NewLogger(logger.LoggingConfig{
Level: "error", // Suppress logs during tests
+37 -7
View File
@@ -128,6 +128,13 @@ type repoStore interface {
sessionExecutorStore
// Additional methods needed by executor
UpdateTaskState(ctx context.Context, id string, state v1.TaskState) error
// UpdateTaskStateIfCurrentIn — see executor.executorStore's doc comment;
// required here because repo (this interface) is what NewService passes
// to executor.NewExecutor.
UpdateTaskStateIfCurrentIn(ctx context.Context, id string, state v1.TaskState, allowed []v1.TaskState) (v1.TaskState, bool, error)
// UpdateTaskStateIfNotArchived — see executor.executorStore's doc comment;
// required here for the same reason as UpdateTaskStateIfCurrentIn above.
UpdateTaskStateIfNotArchived(ctx context.Context, id string, state v1.TaskState) (v1.TaskState, bool, error)
GetPrimaryTaskRepository(ctx context.Context, taskID string) (*models.TaskRepository, error)
ListTaskRepositories(ctx context.Context, taskID string) ([]*models.TaskRepository, error)
CreateTaskSession(ctx context.Context, session *models.TaskSession) error
@@ -541,10 +548,25 @@ func NewService(
// Wire executor state changes through the orchestrator so events are published
// (e.g. WebSocket notifications to the frontend). Must be set after service
// construction so the session callback can reference s.updateTaskSessionState.
// UpdateTaskStateIfNotArchived (not the unconditional UpdateTaskState) so
// every runtime-driven task-state write the executor makes through this
// single callback — IN_PROGRESS on agent start/resume, FAILED on launch
// error — is atomic against archived_at. Each caller's own archived
// guard (e.g. writeTaskInProgressForRuntime) reads the task before this
// fires; ArchiveTask can commit in that window without an intervening
// state write to invalidate the guard, so only this conditional write
// closes the race (PR #1706 review).
exec.SetOnTaskStateChange(func(ctx context.Context, taskID string, state v1.TaskState) error {
if err := taskRepo.UpdateTaskState(ctx, taskID, state); err != nil {
updated, err := taskRepo.UpdateTaskStateIfNotArchived(ctx, taskID, state)
if err != nil {
return err
}
if !updated {
s.logger.Debug("skipping runtime task state write for archived task",
zap.String("task_id", taskID),
zap.String("state", string(state)))
return nil
}
s.processParentChildrenCompletedForTaskState(ctx, taskID, state)
return nil
})
@@ -1311,9 +1333,14 @@ func (s *Service) reconcileOneSessionOnStartup(ctx context.Context, running *mod
// Ensure task is in REVIEW state (not stuck IN_PROGRESS)
if running.TaskID != "" {
task, taskErr := s.taskRepo.GetTask(ctx, running.TaskID)
if taskErr == nil && task != nil && task.State == v1.TaskStateInProgress {
if updateErr := s.taskRepo.UpdateTaskState(ctx, running.TaskID, v1.TaskStateReview); updateErr != nil {
task, taskErr := s.repo.GetTask(ctx, running.TaskID)
if taskErr == nil && task != nil && task.State == v1.TaskStateInProgress && !taskArchived(task) {
// UpdateTaskStateIfCurrentIn (not the unconditional UpdateTaskState)
// so the write is atomic against archived_at: the taskArchived guard
// above reads the row before this call, and ArchiveTask can commit in
// that window without changing task.State, so only an archive-aware
// conditional write closes the race.
if _, updateErr := s.taskRepo.UpdateTaskStateIfCurrentIn(ctx, running.TaskID, v1.TaskStateReview, []v1.TaskState{v1.TaskStateInProgress}); updateErr != nil {
s.logger.Warn("failed to update task to REVIEW on startup",
zap.String("task_id", running.TaskID),
zap.Error(updateErr))
@@ -1422,12 +1449,15 @@ func (s *Service) handleFailedSessionOnStartup(ctx context.Context, session *mod
s.abandonOpenTurnsOnStartup(ctx, sessionID, "failed session cleanup")
// If session failed, ensure task is in REVIEW state (not stuck IN_PROGRESS)
if session.TaskID != "" {
task, taskErr := s.taskRepo.GetTask(ctx, session.TaskID)
if taskErr == nil && task != nil && task.State == v1.TaskStateInProgress {
task, taskErr := s.repo.GetTask(ctx, session.TaskID)
if taskErr == nil && task != nil && task.State == v1.TaskStateInProgress && !taskArchived(task) {
s.logger.Info("fixing task state: session failed but task still IN_PROGRESS",
zap.String("task_id", session.TaskID),
zap.String("session_id", sessionID))
if updateErr := s.taskRepo.UpdateTaskState(ctx, session.TaskID, v1.TaskStateReview); updateErr != nil {
// UpdateTaskStateIfCurrentIn (not the unconditional UpdateTaskState)
// so the write is atomic against archived_at — see the matching
// comment in reconcileOneSessionOnStartup above.
if _, updateErr := s.taskRepo.UpdateTaskStateIfCurrentIn(ctx, session.TaskID, v1.TaskStateReview, []v1.TaskState{v1.TaskStateInProgress}); updateErr != nil {
s.logger.Warn("failed to update task state to REVIEW",
zap.String("task_id", session.TaskID),
zap.Error(updateErr))
@@ -2894,6 +2894,15 @@ func (c *ctxAwareTaskRepo) UpdateTaskStateIfCurrentIn(
return c.inner.UpdateTaskStateIfCurrentIn(ctx, taskID, state, allowed)
}
func (c *ctxAwareTaskRepo) UpdateTaskStateIfNotArchived(
ctx context.Context, taskID string, state v1.TaskState,
) (bool, error) {
if err := ctx.Err(); err != nil {
return false, err
}
return c.inner.UpdateTaskStateIfNotArchived(ctx, taskID, state)
}
// TestResumeTaskSession_FailedStateWriteSurvivesCancelledCallerCtx verifies the
// fix for the WS-disconnect cascade: when the caller's ctx was already
// cancelled (e.g. the user navigated away mid-resume) and the launch then
@@ -3594,6 +3603,131 @@ func TestReconcileSessionsOnStartup(t *testing.T) {
if state != v1.TaskStateReview {
t.Fatalf("expected task state %q, got %q", v1.TaskStateReview, state)
}
// The write must go through the archive-aware UpdateTaskStateIfCurrentIn
// CAS, not the unconditional UpdateTaskState — see the comment on this
// call site in reconcileOneSessionOnStartup. Otherwise an archive that
// commits between the taskArchived guard read and this write could
// still resurrect the task to REVIEW (PR #1706 review finding).
if n := taskRepo.unconditionalWrites["task1"]; n != 0 {
t.Fatalf("expected REVIEW write to use UpdateTaskStateIfCurrentIn, got %d unconditional UpdateTaskState call(s)", n)
}
})
t.Run("archived_task_active_session_state_preserved", func(t *testing.T) {
repo := setupTestRepo(t)
ctx := context.Background()
now := time.Now().UTC()
seedTaskAndSession(t, repo, "task1", "session1", models.TaskSessionStateRunning)
if err := repo.ArchiveTask(ctx, "task1"); err != nil {
t.Fatalf("failed to archive task: %v", err)
}
err := repo.UpsertExecutorRunning(ctx, &models.ExecutorRunning{
ID: "er1",
SessionID: "session1",
TaskID: "task1",
CreatedAt: now,
UpdatedAt: now,
})
if err != nil {
t.Fatalf("failed to upsert executor running: %v", err)
}
taskRepo := newMockTaskRepo()
taskRepo.tasks["task1"] = &v1.Task{
ID: "task1",
State: v1.TaskStateInProgress,
}
svc := createTestServiceWithAgent(repo, newMockStepGetter(), taskRepo, &mockAgentManager{})
svc.reconcileSessionsOnStartup(ctx)
if state, ok := taskRepo.updatedStates["task1"]; ok {
t.Fatalf("expected archived task state to be left untouched, got write to %q", state)
}
})
t.Run("archived_task_failed_session_state_preserved", func(t *testing.T) {
repo := setupTestRepo(t)
ctx := context.Background()
now := time.Now().UTC()
seedTaskAndSession(t, repo, "task1", "session1", models.TaskSessionStateFailed)
if err := repo.ArchiveTask(ctx, "task1"); err != nil {
t.Fatalf("failed to archive task: %v", err)
}
err := repo.UpsertExecutorRunning(ctx, &models.ExecutorRunning{
ID: "er1",
SessionID: "session1",
TaskID: "task1",
ResumeToken: "acp-session-archived",
Resumable: true,
CreatedAt: now,
UpdatedAt: now,
})
if err != nil {
t.Fatalf("failed to upsert executor running: %v", err)
}
taskRepo := newMockTaskRepo()
taskRepo.tasks["task1"] = &v1.Task{
ID: "task1",
State: v1.TaskStateInProgress,
}
svc := createTestServiceWithAgent(repo, newMockStepGetter(), taskRepo, &mockAgentManager{})
svc.reconcileSessionsOnStartup(ctx)
if state, ok := taskRepo.updatedStates["task1"]; ok {
t.Fatalf("expected archived task state to be left untouched, got write to %q", state)
}
})
// TestReconcileSessionsOnStartup/failed_session_moved_to_review covers the
// non-archived REVIEW write in handleFailedSessionOnStartup — previously
// untested (only the archived-guard branch above had coverage). Confirms
// both the resulting state and that the write goes through the
// archive-aware UpdateTaskStateIfCurrentIn CAS, not the unconditional
// UpdateTaskState (PR #1706 review finding).
t.Run("failed_session_moved_to_review", func(t *testing.T) {
repo := setupTestRepo(t)
ctx := context.Background()
now := time.Now().UTC()
seedTaskAndSession(t, repo, "task1", "session1", models.TaskSessionStateFailed)
err := repo.UpsertExecutorRunning(ctx, &models.ExecutorRunning{
ID: "er1",
SessionID: "session1",
TaskID: "task1",
CreatedAt: now,
UpdatedAt: now,
})
if err != nil {
t.Fatalf("failed to upsert executor running: %v", err)
}
taskRepo := newMockTaskRepo()
taskRepo.tasks["task1"] = &v1.Task{
ID: "task1",
State: v1.TaskStateInProgress,
}
svc := createTestServiceWithAgent(repo, newMockStepGetter(), taskRepo, &mockAgentManager{})
svc.reconcileSessionsOnStartup(ctx)
state, ok := taskRepo.updatedStates["task1"]
if !ok {
t.Fatal("expected task state to be updated")
}
if state != v1.TaskStateReview {
t.Fatalf("expected task state %q, got %q", v1.TaskStateReview, state)
}
if n := taskRepo.unconditionalWrites["task1"]; n != 0 {
t.Fatalf("expected REVIEW write to use UpdateTaskStateIfCurrentIn, got %d unconditional UpdateTaskState call(s)", n)
}
})
}
@@ -98,6 +98,9 @@ func (m *mockRepository) UpdateTaskState(ctx context.Context, id string, state v
func (m *mockRepository) UpdateTaskStateIfCurrentIn(_ context.Context, _ string, _ v1.TaskState, _ []v1.TaskState) (v1.TaskState, bool, error) {
return "", false, nil
}
func (m *mockRepository) UpdateTaskStateIfNotArchived(_ context.Context, _ string, _ v1.TaskState) (v1.TaskState, bool, error) {
return "", false, nil
}
func (m *mockRepository) AddTaskToWorkflow(ctx context.Context, taskID, workflowID, columnID string, position int) error {
return nil
}
@@ -64,9 +64,20 @@ type TaskRepository interface {
CountOpenWatcherCreatedTasks(ctx context.Context, metadataKey, watchID string) (int, error)
UpdateTaskState(ctx context.Context, id string, state v1.TaskState) error
// UpdateTaskStateIfCurrentIn atomically transitions state only when the
// task's current state is in allowed. Returns the pre-update state and
// task's current state is in allowed AND the task is not archived
// (archived_at IS NULL). The archived check is enforced inside the same
// UPDATE's WHERE clause, not just by a caller's earlier (non-transactional)
// read, so a late write can never race an ArchiveTask commit that lands
// between that read and this call. Returns the pre-update state and
// whether a row was modified.
UpdateTaskStateIfCurrentIn(ctx context.Context, id string, state v1.TaskState, allowed []v1.TaskState) (v1.TaskState, bool, error)
// UpdateTaskStateIfNotArchived is UpdateTaskStateIfCurrentIn without the
// prior-state constraint — for writers (e.g. IN_PROGRESS reconciliation)
// that legitimately fire from many prior states and only need the
// archived_at IS NULL guarantee. Same TOCTOU-closing semantics: the
// archived check is atomic with the write. Returns the pre-update state
// and whether a row was modified.
UpdateTaskStateIfNotArchived(ctx context.Context, id string, state v1.TaskState) (v1.TaskState, bool, error)
CountTasksByWorkflow(ctx context.Context, workflowID string) (int, error)
CountTasksByWorkflowStep(ctx context.Context, stepID string) (int, error)
AddTaskToWorkflow(ctx context.Context, taskID, workflowID, workflowStepID string, position int) error
@@ -10,6 +10,7 @@ import (
"time"
"github.com/google/uuid"
"github.com/jackc/pgx/v5/pgconn"
"github.com/kandev/kandev/internal/agentctl/tracing"
"github.com/kandev/kandev/internal/db/dialect"
@@ -1166,7 +1167,11 @@ func (r *Repository) UpdateTaskState(ctx context.Context, id string, state v1.Ta
// UpdateTaskStateIfCurrentIn transitions state inside a transaction, re-checking
// the current state on write so concurrent handlers cannot clobber a task that
// moved out of allowed between read and update.
// moved out of allowed between read and update. The write is also scoped to
// archived_at IS NULL: if ArchiveTask commits between the caller's earlier
// archived-state guard and this call, the archived_at check in the UPDATE's
// WHERE clause (not just the state check) makes the no-op atomic, so a late
// runtime write can never resurrect an archived task's state.
func (r *Repository) UpdateTaskStateIfCurrentIn(
ctx context.Context, id string, state v1.TaskState, allowed []v1.TaskState,
) (v1.TaskState, bool, error) {
@@ -1194,7 +1199,7 @@ func (r *Repository) UpdateTaskStateIfCurrentIn(
result, err := tx.ExecContext(ctx, r.db.Rebind(`
UPDATE tasks SET state = ?, updated_at = ?
WHERE id = ? AND state = ?
WHERE id = ? AND state = ? AND archived_at IS NULL
`), state, time.Now().UTC(), id, currentState)
if err != nil {
return "", false, err
@@ -1209,6 +1214,146 @@ func (r *Repository) UpdateTaskStateIfCurrentIn(
return currentState, true, nil
}
// updateTaskStateIfNotArchivedMaxAttempts bounds the optimistic-retry loop in
// UpdateTaskStateIfNotArchived. A retry only fires when another writer
// changed tasks.state (not archived_at) in the gap between this method's
// read and its write; real contention on a single task's state column is
// rare enough that a handful of attempts is generous headroom, not a
// meaningful latency risk. updateTaskStateIfNotArchivedBackoff is a fixed
// delay between attempts, giving the concurrent writer time to finish
// (commit or roll back) before the next attempt re-reads — without it, a
// tight spin-retry can exhaust every attempt in well under a millisecond,
// faster than any real concurrent write completes.
const (
updateTaskStateIfNotArchivedMaxAttempts = 8
updateTaskStateIfNotArchivedBackoff = 15 * time.Millisecond
)
// UpdateTaskStateIfNotArchived atomically transitions state unless the task
// is archived — no prior-state constraint, unlike UpdateTaskStateIfCurrentIn.
// IN_PROGRESS writes (unlike the REVIEW CAS) are legitimately reachable from
// many prior states, so there is no "allowed" set to check; the only
// invariant that must hold atomically is archived_at IS NULL. Closes the
// same TOCTOU window as UpdateTaskStateIfCurrentIn: if ArchiveTask commits
// between a caller's earlier archived-state guard and this call, the
// archived_at check inside the transaction (not just the caller's read)
// makes the no-op atomic. Returns the pre-update state and whether a row
// was modified.
//
// The UPDATE's WHERE clause also pins state = currentState (the value read
// moments earlier in the same transaction), so the returned "pre-update
// state" is never stale: without that pin, a concurrent state write landing
// between the SELECT and the UPDATE would still match archived_at IS NULL
// alone, silently clobbering the newer state while this method kept
// reporting the old (now wrong) value as the pre-update state — which the
// service layer publishes as the task.state_changed event's old_state
// (CodeRabbit review on PR #1706). Losing the race on state (rows==0 while
// the row is still unarchived) means state moved, not that the row is gone
// or archived, so this is optimistic-concurrency-retried rather than
// treated as a no-op — an IN_PROGRESS write has no prior-state restriction
// to honor, so it always still applies once the retry re-reads a fresh
// state.
func (r *Repository) UpdateTaskStateIfNotArchived(
ctx context.Context, id string, state v1.TaskState,
) (v1.TaskState, bool, error) {
for attempt := range updateTaskStateIfNotArchivedMaxAttempts {
if attempt > 0 {
select {
case <-ctx.Done():
return "", false, ctx.Err()
case <-time.After(updateTaskStateIfNotArchivedBackoff):
}
}
currentState, updated, retry, err := r.tryUpdateTaskStateIfNotArchived(ctx, id, state)
if err != nil || !retry {
return currentState, updated, err
}
}
return "", false, fmt.Errorf("update task state if not archived: exceeded %d attempts for task %s",
updateTaskStateIfNotArchivedMaxAttempts, id)
}
// tryUpdateTaskStateIfNotArchived is one attempt of the optimistic-retry loop
// above. retry=true means the state changed concurrently while the row
// stayed unarchived — the caller should re-read and try again.
func (r *Repository) tryUpdateTaskStateIfNotArchived(
ctx context.Context, id string, state v1.TaskState,
) (currentState v1.TaskState, updated, retry bool, err error) {
tx, err := r.db.BeginTx(ctx, nil)
if err != nil {
return "", false, false, err
}
defer func() { _ = tx.Rollback() }()
var archivedAt sql.NullTime
err = tx.QueryRowContext(ctx, r.db.Rebind(`SELECT state, archived_at FROM tasks WHERE id = ?`), id).
Scan(&currentState, &archivedAt)
if err != nil {
if errors.Is(err, sql.ErrNoRows) {
return "", false, false, fmt.Errorf("%w: %s", ErrTaskNotFound, id)
}
return "", false, false, err
}
if archivedAt.Valid {
return currentState, false, false, nil
}
result, err := tx.ExecContext(ctx, r.db.Rebind(`
UPDATE tasks SET state = ?, updated_at = ?
WHERE id = ? AND state = ? AND archived_at IS NULL
`), state, time.Now().UTC(), id, currentState)
if err != nil {
if isRetryableStateRaceError(err) {
// SQLite (under WAL) can refuse to upgrade this transaction's
// already-established read snapshot to a writer once another
// connection committed a conflicting write in between — it
// returns SQLITE_BUSY ("database is locked") on the UPDATE
// itself rather than a clean 0-rows-affected. Same underlying
// condition as the rows==0 branch below: retry with a fresh
// transaction/snapshot instead of surfacing a transient error.
return currentState, false, true, nil
}
return "", false, false, err
}
rows, _ := result.RowsAffected()
if rows == 0 {
// archived_at IS NULL was true moments ago (checked above, inside
// this same transaction) — a concurrent writer changed state, not
// archived_at. Retry with a fresh read rather than reporting a
// stale old-state/no-op.
return currentState, false, true, nil
}
if err := tx.Commit(); err != nil {
return "", false, false, err
}
return currentState, true, false, nil
}
// isRetryableStateRaceError reports whether err is a database-level
// contention error rather than a genuine failure: SQLite's WAL snapshot
// conflict (a write following a read in the same transaction, after
// another connection committed a conflicting write — surfaces as
// SQLITE_BUSY/"database is locked" on the write itself, not a clean
// 0-rows-affected), or a Postgres serialization/deadlock/lock-timeout
// error. UpdateTaskStateIfNotArchived's retry loop treats this the same as
// rows==0: the underlying condition (another writer changed the row) is
// exactly the one it already retries for.
func isRetryableStateRaceError(err error) bool {
if err == nil {
return false
}
var pgErr *pgconn.PgError
if errors.As(err, &pgErr) {
switch pgErr.Code {
case "40001", "40P01", "55P03": // serialization_failure, deadlock_detected, lock_not_available
return true
}
return false
}
s := err.Error()
return strings.Contains(s, "database is locked") || strings.Contains(s, "database table is locked")
}
func taskStateInSet(state v1.TaskState, allowed []v1.TaskState) bool {
for _, candidate := range allowed {
if state == candidate {
@@ -0,0 +1,314 @@
package sqlite
import (
"context"
"path/filepath"
"testing"
"time"
"github.com/jmoiron/sqlx"
"github.com/kandev/kandev/internal/db"
v1 "github.com/kandev/kandev/pkg/api/v1"
)
// TestUpdateTaskStateIfCurrentIn_SkipsArchivedTask reproduces the TOCTOU race
// flagged in PR #1706 review: a caller's earlier archived-state guard (a
// plain GetTask read) can observe archived_at == NULL, then have ArchiveTask
// commit before the caller's CAS write lands. Because state is untouched by
// ArchiveTask, the old `WHERE id = ? AND state = ?` clause alone would still
// match and resurrect the task to REVIEW. The archived_at IS NULL clause
// added to the UPDATE closes that window: the write becomes a no-op even
// though currentState (read moments earlier, inside the same transaction)
// was still in the allowed set.
func TestUpdateTaskStateIfCurrentIn_SkipsArchivedTask(t *testing.T) {
repo := newRepoForHealTests(t)
ctx := context.Background()
insertTask(t, repo.db, "task-archived-race")
if err := repo.UpdateTaskState(ctx, "task-archived-race", v1.TaskStateInProgress); err != nil {
t.Fatalf("seed IN_PROGRESS: %v", err)
}
// Simulates the archive committing in the race window between the
// caller's taskArchived() guard read and this CAS call.
if err := repo.ArchiveTask(ctx, "task-archived-race"); err != nil {
t.Fatalf("archive task: %v", err)
}
gotState, updated, err := repo.UpdateTaskStateIfCurrentIn(
ctx, "task-archived-race", v1.TaskStateReview,
[]v1.TaskState{v1.TaskStateInProgress, v1.TaskStateScheduling},
)
if err != nil {
t.Fatalf("UpdateTaskStateIfCurrentIn: %v", err)
}
if updated {
t.Fatal("expected archived task's state to be left untouched, got updated=true")
}
if gotState != v1.TaskStateInProgress {
t.Errorf("returned currentState = %q, want %q (pre-CAS read)", gotState, v1.TaskStateInProgress)
}
task, err := repo.GetTask(ctx, "task-archived-race")
if err != nil {
t.Fatalf("get task: %v", err)
}
if task.State != v1.TaskStateInProgress {
t.Errorf("persisted state = %q, want %q (archived task must not resurrect to REVIEW)", task.State, v1.TaskStateInProgress)
}
if task.ArchivedAt == nil {
t.Error("expected task to remain archived")
}
}
// TestUpdateTaskStateIfCurrentIn_UpdatesWhenNotArchived is the CAS positive
// path sanity check: a non-archived task whose state is in the allowed set
// still transitions normally. Guards against a too-broad archived_at fix
// (e.g. accidentally scoping the WHERE clause to always require archived_at
// IS NULL on every row, including ones with no archive concept at all) that
// would silently break every ordinary REVIEW/IN_PROGRESS transition.
func TestUpdateTaskStateIfCurrentIn_UpdatesWhenNotArchived(t *testing.T) {
repo := newRepoForHealTests(t)
ctx := context.Background()
insertTask(t, repo.db, "task-normal")
if err := repo.UpdateTaskState(ctx, "task-normal", v1.TaskStateInProgress); err != nil {
t.Fatalf("seed IN_PROGRESS: %v", err)
}
gotState, updated, err := repo.UpdateTaskStateIfCurrentIn(
ctx, "task-normal", v1.TaskStateReview,
[]v1.TaskState{v1.TaskStateInProgress, v1.TaskStateScheduling},
)
if err != nil {
t.Fatalf("UpdateTaskStateIfCurrentIn: %v", err)
}
if !updated {
t.Fatal("expected non-archived task in the allowed set to transition, got updated=false")
}
if gotState != v1.TaskStateInProgress {
t.Errorf("returned currentState = %q, want %q", gotState, v1.TaskStateInProgress)
}
task, err := repo.GetTask(ctx, "task-normal")
if err != nil {
t.Fatalf("get task: %v", err)
}
if task.State != v1.TaskStateReview {
t.Errorf("persisted state = %q, want %q", task.State, v1.TaskStateReview)
}
}
// TestUpdateTaskStateIfNotArchived_SkipsArchivedTask is the
// UpdateTaskStateIfNotArchived analog of TestUpdateTaskStateIfCurrentIn_SkipsArchivedTask
// for the IN_PROGRESS-reconciliation writers (Service/Executor
// writeTaskInProgressForRuntime, review comment on PR #1706): those callers
// have no "allowed" prior-state set to check, only an archived guard, so
// they route through this CAS instead. Same race: ArchiveTask commits
// between the caller's earlier archived-state read and this call; the
// archived_at IS NULL clause inside the UPDATE must still make it a no-op.
func TestUpdateTaskStateIfNotArchived_SkipsArchivedTask(t *testing.T) {
repo := newRepoForHealTests(t)
ctx := context.Background()
insertTask(t, repo.db, "task-archived-race-2")
if err := repo.UpdateTaskState(ctx, "task-archived-race-2", v1.TaskStateWaitingForInput); err != nil {
t.Fatalf("seed WAITING_FOR_INPUT: %v", err)
}
// Simulates the archive committing in the race window between the
// caller's taskArchived() guard read and this CAS call.
if err := repo.ArchiveTask(ctx, "task-archived-race-2"); err != nil {
t.Fatalf("archive task: %v", err)
}
gotState, updated, err := repo.UpdateTaskStateIfNotArchived(ctx, "task-archived-race-2", v1.TaskStateInProgress)
if err != nil {
t.Fatalf("UpdateTaskStateIfNotArchived: %v", err)
}
if updated {
t.Fatal("expected archived task's state to be left untouched, got updated=true")
}
if gotState != v1.TaskStateWaitingForInput {
t.Errorf("returned currentState = %q, want %q (pre-CAS read)", gotState, v1.TaskStateWaitingForInput)
}
task, err := repo.GetTask(ctx, "task-archived-race-2")
if err != nil {
t.Fatalf("get task: %v", err)
}
if task.State != v1.TaskStateWaitingForInput {
t.Errorf("persisted state = %q, want %q (archived task must not resurrect to IN_PROGRESS)", task.State, v1.TaskStateWaitingForInput)
}
if task.ArchivedAt == nil {
t.Error("expected task to remain archived")
}
}
// TestUpdateTaskStateIfNotArchived_UpdatesWhenNotArchived is the CAS
// positive-path sanity check: a non-archived task transitions normally
// regardless of its prior state, since UpdateTaskStateIfNotArchived has no
// "allowed" constraint.
func TestUpdateTaskStateIfNotArchived_UpdatesWhenNotArchived(t *testing.T) {
repo := newRepoForHealTests(t)
ctx := context.Background()
insertTask(t, repo.db, "task-normal-2")
if err := repo.UpdateTaskState(ctx, "task-normal-2", v1.TaskStateWaitingForInput); err != nil {
t.Fatalf("seed WAITING_FOR_INPUT: %v", err)
}
gotState, updated, err := repo.UpdateTaskStateIfNotArchived(ctx, "task-normal-2", v1.TaskStateInProgress)
if err != nil {
t.Fatalf("UpdateTaskStateIfNotArchived: %v", err)
}
if !updated {
t.Fatal("expected non-archived task to transition, got updated=false")
}
if gotState != v1.TaskStateWaitingForInput {
t.Errorf("returned currentState = %q, want %q", gotState, v1.TaskStateWaitingForInput)
}
task, err := repo.GetTask(ctx, "task-normal-2")
if err != nil {
t.Fatalf("get task: %v", err)
}
if task.State != v1.TaskStateInProgress {
t.Errorf("persisted state = %q, want %q", task.State, v1.TaskStateInProgress)
}
}
// TestUpdateTaskStateIfNotArchived_RetriesOnConcurrentStateChange reproduces
// the atomicity gap CodeRabbit flagged on PR #1706: the original
// implementation read currentState, then updated any row with
// archived_at IS NULL — with no predicate pinning the row to the state it
// just read. A concurrent writer that changes state (not archived_at)
// between that SELECT and this method's UPDATE would still match, silently
// overwriting the newer state while this method kept reporting the stale
// first-read value as the "pre-update state" — which the service layer
// publishes as the task.state_changed event's old_state.
//
// Reproduces the actual race (not just sequential ordering) using two
// independent SQLite connections to the same database file under WAL mode.
// A single shared *Repository can't exercise this: db.OpenSQLite caps its
// pool to one Go-level connection (single-writer-connection app design), so
// two calls through the SAME repo would be fully serialized by
// database/sql's connection pool before either reached SQLite's own
// locking — never opening the race window. A second, independent
// connection (as a genuinely concurrent writer — another connection or
// process — would be) models that correctly: the racer opens its own write
// transaction, changes the task's state, and holds it open (uncommitted).
// WAL readers don't block on an uncommitted writer, so the main call's
// SELECT (via the first connection) observes the pre-racer state and
// proceeds to its own UPDATE, which blocks on SQLite's single-writer lock
// until the racer commits. Once it does, the UPDATE's WHERE state = <stale
// value> matches zero rows — this is the exact window the fix's
// optimistic-retry loop exists to close: it must re-read and retry rather
// than reporting the stale value or silently clobbering the racer's write.
func TestUpdateTaskStateIfNotArchived_RetriesOnConcurrentStateChange(t *testing.T) {
tmpDir := t.TempDir()
dbPath := filepath.Join(tmpDir, "test.db")
mainConn, err := db.OpenSQLite(dbPath)
if err != nil {
t.Fatalf("open main connection: %v", err)
}
mainDB := sqlx.NewDb(mainConn, "sqlite3")
t.Cleanup(func() { _ = mainDB.Close() })
repo, err := NewWithDB(mainDB, mainDB, nil)
if err != nil {
t.Fatalf("new repo: %v", err)
}
// racerConn is a second, independent connection to the same file —
// standing in for a genuinely concurrent writer (another connection or
// process), which is what the atomicity gap this test targets actually
// requires.
racerConn, err := db.OpenSQLite(dbPath)
if err != nil {
t.Fatalf("open racer connection: %v", err)
}
racerDB := sqlx.NewDb(racerConn, "sqlite3")
t.Cleanup(func() { _ = racerDB.Close() })
ctx := context.Background()
insertTask(t, repo.db, "task-state-race")
if err := repo.UpdateTaskState(ctx, "task-state-race", v1.TaskStateWaitingForInput); err != nil {
t.Fatalf("seed WAITING_FOR_INPUT: %v", err)
}
racerWrote := make(chan struct{})
releaseRacer := make(chan struct{})
racerDone := make(chan error, 1)
go func() {
tx, err := racerDB.BeginTx(ctx, nil)
if err != nil {
racerDone <- err
return
}
if _, err := tx.ExecContext(ctx, racerDB.Rebind(
`UPDATE tasks SET state = ? WHERE id = ?`,
), v1.TaskStateBlocked, "task-state-race"); err != nil {
_ = tx.Rollback()
racerDone <- err
return
}
close(racerWrote)
<-releaseRacer
racerDone <- tx.Commit()
}()
select {
case <-racerWrote:
case <-time.After(5 * time.Second):
t.Fatal("timed out waiting for racer to write")
}
type callResult struct {
state v1.TaskState
updated bool
err error
}
done := make(chan callResult, 1)
go func() {
state, updated, err := repo.UpdateTaskStateIfNotArchived(ctx, "task-state-race", v1.TaskStateInProgress)
done <- callResult{state, updated, err}
}()
// Give the main call's UPDATE time to reach SQLite's write-lock queue
// (it blocks behind the racer's open transaction) before releasing the
// racer — otherwise the racer could commit before the main call even
// starts, and the race window this test targets would never open.
time.Sleep(50 * time.Millisecond)
close(releaseRacer)
if err := <-racerDone; err != nil {
t.Fatalf("racer transaction failed: %v", err)
}
var result callResult
select {
case result = <-done:
case <-time.After(5 * time.Second):
t.Fatal("timed out waiting for UpdateTaskStateIfNotArchived")
}
if result.err != nil {
t.Fatalf("UpdateTaskStateIfNotArchived: %v", result.err)
}
if !result.updated {
t.Fatal("expected the retry to succeed once the racer's write committed, got updated=false")
}
if result.state != v1.TaskStateBlocked {
t.Errorf("returned pre-update state = %q, want %q (the racer's committed value, not the stale first read)",
result.state, v1.TaskStateBlocked)
}
task, err := repo.GetTask(ctx, "task-state-race")
if err != nil {
t.Fatalf("get task: %v", err)
}
if task.State != v1.TaskStateInProgress {
t.Errorf("persisted state = %q, want %q", task.State, v1.TaskStateInProgress)
}
}
@@ -399,6 +399,10 @@ func (r *phase4TaskRepo) UpdateTaskStateIfCurrentIn(context.Context, string, v1.
r.panicNotUsed("UpdateTaskStateIfCurrentIn")
return "", false, nil
}
func (r *phase4TaskRepo) UpdateTaskStateIfNotArchived(context.Context, string, v1.TaskState) (v1.TaskState, bool, error) {
r.panicNotUsed("UpdateTaskStateIfNotArchived")
return "", false, nil
}
func (r *phase4TaskRepo) CountTasksByWorkflow(context.Context, string) (int, error) {
r.panicNotUsed("CountTasksByWorkflow")
return 0, nil
@@ -221,6 +221,44 @@ func (s *Service) UpdateTaskStateIfCurrentIn(
return true, nil
}
// UpdateTaskStateIfNotArchived is UpdateTaskStateIfCurrentIn without the
// prior-state constraint — for writers (IN_PROGRESS runtime reconciliation)
// that legitimately fire from many prior states and only need the
// archived-task freeze guarantee. Publishes task.state_changed only when a
// row changes.
func (s *Service) UpdateTaskStateIfNotArchived(
ctx context.Context, id string, state v1.TaskState,
) (bool, error) {
oldState, updated, err := s.tasks.UpdateTaskStateIfNotArchived(ctx, id, state)
if err != nil || !updated {
return false, err
}
if oldState == state {
return false, nil
}
task, err := s.tasks.GetTask(ctx, id)
if err != nil {
return true, err
}
// The CAS wrote `state`; pin it on the payload so a concurrent transition
// between commit and read cannot publish a mismatched new_state.
task.State = state
s.logger.Info("task state updated",
zap.String("task_id", id),
zap.String("workflow_step_id", task.WorkflowStepID),
zap.String("state", string(state)))
s.publishTaskEvent(ctx, events.TaskStateChanged, task, &oldState)
s.logger.Info("task state changed",
zap.String("task_id", id),
zap.String("old_state", string(oldState)),
zap.String("new_state", string(state)))
return true, nil
}
// UpdateTaskMetadata updates only the metadata of a task (merges with existing)
func (s *Service) UpdateTaskMetadata(ctx context.Context, id string, metadata map[string]interface{}) (*models.Task, error) {
task, err := s.tasks.GetTask(ctx, id)
@@ -0,0 +1,80 @@
import { test, expect } from "../../fixtures/test-base";
// Regression: archiving a task must freeze its runtime `state`. Several
// backend reconcile paths (turn completion, turn cancel, failed-start
// fallback, and startup/crash reconciliation) used to write tasks.state to
// REVIEW whenever the *session* looked idle/failed, without checking
// whether the *task* had since been archived. Once a task was archived
// while its session was still active, any of those paths — including a
// plain agent.cancel — silently resurrected the archived task's state,
// which is the "archived task comes back with the wrong state after a
// crash/restart" bug. This test reproduces the live (non-restart) trigger
// deterministically: archive a task mid-turn, then cancel the turn via the
// same WS action the chat toolbar's Cancel button sends.
test.describe("Archiving a task freezes its runtime state", () => {
test("agent.cancel after archive does not resurrect task state to REVIEW", async ({
testPage,
apiClient,
seedData,
}) => {
test.setTimeout(60_000);
const taskTitle = "Archive Freezes State";
const created = await apiClient.createTaskWithAgent(
seedData.workspaceId,
taskTitle,
seedData.agentProfileId,
{
description: 'e2e:delay(15000)\ne2e:message("still going")',
workflow_id: seedData.workflowId,
workflow_step_id: seedData.startStepId,
repository_ids: [seedData.repositoryId],
},
);
const taskId = created.id;
const sessionId = created.session_id;
expect(sessionId, "createTaskWithAgent should return a session_id").toBeTruthy();
// Wait for the agent to actually be mid-turn (task promoted to
// IN_PROGRESS) before archiving — this is the state we assert stays
// frozen for the rest of the test.
await expect
.poll(async () => (await apiClient.getTask(taskId)).state, {
timeout: 20_000,
message: "waiting for task to reach IN_PROGRESS before archiving",
})
.toBe("IN_PROGRESS");
// Archive while the agent turn is still in flight (15s delay keeps the
// mock agent from finishing on its own during this test).
await apiClient.archiveTask(taskId);
// Cancel the now-archived task's turn — the same `agent.cancel` WS
// action the chat toolbar's Cancel button sends. Service.CancelAgent
// tolerates a missing/already-stopped execution and still reconciles
// DB state, which is exactly the path that used to write REVIEW onto
// an archived task unconditionally.
await apiClient.wsRequest("agent.cancel", { session_id: sessionId });
const taskAfterCancel = await apiClient.getTask(taskId);
expect(taskAfterCancel.state).toBe("IN_PROGRESS");
// User-facing check: the Tasks list (with archived tasks shown) must
// reflect the same frozen state — the card must not have been
// silently moved into a "needs review" bucket.
await testPage.goto("/tasks");
await testPage.waitForLoadState("networkidle");
await testPage.getByRole("checkbox", { name: "Show archived" }).click();
const row = testPage
.getByTestId("tasks-list-row")
.filter({ has: testPage.getByTestId("tasks-list-row-title").getByText(taskTitle) });
await expect(row).toBeVisible({ timeout: 15_000 });
await expect(row.getByText("Archived")).toBeVisible();
// Re-check via API right after the UI observed the row, closing out
// on the precise contract: still IN_PROGRESS, never REVIEW.
const finalTask = await apiClient.getTask(taskId);
expect(finalTask.state).toBe("IN_PROGRESS");
});
});