fix: freeze archived task state against late REVIEW writes (#1706)
This commit is contained in:
@@ -59,6 +59,14 @@ func (a *taskRepositoryAdapter) UpdateTaskStateIfCurrentIn(
|
||||
return a.svc.UpdateTaskStateIfCurrentIn(ctx, taskID, state, allowed)
|
||||
}
|
||||
|
||||
// UpdateTaskStateIfNotArchived is UpdateTaskStateIfCurrentIn without the
|
||||
// prior-state constraint (see scheduler.TaskRepository doc).
|
||||
func (a *taskRepositoryAdapter) UpdateTaskStateIfNotArchived(
|
||||
ctx context.Context, taskID string, state v1.TaskState,
|
||||
) (bool, error) {
|
||||
return a.svc.UpdateTaskStateIfNotArchived(ctx, taskID, state)
|
||||
}
|
||||
|
||||
// lifecycleAdapter adapts the lifecycle manager as an AgentManagerClient
|
||||
type lifecycleAdapter struct {
|
||||
mgr *lifecycle.Manager
|
||||
|
||||
@@ -73,6 +73,12 @@ func (a *taskRepositoryAdapter) UpdateTaskStateIfCurrentIn(
|
||||
return a.svc.UpdateTaskStateIfCurrentIn(ctx, taskID, state, allowed)
|
||||
}
|
||||
|
||||
func (a *taskRepositoryAdapter) UpdateTaskStateIfNotArchived(
|
||||
ctx context.Context, taskID string, state v1.TaskState,
|
||||
) (bool, error) {
|
||||
return a.svc.UpdateTaskStateIfNotArchived(ctx, taskID, state)
|
||||
}
|
||||
|
||||
// testMessageCreatorAdapter adapts the task service to the orchestrator.MessageCreator interface for tests
|
||||
type testMessageCreatorAdapter struct {
|
||||
svc *taskservice.Service
|
||||
|
||||
@@ -819,6 +819,16 @@ func (s *Service) setSessionWaitingForInput(ctx context.Context, taskID, session
|
||||
s.writeTaskReviewState(ctx, taskID, sessionID)
|
||||
}
|
||||
|
||||
// taskArchived reports whether a task row has been archived. Runtime-state
|
||||
// writes (IN_PROGRESS on session start, REVIEW on turn completion/cancel/
|
||||
// startup reconciliation) must never resurrect an archived task's state —
|
||||
// once archived_at is set the row is frozen from the kanban's perspective,
|
||||
// so every write site here has to skip it instead of reviving stale runtime
|
||||
// state that raced the archive.
|
||||
func taskArchived(task *models.Task) bool {
|
||||
return task != nil && task.ArchivedAt != nil
|
||||
}
|
||||
|
||||
func (s *Service) writeTaskReviewState(ctx context.Context, taskID, completedSessionID string) {
|
||||
// Task lookup errors fail closed so office/archived guards cannot be bypassed
|
||||
// by a transient repository failure.
|
||||
@@ -831,6 +841,10 @@ func (s *Service) writeTaskReviewState(ctx context.Context, taskID, completedSes
|
||||
s.logger.Debug("skipping REVIEW transition for office task",
|
||||
zap.String("task_id", taskID))
|
||||
return
|
||||
} else if taskArchived(dbTask) {
|
||||
s.logger.Debug("skipping REVIEW transition for archived task",
|
||||
zap.String("task_id", taskID))
|
||||
return
|
||||
}
|
||||
|
||||
s.taskRuntimeStateMu.Lock()
|
||||
@@ -921,6 +935,11 @@ func (s *Service) writeTaskReviewStateOnCancel(ctx context.Context, taskID, sess
|
||||
if dbTask.AssigneeAgentProfileID != "" {
|
||||
return
|
||||
}
|
||||
if taskArchived(dbTask) {
|
||||
s.logger.Debug("skipping REVIEW transition after cancel for archived task",
|
||||
zap.String("task_id", taskID))
|
||||
return
|
||||
}
|
||||
|
||||
s.taskRuntimeStateMu.Lock()
|
||||
defer s.taskRuntimeStateMu.Unlock()
|
||||
@@ -1026,7 +1045,7 @@ func (s *Service) writeTaskInProgressForRuntime(ctx context.Context, taskID, ses
|
||||
zap.Error(err))
|
||||
return
|
||||
}
|
||||
if task != nil && task.ArchivedAt != nil {
|
||||
if taskArchived(task) {
|
||||
s.logger.Debug("skipping IN_PROGRESS transition for archived task",
|
||||
zap.String("task_id", taskID))
|
||||
return
|
||||
@@ -1065,11 +1084,17 @@ func (s *Service) writeTaskInProgressForRuntime(ctx context.Context, taskID, ses
|
||||
}
|
||||
}
|
||||
|
||||
if err := s.taskRepo.UpdateTaskState(ctx, taskID, v1.TaskStateInProgress); err != nil {
|
||||
// UpdateTaskStateIfNotArchived (not the unconditional UpdateTaskState) so
|
||||
// the write is atomic against archived_at: the taskArchived guard above
|
||||
// reads the row before this call, and ArchiveTask can commit in that
|
||||
// window without changing task.State, so only an archive-aware
|
||||
// conditional write closes the race (PR #1706 review).
|
||||
updated, err := s.taskRepo.UpdateTaskStateIfNotArchived(ctx, taskID, v1.TaskStateInProgress)
|
||||
if err != nil {
|
||||
s.logger.Error("failed to update task state to IN_PROGRESS",
|
||||
zap.String("task_id", taskID),
|
||||
zap.Error(err))
|
||||
} else {
|
||||
} else if updated {
|
||||
s.logger.Info("task moved to IN_PROGRESS state",
|
||||
zap.String("task_id", taskID))
|
||||
}
|
||||
|
||||
@@ -1204,6 +1204,71 @@ func TestWriteTaskInProgressForRuntimeSkipsArchivedTask(t *testing.T) {
|
||||
"runtime reconciliation must not promote archived tasks to IN_PROGRESS")
|
||||
}
|
||||
|
||||
// TestWriteTaskInProgressForRuntimeUsesArchiveAwareCAS is the TOCTOU
|
||||
// companion to TestWriteTaskInProgressForRuntimeSkipsArchivedTask
|
||||
// (carlosflorencio review on PR #1706): the earlier taskArchived() guard is
|
||||
// a plain, non-transactional read, so an ArchiveTask commit landing in the
|
||||
// window between that read and the write could still resurrect the task's
|
||||
// state if the write itself were the unconditional UpdateTaskState. Asserts
|
||||
// the write goes through UpdateTaskStateIfNotArchived (tracked separately
|
||||
// from the unconditional path via unconditionalWrites) instead.
|
||||
func TestWriteTaskInProgressForRuntimeUsesArchiveAwareCAS(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
repo := setupTestRepo(t)
|
||||
seedSession(t, repo, "t1", "s1", "")
|
||||
|
||||
taskRepo := newMockTaskRepo()
|
||||
svc := createTestService(repo, newMockStepGetter(), taskRepo)
|
||||
|
||||
svc.writeTaskInProgressForRuntime(ctx, "t1", "s1")
|
||||
|
||||
require.Equal(t, 1, taskRepo.stateWrites["t1"],
|
||||
"runtime reconciliation must still promote a non-archived task to IN_PROGRESS")
|
||||
require.Equal(t, v1.TaskStateInProgress, taskRepo.updatedStates["t1"])
|
||||
require.Zero(t, taskRepo.unconditionalWrites["t1"],
|
||||
"IN_PROGRESS write must use UpdateTaskStateIfNotArchived, not the unconditional UpdateTaskState")
|
||||
}
|
||||
|
||||
func TestWriteTaskReviewStateSkipsArchivedTask(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
repo := setupTestRepo(t)
|
||||
seedSession(t, repo, "t1", "s1", "")
|
||||
session, err := repo.GetTaskSession(ctx, "s1")
|
||||
require.NoError(t, err)
|
||||
session.State = models.TaskSessionStateWaitingForInput
|
||||
require.NoError(t, repo.UpdateTaskSession(ctx, session))
|
||||
require.NoError(t, repo.ArchiveTask(ctx, "t1"))
|
||||
|
||||
taskRepo := newMockTaskRepo()
|
||||
taskRepo.tasks["t1"] = &v1.Task{ID: "t1", State: v1.TaskStateInProgress}
|
||||
svc := createTestService(repo, newMockStepGetter(), taskRepo)
|
||||
|
||||
svc.writeTaskReviewState(ctx, "t1", "s1")
|
||||
|
||||
require.Zero(t, taskRepo.stateWrites["t1"],
|
||||
"REVIEW reconcile must not resurrect an archived task's state")
|
||||
}
|
||||
|
||||
func TestWriteTaskReviewStateOnCancelSkipsArchivedTask(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
repo := setupTestRepo(t)
|
||||
seedSession(t, repo, "t1", "s1", "")
|
||||
session, err := repo.GetTaskSession(ctx, "s1")
|
||||
require.NoError(t, err)
|
||||
session.State = models.TaskSessionStateWaitingForInput
|
||||
require.NoError(t, repo.UpdateTaskSession(ctx, session))
|
||||
require.NoError(t, repo.ArchiveTask(ctx, "t1"))
|
||||
|
||||
taskRepo := newMockTaskRepo()
|
||||
taskRepo.tasks["t1"] = &v1.Task{ID: "t1", State: v1.TaskStateInProgress}
|
||||
svc := createTestService(repo, newMockStepGetter(), taskRepo)
|
||||
|
||||
svc.writeTaskReviewStateOnCancel(ctx, "t1", "s1")
|
||||
|
||||
require.Zero(t, taskRepo.stateWrites["t1"],
|
||||
"cancel reconcile must not resurrect an archived task's state")
|
||||
}
|
||||
|
||||
func TestWriteTaskReviewStateOnCancelSkipsWhenSessionActiveAgain(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
repo := setupTestRepo(t)
|
||||
|
||||
@@ -96,15 +96,23 @@ type mockTaskRepo struct {
|
||||
// dispatch's IN_PROGRESS write) — tests asserting a state was NEVER
|
||||
// written at any point must check stateHistory, not updatedStates.
|
||||
stateHistory map[string][]v1.TaskState
|
||||
getTaskErr error // if set, GetTask returns this error
|
||||
// unconditionalWrites counts UpdateTaskState (the non-CAS write) calls per
|
||||
// task, tracked separately from stateWrites (which both UpdateTaskState
|
||||
// and UpdateTaskStateIfCurrentIn increment). Startup/crash reconciliation
|
||||
// callers must route guarded REVIEW writes through the CAS method so an
|
||||
// archive can't race a late write; tests assert this stays 0 for those
|
||||
// paths instead of just checking the resulting state.
|
||||
unconditionalWrites map[string]int
|
||||
getTaskErr error // if set, GetTask returns this error
|
||||
}
|
||||
|
||||
func newMockTaskRepo() *mockTaskRepo {
|
||||
return &mockTaskRepo{
|
||||
tasks: make(map[string]*v1.Task),
|
||||
updatedStates: make(map[string]v1.TaskState),
|
||||
stateWrites: make(map[string]int),
|
||||
stateHistory: make(map[string][]v1.TaskState),
|
||||
tasks: make(map[string]*v1.Task),
|
||||
updatedStates: make(map[string]v1.TaskState),
|
||||
stateWrites: make(map[string]int),
|
||||
stateHistory: make(map[string][]v1.TaskState),
|
||||
unconditionalWrites: make(map[string]int),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -130,6 +138,7 @@ func (m *mockTaskRepo) UpdateTaskState(_ context.Context, taskID string, state v
|
||||
m.updatedStates[taskID] = state
|
||||
m.stateWrites[taskID]++
|
||||
m.stateHistory[taskID] = append(m.stateHistory[taskID], state)
|
||||
m.unconditionalWrites[taskID]++
|
||||
if t, ok := m.tasks[taskID]; ok {
|
||||
t.State = state
|
||||
}
|
||||
@@ -158,6 +167,30 @@ func (m *mockTaskRepo) UpdateTaskStateIfCurrentIn(
|
||||
return false, nil
|
||||
}
|
||||
|
||||
// UpdateTaskStateIfNotArchived has no "allowed" precondition to check, so
|
||||
// (unlike UpdateTaskStateIfCurrentIn above) this mock cannot model the
|
||||
// archived_at race itself — v1.Task carries no ArchivedAt field for it to
|
||||
// consult. It behaves like UpdateTaskState above: unconditional and
|
||||
// always tracked, mutating the seeded task's State only when present —
|
||||
// existing callers seed via seedMockTaskState only when they need to read
|
||||
// the state back, not merely to assert a write happened.
|
||||
// Real archived-freeze coverage for this CAS lives at the sqlite layer
|
||||
// (task_state_cas_test.go) and the executor-layer mock (models.Task, which
|
||||
// does carry ArchivedAt).
|
||||
func (m *mockTaskRepo) UpdateTaskStateIfNotArchived(
|
||||
_ context.Context, taskID string, state v1.TaskState,
|
||||
) (bool, error) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
m.updatedStates[taskID] = state
|
||||
m.stateWrites[taskID]++
|
||||
m.stateHistory[taskID] = append(m.stateHistory[taskID], state)
|
||||
if t, ok := m.tasks[taskID]; ok {
|
||||
t.State = state
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// mockAgentManager is a minimal mock of executor.AgentManagerClient for testing.
|
||||
type mockAgentManager struct {
|
||||
isPassthrough bool
|
||||
|
||||
@@ -23,7 +23,21 @@ import (
|
||||
type executorStore interface {
|
||||
// Task
|
||||
GetTask(ctx context.Context, id string) (*models.Task, error)
|
||||
UpdateTaskState(ctx context.Context, id string, state v1.TaskState) error
|
||||
// UpdateTaskStateIfNotArchived atomically transitions state unless the
|
||||
// task is archived (archived_at IS NULL). Used instead of a plain
|
||||
// unconditional UpdateTaskState for every runtime-driven task-state write
|
||||
// (IN_PROGRESS on start/resume, FAILED on launch error) so a late write
|
||||
// can never race an archive that lands after an earlier (non-transactional)
|
||||
// archived-state check. Returns the pre-update state and whether a row
|
||||
// was modified.
|
||||
UpdateTaskStateIfNotArchived(ctx context.Context, id string, state v1.TaskState) (v1.TaskState, bool, error)
|
||||
// UpdateTaskStateIfCurrentIn atomically transitions state only when the
|
||||
// current state is in allowed AND the task is not archived (archived_at
|
||||
// IS NULL). Used instead of UpdateTaskState for guarded REVIEW writes so
|
||||
// a late write can never race an archive that lands after an earlier
|
||||
// (non-transactional) archived-state check. Returns the pre-update state
|
||||
// and whether a row was modified.
|
||||
UpdateTaskStateIfCurrentIn(ctx context.Context, id string, state v1.TaskState, allowed []v1.TaskState) (v1.TaskState, bool, error)
|
||||
// Task↔repo junction
|
||||
GetPrimaryTaskRepository(ctx context.Context, taskID string) (*models.TaskRepository, error)
|
||||
ListTaskRepositories(ctx context.Context, taskID string) ([]*models.TaskRepository, error)
|
||||
|
||||
@@ -148,7 +148,23 @@ func (e *Executor) writeTaskReviewStateIfNoWorkingSessions(ctx context.Context,
|
||||
if e.hasOtherWorkingSessions(ctx, taskID, failedSessionID) {
|
||||
return
|
||||
}
|
||||
if updateErr := e.updateTaskState(ctx, taskID, v1.TaskStateReview); updateErr != nil {
|
||||
// When onTaskStateChange is configured, it owns event publishing for
|
||||
// this write (see its doc comment) — keep routing through it rather
|
||||
// than bypassing it. Only when NEITHER callback is set (no orchestrator
|
||||
// wiring at all — production always wires both, see service.go's
|
||||
// exec.SetOnTaskStateChange/SetOnTaskReviewStateReconcile) do we fall
|
||||
// back to the archive-aware UpdateTaskStateIfCurrentIn CAS directly on
|
||||
// the repository, so even that raw path can't race an archive that
|
||||
// commits between shouldSkipFailedStartReviewForTask's read and this write.
|
||||
if e.onTaskStateChange != nil {
|
||||
if updateErr := e.onTaskStateChange(ctx, taskID, v1.TaskStateReview); updateErr != nil {
|
||||
e.logger.Warn("failed to update task state to REVIEW after start error",
|
||||
zap.String("task_id", taskID),
|
||||
zap.Error(updateErr))
|
||||
}
|
||||
return
|
||||
}
|
||||
if _, _, updateErr := e.repo.UpdateTaskStateIfCurrentIn(ctx, taskID, v1.TaskStateReview, []v1.TaskState{v1.TaskStateInProgress, v1.TaskStateScheduling}); updateErr != nil {
|
||||
e.logger.Warn("failed to update task state to REVIEW after start error",
|
||||
zap.String("task_id", taskID),
|
||||
zap.Error(updateErr))
|
||||
@@ -170,6 +186,12 @@ func (e *Executor) shouldSkipFailedStartReviewForTask(ctx context.Context, taskI
|
||||
zap.String("session_id", failedSessionID))
|
||||
return true
|
||||
}
|
||||
if task != nil && task.ArchivedAt != nil {
|
||||
e.logger.Debug("skipping failed-start task REVIEW state for archived task",
|
||||
zap.String("task_id", taskID),
|
||||
zap.String("session_id", failedSessionID))
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
@@ -227,12 +249,15 @@ func isRuntimeWorkingSessionState(state models.TaskSessionState) bool {
|
||||
}
|
||||
|
||||
// updateTaskState updates a task's state, using the callback if set for event publishing,
|
||||
// or falling back to the raw repository.
|
||||
// or falling back to the archive-aware CAS (UpdateTaskStateIfNotArchived) directly. Every
|
||||
// call site here is a runtime-driven write (IN_PROGRESS on start/resume, FAILED on launch
|
||||
// error) that must never resurrect an archived task's state (PR #1706 review).
|
||||
func (e *Executor) updateTaskState(ctx context.Context, taskID string, state v1.TaskState) error {
|
||||
if e.onTaskStateChange != nil {
|
||||
return e.onTaskStateChange(ctx, taskID, state)
|
||||
}
|
||||
return e.repo.UpdateTaskState(ctx, taskID, state)
|
||||
_, _, err := e.repo.UpdateTaskStateIfNotArchived(ctx, taskID, state)
|
||||
return err
|
||||
}
|
||||
|
||||
// updateSessionState updates a session's state, using the callback if set for event publishing,
|
||||
|
||||
@@ -258,14 +258,48 @@ type mockRepository struct {
|
||||
// transient DB error); if nil, the default map lookup is used.
|
||||
getTaskSessionFunc func(ctx context.Context, id string) (*models.TaskSession, error)
|
||||
createTaskSessionFunc func(ctx context.Context, session *models.TaskSession) error
|
||||
// Optional hook invoked at the top of UpdateTaskStateIfCurrentIn, before
|
||||
// it reads task state/archived_at. Lets tests simulate the exact TOCTOU
|
||||
// window this CAS closes: an earlier (non-transactional) archived-state
|
||||
// guard already passed, then an archive commits in the gap before this
|
||||
// call — the hook mutates the task here to model that gap. If nil, the
|
||||
// CAS runs immediately against the task as currently seeded.
|
||||
preCASHook func(taskID string)
|
||||
// updateTaskStateIfNotArchivedCh, when non-nil, receives a non-blocking
|
||||
// signal every time UpdateTaskStateIfNotArchived records a call — lets
|
||||
// tests wait via select instead of polling with time.Sleep for a
|
||||
// background goroutine's call to land. Buffered so a signal is never
|
||||
// lost even if the test hasn't started waiting yet.
|
||||
updateTaskStateIfNotArchivedCh chan struct{}
|
||||
|
||||
// Track calls for verification
|
||||
createTaskSessionCalls []*models.TaskSession
|
||||
updateTaskSessionCalls []*models.TaskSession
|
||||
setSessionMetadataKeyCalls []setSessionMetadataKeyCall
|
||||
setSessionPrimaryCalls []string
|
||||
createTaskEnvironmentCalls []*models.TaskEnvironment
|
||||
updateTaskEnvironmentCalls []*models.TaskEnvironment
|
||||
createTaskSessionCalls []*models.TaskSession
|
||||
updateTaskSessionCalls []*models.TaskSession
|
||||
setSessionMetadataKeyCalls []setSessionMetadataKeyCall
|
||||
setSessionPrimaryCalls []string
|
||||
createTaskEnvironmentCalls []*models.TaskEnvironment
|
||||
updateTaskEnvironmentCalls []*models.TaskEnvironment
|
||||
updateTaskStateIfCurrentInCalls []updateTaskStateIfCurrentInCall
|
||||
updateTaskStateIfNotArchivedCalls []updateTaskStateIfNotArchivedCall
|
||||
}
|
||||
|
||||
// updateTaskStateIfCurrentInCall records one UpdateTaskStateIfCurrentIn
|
||||
// invocation so tests can assert callers route guarded REVIEW writes
|
||||
// through the archive-aware CAS instead of the unconditional
|
||||
// UpdateTaskState.
|
||||
type updateTaskStateIfCurrentInCall struct {
|
||||
TaskID string
|
||||
State v1.TaskState
|
||||
Allowed []v1.TaskState
|
||||
}
|
||||
|
||||
// updateTaskStateIfNotArchivedCall records one UpdateTaskStateIfNotArchived
|
||||
// invocation — the IN_PROGRESS/FAILED-writer analog of
|
||||
// updateTaskStateIfCurrentInCall, used to assert those callers route
|
||||
// through the archive-aware CAS instead of the unconditional UpdateTaskState.
|
||||
type updateTaskStateIfNotArchivedCall struct {
|
||||
TaskID string
|
||||
State v1.TaskState
|
||||
}
|
||||
|
||||
type setSessionMetadataKeyCall struct {
|
||||
@@ -276,15 +310,16 @@ type setSessionMetadataKeyCall struct {
|
||||
|
||||
func newMockRepository() *mockRepository {
|
||||
return &mockRepository{
|
||||
sessions: make(map[string]*models.TaskSession),
|
||||
tasks: make(map[string]*models.Task),
|
||||
taskRepositories: make(map[string]*models.TaskRepository),
|
||||
repositories: make(map[string]*models.Repository),
|
||||
executors: make(map[string]*models.Executor),
|
||||
executorProfiles: make(map[string]*models.ExecutorProfile),
|
||||
executorsRunning: make(map[string]*models.ExecutorRunning),
|
||||
taskEnvironments: make(map[string]*models.TaskEnvironment),
|
||||
taskEnvironmentRepos: make(map[string][]*models.TaskEnvironmentRepo),
|
||||
sessions: make(map[string]*models.TaskSession),
|
||||
tasks: make(map[string]*models.Task),
|
||||
taskRepositories: make(map[string]*models.TaskRepository),
|
||||
repositories: make(map[string]*models.Repository),
|
||||
executors: make(map[string]*models.Executor),
|
||||
executorProfiles: make(map[string]*models.ExecutorProfile),
|
||||
executorsRunning: make(map[string]*models.ExecutorRunning),
|
||||
taskEnvironments: make(map[string]*models.TaskEnvironment),
|
||||
taskEnvironmentRepos: make(map[string][]*models.TaskEnvironmentRepo),
|
||||
updateTaskStateIfNotArchivedCh: make(chan struct{}, 8),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -388,6 +423,80 @@ func (m *mockRepository) CreateTaskSessionWorktree(_ context.Context, worktree *
|
||||
func (m *mockRepository) UpdateTaskState(ctx context.Context, taskID string, state v1.TaskState) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// UpdateTaskStateIfCurrentIn mirrors the real repository's archive-aware CAS
|
||||
// semantics (task.go's UpdateTaskStateIfCurrentIn): the write only lands when
|
||||
// the task's current state is in allowed AND the task is not archived.
|
||||
func (m *mockRepository) UpdateTaskStateIfCurrentIn(
|
||||
ctx context.Context, taskID string, state v1.TaskState, allowed []v1.TaskState,
|
||||
) (v1.TaskState, bool, error) {
|
||||
if m.preCASHook != nil {
|
||||
m.preCASHook(taskID)
|
||||
}
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
m.updateTaskStateIfCurrentInCalls = append(m.updateTaskStateIfCurrentInCalls, updateTaskStateIfCurrentInCall{
|
||||
TaskID: taskID, State: state, Allowed: allowed,
|
||||
})
|
||||
task, ok := m.tasks[taskID]
|
||||
if !ok {
|
||||
return "", false, fmt.Errorf("task not found: %s", taskID)
|
||||
}
|
||||
currentState := task.State
|
||||
if task.ArchivedAt != nil {
|
||||
return currentState, false, nil
|
||||
}
|
||||
for _, candidate := range allowed {
|
||||
if currentState == candidate {
|
||||
task.State = state
|
||||
return currentState, true, nil
|
||||
}
|
||||
}
|
||||
return currentState, false, nil
|
||||
}
|
||||
|
||||
// UpdateTaskStateIfNotArchived mirrors the real repository's archive-aware
|
||||
// CAS semantics (task.go's UpdateTaskStateIfNotArchived): unlike
|
||||
// UpdateTaskStateIfCurrentIn, there is no "allowed" prior-state set — the
|
||||
// write lands whenever the task is not archived. Reuses preCASHook so tests
|
||||
// can model the same TOCTOU window (archive commits between an earlier
|
||||
// non-transactional guard read and this call) for IN_PROGRESS/FAILED writers.
|
||||
func (m *mockRepository) UpdateTaskStateIfNotArchived(
|
||||
ctx context.Context, taskID string, state v1.TaskState,
|
||||
) (v1.TaskState, bool, error) {
|
||||
if m.preCASHook != nil {
|
||||
m.preCASHook(taskID)
|
||||
}
|
||||
defer m.notifyUpdateTaskStateIfNotArchived()
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
m.updateTaskStateIfNotArchivedCalls = append(m.updateTaskStateIfNotArchivedCalls, updateTaskStateIfNotArchivedCall{
|
||||
TaskID: taskID, State: state,
|
||||
})
|
||||
task, ok := m.tasks[taskID]
|
||||
if !ok {
|
||||
return "", false, fmt.Errorf("task not found: %s", taskID)
|
||||
}
|
||||
currentState := task.State
|
||||
if task.ArchivedAt != nil {
|
||||
return currentState, false, nil
|
||||
}
|
||||
task.State = state
|
||||
return currentState, true, nil
|
||||
}
|
||||
|
||||
// notifyUpdateTaskStateIfNotArchived signals updateTaskStateIfNotArchivedCh
|
||||
// (if the test wired one) that a call just landed. Non-blocking so it never
|
||||
// stalls the caller when nothing is listening.
|
||||
func (m *mockRepository) notifyUpdateTaskStateIfNotArchived() {
|
||||
if m.updateTaskStateIfNotArchivedCh == nil {
|
||||
return
|
||||
}
|
||||
select {
|
||||
case m.updateTaskStateIfNotArchivedCh <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
}
|
||||
func (m *mockRepository) ArchiveTask(ctx context.Context, id string) error { return nil }
|
||||
func (m *mockRepository) ListTasksForAutoArchive(ctx context.Context) ([]*models.Task, error) {
|
||||
return nil, nil
|
||||
|
||||
@@ -1358,6 +1358,215 @@ func TestRunAgentProcessAsync_ResumeDoesNotEscalateTaskState(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestRunAgentProcessAsync_ResumeFailureUsesRawCASWithoutCallbacks covers the
|
||||
// true raw-fallback branch: neither onTaskReviewStateReconcile nor
|
||||
// onTaskStateChange is configured (a standalone Executor with no orchestrator
|
||||
// wiring at all — never the case in production, see service.go's
|
||||
// exec.SetOnTaskStateChange/SetOnTaskReviewStateReconcile). The REVIEW write
|
||||
// must go straight through the archive-aware UpdateTaskStateIfCurrentIn CAS
|
||||
// on the repository rather than the unconditional UpdateTaskState, so a late
|
||||
// write here still can't race an archive.
|
||||
func TestRunAgentProcessAsync_ResumeFailureUsesRawCASWithoutCallbacks(t *testing.T) {
|
||||
repo := newMockRepository()
|
||||
repo.sessions["session-123"] = &models.TaskSession{
|
||||
ID: "session-123", TaskID: "task-123", State: models.TaskSessionStateStarting,
|
||||
}
|
||||
repo.tasks["task-123"] = &models.Task{ID: "task-123", State: v1.TaskStateInProgress}
|
||||
stopCh := make(chan struct{})
|
||||
agentManager := &mockAgentManager{
|
||||
startAgentProcessFunc: func(ctx context.Context, agentExecutionID string) error {
|
||||
return fmt.Errorf("ACP initialize handshake failed: context deadline exceeded")
|
||||
},
|
||||
stopAgentFunc: func(ctx context.Context, agentExecutionID string, force bool) error {
|
||||
close(stopCh)
|
||||
return nil
|
||||
},
|
||||
}
|
||||
exec := newTestExecutor(t, agentManager, repo)
|
||||
// Deliberately no SetOnTaskStateChange / SetOnTaskReviewStateReconcile.
|
||||
|
||||
exec.runAgentProcessAsync(context.Background(), "task-123", "session-123", "exec-456",
|
||||
func(ctx context.Context) { t.Error("onSuccess should not run on failure") },
|
||||
false, true) // resume path: no escalation, fromResume=true
|
||||
|
||||
select {
|
||||
case <-stopCh:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("timed out waiting for StopAgent to be called")
|
||||
}
|
||||
|
||||
if got := repo.tasks["task-123"].State; got != v1.TaskStateReview {
|
||||
t.Errorf("expected resume failure to reconcile task state to REVIEW via the raw CAS, got %q", got)
|
||||
}
|
||||
if len(repo.updateTaskStateIfCurrentInCalls) != 1 {
|
||||
t.Fatalf("expected exactly 1 UpdateTaskStateIfCurrentIn call, got %d: %+v", len(repo.updateTaskStateIfCurrentInCalls), repo.updateTaskStateIfCurrentInCalls)
|
||||
}
|
||||
call := repo.updateTaskStateIfCurrentInCalls[0]
|
||||
if call.TaskID != "task-123" || call.State != v1.TaskStateReview {
|
||||
t.Errorf("unexpected CAS call: %+v", call)
|
||||
}
|
||||
if len(call.Allowed) != 2 || call.Allowed[0] != v1.TaskStateInProgress || call.Allowed[1] != v1.TaskStateScheduling {
|
||||
t.Errorf("expected CAS allowed=[IN_PROGRESS, SCHEDULING], got %v", call.Allowed)
|
||||
}
|
||||
}
|
||||
|
||||
// TestRunAgentProcessAsync_ResumeFailureSkipsArchivedTaskRacingCAS is the
|
||||
// TOCTOU companion to TestRunAgentProcessAsync_ResumeFailureUsesRawCASWithoutCallbacks
|
||||
// (cubic review finding on PR #1706): the task is NOT archived when
|
||||
// shouldSkipFailedStartReviewForTask's earlier (non-transactional) guard
|
||||
// runs — that guard passes — and only archives via preCASHook right as
|
||||
// UpdateTaskStateIfCurrentIn is invoked, modeling ArchiveTask committing in
|
||||
// the exact gap between the guard read and the atomic write. Without the
|
||||
// archived_at check inside the CAS itself (not just the earlier guard),
|
||||
// this write would still land and resurrect the task to REVIEW.
|
||||
func TestRunAgentProcessAsync_ResumeFailureSkipsArchivedTaskRacingCAS(t *testing.T) {
|
||||
repo := newMockRepository()
|
||||
repo.sessions["session-123"] = &models.TaskSession{
|
||||
ID: "session-123", TaskID: "task-123", State: models.TaskSessionStateStarting,
|
||||
}
|
||||
repo.tasks["task-123"] = &models.Task{ID: "task-123", State: v1.TaskStateInProgress}
|
||||
repo.preCASHook = func(taskID string) {
|
||||
if task, ok := repo.tasks[taskID]; ok {
|
||||
archivedAt := time.Now().UTC()
|
||||
task.ArchivedAt = &archivedAt
|
||||
}
|
||||
}
|
||||
stopCh := make(chan struct{})
|
||||
agentManager := &mockAgentManager{
|
||||
startAgentProcessFunc: func(ctx context.Context, agentExecutionID string) error {
|
||||
return fmt.Errorf("ACP initialize handshake failed: context deadline exceeded")
|
||||
},
|
||||
stopAgentFunc: func(ctx context.Context, agentExecutionID string, force bool) error {
|
||||
close(stopCh)
|
||||
return nil
|
||||
},
|
||||
}
|
||||
exec := newTestExecutor(t, agentManager, repo)
|
||||
// Deliberately no SetOnTaskStateChange / SetOnTaskReviewStateReconcile.
|
||||
|
||||
exec.runAgentProcessAsync(context.Background(), "task-123", "session-123", "exec-456",
|
||||
func(ctx context.Context) { t.Error("onSuccess should not run on failure") },
|
||||
false, true) // resume path: no escalation, fromResume=true
|
||||
|
||||
select {
|
||||
case <-stopCh:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("timed out waiting for StopAgent to be called")
|
||||
}
|
||||
|
||||
if got := repo.tasks["task-123"].State; got != v1.TaskStateInProgress {
|
||||
t.Errorf("expected archive racing the CAS to leave state untouched, got %q", got)
|
||||
}
|
||||
if len(repo.updateTaskStateIfCurrentInCalls) != 1 {
|
||||
t.Fatalf("expected the CAS to still be called (guard passed pre-archive), got %+v", repo.updateTaskStateIfCurrentInCalls)
|
||||
}
|
||||
}
|
||||
|
||||
// waitForUpdateTaskStateIfNotArchivedCall blocks (via a channel signal, not
|
||||
// polling) until startAgentProcessAsync's background goroutine has recorded
|
||||
// its UpdateTaskStateIfNotArchived call. Needed because — unlike the
|
||||
// resume-failure tests above, where StopAgent (closing the sync channel)
|
||||
// runs strictly after the state write — startAgentProcessAsync's onSuccess
|
||||
// callback has nothing observable after the write itself.
|
||||
func waitForUpdateTaskStateIfNotArchivedCall(t *testing.T, repo *mockRepository) {
|
||||
t.Helper()
|
||||
select {
|
||||
case <-repo.updateTaskStateIfNotArchivedCh:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("timed out waiting for UpdateTaskStateIfNotArchived call")
|
||||
}
|
||||
}
|
||||
|
||||
// TestStartAgentProcessAsync_UsesRawCASWithoutCallbacks is the IN_PROGRESS
|
||||
// analog of TestRunAgentProcessAsync_ResumeFailureUsesRawCASWithoutCallbacks:
|
||||
// with no SetOnTaskStateChange callback wired, startAgentProcessAsync's
|
||||
// post-launch IN_PROGRESS write must go through the archive-aware
|
||||
// UpdateTaskStateIfNotArchived CAS on the repository rather than the
|
||||
// unconditional UpdateTaskState (carlosflorencio review on PR #1706:
|
||||
// writeTaskInProgressForRuntime's ArchivedAt guard was followed by an
|
||||
// unconditional write, leaving the same TOCTOU window open as the REVIEW
|
||||
// writers had before the CAS fix).
|
||||
func TestStartAgentProcessAsync_UsesRawCASWithoutCallbacks(t *testing.T) {
|
||||
repo := newMockRepository()
|
||||
repo.sessions["session-123"] = &models.TaskSession{
|
||||
ID: "session-123", TaskID: "task-123", State: models.TaskSessionStateStarting,
|
||||
}
|
||||
repo.tasks["task-123"] = &models.Task{ID: "task-123", State: v1.TaskStateWaitingForInput}
|
||||
startedCh := make(chan struct{})
|
||||
agentManager := &mockAgentManager{
|
||||
startAgentProcessFunc: func(ctx context.Context, agentExecutionID string) error {
|
||||
close(startedCh)
|
||||
return nil
|
||||
},
|
||||
}
|
||||
exec := newTestExecutor(t, agentManager, repo)
|
||||
// Deliberately no SetOnTaskStateChange.
|
||||
|
||||
exec.startAgentProcessAsync(context.Background(), "task-123", "session-123", "exec-456")
|
||||
|
||||
select {
|
||||
case <-startedCh:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("timed out waiting for StartAgentProcess to be called")
|
||||
}
|
||||
|
||||
waitForUpdateTaskStateIfNotArchivedCall(t, repo)
|
||||
|
||||
if got := repo.tasks["task-123"].State; got != v1.TaskStateInProgress {
|
||||
t.Errorf("expected agent start success to promote task state to IN_PROGRESS via the raw CAS, got %q", got)
|
||||
}
|
||||
call := repo.updateTaskStateIfNotArchivedCalls[0]
|
||||
if call.TaskID != "task-123" || call.State != v1.TaskStateInProgress {
|
||||
t.Errorf("unexpected CAS call: %+v", call)
|
||||
}
|
||||
}
|
||||
|
||||
// TestStartAgentProcessAsync_SkipsArchivedTaskRacingCAS is the TOCTOU
|
||||
// companion to TestStartAgentProcessAsync_UsesRawCASWithoutCallbacks: the
|
||||
// task is NOT archived when startAgentProcessAsync's earlier archived guard
|
||||
// (inside its onSuccess callback, via updateTaskState → the repo fallback)
|
||||
// would have run, and only archives via preCASHook right as
|
||||
// UpdateTaskStateIfNotArchived is invoked, modeling ArchiveTask committing
|
||||
// in the exact gap between an earlier archived-state read and the atomic
|
||||
// write. Without the archived_at check inside the CAS itself, this write
|
||||
// would still land and resurrect the task to IN_PROGRESS.
|
||||
func TestStartAgentProcessAsync_SkipsArchivedTaskRacingCAS(t *testing.T) {
|
||||
repo := newMockRepository()
|
||||
repo.sessions["session-123"] = &models.TaskSession{
|
||||
ID: "session-123", TaskID: "task-123", State: models.TaskSessionStateStarting,
|
||||
}
|
||||
repo.tasks["task-123"] = &models.Task{ID: "task-123", State: v1.TaskStateWaitingForInput}
|
||||
repo.preCASHook = func(taskID string) {
|
||||
if task, ok := repo.tasks[taskID]; ok {
|
||||
archivedAt := time.Now().UTC()
|
||||
task.ArchivedAt = &archivedAt
|
||||
}
|
||||
}
|
||||
startedCh := make(chan struct{})
|
||||
agentManager := &mockAgentManager{
|
||||
startAgentProcessFunc: func(ctx context.Context, agentExecutionID string) error {
|
||||
close(startedCh)
|
||||
return nil
|
||||
},
|
||||
}
|
||||
exec := newTestExecutor(t, agentManager, repo)
|
||||
// Deliberately no SetOnTaskStateChange.
|
||||
|
||||
exec.startAgentProcessAsync(context.Background(), "task-123", "session-123", "exec-456")
|
||||
|
||||
select {
|
||||
case <-startedCh:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("timed out waiting for StartAgentProcess to be called")
|
||||
}
|
||||
|
||||
waitForUpdateTaskStateIfNotArchivedCall(t, repo)
|
||||
|
||||
if got := repo.tasks["task-123"].State; got != v1.TaskStateWaitingForInput {
|
||||
t.Errorf("expected archive racing the CAS to leave state untouched, got %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunAgentProcessAsync_ResumeFailureDoesNotReviewWhileSiblingWorks(t *testing.T) {
|
||||
f := newRunAgentProcessAsyncFailureFixture(t)
|
||||
f.repo.sessions["session-456"] = &models.TaskSession{
|
||||
@@ -1459,6 +1668,30 @@ func TestWriteTaskReviewStateIfNoWorkingSessionsSkipsOfficeTask(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestWriteTaskReviewStateIfNoWorkingSessionsSkipsArchivedTask(t *testing.T) {
|
||||
repo := newMockRepository()
|
||||
archivedAt := time.Now().UTC()
|
||||
repo.tasks["task-123"] = &models.Task{
|
||||
ID: "task-123",
|
||||
ArchivedAt: &archivedAt,
|
||||
}
|
||||
repo.sessions["session-123"] = &models.TaskSession{
|
||||
ID: "session-123", TaskID: "task-123", State: models.TaskSessionStateFailed,
|
||||
}
|
||||
exec := newTestExecutor(t, &mockAgentManager{}, repo)
|
||||
var taskStateUpdates []v1.TaskState
|
||||
exec.SetOnTaskStateChange(func(ctx context.Context, taskID string, state v1.TaskState) error {
|
||||
taskStateUpdates = append(taskStateUpdates, state)
|
||||
return nil
|
||||
})
|
||||
|
||||
exec.writeTaskReviewStateIfNoWorkingSessions(context.Background(), "task-123", "session-123")
|
||||
|
||||
if len(taskStateUpdates) != 0 {
|
||||
t.Errorf("expected archived task to keep its frozen state, got %v", taskStateUpdates)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStartAgentProcessOnResumePromotesTaskAfterSuccess(t *testing.T) {
|
||||
repo := newMockRepository()
|
||||
session := &models.TaskSession{
|
||||
|
||||
@@ -42,7 +42,18 @@ func DefaultSchedulerConfig() SchedulerConfig {
|
||||
type TaskRepository interface {
|
||||
GetTask(ctx context.Context, taskID string) (*v1.Task, error)
|
||||
UpdateTaskState(ctx context.Context, taskID string, state v1.TaskState) error
|
||||
// UpdateTaskStateIfCurrentIn atomically transitions state only when the
|
||||
// task's current state is in allowed AND the task is not archived
|
||||
// (archived_at IS NULL, enforced inside the write's own transaction —
|
||||
// not just by a caller's earlier, non-transactional archived-state
|
||||
// check). Returns whether a row was modified.
|
||||
UpdateTaskStateIfCurrentIn(ctx context.Context, taskID string, state v1.TaskState, allowed []v1.TaskState) (bool, error)
|
||||
// UpdateTaskStateIfNotArchived is UpdateTaskStateIfCurrentIn without the
|
||||
// prior-state constraint — for writers (e.g. IN_PROGRESS runtime
|
||||
// reconciliation) that legitimately fire from many prior states and only
|
||||
// need the archived_at IS NULL guarantee. Returns whether a row was
|
||||
// modified.
|
||||
UpdateTaskStateIfNotArchived(ctx context.Context, taskID string, state v1.TaskState) (bool, error)
|
||||
}
|
||||
|
||||
// QueueStatus contains queue statistics
|
||||
|
||||
@@ -233,6 +233,19 @@ func (r *testTaskRepository) UpdateTaskStateIfCurrentIn(
|
||||
return false, nil
|
||||
}
|
||||
|
||||
func (r *testTaskRepository) UpdateTaskStateIfNotArchived(
|
||||
_ context.Context, taskID string, state v1.TaskState,
|
||||
) (bool, error) {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
task, exists := r.tasks[taskID]
|
||||
if !exists {
|
||||
return false, fmt.Errorf("%w: %s", taskrepo.ErrTaskNotFound, taskID)
|
||||
}
|
||||
task.State = state
|
||||
return true, nil
|
||||
}
|
||||
|
||||
func createTestLogger() *logger.Logger {
|
||||
log, _ := logger.NewLogger(logger.LoggingConfig{
|
||||
Level: "error", // Suppress logs during tests
|
||||
|
||||
@@ -128,6 +128,13 @@ type repoStore interface {
|
||||
sessionExecutorStore
|
||||
// Additional methods needed by executor
|
||||
UpdateTaskState(ctx context.Context, id string, state v1.TaskState) error
|
||||
// UpdateTaskStateIfCurrentIn — see executor.executorStore's doc comment;
|
||||
// required here because repo (this interface) is what NewService passes
|
||||
// to executor.NewExecutor.
|
||||
UpdateTaskStateIfCurrentIn(ctx context.Context, id string, state v1.TaskState, allowed []v1.TaskState) (v1.TaskState, bool, error)
|
||||
// UpdateTaskStateIfNotArchived — see executor.executorStore's doc comment;
|
||||
// required here for the same reason as UpdateTaskStateIfCurrentIn above.
|
||||
UpdateTaskStateIfNotArchived(ctx context.Context, id string, state v1.TaskState) (v1.TaskState, bool, error)
|
||||
GetPrimaryTaskRepository(ctx context.Context, taskID string) (*models.TaskRepository, error)
|
||||
ListTaskRepositories(ctx context.Context, taskID string) ([]*models.TaskRepository, error)
|
||||
CreateTaskSession(ctx context.Context, session *models.TaskSession) error
|
||||
@@ -541,10 +548,25 @@ func NewService(
|
||||
// Wire executor state changes through the orchestrator so events are published
|
||||
// (e.g. WebSocket notifications to the frontend). Must be set after service
|
||||
// construction so the session callback can reference s.updateTaskSessionState.
|
||||
// UpdateTaskStateIfNotArchived (not the unconditional UpdateTaskState) so
|
||||
// every runtime-driven task-state write the executor makes through this
|
||||
// single callback — IN_PROGRESS on agent start/resume, FAILED on launch
|
||||
// error — is atomic against archived_at. Each caller's own archived
|
||||
// guard (e.g. writeTaskInProgressForRuntime) reads the task before this
|
||||
// fires; ArchiveTask can commit in that window without an intervening
|
||||
// state write to invalidate the guard, so only this conditional write
|
||||
// closes the race (PR #1706 review).
|
||||
exec.SetOnTaskStateChange(func(ctx context.Context, taskID string, state v1.TaskState) error {
|
||||
if err := taskRepo.UpdateTaskState(ctx, taskID, state); err != nil {
|
||||
updated, err := taskRepo.UpdateTaskStateIfNotArchived(ctx, taskID, state)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !updated {
|
||||
s.logger.Debug("skipping runtime task state write for archived task",
|
||||
zap.String("task_id", taskID),
|
||||
zap.String("state", string(state)))
|
||||
return nil
|
||||
}
|
||||
s.processParentChildrenCompletedForTaskState(ctx, taskID, state)
|
||||
return nil
|
||||
})
|
||||
@@ -1311,9 +1333,14 @@ func (s *Service) reconcileOneSessionOnStartup(ctx context.Context, running *mod
|
||||
|
||||
// Ensure task is in REVIEW state (not stuck IN_PROGRESS)
|
||||
if running.TaskID != "" {
|
||||
task, taskErr := s.taskRepo.GetTask(ctx, running.TaskID)
|
||||
if taskErr == nil && task != nil && task.State == v1.TaskStateInProgress {
|
||||
if updateErr := s.taskRepo.UpdateTaskState(ctx, running.TaskID, v1.TaskStateReview); updateErr != nil {
|
||||
task, taskErr := s.repo.GetTask(ctx, running.TaskID)
|
||||
if taskErr == nil && task != nil && task.State == v1.TaskStateInProgress && !taskArchived(task) {
|
||||
// UpdateTaskStateIfCurrentIn (not the unconditional UpdateTaskState)
|
||||
// so the write is atomic against archived_at: the taskArchived guard
|
||||
// above reads the row before this call, and ArchiveTask can commit in
|
||||
// that window without changing task.State, so only an archive-aware
|
||||
// conditional write closes the race.
|
||||
if _, updateErr := s.taskRepo.UpdateTaskStateIfCurrentIn(ctx, running.TaskID, v1.TaskStateReview, []v1.TaskState{v1.TaskStateInProgress}); updateErr != nil {
|
||||
s.logger.Warn("failed to update task to REVIEW on startup",
|
||||
zap.String("task_id", running.TaskID),
|
||||
zap.Error(updateErr))
|
||||
@@ -1422,12 +1449,15 @@ func (s *Service) handleFailedSessionOnStartup(ctx context.Context, session *mod
|
||||
s.abandonOpenTurnsOnStartup(ctx, sessionID, "failed session cleanup")
|
||||
// If session failed, ensure task is in REVIEW state (not stuck IN_PROGRESS)
|
||||
if session.TaskID != "" {
|
||||
task, taskErr := s.taskRepo.GetTask(ctx, session.TaskID)
|
||||
if taskErr == nil && task != nil && task.State == v1.TaskStateInProgress {
|
||||
task, taskErr := s.repo.GetTask(ctx, session.TaskID)
|
||||
if taskErr == nil && task != nil && task.State == v1.TaskStateInProgress && !taskArchived(task) {
|
||||
s.logger.Info("fixing task state: session failed but task still IN_PROGRESS",
|
||||
zap.String("task_id", session.TaskID),
|
||||
zap.String("session_id", sessionID))
|
||||
if updateErr := s.taskRepo.UpdateTaskState(ctx, session.TaskID, v1.TaskStateReview); updateErr != nil {
|
||||
// UpdateTaskStateIfCurrentIn (not the unconditional UpdateTaskState)
|
||||
// so the write is atomic against archived_at — see the matching
|
||||
// comment in reconcileOneSessionOnStartup above.
|
||||
if _, updateErr := s.taskRepo.UpdateTaskStateIfCurrentIn(ctx, session.TaskID, v1.TaskStateReview, []v1.TaskState{v1.TaskStateInProgress}); updateErr != nil {
|
||||
s.logger.Warn("failed to update task state to REVIEW",
|
||||
zap.String("task_id", session.TaskID),
|
||||
zap.Error(updateErr))
|
||||
|
||||
@@ -2894,6 +2894,15 @@ func (c *ctxAwareTaskRepo) UpdateTaskStateIfCurrentIn(
|
||||
return c.inner.UpdateTaskStateIfCurrentIn(ctx, taskID, state, allowed)
|
||||
}
|
||||
|
||||
func (c *ctxAwareTaskRepo) UpdateTaskStateIfNotArchived(
|
||||
ctx context.Context, taskID string, state v1.TaskState,
|
||||
) (bool, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return false, err
|
||||
}
|
||||
return c.inner.UpdateTaskStateIfNotArchived(ctx, taskID, state)
|
||||
}
|
||||
|
||||
// TestResumeTaskSession_FailedStateWriteSurvivesCancelledCallerCtx verifies the
|
||||
// fix for the WS-disconnect cascade: when the caller's ctx was already
|
||||
// cancelled (e.g. the user navigated away mid-resume) and the launch then
|
||||
@@ -3594,6 +3603,131 @@ func TestReconcileSessionsOnStartup(t *testing.T) {
|
||||
if state != v1.TaskStateReview {
|
||||
t.Fatalf("expected task state %q, got %q", v1.TaskStateReview, state)
|
||||
}
|
||||
// The write must go through the archive-aware UpdateTaskStateIfCurrentIn
|
||||
// CAS, not the unconditional UpdateTaskState — see the comment on this
|
||||
// call site in reconcileOneSessionOnStartup. Otherwise an archive that
|
||||
// commits between the taskArchived guard read and this write could
|
||||
// still resurrect the task to REVIEW (PR #1706 review finding).
|
||||
if n := taskRepo.unconditionalWrites["task1"]; n != 0 {
|
||||
t.Fatalf("expected REVIEW write to use UpdateTaskStateIfCurrentIn, got %d unconditional UpdateTaskState call(s)", n)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("archived_task_active_session_state_preserved", func(t *testing.T) {
|
||||
repo := setupTestRepo(t)
|
||||
ctx := context.Background()
|
||||
now := time.Now().UTC()
|
||||
|
||||
seedTaskAndSession(t, repo, "task1", "session1", models.TaskSessionStateRunning)
|
||||
if err := repo.ArchiveTask(ctx, "task1"); err != nil {
|
||||
t.Fatalf("failed to archive task: %v", err)
|
||||
}
|
||||
|
||||
err := repo.UpsertExecutorRunning(ctx, &models.ExecutorRunning{
|
||||
ID: "er1",
|
||||
SessionID: "session1",
|
||||
TaskID: "task1",
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("failed to upsert executor running: %v", err)
|
||||
}
|
||||
|
||||
taskRepo := newMockTaskRepo()
|
||||
taskRepo.tasks["task1"] = &v1.Task{
|
||||
ID: "task1",
|
||||
State: v1.TaskStateInProgress,
|
||||
}
|
||||
|
||||
svc := createTestServiceWithAgent(repo, newMockStepGetter(), taskRepo, &mockAgentManager{})
|
||||
svc.reconcileSessionsOnStartup(ctx)
|
||||
|
||||
if state, ok := taskRepo.updatedStates["task1"]; ok {
|
||||
t.Fatalf("expected archived task state to be left untouched, got write to %q", state)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("archived_task_failed_session_state_preserved", func(t *testing.T) {
|
||||
repo := setupTestRepo(t)
|
||||
ctx := context.Background()
|
||||
now := time.Now().UTC()
|
||||
|
||||
seedTaskAndSession(t, repo, "task1", "session1", models.TaskSessionStateFailed)
|
||||
if err := repo.ArchiveTask(ctx, "task1"); err != nil {
|
||||
t.Fatalf("failed to archive task: %v", err)
|
||||
}
|
||||
|
||||
err := repo.UpsertExecutorRunning(ctx, &models.ExecutorRunning{
|
||||
ID: "er1",
|
||||
SessionID: "session1",
|
||||
TaskID: "task1",
|
||||
ResumeToken: "acp-session-archived",
|
||||
Resumable: true,
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("failed to upsert executor running: %v", err)
|
||||
}
|
||||
|
||||
taskRepo := newMockTaskRepo()
|
||||
taskRepo.tasks["task1"] = &v1.Task{
|
||||
ID: "task1",
|
||||
State: v1.TaskStateInProgress,
|
||||
}
|
||||
|
||||
svc := createTestServiceWithAgent(repo, newMockStepGetter(), taskRepo, &mockAgentManager{})
|
||||
svc.reconcileSessionsOnStartup(ctx)
|
||||
|
||||
if state, ok := taskRepo.updatedStates["task1"]; ok {
|
||||
t.Fatalf("expected archived task state to be left untouched, got write to %q", state)
|
||||
}
|
||||
})
|
||||
|
||||
// TestReconcileSessionsOnStartup/failed_session_moved_to_review covers the
|
||||
// non-archived REVIEW write in handleFailedSessionOnStartup — previously
|
||||
// untested (only the archived-guard branch above had coverage). Confirms
|
||||
// both the resulting state and that the write goes through the
|
||||
// archive-aware UpdateTaskStateIfCurrentIn CAS, not the unconditional
|
||||
// UpdateTaskState (PR #1706 review finding).
|
||||
t.Run("failed_session_moved_to_review", func(t *testing.T) {
|
||||
repo := setupTestRepo(t)
|
||||
ctx := context.Background()
|
||||
now := time.Now().UTC()
|
||||
|
||||
seedTaskAndSession(t, repo, "task1", "session1", models.TaskSessionStateFailed)
|
||||
|
||||
err := repo.UpsertExecutorRunning(ctx, &models.ExecutorRunning{
|
||||
ID: "er1",
|
||||
SessionID: "session1",
|
||||
TaskID: "task1",
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("failed to upsert executor running: %v", err)
|
||||
}
|
||||
|
||||
taskRepo := newMockTaskRepo()
|
||||
taskRepo.tasks["task1"] = &v1.Task{
|
||||
ID: "task1",
|
||||
State: v1.TaskStateInProgress,
|
||||
}
|
||||
|
||||
svc := createTestServiceWithAgent(repo, newMockStepGetter(), taskRepo, &mockAgentManager{})
|
||||
svc.reconcileSessionsOnStartup(ctx)
|
||||
|
||||
state, ok := taskRepo.updatedStates["task1"]
|
||||
if !ok {
|
||||
t.Fatal("expected task state to be updated")
|
||||
}
|
||||
if state != v1.TaskStateReview {
|
||||
t.Fatalf("expected task state %q, got %q", v1.TaskStateReview, state)
|
||||
}
|
||||
if n := taskRepo.unconditionalWrites["task1"]; n != 0 {
|
||||
t.Fatalf("expected REVIEW write to use UpdateTaskStateIfCurrentIn, got %d unconditional UpdateTaskState call(s)", n)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -98,6 +98,9 @@ func (m *mockRepository) UpdateTaskState(ctx context.Context, id string, state v
|
||||
func (m *mockRepository) UpdateTaskStateIfCurrentIn(_ context.Context, _ string, _ v1.TaskState, _ []v1.TaskState) (v1.TaskState, bool, error) {
|
||||
return "", false, nil
|
||||
}
|
||||
func (m *mockRepository) UpdateTaskStateIfNotArchived(_ context.Context, _ string, _ v1.TaskState) (v1.TaskState, bool, error) {
|
||||
return "", false, nil
|
||||
}
|
||||
func (m *mockRepository) AddTaskToWorkflow(ctx context.Context, taskID, workflowID, columnID string, position int) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -64,9 +64,20 @@ type TaskRepository interface {
|
||||
CountOpenWatcherCreatedTasks(ctx context.Context, metadataKey, watchID string) (int, error)
|
||||
UpdateTaskState(ctx context.Context, id string, state v1.TaskState) error
|
||||
// UpdateTaskStateIfCurrentIn atomically transitions state only when the
|
||||
// task's current state is in allowed. Returns the pre-update state and
|
||||
// task's current state is in allowed AND the task is not archived
|
||||
// (archived_at IS NULL). The archived check is enforced inside the same
|
||||
// UPDATE's WHERE clause, not just by a caller's earlier (non-transactional)
|
||||
// read, so a late write can never race an ArchiveTask commit that lands
|
||||
// between that read and this call. Returns the pre-update state and
|
||||
// whether a row was modified.
|
||||
UpdateTaskStateIfCurrentIn(ctx context.Context, id string, state v1.TaskState, allowed []v1.TaskState) (v1.TaskState, bool, error)
|
||||
// UpdateTaskStateIfNotArchived is UpdateTaskStateIfCurrentIn without the
|
||||
// prior-state constraint — for writers (e.g. IN_PROGRESS reconciliation)
|
||||
// that legitimately fire from many prior states and only need the
|
||||
// archived_at IS NULL guarantee. Same TOCTOU-closing semantics: the
|
||||
// archived check is atomic with the write. Returns the pre-update state
|
||||
// and whether a row was modified.
|
||||
UpdateTaskStateIfNotArchived(ctx context.Context, id string, state v1.TaskState) (v1.TaskState, bool, error)
|
||||
CountTasksByWorkflow(ctx context.Context, workflowID string) (int, error)
|
||||
CountTasksByWorkflowStep(ctx context.Context, stepID string) (int, error)
|
||||
AddTaskToWorkflow(ctx context.Context, taskID, workflowID, workflowStepID string, position int) error
|
||||
|
||||
@@ -10,6 +10,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/jackc/pgx/v5/pgconn"
|
||||
|
||||
"github.com/kandev/kandev/internal/agentctl/tracing"
|
||||
"github.com/kandev/kandev/internal/db/dialect"
|
||||
@@ -1166,7 +1167,11 @@ func (r *Repository) UpdateTaskState(ctx context.Context, id string, state v1.Ta
|
||||
|
||||
// UpdateTaskStateIfCurrentIn transitions state inside a transaction, re-checking
|
||||
// the current state on write so concurrent handlers cannot clobber a task that
|
||||
// moved out of allowed between read and update.
|
||||
// moved out of allowed between read and update. The write is also scoped to
|
||||
// archived_at IS NULL: if ArchiveTask commits between the caller's earlier
|
||||
// archived-state guard and this call, the archived_at check in the UPDATE's
|
||||
// WHERE clause (not just the state check) makes the no-op atomic, so a late
|
||||
// runtime write can never resurrect an archived task's state.
|
||||
func (r *Repository) UpdateTaskStateIfCurrentIn(
|
||||
ctx context.Context, id string, state v1.TaskState, allowed []v1.TaskState,
|
||||
) (v1.TaskState, bool, error) {
|
||||
@@ -1194,7 +1199,7 @@ func (r *Repository) UpdateTaskStateIfCurrentIn(
|
||||
|
||||
result, err := tx.ExecContext(ctx, r.db.Rebind(`
|
||||
UPDATE tasks SET state = ?, updated_at = ?
|
||||
WHERE id = ? AND state = ?
|
||||
WHERE id = ? AND state = ? AND archived_at IS NULL
|
||||
`), state, time.Now().UTC(), id, currentState)
|
||||
if err != nil {
|
||||
return "", false, err
|
||||
@@ -1209,6 +1214,146 @@ func (r *Repository) UpdateTaskStateIfCurrentIn(
|
||||
return currentState, true, nil
|
||||
}
|
||||
|
||||
// updateTaskStateIfNotArchivedMaxAttempts bounds the optimistic-retry loop in
|
||||
// UpdateTaskStateIfNotArchived. A retry only fires when another writer
|
||||
// changed tasks.state (not archived_at) in the gap between this method's
|
||||
// read and its write; real contention on a single task's state column is
|
||||
// rare enough that a handful of attempts is generous headroom, not a
|
||||
// meaningful latency risk. updateTaskStateIfNotArchivedBackoff is a fixed
|
||||
// delay between attempts, giving the concurrent writer time to finish
|
||||
// (commit or roll back) before the next attempt re-reads — without it, a
|
||||
// tight spin-retry can exhaust every attempt in well under a millisecond,
|
||||
// faster than any real concurrent write completes.
|
||||
const (
|
||||
updateTaskStateIfNotArchivedMaxAttempts = 8
|
||||
updateTaskStateIfNotArchivedBackoff = 15 * time.Millisecond
|
||||
)
|
||||
|
||||
// UpdateTaskStateIfNotArchived atomically transitions state unless the task
|
||||
// is archived — no prior-state constraint, unlike UpdateTaskStateIfCurrentIn.
|
||||
// IN_PROGRESS writes (unlike the REVIEW CAS) are legitimately reachable from
|
||||
// many prior states, so there is no "allowed" set to check; the only
|
||||
// invariant that must hold atomically is archived_at IS NULL. Closes the
|
||||
// same TOCTOU window as UpdateTaskStateIfCurrentIn: if ArchiveTask commits
|
||||
// between a caller's earlier archived-state guard and this call, the
|
||||
// archived_at check inside the transaction (not just the caller's read)
|
||||
// makes the no-op atomic. Returns the pre-update state and whether a row
|
||||
// was modified.
|
||||
//
|
||||
// The UPDATE's WHERE clause also pins state = currentState (the value read
|
||||
// moments earlier in the same transaction), so the returned "pre-update
|
||||
// state" is never stale: without that pin, a concurrent state write landing
|
||||
// between the SELECT and the UPDATE would still match archived_at IS NULL
|
||||
// alone, silently clobbering the newer state while this method kept
|
||||
// reporting the old (now wrong) value as the pre-update state — which the
|
||||
// service layer publishes as the task.state_changed event's old_state
|
||||
// (CodeRabbit review on PR #1706). Losing the race on state (rows==0 while
|
||||
// the row is still unarchived) means state moved, not that the row is gone
|
||||
// or archived, so this is optimistic-concurrency-retried rather than
|
||||
// treated as a no-op — an IN_PROGRESS write has no prior-state restriction
|
||||
// to honor, so it always still applies once the retry re-reads a fresh
|
||||
// state.
|
||||
func (r *Repository) UpdateTaskStateIfNotArchived(
|
||||
ctx context.Context, id string, state v1.TaskState,
|
||||
) (v1.TaskState, bool, error) {
|
||||
for attempt := range updateTaskStateIfNotArchivedMaxAttempts {
|
||||
if attempt > 0 {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return "", false, ctx.Err()
|
||||
case <-time.After(updateTaskStateIfNotArchivedBackoff):
|
||||
}
|
||||
}
|
||||
currentState, updated, retry, err := r.tryUpdateTaskStateIfNotArchived(ctx, id, state)
|
||||
if err != nil || !retry {
|
||||
return currentState, updated, err
|
||||
}
|
||||
}
|
||||
return "", false, fmt.Errorf("update task state if not archived: exceeded %d attempts for task %s",
|
||||
updateTaskStateIfNotArchivedMaxAttempts, id)
|
||||
}
|
||||
|
||||
// tryUpdateTaskStateIfNotArchived is one attempt of the optimistic-retry loop
|
||||
// above. retry=true means the state changed concurrently while the row
|
||||
// stayed unarchived — the caller should re-read and try again.
|
||||
func (r *Repository) tryUpdateTaskStateIfNotArchived(
|
||||
ctx context.Context, id string, state v1.TaskState,
|
||||
) (currentState v1.TaskState, updated, retry bool, err error) {
|
||||
tx, err := r.db.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return "", false, false, err
|
||||
}
|
||||
defer func() { _ = tx.Rollback() }()
|
||||
|
||||
var archivedAt sql.NullTime
|
||||
err = tx.QueryRowContext(ctx, r.db.Rebind(`SELECT state, archived_at FROM tasks WHERE id = ?`), id).
|
||||
Scan(¤tState, &archivedAt)
|
||||
if err != nil {
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return "", false, false, fmt.Errorf("%w: %s", ErrTaskNotFound, id)
|
||||
}
|
||||
return "", false, false, err
|
||||
}
|
||||
if archivedAt.Valid {
|
||||
return currentState, false, false, nil
|
||||
}
|
||||
|
||||
result, err := tx.ExecContext(ctx, r.db.Rebind(`
|
||||
UPDATE tasks SET state = ?, updated_at = ?
|
||||
WHERE id = ? AND state = ? AND archived_at IS NULL
|
||||
`), state, time.Now().UTC(), id, currentState)
|
||||
if err != nil {
|
||||
if isRetryableStateRaceError(err) {
|
||||
// SQLite (under WAL) can refuse to upgrade this transaction's
|
||||
// already-established read snapshot to a writer once another
|
||||
// connection committed a conflicting write in between — it
|
||||
// returns SQLITE_BUSY ("database is locked") on the UPDATE
|
||||
// itself rather than a clean 0-rows-affected. Same underlying
|
||||
// condition as the rows==0 branch below: retry with a fresh
|
||||
// transaction/snapshot instead of surfacing a transient error.
|
||||
return currentState, false, true, nil
|
||||
}
|
||||
return "", false, false, err
|
||||
}
|
||||
rows, _ := result.RowsAffected()
|
||||
if rows == 0 {
|
||||
// archived_at IS NULL was true moments ago (checked above, inside
|
||||
// this same transaction) — a concurrent writer changed state, not
|
||||
// archived_at. Retry with a fresh read rather than reporting a
|
||||
// stale old-state/no-op.
|
||||
return currentState, false, true, nil
|
||||
}
|
||||
if err := tx.Commit(); err != nil {
|
||||
return "", false, false, err
|
||||
}
|
||||
return currentState, true, false, nil
|
||||
}
|
||||
|
||||
// isRetryableStateRaceError reports whether err is a database-level
|
||||
// contention error rather than a genuine failure: SQLite's WAL snapshot
|
||||
// conflict (a write following a read in the same transaction, after
|
||||
// another connection committed a conflicting write — surfaces as
|
||||
// SQLITE_BUSY/"database is locked" on the write itself, not a clean
|
||||
// 0-rows-affected), or a Postgres serialization/deadlock/lock-timeout
|
||||
// error. UpdateTaskStateIfNotArchived's retry loop treats this the same as
|
||||
// rows==0: the underlying condition (another writer changed the row) is
|
||||
// exactly the one it already retries for.
|
||||
func isRetryableStateRaceError(err error) bool {
|
||||
if err == nil {
|
||||
return false
|
||||
}
|
||||
var pgErr *pgconn.PgError
|
||||
if errors.As(err, &pgErr) {
|
||||
switch pgErr.Code {
|
||||
case "40001", "40P01", "55P03": // serialization_failure, deadlock_detected, lock_not_available
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
s := err.Error()
|
||||
return strings.Contains(s, "database is locked") || strings.Contains(s, "database table is locked")
|
||||
}
|
||||
|
||||
func taskStateInSet(state v1.TaskState, allowed []v1.TaskState) bool {
|
||||
for _, candidate := range allowed {
|
||||
if state == candidate {
|
||||
|
||||
@@ -0,0 +1,314 @@
|
||||
package sqlite
|
||||
|
||||
import (
|
||||
"context"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/jmoiron/sqlx"
|
||||
|
||||
"github.com/kandev/kandev/internal/db"
|
||||
v1 "github.com/kandev/kandev/pkg/api/v1"
|
||||
)
|
||||
|
||||
// TestUpdateTaskStateIfCurrentIn_SkipsArchivedTask reproduces the TOCTOU race
|
||||
// flagged in PR #1706 review: a caller's earlier archived-state guard (a
|
||||
// plain GetTask read) can observe archived_at == NULL, then have ArchiveTask
|
||||
// commit before the caller's CAS write lands. Because state is untouched by
|
||||
// ArchiveTask, the old `WHERE id = ? AND state = ?` clause alone would still
|
||||
// match and resurrect the task to REVIEW. The archived_at IS NULL clause
|
||||
// added to the UPDATE closes that window: the write becomes a no-op even
|
||||
// though currentState (read moments earlier, inside the same transaction)
|
||||
// was still in the allowed set.
|
||||
func TestUpdateTaskStateIfCurrentIn_SkipsArchivedTask(t *testing.T) {
|
||||
repo := newRepoForHealTests(t)
|
||||
ctx := context.Background()
|
||||
insertTask(t, repo.db, "task-archived-race")
|
||||
|
||||
if err := repo.UpdateTaskState(ctx, "task-archived-race", v1.TaskStateInProgress); err != nil {
|
||||
t.Fatalf("seed IN_PROGRESS: %v", err)
|
||||
}
|
||||
|
||||
// Simulates the archive committing in the race window between the
|
||||
// caller's taskArchived() guard read and this CAS call.
|
||||
if err := repo.ArchiveTask(ctx, "task-archived-race"); err != nil {
|
||||
t.Fatalf("archive task: %v", err)
|
||||
}
|
||||
|
||||
gotState, updated, err := repo.UpdateTaskStateIfCurrentIn(
|
||||
ctx, "task-archived-race", v1.TaskStateReview,
|
||||
[]v1.TaskState{v1.TaskStateInProgress, v1.TaskStateScheduling},
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("UpdateTaskStateIfCurrentIn: %v", err)
|
||||
}
|
||||
if updated {
|
||||
t.Fatal("expected archived task's state to be left untouched, got updated=true")
|
||||
}
|
||||
if gotState != v1.TaskStateInProgress {
|
||||
t.Errorf("returned currentState = %q, want %q (pre-CAS read)", gotState, v1.TaskStateInProgress)
|
||||
}
|
||||
|
||||
task, err := repo.GetTask(ctx, "task-archived-race")
|
||||
if err != nil {
|
||||
t.Fatalf("get task: %v", err)
|
||||
}
|
||||
if task.State != v1.TaskStateInProgress {
|
||||
t.Errorf("persisted state = %q, want %q (archived task must not resurrect to REVIEW)", task.State, v1.TaskStateInProgress)
|
||||
}
|
||||
if task.ArchivedAt == nil {
|
||||
t.Error("expected task to remain archived")
|
||||
}
|
||||
}
|
||||
|
||||
// TestUpdateTaskStateIfCurrentIn_UpdatesWhenNotArchived is the CAS positive
|
||||
// path sanity check: a non-archived task whose state is in the allowed set
|
||||
// still transitions normally. Guards against a too-broad archived_at fix
|
||||
// (e.g. accidentally scoping the WHERE clause to always require archived_at
|
||||
// IS NULL on every row, including ones with no archive concept at all) that
|
||||
// would silently break every ordinary REVIEW/IN_PROGRESS transition.
|
||||
func TestUpdateTaskStateIfCurrentIn_UpdatesWhenNotArchived(t *testing.T) {
|
||||
repo := newRepoForHealTests(t)
|
||||
ctx := context.Background()
|
||||
insertTask(t, repo.db, "task-normal")
|
||||
|
||||
if err := repo.UpdateTaskState(ctx, "task-normal", v1.TaskStateInProgress); err != nil {
|
||||
t.Fatalf("seed IN_PROGRESS: %v", err)
|
||||
}
|
||||
|
||||
gotState, updated, err := repo.UpdateTaskStateIfCurrentIn(
|
||||
ctx, "task-normal", v1.TaskStateReview,
|
||||
[]v1.TaskState{v1.TaskStateInProgress, v1.TaskStateScheduling},
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("UpdateTaskStateIfCurrentIn: %v", err)
|
||||
}
|
||||
if !updated {
|
||||
t.Fatal("expected non-archived task in the allowed set to transition, got updated=false")
|
||||
}
|
||||
if gotState != v1.TaskStateInProgress {
|
||||
t.Errorf("returned currentState = %q, want %q", gotState, v1.TaskStateInProgress)
|
||||
}
|
||||
|
||||
task, err := repo.GetTask(ctx, "task-normal")
|
||||
if err != nil {
|
||||
t.Fatalf("get task: %v", err)
|
||||
}
|
||||
if task.State != v1.TaskStateReview {
|
||||
t.Errorf("persisted state = %q, want %q", task.State, v1.TaskStateReview)
|
||||
}
|
||||
}
|
||||
|
||||
// TestUpdateTaskStateIfNotArchived_SkipsArchivedTask is the
|
||||
// UpdateTaskStateIfNotArchived analog of TestUpdateTaskStateIfCurrentIn_SkipsArchivedTask
|
||||
// for the IN_PROGRESS-reconciliation writers (Service/Executor
|
||||
// writeTaskInProgressForRuntime, review comment on PR #1706): those callers
|
||||
// have no "allowed" prior-state set to check, only an archived guard, so
|
||||
// they route through this CAS instead. Same race: ArchiveTask commits
|
||||
// between the caller's earlier archived-state read and this call; the
|
||||
// archived_at IS NULL clause inside the UPDATE must still make it a no-op.
|
||||
func TestUpdateTaskStateIfNotArchived_SkipsArchivedTask(t *testing.T) {
|
||||
repo := newRepoForHealTests(t)
|
||||
ctx := context.Background()
|
||||
insertTask(t, repo.db, "task-archived-race-2")
|
||||
|
||||
if err := repo.UpdateTaskState(ctx, "task-archived-race-2", v1.TaskStateWaitingForInput); err != nil {
|
||||
t.Fatalf("seed WAITING_FOR_INPUT: %v", err)
|
||||
}
|
||||
|
||||
// Simulates the archive committing in the race window between the
|
||||
// caller's taskArchived() guard read and this CAS call.
|
||||
if err := repo.ArchiveTask(ctx, "task-archived-race-2"); err != nil {
|
||||
t.Fatalf("archive task: %v", err)
|
||||
}
|
||||
|
||||
gotState, updated, err := repo.UpdateTaskStateIfNotArchived(ctx, "task-archived-race-2", v1.TaskStateInProgress)
|
||||
if err != nil {
|
||||
t.Fatalf("UpdateTaskStateIfNotArchived: %v", err)
|
||||
}
|
||||
if updated {
|
||||
t.Fatal("expected archived task's state to be left untouched, got updated=true")
|
||||
}
|
||||
if gotState != v1.TaskStateWaitingForInput {
|
||||
t.Errorf("returned currentState = %q, want %q (pre-CAS read)", gotState, v1.TaskStateWaitingForInput)
|
||||
}
|
||||
|
||||
task, err := repo.GetTask(ctx, "task-archived-race-2")
|
||||
if err != nil {
|
||||
t.Fatalf("get task: %v", err)
|
||||
}
|
||||
if task.State != v1.TaskStateWaitingForInput {
|
||||
t.Errorf("persisted state = %q, want %q (archived task must not resurrect to IN_PROGRESS)", task.State, v1.TaskStateWaitingForInput)
|
||||
}
|
||||
if task.ArchivedAt == nil {
|
||||
t.Error("expected task to remain archived")
|
||||
}
|
||||
}
|
||||
|
||||
// TestUpdateTaskStateIfNotArchived_UpdatesWhenNotArchived is the CAS
|
||||
// positive-path sanity check: a non-archived task transitions normally
|
||||
// regardless of its prior state, since UpdateTaskStateIfNotArchived has no
|
||||
// "allowed" constraint.
|
||||
func TestUpdateTaskStateIfNotArchived_UpdatesWhenNotArchived(t *testing.T) {
|
||||
repo := newRepoForHealTests(t)
|
||||
ctx := context.Background()
|
||||
insertTask(t, repo.db, "task-normal-2")
|
||||
|
||||
if err := repo.UpdateTaskState(ctx, "task-normal-2", v1.TaskStateWaitingForInput); err != nil {
|
||||
t.Fatalf("seed WAITING_FOR_INPUT: %v", err)
|
||||
}
|
||||
|
||||
gotState, updated, err := repo.UpdateTaskStateIfNotArchived(ctx, "task-normal-2", v1.TaskStateInProgress)
|
||||
if err != nil {
|
||||
t.Fatalf("UpdateTaskStateIfNotArchived: %v", err)
|
||||
}
|
||||
if !updated {
|
||||
t.Fatal("expected non-archived task to transition, got updated=false")
|
||||
}
|
||||
if gotState != v1.TaskStateWaitingForInput {
|
||||
t.Errorf("returned currentState = %q, want %q", gotState, v1.TaskStateWaitingForInput)
|
||||
}
|
||||
|
||||
task, err := repo.GetTask(ctx, "task-normal-2")
|
||||
if err != nil {
|
||||
t.Fatalf("get task: %v", err)
|
||||
}
|
||||
if task.State != v1.TaskStateInProgress {
|
||||
t.Errorf("persisted state = %q, want %q", task.State, v1.TaskStateInProgress)
|
||||
}
|
||||
}
|
||||
|
||||
// TestUpdateTaskStateIfNotArchived_RetriesOnConcurrentStateChange reproduces
|
||||
// the atomicity gap CodeRabbit flagged on PR #1706: the original
|
||||
// implementation read currentState, then updated any row with
|
||||
// archived_at IS NULL — with no predicate pinning the row to the state it
|
||||
// just read. A concurrent writer that changes state (not archived_at)
|
||||
// between that SELECT and this method's UPDATE would still match, silently
|
||||
// overwriting the newer state while this method kept reporting the stale
|
||||
// first-read value as the "pre-update state" — which the service layer
|
||||
// publishes as the task.state_changed event's old_state.
|
||||
//
|
||||
// Reproduces the actual race (not just sequential ordering) using two
|
||||
// independent SQLite connections to the same database file under WAL mode.
|
||||
// A single shared *Repository can't exercise this: db.OpenSQLite caps its
|
||||
// pool to one Go-level connection (single-writer-connection app design), so
|
||||
// two calls through the SAME repo would be fully serialized by
|
||||
// database/sql's connection pool before either reached SQLite's own
|
||||
// locking — never opening the race window. A second, independent
|
||||
// connection (as a genuinely concurrent writer — another connection or
|
||||
// process — would be) models that correctly: the racer opens its own write
|
||||
// transaction, changes the task's state, and holds it open (uncommitted).
|
||||
// WAL readers don't block on an uncommitted writer, so the main call's
|
||||
// SELECT (via the first connection) observes the pre-racer state and
|
||||
// proceeds to its own UPDATE, which blocks on SQLite's single-writer lock
|
||||
// until the racer commits. Once it does, the UPDATE's WHERE state = <stale
|
||||
// value> matches zero rows — this is the exact window the fix's
|
||||
// optimistic-retry loop exists to close: it must re-read and retry rather
|
||||
// than reporting the stale value or silently clobbering the racer's write.
|
||||
func TestUpdateTaskStateIfNotArchived_RetriesOnConcurrentStateChange(t *testing.T) {
|
||||
tmpDir := t.TempDir()
|
||||
dbPath := filepath.Join(tmpDir, "test.db")
|
||||
|
||||
mainConn, err := db.OpenSQLite(dbPath)
|
||||
if err != nil {
|
||||
t.Fatalf("open main connection: %v", err)
|
||||
}
|
||||
mainDB := sqlx.NewDb(mainConn, "sqlite3")
|
||||
t.Cleanup(func() { _ = mainDB.Close() })
|
||||
repo, err := NewWithDB(mainDB, mainDB, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("new repo: %v", err)
|
||||
}
|
||||
|
||||
// racerConn is a second, independent connection to the same file —
|
||||
// standing in for a genuinely concurrent writer (another connection or
|
||||
// process), which is what the atomicity gap this test targets actually
|
||||
// requires.
|
||||
racerConn, err := db.OpenSQLite(dbPath)
|
||||
if err != nil {
|
||||
t.Fatalf("open racer connection: %v", err)
|
||||
}
|
||||
racerDB := sqlx.NewDb(racerConn, "sqlite3")
|
||||
t.Cleanup(func() { _ = racerDB.Close() })
|
||||
|
||||
ctx := context.Background()
|
||||
insertTask(t, repo.db, "task-state-race")
|
||||
if err := repo.UpdateTaskState(ctx, "task-state-race", v1.TaskStateWaitingForInput); err != nil {
|
||||
t.Fatalf("seed WAITING_FOR_INPUT: %v", err)
|
||||
}
|
||||
|
||||
racerWrote := make(chan struct{})
|
||||
releaseRacer := make(chan struct{})
|
||||
racerDone := make(chan error, 1)
|
||||
go func() {
|
||||
tx, err := racerDB.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
racerDone <- err
|
||||
return
|
||||
}
|
||||
if _, err := tx.ExecContext(ctx, racerDB.Rebind(
|
||||
`UPDATE tasks SET state = ? WHERE id = ?`,
|
||||
), v1.TaskStateBlocked, "task-state-race"); err != nil {
|
||||
_ = tx.Rollback()
|
||||
racerDone <- err
|
||||
return
|
||||
}
|
||||
close(racerWrote)
|
||||
<-releaseRacer
|
||||
racerDone <- tx.Commit()
|
||||
}()
|
||||
|
||||
select {
|
||||
case <-racerWrote:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("timed out waiting for racer to write")
|
||||
}
|
||||
|
||||
type callResult struct {
|
||||
state v1.TaskState
|
||||
updated bool
|
||||
err error
|
||||
}
|
||||
done := make(chan callResult, 1)
|
||||
go func() {
|
||||
state, updated, err := repo.UpdateTaskStateIfNotArchived(ctx, "task-state-race", v1.TaskStateInProgress)
|
||||
done <- callResult{state, updated, err}
|
||||
}()
|
||||
|
||||
// Give the main call's UPDATE time to reach SQLite's write-lock queue
|
||||
// (it blocks behind the racer's open transaction) before releasing the
|
||||
// racer — otherwise the racer could commit before the main call even
|
||||
// starts, and the race window this test targets would never open.
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
close(releaseRacer)
|
||||
|
||||
if err := <-racerDone; err != nil {
|
||||
t.Fatalf("racer transaction failed: %v", err)
|
||||
}
|
||||
|
||||
var result callResult
|
||||
select {
|
||||
case result = <-done:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("timed out waiting for UpdateTaskStateIfNotArchived")
|
||||
}
|
||||
if result.err != nil {
|
||||
t.Fatalf("UpdateTaskStateIfNotArchived: %v", result.err)
|
||||
}
|
||||
if !result.updated {
|
||||
t.Fatal("expected the retry to succeed once the racer's write committed, got updated=false")
|
||||
}
|
||||
if result.state != v1.TaskStateBlocked {
|
||||
t.Errorf("returned pre-update state = %q, want %q (the racer's committed value, not the stale first read)",
|
||||
result.state, v1.TaskStateBlocked)
|
||||
}
|
||||
|
||||
task, err := repo.GetTask(ctx, "task-state-race")
|
||||
if err != nil {
|
||||
t.Fatalf("get task: %v", err)
|
||||
}
|
||||
if task.State != v1.TaskStateInProgress {
|
||||
t.Errorf("persisted state = %q, want %q", task.State, v1.TaskStateInProgress)
|
||||
}
|
||||
}
|
||||
@@ -399,6 +399,10 @@ func (r *phase4TaskRepo) UpdateTaskStateIfCurrentIn(context.Context, string, v1.
|
||||
r.panicNotUsed("UpdateTaskStateIfCurrentIn")
|
||||
return "", false, nil
|
||||
}
|
||||
func (r *phase4TaskRepo) UpdateTaskStateIfNotArchived(context.Context, string, v1.TaskState) (v1.TaskState, bool, error) {
|
||||
r.panicNotUsed("UpdateTaskStateIfNotArchived")
|
||||
return "", false, nil
|
||||
}
|
||||
func (r *phase4TaskRepo) CountTasksByWorkflow(context.Context, string) (int, error) {
|
||||
r.panicNotUsed("CountTasksByWorkflow")
|
||||
return 0, nil
|
||||
|
||||
@@ -221,6 +221,44 @@ func (s *Service) UpdateTaskStateIfCurrentIn(
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// UpdateTaskStateIfNotArchived is UpdateTaskStateIfCurrentIn without the
|
||||
// prior-state constraint — for writers (IN_PROGRESS runtime reconciliation)
|
||||
// that legitimately fire from many prior states and only need the
|
||||
// archived-task freeze guarantee. Publishes task.state_changed only when a
|
||||
// row changes.
|
||||
func (s *Service) UpdateTaskStateIfNotArchived(
|
||||
ctx context.Context, id string, state v1.TaskState,
|
||||
) (bool, error) {
|
||||
oldState, updated, err := s.tasks.UpdateTaskStateIfNotArchived(ctx, id, state)
|
||||
if err != nil || !updated {
|
||||
return false, err
|
||||
}
|
||||
if oldState == state {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
task, err := s.tasks.GetTask(ctx, id)
|
||||
if err != nil {
|
||||
return true, err
|
||||
}
|
||||
// The CAS wrote `state`; pin it on the payload so a concurrent transition
|
||||
// between commit and read cannot publish a mismatched new_state.
|
||||
task.State = state
|
||||
|
||||
s.logger.Info("task state updated",
|
||||
zap.String("task_id", id),
|
||||
zap.String("workflow_step_id", task.WorkflowStepID),
|
||||
zap.String("state", string(state)))
|
||||
|
||||
s.publishTaskEvent(ctx, events.TaskStateChanged, task, &oldState)
|
||||
s.logger.Info("task state changed",
|
||||
zap.String("task_id", id),
|
||||
zap.String("old_state", string(oldState)),
|
||||
zap.String("new_state", string(state)))
|
||||
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// UpdateTaskMetadata updates only the metadata of a task (merges with existing)
|
||||
func (s *Service) UpdateTaskMetadata(ctx context.Context, id string, metadata map[string]interface{}) (*models.Task, error) {
|
||||
task, err := s.tasks.GetTask(ctx, id)
|
||||
|
||||
@@ -0,0 +1,80 @@
|
||||
import { test, expect } from "../../fixtures/test-base";
|
||||
|
||||
// Regression: archiving a task must freeze its runtime `state`. Several
|
||||
// backend reconcile paths (turn completion, turn cancel, failed-start
|
||||
// fallback, and startup/crash reconciliation) used to write tasks.state to
|
||||
// REVIEW whenever the *session* looked idle/failed, without checking
|
||||
// whether the *task* had since been archived. Once a task was archived
|
||||
// while its session was still active, any of those paths — including a
|
||||
// plain agent.cancel — silently resurrected the archived task's state,
|
||||
// which is the "archived task comes back with the wrong state after a
|
||||
// crash/restart" bug. This test reproduces the live (non-restart) trigger
|
||||
// deterministically: archive a task mid-turn, then cancel the turn via the
|
||||
// same WS action the chat toolbar's Cancel button sends.
|
||||
test.describe("Archiving a task freezes its runtime state", () => {
|
||||
test("agent.cancel after archive does not resurrect task state to REVIEW", async ({
|
||||
testPage,
|
||||
apiClient,
|
||||
seedData,
|
||||
}) => {
|
||||
test.setTimeout(60_000);
|
||||
|
||||
const taskTitle = "Archive Freezes State";
|
||||
const created = await apiClient.createTaskWithAgent(
|
||||
seedData.workspaceId,
|
||||
taskTitle,
|
||||
seedData.agentProfileId,
|
||||
{
|
||||
description: 'e2e:delay(15000)\ne2e:message("still going")',
|
||||
workflow_id: seedData.workflowId,
|
||||
workflow_step_id: seedData.startStepId,
|
||||
repository_ids: [seedData.repositoryId],
|
||||
},
|
||||
);
|
||||
const taskId = created.id;
|
||||
const sessionId = created.session_id;
|
||||
expect(sessionId, "createTaskWithAgent should return a session_id").toBeTruthy();
|
||||
|
||||
// Wait for the agent to actually be mid-turn (task promoted to
|
||||
// IN_PROGRESS) before archiving — this is the state we assert stays
|
||||
// frozen for the rest of the test.
|
||||
await expect
|
||||
.poll(async () => (await apiClient.getTask(taskId)).state, {
|
||||
timeout: 20_000,
|
||||
message: "waiting for task to reach IN_PROGRESS before archiving",
|
||||
})
|
||||
.toBe("IN_PROGRESS");
|
||||
|
||||
// Archive while the agent turn is still in flight (15s delay keeps the
|
||||
// mock agent from finishing on its own during this test).
|
||||
await apiClient.archiveTask(taskId);
|
||||
|
||||
// Cancel the now-archived task's turn — the same `agent.cancel` WS
|
||||
// action the chat toolbar's Cancel button sends. Service.CancelAgent
|
||||
// tolerates a missing/already-stopped execution and still reconciles
|
||||
// DB state, which is exactly the path that used to write REVIEW onto
|
||||
// an archived task unconditionally.
|
||||
await apiClient.wsRequest("agent.cancel", { session_id: sessionId });
|
||||
|
||||
const taskAfterCancel = await apiClient.getTask(taskId);
|
||||
expect(taskAfterCancel.state).toBe("IN_PROGRESS");
|
||||
|
||||
// User-facing check: the Tasks list (with archived tasks shown) must
|
||||
// reflect the same frozen state — the card must not have been
|
||||
// silently moved into a "needs review" bucket.
|
||||
await testPage.goto("/tasks");
|
||||
await testPage.waitForLoadState("networkidle");
|
||||
await testPage.getByRole("checkbox", { name: "Show archived" }).click();
|
||||
|
||||
const row = testPage
|
||||
.getByTestId("tasks-list-row")
|
||||
.filter({ has: testPage.getByTestId("tasks-list-row-title").getByText(taskTitle) });
|
||||
await expect(row).toBeVisible({ timeout: 15_000 });
|
||||
await expect(row.getByText("Archived")).toBeVisible();
|
||||
|
||||
// Re-check via API right after the UI observed the row, closing out
|
||||
// on the precise contract: still IN_PROGRESS, never REVIEW.
|
||||
const finalTask = await apiClient.getTask(taskId);
|
||||
expect(finalTask.state).toBe("IN_PROGRESS");
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user